Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Try updating tests to zarr-python 3 #177

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Try updating tests to zarr-python 3 #177

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion test/CondaPkg.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[deps]
zarr = ">=2.13,<3"
zarr = "3.*"
python = ">=3.7,<4"
numcodecs = "0.15.*"
79 changes: 48 additions & 31 deletions test/python.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,15 @@ numeric_dtypes = (UInt8, UInt16, UInt32, UInt64,
Float16, Float32, Float64,
Complex{Float32}, Complex{Float64},
Bool,)
dtypes = numeric_dtypes
dtypes = (numeric_dtypes...,
MaxLengthString{10,UInt8},MaxLengthString{10,UInt32},
String)
dtypesp = ("uint8","uint16","uint32","uint64",
"int8","int16","int32","int64",
"float16","float32","float64",
"complex64", "complex128","bool", "S10","U10", "O")

compressors = (
"no"=>NoCompressor(),
"blosc"=>BloscCompressor(cname="zstd"),
Expand Down Expand Up @@ -81,18 +87,18 @@ end

# Test reading in python
for julia_path in (pjulia, pjulia*".zip")
g = zarr.open_group(julia_path)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This change can be reverted once a bug in zarr-python is fixed. zarr-developers/zarr-python#2856

g = if endswith(julia_path, ".zip")
zarr.open_group(zarr.storage.ZipStore(julia_path); mode="r", zarr_format=2)
else
zarr.open_group(julia_path; mode="r", zarr_format=2)
end
gatts = pyconvert(Any, g.attrs)

#Test group attributes
@test gatts["String attribute"] == "One"
@test gatts["Int attribute"] == 5
@test gatts["Float attribute"] == 10.5

dtypesp = ("uint8","uint16","uint32","uint64",
"int8","int16","int32","int64",
"float16","float32","float64",
"complex64", "complex128","bool","S10","U10", "O")

#Test accessing arrays from python and reading data
for i=1:length(dtypes), co in compressors
Expand All @@ -108,9 +114,11 @@ for julia_path in (pjulia, pjulia*".zip")
if t<:MaxLengthString || t<:String
jar = [
if tp == "S10"
pyconvert(String, ar[k, j, i].decode())
pyconvert(String, ar[k, j, i][].decode())
elseif tp == "U10"
pyconvert(String, ar[k, j, i][])
else
pyconvert(String, ar[k, j, i])
pyconvert(String, ar[k, j, i][][])
end
for i in 0:9, j in 0:5, k in 0:1
]
Expand Down Expand Up @@ -162,9 +170,11 @@ for julia_path in (pjulia, pjulia*".zip")
@test pyconvert(Tuple, ar.shape) == ()
if t<:MaxLengthString || t<:String
local x = if tp == "S10"
pyconvert(String, ar[()].decode())
pyconvert(String, ar[()][].decode())
elseif tp == "U10"
pyconvert(String, ar[()][])
else
pyconvert(String, ar[()])
pyconvert(String, ar[()][][])
end
@test x == testzerodimarrays[t]
else
Expand All @@ -179,15 +189,17 @@ data = rand(Int32,2,6,10)

numpy = pyimport("numpy")
numcodecs = pyimport("numcodecs")
g = zarr.group(ppython)
# TODO test zarr_format=3 when supported
g = zarr.group(ppython; zarr_format=2)
g.attrs["groupatt"] = "Hi"
z1 = g.create_dataset("a1", shape=(2,6,10),chunks=(1,2,3), dtype="i4")
z1 = g.create_array("a1", shape=(2,6,10),chunks=(1,2,3), dtype="i4")
z1[pybuiltins.Ellipsis] = numpy.array(data)
z1.attrs["test"] = pydict(Dict("b"=>6))
z2 = g.create_dataset("a2", shape=(5,),chunks=(5,), dtype="S1", compressor=numcodecs.Zlib())
z2[pybuiltins.Ellipsis] = pylist([k for k in "hallo"])
z3 = g.create_dataset("a3", shape=(2,), dtype=pybuiltins.str)
z3[pybuiltins.Ellipsis]=numpy.asarray(["test1", "test234"], dtype="O")
# TODO fix strings for zarr-python v3
# z2 = g.create_array("a2", shape=(5,),chunks=(5,), dtype="S1", compressor=numcodecs.Zlib())
# z2[pybuiltins.Ellipsis] = pylist([k for k in "hallo"])
# z3 = g.create_array("a3", shape=(2,), dtype=pybuiltins.str)
# z3[pybuiltins.Ellipsis]=numpy.asarray(["test1", "test234"], dtype="O")
zarr.consolidate_metadata(ppython)

#Open in Julia
Expand All @@ -199,15 +211,16 @@ a1 = g["a1"]
@test a1[:,:,:]==permutedims(data,(3,2,1))
@test a1.attrs["test"]==Dict("b"=>6)
# Test reading the string array
@test String(g["a2"][:])=="hallo"
@test g["a3"] == ["test1", "test234"]
# TODO fix strings for zarr-python v3
# @test String(g["a2"][:])=="hallo"
# @test g["a3"] == ["test1", "test234"]

# And test for consolidated metadata
# Delete files so we make sure they are not accessed
rm(joinpath(ppython,".zattrs"))
rm(joinpath(ppython,"a1",".zattrs"))
rm(joinpath(ppython,"a1",".zarray"))
rm(joinpath(ppython,"a2",".zarray"))
rm(joinpath(ppython,".zattrs"); force=true)
rm(joinpath(ppython,"a1",".zattrs"); force=true)
rm(joinpath(ppython,"a1",".zarray"); force=true)
rm(joinpath(ppython,"a2",".zarray"); force=true)
g = zopen(ppython, "w", consolidated=true)
@test g isa Zarr.ZGroup
@test g.attrs["groupatt"] == "Hi"
Expand All @@ -220,21 +233,24 @@ a1 = g["a1"]
a1[:,1,1] = 1:10
@test a1[:,1,1] == 1:10
# Test reading the string array
@test String(g["a2"][:])=="hallo"
# TODO fix strings for zarr-python v3
# @test String(g["a2"][:])=="hallo"


# Test zip file can be read
ppythonzip = ppython*".zip"
store = zarr.ZipStore(ppythonzip, mode="w")
g = zarr.group(store=store)
store = zarr.storage.ZipStore(ppythonzip, mode="w")
# TODO test zarr_format=3 when supported
g = zarr.group(;store=store, zarr_format=2)
g.attrs["groupatt"] = "Hi"
z1 = g.create_dataset("a1", shape=(2,6,10),chunks=(1,2,3), dtype="i4")
z1 = g.create_array("a1", shape=(2,6,10),chunks=(1,2,3), dtype="i4")
z1[pybuiltins.Ellipsis] = numpy.array(data)
z1.attrs["test"] = pydict(Dict("b"=>6))
z2 = g.create_dataset("a2", shape=(5,),chunks=(5,), dtype="S1", compressor=numcodecs.Zlib())
z2[pybuiltins.Ellipsis] = pylist([k for k in "hallo"])
z3 = g.create_dataset("a3", shape=(2,), dtype=pybuiltins.str)
z3[pybuiltins.Ellipsis] = numpy.asarray(["test1", "test234"], dtype="O")
# TODO fix strings for zarr-python v3
# z2 = g.create_array("a2", shape=(5,),chunks=(5,), dtype="S1", compressor=numcodecs.Zlib())
# z2[pybuiltins.Ellipsis] = pylist([k for k in "hallo"])
# z3 = g.create_array("a3", shape=(2,), dtype=pybuiltins.str)
# z3[pybuiltins.Ellipsis] = numpy.asarray(["test1", "test234"], dtype="O")
store.close()

g = zopen(Zarr.ZipStore(Mmap.mmap(ppythonzip)))
Expand All @@ -245,8 +261,9 @@ a1 = g["a1"]
@test a1[:,:,:]==permutedims(data,(3,2,1))
@test a1.attrs["test"]==Dict("b"=>6)
# Test reading the string array
@test String(g["a2"][:])=="hallo"
@test g["a3"] == ["test1", "test234"]
# TODO fix strings for zarr-python v3
# @test String(g["a2"][:])=="hallo"
# @test g["a3"] == ["test1", "test234"]

end

Expand Down
3 changes: 2 additions & 1 deletion test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ using PythonCall
using CondaPkg
using Dates

CondaPkg.add("zarr"; version="2.*")
CondaPkg.add("zarr"; version="3.*")
CondaPkg.add("numcodecs"; version="0.15.*")

@testset "Zarr" begin

Expand Down
Loading