Skip to content

Commit

Permalink
make mypy happy
Browse files Browse the repository at this point in the history
  • Loading branch information
norlandrhagen committed Dec 17, 2024
1 parent 31aacf9 commit 5d14b20
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 23 deletions.
2 changes: 1 addition & 1 deletion virtualizarr/readers/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ def open_loadable_vars_and_indexes(

# Updates the Xarray open_dataset kwargs if Zarr

if fpath.filepath.suffix == ".zarr": # type: ignore
if fpath.upath.suffix == ".zarr":
engine = "zarr"
xr_input = fpath.filepath

Expand Down
38 changes: 20 additions & 18 deletions virtualizarr/readers/zarr.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,6 @@
from virtualizarr.zarr import ZArray

if TYPE_CHECKING:
from pathlib import PosixPath

import upath
import zarr

Expand Down Expand Up @@ -232,12 +230,12 @@ def virtual_dataset_from_zarr_group(
)


async def get_chunk_size(zarr_group: zarr.Group, chunk_key: PosixPath) -> int:
async def get_chunk_size(zarr_group: zarr.Group, chunk_key: str) -> int:
# User zarr-pythons `getsize` method to get bytes per chunk
return await zarr_group.store.getsize(chunk_key)


async def chunk_exists(zarr_group: zarr.Group, chunk_key: PosixPath) -> bool:
async def chunk_exists(zarr_group: zarr.Group, chunk_key: str) -> bool:
# calls zarr-pythons `exists` to check for a chunk
return await zarr_group.store.exists(chunk_key)

Expand Down Expand Up @@ -300,12 +298,12 @@ def construct_virtual_array(
attrs = zarr_array.metadata.attributes

if zarr_array.metadata.zarr_format == 2:
array_zarray = _parse_zarr_v2_metadata(zarr_array=zarr_array)
array_zarray = _parse_zarr_v2_metadata(zarr_array=zarr_array) # type: ignore[arg-type]
array_dims = attrs["_ARRAY_DIMENSIONS"]

elif zarr_array.metadata.zarr_format == 3:
array_zarray = _parse_zarr_v3_metadata(zarr_array=zarr_array)
array_dims = zarr_array.metadata.dimension_names
array_zarray = _parse_zarr_v3_metadata(zarr_array=zarr_array) # type: ignore[arg-type]
array_dims = zarr_array.metadata.dimension_names # type: ignore[union-attr]

else:
raise NotImplementedError("Zarr format is not recognized as v2 or v3.")
Expand Down Expand Up @@ -333,20 +331,20 @@ def construct_virtual_array(
return array_variable


def _parse_zarr_v2_metadata(zarr_array: zarr.core.array.Array) -> ZArray:
def _parse_zarr_v2_metadata(zarr_array: zarr.Array) -> ZArray:
return ZArray(
shape=zarr_array.metadata.shape,
chunks=zarr_array.metadata.chunks,
chunks=zarr_array.metadata.chunks, # type: ignore[union-attr]
dtype=zarr_array.metadata.dtype,
fill_value=zarr_array.metadata.fill_value,
fill_value=zarr_array.metadata.fill_value, # type: ignore[arg-type]
order="C",
compressor=zarr_array.metadata.compressor,
filters=zarr_array.metadata.filters,
compressor=zarr_array.metadata.compressor, # type: ignore[union-attr]
filters=zarr_array.metadata.filters, # type: ignore
zarr_format=zarr_array.metadata.zarr_format,
)


def _parse_zarr_v3_metadata(zarr_array: zarr.core.array.Array) -> ZArray:
def _parse_zarr_v3_metadata(zarr_array: zarr.Array) -> ZArray:
from virtualizarr.codecs import get_codecs

if zarr_array.metadata.fill_value is None:
Expand All @@ -360,14 +358,18 @@ def _parse_zarr_v3_metadata(zarr_array: zarr.core.array.Array) -> ZArray:
# Questions: What do we do with endian info?
codecs = get_codecs(zarr_array)

compressor = getattr(codecs[0], "compressor", None)
filters = getattr(codecs[0], "filters", None)
# Question: How should we parse the values from get_codecs?
# typing: Union[Codec, tuple["ArrayArrayCodec | ArrayBytesCodec | BytesBytesCodec", ...]]
# mypy: ... is not indexable [index]
# added tmp bypyass for mypy
compressor = getattr(codecs[0], "compressor", None) # type: ignore
filters = getattr(codecs[0], "filters", None) # type: ignore

return ZArray(
chunks=zarr_array.metadata.chunk_grid.chunk_shape,
chunks=zarr_array.metadata.chunk_grid.chunk_shape, # type: ignore[attr-defined]
compressor=compressor,
dtype=zarr_array.metadata.data_type.name,
fill_value=fill_value,
dtype=zarr_array.metadata.data_type.name, # type: ignore
fill_value=fill_value, # type: ignore[arg-type]
filters=filters,
order="C",
shape=zarr_array.metadata.shape,
Expand Down
11 changes: 7 additions & 4 deletions virtualizarr/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,10 @@ class _FsspecFSFromFilepath:
"""

filepath: str | upath.core.UPath
filepath: str
reader_options: Optional[dict] = field(default_factory=dict)
fs: fsspec.AbstractFileSystem = field(init=False)
upath: upath.core.UPath = field(init=False)

def open_file(self) -> OpenFileType:
"""Calls `.open` on fsspec.Filesystem instantiation using self.filepath as an input.
Expand All @@ -61,14 +62,16 @@ def __post_init__(self) -> None:
from upath import UPath

if not isinstance(self.filepath, UPath):
self.filepath = UPath(self.filepath)
upath = UPath(self.filepath)

protocol = self.filepath.protocol
self.upath = upath
self.protocol = upath.protocol
self.filepath = upath.as_uri()

self.reader_options = self.reader_options or {}
storage_options = self.reader_options.get("storage_options", {}) # type: ignore

self.fs = fsspec.filesystem(protocol, **storage_options)
self.fs = fsspec.filesystem(self.protocol, **storage_options)


def check_for_collisions(
Expand Down

0 comments on commit 5d14b20

Please sign in to comment.