Skip to content

Commit

Permalink
Cancel usage of chunk_number if loading the whole dependency (#894)
Browse files Browse the repository at this point in the history
* Cancel usage of `chunk_number` if loading the whole dependency

* Add argument of whether check data_type is stored

* Add `strax_version`
  • Loading branch information
dachengx committed Sep 25, 2024
1 parent 4b823ac commit ef2861e
Showing 1 changed file with 9 additions and 3 deletions.
12 changes: 9 additions & 3 deletions strax/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -2384,22 +2384,27 @@ def merge_per_chunk_storage(
rechunk=True,
chunk_number_group: ty.Optional[ty.List[ty.List[int]]] = None,
target_frontend_id: ty.Optional[int] = None,
check_is_stored: bool = True,
):
"""Merge the per-chunked data from the per-chunked dependency into the target storage."""

if self.is_stored(run_id, target):
if check_is_stored and self.is_stored(run_id, target):
raise ValueError(f"Data {target} for {run_id} already exists.")

self._check_merge_per_chunk_storage_kwargs(run_id, target, target_frontend_id)

chunks = self.get_metadata(run_id, per_chunked_dependency)["chunks"]
if chunk_number_group is not None:
combined_chunk_numbers = list(itertools.chain(*chunk_number_group))
if len(combined_chunk_numbers) != len(set(combined_chunk_numbers)):
raise ValueError(f"Duplicate chunk numbers found in {chunk_number_group}")
_chunk_number = {per_chunked_dependency: combined_chunk_numbers}
if min(combined_chunk_numbers) == 0 and max(combined_chunk_numbers) == len(chunks) - 1:
# If the chunks are all the chunks of the dependency, we can drop the chunk_number
_chunk_number = None
else:
_chunk_number = {per_chunked_dependency: combined_chunk_numbers}
else:
# if no chunk numbers are given, use information from the dependency
chunks = self.get_metadata(run_id, per_chunked_dependency)["chunks"]
chunk_number_group = [[c["chunk_i"]] for c in chunks]
_chunk_number = None

Expand Down Expand Up @@ -2449,6 +2454,7 @@ def wrapped_loader():
data_key = self.key_for(run_id, target, chunk_number=_chunk_number)
target_plugin = self.__get_plugin(run_id, target, chunk_number=_chunk_number)
target_md = target_plugin.metadata(run_id, target)
target_md["strax_version"] = strax.__version__
# Copied from StorageBackend.saver
if "dtype" in target_md:
target_md["dtype"] = target_md["dtype"].descr.__repr__()
Expand Down

0 comments on commit ef2861e

Please sign in to comment.