Skip to content

Commit

Permalink
try to fix warning about large cubes: dask cubes were warning with no…
Browse files Browse the repository at this point in the history
…n-dask keywords, likely caused by wrong inheritance order in the composite (the mixin is overridden by the base SpectralCube)
  • Loading branch information
keflavich committed Jun 27, 2024
1 parent 4d7182f commit 0da6c93
Show file tree
Hide file tree
Showing 3 changed files with 21 additions and 23 deletions.
9 changes: 0 additions & 9 deletions spectral_cube/dask_spectral_cube.py
Original file line number Diff line number Diff line change
Expand Up @@ -287,15 +287,6 @@ def __exit__(self, *args):
def _compute(self, array):
return array.compute(**self._scheduler_kwargs)

def _warn_slow(self, funcname):
if self._is_huge and not self.allow_huge_operations:
raise ValueError("This function ({0}) requires loading the entire "
"cube into memory, and the cube is large ({1} "
"pixels), so by default we disable this operation. "
"To enable the operation, set "
"`cube.allow_huge_operations=True` and try again."
.format(funcname, self.size))

def _get_filled_data(self, view=(), fill=np.nan, check_endian=None, use_memmap=None):

if check_endian:
Expand Down
2 changes: 1 addition & 1 deletion spectral_cube/spectral_cube.py
Original file line number Diff line number Diff line change
Expand Up @@ -1204,7 +1204,7 @@ def median(self, axis=None, iterate_rays=False, **kwargs):
log.debug("Using numpy nanmedian")
result = self.apply_numpy_function(np.nanmedian, axis=axis,
projection=True, unit=self.unit,
how='cube',**kwargs)
how='cube', **kwargs)
elif iterate_rays:
result = self.apply_numpy_function(
nanmedian if bnok else np.nanmedian if hasattr(np, 'nanmedian') else np.median,
Expand Down
33 changes: 20 additions & 13 deletions spectral_cube/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,19 +27,26 @@ def warn_slow(function):
@wraps(function)
def wrapper(self, *args, **kwargs):
# if the function accepts a 'how', the 'cube' approach requires the whole cube in memory
warn_how = (kwargs.get('how') == 'cube') or 'how' not in kwargs
if self._is_huge and not self.allow_huge_operations and warn_how:
raise ValueError("This function ({0}) requires loading the entire "
"cube into memory, and the cube is large ({1} "
"pixels), so by default we disable this operation. "
"To enable the operation, set "
"`cube.allow_huge_operations=True` and try again. "
"Alternatively, you may want to consider using an "
"approach that does not load the whole cube into "
"memory by specifying how='slice' or how='ray'. "
"See {bigdataurl} for details."
.format(str(function), self.size,
bigdataurl=bigdataurl))
argspec = inspect.getfullargspec(function)
accepts_how_keyword = 'how' in argspec.args or argspec.varkw == 'how'

warn_how = accepts_how_keyword and ((kwargs.get('how') == 'cube') or 'how' not in kwargs)

if self._is_huge and not self.allow_huge_operations:
warn_message = ("This function ({0}) requires loading the entire "
"cube into memory, and the cube is large ({1} "
"pixels), so by default we disable this operation. "
"To enable the operation, set "
"`cube.allow_huge_operations=True` and try again. ").format(str(function), self.size)

if warn_how:
warn_message += ("Alternatively, you may want to consider using an "
"approach that does not load the whole cube into "
"memory by specifying how='slice' or how='ray'. ")

warn_message += ("See {bigdataurl} for details.".format(bigdataurl=bigdataurl))

raise ValueError(warn_message)
elif warn_how and not self._is_huge:
# TODO: add check for whether cube has been loaded into memory
warnings.warn("This function ({0}) requires loading the entire cube into "
Expand Down

0 comments on commit 0da6c93

Please sign in to comment.