Skip to content

Commit

Permalink
Skip preloading spatial datasets with doubtful coordinates
Browse files Browse the repository at this point in the history
Instead of raising an error, allow non-on-sky visits but
log an error and do not use those coordinates and camera
angles.
  • Loading branch information
hsinfang committed Aug 13, 2024
1 parent 6bd89f8 commit 06a13c7
Showing 1 changed file with 22 additions and 13 deletions.
35 changes: 22 additions & 13 deletions python/activator/middleware_interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,12 +283,6 @@ class MiddlewareInterface:
constructing URIs to retrieve incoming files. The default is
appropriate for use in the USDF environment; typically only
change this when running local tests.
Raises
------
ValueError
Raised if ``visit`` does not have equatorial coordinates and sky
rotation angles.
"""
DATASET_IDENTIFIER = "Live"
"""The dataset ID used for Sasquatch uploads.
Expand All @@ -313,6 +307,7 @@ def _collection_template(self):
# Class invariants:
# self.image_host is a valid URI with non-empty path and no query or fragment.
# self._download_store is None if and only if self.image_host is a local URI.
# self._skip_spatial_preload indicates that the coordinates cannot be trusted.
# self.visit, self.instrument, self.camera, self.skymap, self._deployment
# self._day_obs do not change after __init__.
# self.butler defaults to the "defaults" chained collection, which contains
Expand All @@ -327,12 +322,22 @@ def __init__(self, central_butler: Butler, image_bucket: str, visit: FannedOutVi
skymap: str, local_repo: str, local_cache: DatasetCache,
prefix: str = "s3://"):
self.visit = visit
# Usually prompt processing only cares about on-sky images and expects all of
# them to have equatorial coordinates and sky rotation angles. For other images,
# ISR can be done but sensible results from further pipelines cannot be expected.
# Continue the processing but using the coordinates to preload spatial datasets
# will be skipped. This allows cases where only ISR is wanted.
self._skip_spatial_preload = False
if self.visit.coordinateSystem != FannedOutVisit.CoordSys.ICRS:
raise ValueError("Only ICRS coordinates are supported in Visit, "
f"got {self.visit.coordinateSystem!r} instead.")
self._skip_spatial_preload = True
_log.error("Only ICRS coordinates are fully supported. "
f"Got {self.visit.coordinateSystem!r} instead in {self.visit}. "
"Spatial datasets won't be loaded.")
if self.visit.rotationSystem != FannedOutVisit.RotSys.SKY:
raise ValueError("Only sky camera rotations are supported in Visit, "
f"got {self.visit.rotationSystem!r} instead.")
self._skip_spatial_preload = True
_log.error("Only sky camera rotations are fully supported. "
f"Got {self.visit.rotationSystem!r} instead in {self.visit}. "
"Spatial datasets won't be loaded.")

# Deployment/version ID -- potentially expensive to generate.
self._deployment = self._get_deployment()
Expand Down Expand Up @@ -535,7 +540,8 @@ def prep_butler(self) -> None:
with lsst.utils.timer.time_this(_log, msg="prep_butler", level=logging.DEBUG):
_log.info(f"Preparing Butler for visit {self.visit!r}")

self._write_region_time() # Must be done before preprocessing pipeline
if not self._skip_spatial_preload:
self._write_region_time() # Must be done before preprocessing pipeline

# repos may have been modified by other MWI instances.
# TODO: get a proper synchronization API for Butler
Expand Down Expand Up @@ -583,6 +589,11 @@ def _find_data_to_preload(self):
calibs : set [`~lsst.daf.butler.DatasetRef`]
The subset of ``datasets`` representing calibs.
"""
with lsst.utils.timer.time_this(_log, msg="prep_butler (find calibs)", level=logging.DEBUG):
calib_datasets = set(self._export_calibs(self.visit.detector, self.visit.filters))
if self._skip_spatial_preload:
return (calib_datasets, calib_datasets)

detector = self.camera[self.visit.detector]
wcs = self._predict_wcs(detector)
center, radius = self._detector_bounding_circle(detector, wcs)
Expand All @@ -592,8 +603,6 @@ def _find_data_to_preload(self):
with lsst.utils.timer.time_this(_log, msg="prep_butler (find templates)", level=logging.DEBUG):
template_datasets = set(self._export_skymap_and_templates(
center, detector, wcs, self.visit.filters))
with lsst.utils.timer.time_this(_log, msg="prep_butler (find calibs)", level=logging.DEBUG):
calib_datasets = set(self._export_calibs(self.visit.detector, self.visit.filters))
with lsst.utils.timer.time_this(_log, msg="prep_butler (find ML models)", level=logging.DEBUG):
model_datasets = set(self._export_ml_models())
return (refcat_datasets | template_datasets | calib_datasets | model_datasets,
Expand Down

0 comments on commit 06a13c7

Please sign in to comment.