Merge pull request #996 from lsst/tickets/DM-46966

DM-46966: Make calibrating pixel values optional in CalibrateImageTask

python/lsst/pipe/tasks/calibrateImage.py

@@ -120,7 +120,10 @@ class CalibrateImageConnections(pipeBase.PipelineTaskConnections,
         dimensions=["instrument", "visit", "detector"],
     )
     applied_photo_calib = connectionTypes.Output(
-        doc="Photometric calibration that was applied to exposure.",
+        doc=(
+            "Photometric calibration that was applied to exposure's pixels. "
+            "This connection is disabled when do_calibrate_pixels=False."
+        ),
         name="initial_photoCalib_detector",
         storageClass="PhotoCalib",
         dimensions=("instrument", "visit", "detector"),
@@ -169,6 +172,8 @@ def __init__(self, *, config=None):
             del self.astrometry_matches
         if config.optional_outputs is None or "photometry_matches" not in config.optional_outputs:
             del self.photometry_matches
+        if not config.do_calibrate_pixels:
+            del self.applied_photo_calib
 
 
 class CalibrateImageConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateImageConnections):
@@ -292,6 +297,18 @@ class CalibrateImageConfig(pipeBase.PipelineTaskConfig, pipelineConnections=Cali
         doc="Task to to compute summary statistics on the calibrated exposure."
     )
 
+    do_calibrate_pixels = pexConfig.Field(
+        dtype=bool,
+        default=True,
+        doc=(
+            "If True, apply the photometric calibration to the image pixels "
+            "and background model, and attach an identity PhotoCalib to "
+            "the output image to reflect this.  If False`, leave the image "
+            "and background uncalibrated and attach the PhotoCalib that maps "
+            "them to physical units."
+        )
+    )
+
     def setDefaults(self):
         super().setDefaults()
 
@@ -571,6 +588,7 @@ def run(self, *, exposures, id_generator=None, result=None):
             ``applied_photo_calib``
                 Photometric calibration that was fit to the star catalog and
                 applied to the exposure. (`lsst.afw.image.PhotoCalib`)
+                This is `None` if ``config.do_calibrate_pixels`` is `False`.
             ``astrometry_matches``
                 Reference catalog stars matches used in the astrometric fit.
                 (`list` [`lsst.afw.table.ReferenceMatch`] or `lsst.afw.table.BaseCatalog`)
@@ -604,9 +622,7 @@ def run(self, *, exposures, id_generator=None, result=None):
                                                                             astrometry_meta)
 
         result.stars_footprints, photometry_matches, \
-            photometry_meta, result.applied_photo_calib = self._fit_photometry(result.exposure,
-                                                                               result.stars_footprints,
-                                                                               result.background)
+            photometry_meta, photo_calib = self._fit_photometry(result.exposure, result.stars_footprints)
         self.metadata["photometry_matches_count"] = len(photometry_matches)
         # fit_photometry returns a new catalog, so we need a new astropy table view.
         result.stars = result.stars_footprints.asAstropy()
@@ -615,7 +631,11 @@ def run(self, *, exposures, id_generator=None, result=None):
                                                                             photometry_meta)
 
         self._summarize(result.exposure, result.stars_footprints, result.background)
-
+        if self.config.do_calibrate_pixels:
+            self._apply_photometry(result.exposure, result.background)
+            result.applied_photo_calib = photo_calib
+        else:
+            result.applied_photo_calib = None
         return result
 
     def _compute_psf(self, exposure, id_generator):
@@ -880,45 +900,63 @@ def _fit_astrometry(self, exposure, stars):
         result = self.astrometry.run(stars, exposure)
         return result.matches, result.matchMeta
 
-    def _fit_photometry(self, exposure, stars, background):
+    def _fit_photometry(self, exposure, stars):
         """Fit a photometric model to the data and return the reference
         matches used in the fit, and the fitted PhotoCalib.
 
         Parameters
         ----------
         exposure : `lsst.afw.image.Exposure`
             Exposure that is being fit, to get PSF and other metadata from.
-            Modified to be in nanojanksy units, with an assigned photoCalib
-            identically 1.
+            Has the fit `lsst.afw.image.PhotoCalib` attached, with pixel values
+            unchanged.
         stars : `lsst.afw.table.SourceCatalog`
             Good stars selected for use in calibration.
 
         Returns
         -------
         calibrated_stars : `lsst.afw.table.SourceCatalog`
             Star catalog with flux/magnitude columns computed from the fitted
-            photoCalib.
+            photoCalib (instFlux columns are retained as well).
         matches : `list` [`lsst.afw.table.ReferenceMatch`]
             Reference/stars matches used in the fit.
-        photoCalib : `lsst.afw.image.PhotoCalib`
+        matchMeta : `lsst.daf.base.PropertyList`
+            Metadata needed to unpersist matches, as returned by the matcher.
+        photo_calib : `lsst.afw.image.PhotoCalib`
             Photometric calibration that was fit to the star catalog.
         """
         result = self.photometry.run(exposure, stars)
         calibrated_stars = result.photoCalib.calibrateCatalog(stars)
-        exposure.maskedImage = result.photoCalib.calibrateImage(exposure.maskedImage)
+        exposure.setPhotoCalib(result.photoCalib)
+        return calibrated_stars, result.matches, result.matchMeta, result.photoCalib
+
+    def _apply_photometry(self, exposure, background):
+        """Apply the photometric model attached to the exposure to the
+        exposure's pixels and an associated background model.
+
+        Parameters
+        ----------
+        exposure : `lsst.afw.image.Exposure`
+            Exposure with the target `lsst.afw.image.PhotoCalib` attached.
+            On return, pixel values will be calibrated and an identity
+            photometric transform will be attached.
+        background : `lsst.afw.math.BackgroundList`
+            Background model to convert to nanojansky units in place.
+        """
+        photo_calib = exposure.getPhotoCalib()
+        exposure.maskedImage = photo_calib.calibrateImage(exposure.maskedImage)
         identity = afwImage.PhotoCalib(1.0,
-                                       result.photoCalib.getCalibrationErr(),
+                                       photo_calib.getCalibrationErr(),
                                        bbox=exposure.getBBox())
         exposure.setPhotoCalib(identity)
+        exposure.metadata["BUNIT"] = "nJy"
 
-        assert result.photoCalib._isConstant, \
+        assert photo_calib._isConstant, \
             "Background calibration assumes a constant PhotoCalib; PhotoCalTask should always return that."
         for bg in background:
             # The statsImage is a view, but we can't assign to a function call in python.
             binned_image = bg[0].getStatsImage()
-            binned_image *= result.photoCalib.getCalibrationMean()
-
-        return calibrated_stars, result.matches, result.matchMeta, result.photoCalib
+            binned_image *= photo_calib.getCalibrationMean()
 
     def _summarize(self, exposure, stars, background):
         """Compute summary statistics on the exposure and update in-place the
@@ -928,17 +966,15 @@ def _summarize(self, exposure, stars, background):
         ----------
         exposure : `lsst.afw.image.Exposure`
             Exposure that was calibrated, to get PSF and other metadata from.
+            Should be in instrumental units with the photometric calibration
+            attached.
             Modified to contain the computed summary statistics.
         stars : `SourceCatalog`
             Good stars selected used in calibration.
         background : `lsst.afw.math.BackgroundList`
             Background that was fit to the exposure during detection of the
-            above stars.
+            above stars.  Should be in instrumental units.
         """
-        # TODO investigation: because this takes the photoCalib from the
-        # exposure, photometric summary values may be "incorrect" (i.e. they
-        # will reflect the ==1 nJy calibration on the exposure, not the
-        # applied calibration). This needs to be checked.
         summary = self.compute_summary_stats.run(exposure, stars, background)
         exposure.info.setSummaryStats(summary)
 

tests/test_calibrateImage.py

@@ -141,7 +141,7 @@ def setUp(self):
         # Something about this test dataset prefers a larger threshold here.
         self.config.star_selector["science"].unresolved.maximum = 0.2
 
-    def _check_run(self, calibrate, result):
+    def _check_run(self, calibrate, result, expect_calibrated_pixels: bool = True):
         """Test the result of CalibrateImage.run().
 
         Parameters
@@ -150,6 +150,8 @@ def _check_run(self, calibrate, result):
             Configured task that had `run` called on it.
         result : `lsst.pipe.base.Struct`
             Result of calling calibrate.run().
+        expect_calibrated_pixels : `bool`, optional
+            Whether to expect image and background pixels to be calibrated.
         """
         # Background should have 4 elements: 3 from compute_psf and one from
         # re-estimation during source detection.
@@ -169,11 +171,17 @@ def _check_run(self, calibrate, result):
         self.assertTrue(np.isfinite(result.stars_footprints["coord_ra"]).all())
         self.assertTrue(np.isfinite(result.stars["coord_ra"]).all())
 
-        # Returned photoCalib should be the applied value, not the ==1 one on the exposure.
-        # Note that this is very approximate because we are basing this comparison
-        # on just 2-3 stars.
-        self.assertFloatsAlmostEqual(result.applied_photo_calib.getCalibrationMean(),
-                                     self.photo_calib, rtol=1e-2)
+        if expect_calibrated_pixels:
+            # Fit photoCalib should be the applied value if we calibrated
+            # pixels, not the ==1 one on the exposure.
+            photo_calib = result.applied_photo_calib
+            self.assertEqual(result.exposure.photoCalib.getCalibrationMean(), 1.0)
+        else:
+            self.assertIsNone(result.applied_photo_calib)
+            photo_calib = result.exposure.photoCalib
+        # PhotoCalib comparison is very approximate because we are basing this
+        # comparison on just 2-3 stars.
+        self.assertFloatsAlmostEqual(photo_calib.getCalibrationMean(), self.photo_calib, rtol=1e-2)
         # Should have calibrated flux/magnitudes in the afw and astropy catalogs
         self.assertIn("slot_PsfFlux_flux", result.stars_footprints.schema)
         self.assertIn("slot_PsfFlux_mag", result.stars_footprints.schema)
@@ -228,6 +236,18 @@ def test_run_no_optionals(self):
         self.assertNotIn("astrometry_matches", result.getDict())
         self.assertNotIn("photometry_matches", result.getDict())
 
+    def test_run_no_calibrate_pixels(self):
+        """Test that run() returns reasonable values when
+        do_calibrate_pixels=False.
+        """
+        self.config.do_calibrate_pixels = False
+        calibrate = CalibrateImageTask(config=self.config)
+        calibrate.astrometry.setRefObjLoader(self.ref_loader)
+        calibrate.photometry.match.setRefObjLoader(self.ref_loader)
+        result = calibrate.run(exposures=self.exposure)
+
+        self._check_run(calibrate, result, expect_calibrated_pixels=False)
+
     def test_compute_psf(self):
         """Test that our brightest sources are found by _compute_psf(),
         that a PSF is assigned to the expopsure.
@@ -338,7 +358,8 @@ def test_photometry(self):
         stars = calibrate._find_stars(self.exposure, background, self.id_generator)
         calibrate._fit_astrometry(self.exposure, stars)
 
-        stars, matches, meta, photoCalib = calibrate._fit_photometry(self.exposure, stars, background)
+        stars, matches, meta, photoCalib = calibrate._fit_photometry(self.exposure, stars)
+        calibrate._apply_photometry(self.exposure, background)
 
         # NOTE: With this test data, PhotoCalTask returns calibrationErr==0,
         # so we can't check that the photoCal error has been set.
@@ -595,6 +616,40 @@ def test_runQuantum_no_optional_outputs(self):
             # Restore the one we removed for the next test.
             connections[optional] = temp
 
+    def test_runQuantum_no_calibrate_pixels(self):
+        """Test that the the task runs when calibrating pixels is disabled,
+        and that this results in the ``applied_photo_calib`` output being
+        removed.
+        """
+        config = CalibrateImageTask.ConfigClass()
+        config.do_calibrate_pixels = False
+        task = CalibrateImageTask(config=config)
+        lsst.pipe.base.testUtils.assertValidInitOutput(task)
+
+        quantum = lsst.pipe.base.testUtils.makeQuantum(
+            task, self.butler, self.visit_id,
+            {"exposures": [self.exposure0_id],
+             "astrometry_ref_cat": [self.htm_id],
+             "photometry_ref_cat": [self.htm_id],
+             # outputs
+             "exposure": self.visit_id,
+             "stars": self.visit_id,
+             "stars_footprints": self.visit_id,
+             "background": self.visit_id,
+             "psf_stars": self.visit_id,
+             "psf_stars_footprints": self.visit_id,
+             "initial_pvi_background": self.visit_id,
+             "astrometry_matches": self.visit_id,
+             "photometry_matches": self.visit_id,
+             })
+        mock_run = lsst.pipe.base.testUtils.runTestQuantum(task, self.butler, quantum)
+
+        # Ensure the reference loaders have been configured.
+        self.assertEqual(task.astrometry.refObjLoader.name, "gaia_dr3_20230707")
+        self.assertEqual(task.photometry.match.refObjLoader.name, "ps1_pv3_3pi_20170110")
+        # Check that the proper kwargs are passed to run().
+        self.assertEqual(mock_run.call_args.kwargs.keys(), {"exposures", "result", "id_generator"})
+
     def test_lintConnections(self):
         """Check that the connections are self-consistent.
         """