Merge pull request #294 from lsst/tickets/DM-46684

DM-46684: Add Metrics to calibrateImage task metadata

pipelines/calexpMetrics.yaml

@@ -1,5 +1,9 @@
 description: |
   Metrics created from the analysis of calibrated exposures.
+  DEPRECATED : The task in this yaml file -- calexpSummary --
+  has been moved to visitQualityCore.yaml.
+  See ticket DM-46965 : This yaml file will be removed on or
+  soon after 13 Dec 2024.
 tasks:
   calexpSummary:
     class: lsst.analysis.tools.tasks.CalexpSummaryAnalysisTask

pipelines/exposureQualityCore.yaml

@@ -9,9 +9,10 @@ tasks:
       atools.isrAmpOffsetMetadataMetric: MetadataMetricTool
       atools.isrAmpOffsetMetadataMetric.parameterizedBand: false
       atools.isrAmpOffsetMetadataMetric.taskName: isr
-      atools.isrAmpOffsetMetadataMetric.subTaskName: ampOffset
       atools.isrAmpOffsetMetadataMetric.metrics:
         # Format is "metric name in the metadata": units
         AMPOFFSET_PEDESTALS: adu
+      atools.isrAmpOffsetMetadataMetric.subTaskNames:
+        AMPOFFSET_PEDESTALS: ampOffset
       python: |
         from lsst.analysis.tools.atools import MetadataMetricTool

pipelines/pviMetrics.yaml

@@ -0,0 +1,58 @@
+description: |
+  Placeholder yaml file to house tasks that will analyse output
+  from calibrateImageTask.
+  TO DO : DM-46931
+  These tasks should be moved to (likely)
+  either visitQualityCore or visitQualityExtended, and this
+  file deleted once the Great Calibrate Refactor has completed.
+tasks:
+  analyzeCalibrateImageMetadata:
+    class: lsst.analysis.tools.tasks.MetadataAnalysisTask
+    config:
+      connections.taskName: calibrateImage
+      connections.outputName: calibrateImage_metadata
+      metadataDimensions: ["instrument", "visit", "detector"]
+      atools.calibrateImageMetadataMetrics: MetadataMetricTool
+      atools.calibrateImageMetadataMetrics.taskName: calibrateImage
+      atools.calibrateImageMetadataMetrics.metrics:
+        initial_psf_positive_footprint_count: ct
+        initial_psf_negative_footprint_count: ct
+        initial_psf_positive_peak_count: ct
+        initial_psf_negative_peak_count: ct
+        simple_psf_positive_footprint_count: ct
+        simple_psf_negative_footprint_count: ct
+        simple_psf_positive_peak_count: ct
+        simple_psf_negative_peak_count: ct
+        bad_mask_fraction: ""
+        cr_mask_fraction: ""
+        crosstalk_mask_fraction: ""
+        detected_mask_fraction: ""
+        detected_negative_mask_fraction: ""
+        edge_mask_fraction: ""
+        intrp_mask_fraction: ""
+        no_data_mask_fraction: ""
+        sat_mask_fraction: ""
+        suspect_mask_fraction: ""
+        unmaskednan_mask_fraction: ""
+        numAvailStars: ct
+        numGoodStars: ct
+        sky_footprint_count: ct
+        post_deblend_source_count: ct
+        star_count: ct
+        saturated_source_count: ct
+        bad_source_count: ct
+        cosmic_ray_count: ct
+        matched_psf_star_count: ct
+        final_psf_sigma: pixel
+        astrometry_matches_count: ct
+        photometry_matches_count: ct
+      atools.calibrateImageMetadataMetrics.subTaskNames:
+        numAvailStars: psf_measure_psf
+        numGoodStars: psf_measure_psf
+        sky_footprint_count: star_sky_sources
+        cosmic_ray_count: psf_repair
+      atools.calibrateImageMetadataMetrics.newNames:
+        numAvailStars: psf_available_star_count
+        numGoodStars: psf_good_star_count
+      python: |
+        from lsst.analysis.tools.atools import MetadataMetricTool

pipelines/visitQualityCore.yaml

@@ -46,3 +46,38 @@ tasks:
     config:
       atools.astromColorDiffMetrics: TargetRefCatDeltaColorMetrics
       python: from lsst.analysis.tools.atools import *
+  # TO DO: DM-46932: Remove/edit/rename the following two tasks for DM-43077.
+  calexpSummary:
+    class: lsst.analysis.tools.tasks.CalexpSummaryAnalysisTask
+    config:
+      atools.calexpSummaryMetrics: CalexpSummaryMetrics
+      python: from lsst.analysis.tools.atools import *
+  analyzeCalibrateMetadata:
+    class: lsst.analysis.tools.tasks.MetadataAnalysisTask
+    config:
+      connections.taskName: calibrate
+      connections.outputName: calibrate_metadata
+      metadataDimensions: ["instrument", "visit", "detector"]
+      atools.calexpMetadataMetrics: MetadataMetricTool
+      atools.calexpMetadataMetrics.taskName: calibrate
+      atools.calexpMetadataMetrics.metrics:
+        positive_footprint_count: ct
+        negative_footprint_count: ct
+        source_count: ct
+        sky_footprint_count: ct
+        saturated_source_count: ct
+        bad_source_count: ct
+        bad_mask_fraction: ""
+        cr_mask_fraction: ""
+        crosstalk_mask_fraction: ""
+        edge_mask_fraction: ""
+        intrp_mask_fraction: ""
+        no_data_mask_fraction: ""
+        detected_mask_fraction: ""
+        detected_negative_mask_fraction: ""
+        sat_mask_fraction: ""
+        streak_mask_fraction: ""
+        suspect_mask_fraction: ""
+        unmaskednan_mask_fraction: ""
+      python: |
+        from lsst.analysis.tools.atools import MetadataMetricTool

python/lsst/analysis/tools/atools/metadataMetrics.py

@@ -41,9 +41,11 @@ class MetadataMetricTool(AnalysisTool):
         default=None,
     )
 
-    subTaskName = Field[str](
-        doc="The name of the subtask to extract metadata from. "
-        "If None, the entire task metadata will be used.",
+    subTaskNames = DictField[str, str](
+        doc="The names of the subtasks to extract metadata from. "
+        "If the metric name is identified as one of the keys, then "
+        "the corresponding value is taken as the subTask metadata "
+        "from which to extract metadata.",
         default=None,
         optional=True,
     )
@@ -52,13 +54,23 @@ class MetadataMetricTool(AnalysisTool):
         doc="The metrics to extract from the task metadata and their respective units."
     )
 
+    newNames = DictField[str, str](
+        doc="New names to allocate to the extracted metrics. Keys are the current "
+        "names, values are the new names.",
+        default=None,
+        optional=True,
+    )
+
     def finalize(self):
-        if self.subTaskName:
-            taskFullName = f"{self.taskName}:{self.subTaskName}"
-        else:
-            taskFullName = self.taskName
         for metric, unit in self.metrics.items():
+            if self.subTaskNames is not None and metric in self.subTaskNames:
+                taskFullName = f"{self.taskName}:{self.subTaskNames[metric]}"
+            else:
+                taskFullName = self.taskName
             setattr(
                 self.process.filterActions, f"{metric}", KeyedDataKeyAccessAction(topLevelKey=taskFullName)
             )
         self.produce.metric.units = dict(self.metrics.items())
+
+        if self.newNames is not None:
+            self.produce.metric.newNames = dict(self.newNames.items())