From f07fb4c2534dc26717a55212cefaf99a58b4d3b0 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Tue, 5 Nov 2024 15:50:26 -0500 Subject: [PATCH] Remove already-deprecated PropagateVisitFlagsTask. This also deprecates a connection in MeasureMergedCoaddSourcesTask that only exists for this task and should have been deprecated earlier. --- ...gateVisitFlags.PropagateVisitFlagsTask.rst | 26 -- python/lsst/pipe/tasks/multiBand.py | 134 +++------- python/lsst/pipe/tasks/propagateVisitFlags.py | 235 ------------------ 3 files changed, 39 insertions(+), 356 deletions(-) delete mode 100644 doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.propagateVisitFlags.PropagateVisitFlagsTask.rst delete mode 100644 python/lsst/pipe/tasks/propagateVisitFlags.py diff --git a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.propagateVisitFlags.PropagateVisitFlagsTask.rst b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.propagateVisitFlags.PropagateVisitFlagsTask.rst deleted file mode 100644 index 454526c11..000000000 --- a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.propagateVisitFlags.PropagateVisitFlagsTask.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. lsst-task-topic:: lsst.pipe.tasks.propagateVisitFlags.PropagateVisitFlagsTask - -####################### -PropagateVisitFlagsTask -####################### - -.. _lsst.pipe.tasks.propagateVisitFlags.PropagateVisitFlagsTask-api: - -Python API summary -================== - -.. lsst-task-api-summary:: lsst.pipe.tasks.propagateVisitFlags.PropagateVisitFlagsTask - -.. _lsst.pipe.tasks.propagateVisitFlags.PropagateVisitFlagsTask-subtasks: - -Retargetable subtasks -===================== - -.. lsst-task-config-subtasks:: lsst.pipe.tasks.propagateVisitFlags.PropagateVisitFlagsTask - -.. _lsst.pipe.tasks.propagateVisitFlags.PropagateVisitFlagsTask-configs: - -Configuration fields -==================== - -.. lsst-task-config-fields:: lsst.pipe.tasks.propagateVisitFlags.PropagateVisitFlagsTask diff --git a/python/lsst/pipe/tasks/multiBand.py b/python/lsst/pipe/tasks/multiBand.py index 55386fcee..acdf73557 100644 --- a/python/lsst/pipe/tasks/multiBand.py +++ b/python/lsst/pipe/tasks/multiBand.py @@ -256,14 +256,14 @@ class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections, storageClass="SkyMap", dimensions=("skymap",), ) + # TODO[DM-47424]: remove this deprecated connection. visitCatalogs = cT.Input( - doc="Source catalogs for visits which overlap input tract, patch, band. Will be " - "further filtered in the task for the purpose of propagating flags from image calibration " - "and characterization to coadd objects. Only used in legacy PropagateVisitFlagsTask.", + doc="Deprecated and unused.", name="src", dimensions=("instrument", "visit", "detector"), storageClass="SourceCatalog", - multiple=True + multiple=True, + deprecated="Deprecated and unused. Will be removed after v29.", ) sourceTableHandles = cT.Input( doc=("Source tables that are derived from the ``CalibrateTask`` sources. " @@ -329,38 +329,30 @@ class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections, def __init__(self, *, config=None): super().__init__(config=config) - if config.doPropagateFlags is False: - self.inputs -= set(("visitCatalogs",)) - self.inputs -= set(("sourceTableHandles",)) - self.inputs -= set(("finalizedSourceTableHandles",)) - elif config.propagateFlags.target == PropagateSourceFlagsTask: - # New PropagateSourceFlagsTask does not use visitCatalogs. - self.inputs -= set(("visitCatalogs",)) + del self.visitCatalogs + if not config.doPropagateFlags: + del self.sourceTableHandles + del self.finalizedSourceTableHandles + else: # Check for types of flags required. if not config.propagateFlags.source_flags: - self.inputs -= set(("sourceTableHandles",)) + del self.sourceTableHandles if not config.propagateFlags.finalized_source_flags: - self.inputs -= set(("finalizedSourceTableHandles",)) - else: - # Deprecated PropagateVisitFlagsTask uses visitCatalogs. - self.inputs -= set(("sourceTableHandles",)) - self.inputs -= set(("finalizedSourceTableHandles",)) - + del self.finalizedSourceTableHandles if config.inputCatalog == "deblendedCatalog": - self.inputs -= set(("inputCatalog",)) - + del self.inputCatalog if not config.doAddFootprints: - self.inputs -= set(("scarletModels",)) + del self.scarletModels else: - self.inputs -= set(("deblendedCatalog")) - self.inputs -= set(("scarletModels",)) + del self.deblendedCatalog + del self.scarletModels - if config.doMatchSources is False: - self.prerequisiteInputs -= set(("refCat",)) - self.outputs -= set(("matchResult",)) + if not config.doMatchSources: + del self.refCat + del self.matchResult - if config.doWriteMatchesDenormalized is False: - self.outputs -= set(("denormMatches",)) + if not config.doWriteMatchesDenormalized: + del self.denormMatches class MeasureMergedCoaddSourcesConfig(PipelineTaskConfig, @@ -601,52 +593,25 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): inputs['skyInfo'] = skyInfo if self.config.doPropagateFlags: - if self.config.propagateFlags.target == PropagateSourceFlagsTask: - # New version - ccdInputs = inputs["exposure"].getInfo().getCoaddInputs().ccds - inputs["ccdInputs"] = ccdInputs - - if "sourceTableHandles" in inputs: - sourceTableHandles = inputs.pop("sourceTableHandles") - sourceTableHandleDict = {handle.dataId["visit"]: handle - for handle in sourceTableHandles} - inputs["sourceTableHandleDict"] = sourceTableHandleDict - if "finalizedSourceTableHandles" in inputs: - finalizedSourceTableHandles = inputs.pop("finalizedSourceTableHandles") - finalizedSourceTableHandleDict = {handle.dataId["visit"]: handle - for handle in finalizedSourceTableHandles} - inputs["finalizedSourceTableHandleDict"] = finalizedSourceTableHandleDict - else: - # Deprecated legacy version - # Filter out any visit catalog that is not coadd inputs - ccdInputs = inputs['exposure'].getInfo().getCoaddInputs().ccds - visitKey = ccdInputs.schema.find("visit").key - ccdKey = ccdInputs.schema.find("ccd").key - inputVisitIds = set() - ccdRecordsWcs = {} - for ccdRecord in ccdInputs: - visit = ccdRecord.get(visitKey) - ccd = ccdRecord.get(ccdKey) - inputVisitIds.add((visit, ccd)) - ccdRecordsWcs[(visit, ccd)] = ccdRecord.getWcs() - - inputCatalogsToKeep = [] - inputCatalogWcsUpdate = [] - for i, dataRef in enumerate(inputRefs.visitCatalogs): - key = (dataRef.dataId['visit'], dataRef.dataId['detector']) - if key in inputVisitIds: - inputCatalogsToKeep.append(inputs['visitCatalogs'][i]) - inputCatalogWcsUpdate.append(ccdRecordsWcs[key]) - inputs['visitCatalogs'] = inputCatalogsToKeep - inputs['wcsUpdates'] = inputCatalogWcsUpdate - inputs['ccdInputs'] = ccdInputs + ccdInputs = inputs["exposure"].getInfo().getCoaddInputs().ccds + inputs["ccdInputs"] = ccdInputs + + if "sourceTableHandles" in inputs: + sourceTableHandles = inputs.pop("sourceTableHandles") + sourceTableHandleDict = {handle.dataId["visit"]: handle for handle in sourceTableHandles} + inputs["sourceTableHandleDict"] = sourceTableHandleDict + if "finalizedSourceTableHandles" in inputs: + finalizedSourceTableHandles = inputs.pop("finalizedSourceTableHandles") + finalizedSourceTableHandleDict = {handle.dataId["visit"]: handle + for handle in finalizedSourceTableHandles} + inputs["finalizedSourceTableHandleDict"] = finalizedSourceTableHandleDict outputs = self.run(**inputs) # Strip HeavyFootprints to save space on disk sources = outputs.outputSources butlerQC.put(outputs, outputRefs) - def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, visitCatalogs=None, wcsUpdates=None, + def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, sourceTableHandleDict=None, finalizedSourceTableHandleDict=None): """Run measurement algorithms on the input exposure, and optionally populate the resulting catalog with extra information. @@ -666,16 +631,6 @@ def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, visitCatal ccdInputs : `lsst.afw.table.ExposureCatalog`, optional Catalog containing information on the individual visits which went into making the coadd. - visitCatalogs : `list` of `lsst.afw.table.SourceCatalogs`, optional - A list of source catalogs corresponding to measurements made on the individual - visits which went into the input exposure. If None and butler is `None` then - the task cannot propagate visit flags to the output catalog. - Deprecated, to be removed with PropagateVisitFlagsTask. - wcsUpdates : `list` of `lsst.afw.geom.SkyWcs`, optional - If visitCatalogs is not `None` this should be a list of wcs objects which correspond - to the input visits. Used to put all coordinates to common system. If `None` and - butler is `None` then the task cannot propagate visit flags to the output catalog. - Deprecated, to be removed with PropagateVisitFlagsTask. sourceTableHandleDict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`], optional Dict for sourceTable_visit handles (key is visit) for propagating flags. These tables are derived from the ``CalibrateTask`` sources, and contain @@ -714,23 +669,12 @@ def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, visitCatal self.setPrimaryFlags.run(sources, skyMap=skyInfo.skyMap, tractInfo=skyInfo.tractInfo, patchInfo=skyInfo.patchInfo) if self.config.doPropagateFlags: - if self.config.propagateFlags.target == PropagateSourceFlagsTask: - # New version - self.propagateFlags.run( - sources, - ccdInputs, - sourceTableHandleDict, - finalizedSourceTableHandleDict - ) - else: - # Legacy deprecated version - self.propagateFlags.run( - sources, - ccdInputs, - exposure.getWcs(), - visitCatalogs, - wcsUpdates - ) + self.propagateFlags.run( + sources, + ccdInputs, + sourceTableHandleDict, + finalizedSourceTableHandleDict + ) results = Struct() diff --git a/python/lsst/pipe/tasks/propagateVisitFlags.py b/python/lsst/pipe/tasks/propagateVisitFlags.py deleted file mode 100644 index e42ef21f9..000000000 --- a/python/lsst/pipe/tasks/propagateVisitFlags.py +++ /dev/null @@ -1,235 +0,0 @@ -# This file is part of pipe_tasks. -# -# Developed for the LSST Data Management System. -# This product includes software developed by the LSST Project -# (https://www.lsst.org). -# See the COPYRIGHT file at the top-level directory of this distribution -# for details of code ownership. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -__all__ = ["PropagateVisitFlagsConfig", "PropagateVisitFlagsTask"] -import numpy -from lsst.pex.config import Config, Field, DictField -from lsst.pipe.base import Task -import lsst.geom as geom -import lsst.afw.table as afwTable -import lsst.pex.exceptions as pexExceptions -from deprecated.sphinx import deprecated - - -class PropagateVisitFlagsConfig(Config): - """Configuration for propagating flags to coadd.""" - - flags = DictField(keytype=str, itemtype=float, - default={"calib_psf_candidate": 0.2, "calib_psf_used": 0.2, "calib_psf_reserved": 0.2, - "calib_astrometry_used": 0.2, "calib_photometry_used": 0.2, - "calib_photometry_reserved": 0.2, }, - doc=("Source catalog flags to propagate, with the threshold of relative occurrence " - "(valid range: [0-1], default is 0.2). Coadd object will have flag set if the " - "fraction of input visits in which it is flagged is greater than the threshold.")) - matchRadius = Field(dtype=float, default=0.2, doc="Source matching radius (arcsec)") - ccdName = Field(dtype=str, default='ccd', doc="Name of ccd to give to butler") - - -@deprecated(reason="This task has been replaced with PropagateSourceFlagsTask", - version="v24.0", category=FutureWarning) -class PropagateVisitFlagsTask(Task): - """Task to propagate flags from single-frame measurements to coadd measurements. - - Parameters - ---------- - schema : `lsst.afw.table.Schema` - The input schema for the reference source catalog, used to initialize - the output schema. - **kwargs - Additional keyword arguments. - - Notes - ----- - We want to be able to set a flag for sources on the coadds using flags - that were determined from the individual visits. A common example is sources - that were used for PSF determination, since we do not do any PSF determination - on the coadd but use the individual visits. This requires matching the coadd - source catalog to each of the catalogs from the inputs (see - PropagateVisitFlagsConfig.matchRadius), and thresholding on the number of - times a source is flagged on the input catalog. - - An important consideration in this is that the flagging of sources in the - individual visits can be somewhat stochastic, e.g., the same stars may not - always be used for PSF determination because the field of view moves slightly - between visits, or the seeing changed. We there threshold on the relative - occurrence of the flag in the visits (see PropagateVisitFlagsConfig.flags). - Flagging a source that is always flagged in inputs corresponds to a threshold - of 1, while flagging a source that is flagged in any of the input corresponds - to a threshold of 0. But neither of these extrema are really useful in - practise. - - Setting the threshold too high means that sources that are not consistently - flagged (e.g., due to chip gaps) will not have the flag propagated. Setting - that threshold too low means that random sources which are falsely flagged in - the inputs will start to dominate. If in doubt, we suggest making this - threshold relatively low, but not zero (e.g., 0.1 to 0.2 or so). The more - confidence in the quality of the flagging, the lower the threshold can be. - The relative occurrence accounts for the edge of the field-of-view of the - camera, but does not include chip gaps, bad or saturated pixels, etc. - - Initialization - - Beyond the usual Task initialization, PropagateVisitFlagsTask also requires - a schema for the catalog that is being constructed. - - The 'run' method (described below) is the entry-point for operations. The - 'getCcdInputs' staticmethod is provided as a convenience for retrieving the - 'ccdInputs' (CCD inputs table) from an Exposure. - - .. code-block :: none - - # Requires: - # * butler: data butler, for retrieving the CCD catalogs - # * coaddCatalog: catalog of source measurements on the coadd (lsst.afw.table.SourceCatalog) - # * coaddExposure: coadd (lsst.afw.image.Exposure) - from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask, PropagateVisitFlagsConfig - config = PropagateVisitFlagsConfig() - config.flags["calib_psf_used"] = 0.3 # Relative threshold for this flag - config.matchRadius = 0.5 # Matching radius in arcsec - task = PropagateVisitFlagsTask(coaddCatalog.schema, config=config) - ccdInputs = task.getCcdInputs(coaddExposure) - task.run(butler, coaddCatalog, ccdInputs, coaddExposure.getWcs()) - """ - - ConfigClass = PropagateVisitFlagsConfig - - def __init__(self, schema, **kwargs): - Task.__init__(self, **kwargs) - self.schema = schema - self._keys = dict((f, self.schema.addField(f, type="Flag", doc="Propagated from visits")) for - f in self.config.flags) - - @staticmethod - def getCcdInputs(coaddExposure): - """Convenience method to retrieve the CCD inputs table from a coadd exposure. - - Parameters - ---------- - coaddExposure : `lsst.afw.image.Exposure` - The exposure we need to retrieve the CCD inputs table from. - - Returns - ------- - ccdInputs : `` - CCD inputs table from a coadd exposure. - """ - return coaddExposure.getInfo().getCoaddInputs().ccds - - def run(self, butler, coaddSources, ccdInputs, coaddWcs, visitCatalogs=None, wcsUpdates=None): - """Propagate flags from individual visit measurements to coadd. - - This requires matching the coadd source catalog to each of the catalogs - from the inputs, and thresholding on the number of times a source is - flagged on the input catalog. The threshold is made on the relative - occurrence of the flag in each source. Flagging a source that is always - flagged in inputs corresponds to a threshold of 1, while flagging a - source that is flagged in any of the input corresponds to a threshold of - 0. But neither of these extrema are really useful in practise. - - Setting the threshold too high means that sources that are not consistently - flagged (e.g., due to chip gaps) will not have the flag propagated. Setting - that threshold too low means that random sources which are falsely flagged in - the inputs will start to dominate. If in doubt, we suggest making this threshold - relatively low, but not zero (e.g., 0.1 to 0.2 or so). The more confidence in - the quality of the flagging, the lower the threshold can be. - - The relative occurrence accounts for the edge of the field-of-view of - the camera, but does not include chip gaps, bad or saturated pixels, etc. - - Parameters - ---------- - butler : `Unknown` - Data butler, for retrieving the input source catalogs. - coaddSources : `lsst.afw.image.SourceCatalog` - Source catalog from the coadd. - ccdInputs : `lsst.afw.table.ExposureCatalog` - Table of CCDs that contribute to the coadd. - coaddWcs : `lsst.afw.geom.SkyWcs` - Wcs for coadd. - visitCatalogs : `list` of `lsst.afw.image.SourceCatalog`, optional - List of loaded source catalogs for each input ccd in - the coadd. If provided this is used instead of this - method loading in the catalogs itself. - wcsUpdates : `list` of `lsst.afw.geom.SkyWcs`, optional - If visitCatalogs is a list of ccd catalogs, this - should be a list of updated wcs to apply. - - Raises - ------ - ValueError - Raised if any of the following occur: - - A list of wcs updates for each catalog is not supplied in the wcsUpdates parameter - and ccdInputs is a list of src catalogs. - - The visitCatalogs and ccdInput parameters are both `None`. - """ - if len(self.config.flags) == 0: - return - - flags = self._keys.keys() - counts = dict((f, numpy.zeros(len(coaddSources), dtype=int)) for f in flags) - indices = numpy.array([s.getId() for s in coaddSources]) # Allowing for non-contiguous data - radius = self.config.matchRadius*geom.arcseconds - - def processCcd(ccdSources, wcsUpdate): - for sourceRecord in ccdSources: - sourceRecord.updateCoord(wcsUpdate) - for flag in flags: - # We assume that the flags will be relatively rare, so it is more efficient to match - # against a subset of the input catalog for each flag than it is to match once against - # the entire catalog. It would be best to have built a kd-tree on coaddSources and - # keep reusing that for the matching, but we don't have a suitable implementation. - mc = afwTable.MatchControl() - mc.findOnlyClosest = False - matches = afwTable.matchRaDec(coaddSources, ccdSources[ccdSources.get(flag)], radius, mc) - for m in matches: - index = (numpy.where(indices == m.first.getId()))[0][0] - counts[flag][index] += 1 - - if visitCatalogs is not None: - if wcsUpdates is None: - raise pexExceptions.ValueError("If ccdInputs is a list of src catalogs, a list of wcs" - " updates for each catalog must be supplied in the " - "wcsUpdates parameter") - for i, ccdSource in enumerate(visitCatalogs): - processCcd(ccdSource, wcsUpdates[i]) - else: - if ccdInputs is None: - raise pexExceptions.ValueError("The visitCatalogs and ccdInput parameters can't both be None") - visitKey = ccdInputs.schema.find("visit").key - ccdKey = ccdInputs.schema.find("ccd").key - - self.log.info("Propagating flags %s from inputs", flags) - - # Accumulate counts of flags being set - for ccdRecord in ccdInputs: - v = ccdRecord.get(visitKey) - c = ccdRecord.get(ccdKey) - dataId = {"visit": int(v), self.config.ccdName: int(c)} - ccdSources = butler.get("src", dataId=dataId) - processCcd(ccdSources, ccdRecord.getWcs()) - - # Apply threshold - for f in flags: - key = self._keys[f] - for s, num in zip(coaddSources, counts[f]): - numOverlaps = len(ccdInputs.subsetContaining(s.getCentroid(), coaddWcs, True)) - s.setFlag(key, bool(num > numOverlaps*self.config.flags[f])) - self.log.info("Propagated %d sources with flag %s", sum(s.get(key) for s in coaddSources), f)