Merge pull request #1012 from lsst/tickets/DM-47796

DM-47796: Deprecate some coadd-processing connections and configs.

python/lsst/pipe/tasks/deblendCoaddSourcesPipeline.py

@@ -24,6 +24,8 @@
 
 import numpy as np
 
+from deprecated.sphinx import deprecated
+
 from lsst.pipe.base import (Struct, PipelineTask, PipelineTaskConfig, PipelineTaskConnections)
 import lsst.pipe.base.connectionTypes as cT
 
@@ -169,6 +171,13 @@ class DeblendCoaddSourcesMultiConfig(PipelineTaskConfig,
     idGenerator = SkyMapIdGeneratorConfig.make_field()
 
 
+# TODO[DM-47797] Remove this task.
+@deprecated(
+    "Support for the old single-band deblender on coadds will be removed "
+    "after v29.",
+    version="v29",
+    category=FutureWarning
+)
 class DeblendCoaddSourcesBaseTask(PipelineTask):
     def __init__(self, initInputs, **kwargs):
         super().__init__(initInputs=initInputs, **kwargs)
@@ -196,6 +205,13 @@ def _makeSourceCatalog(self, mergedDetections, idFactory):
         return sources
 
 
+# TODO[DM-47797] Remove this task, its connections, and its config.
+@deprecated(
+    "Support for the old single-band deblender on coadds will be removed "
+    "after v29.",
+    version="v29",
+    category=FutureWarning
+)
 class DeblendCoaddSourcesSingleTask(DeblendCoaddSourcesBaseTask):
     ConfigClass = DeblendCoaddSourcesSingleConfig
     _DefaultName = "deblendCoaddSourcesSingle"
@@ -213,12 +229,17 @@ def run(self, coadd, mergedDetections, idFactory):
         return Struct(measureCatalog=sources)
 
 
-class DeblendCoaddSourcesMultiTask(DeblendCoaddSourcesBaseTask):
+class DeblendCoaddSourcesMultiTask(PipelineTask):
     ConfigClass = DeblendCoaddSourcesMultiConfig
     _DefaultName = "deblendCoaddSourcesMulti"
 
     def __init__(self, initInputs, **kwargs):
         super().__init__(initInputs=initInputs, **kwargs)
+        schema = initInputs["inputSchema"].schema
+        self.peakSchema = initInputs["peakSchema"].schema
+        self.schemaMapper = afwTable.SchemaMapper(schema)
+        self.schemaMapper.addMinimalSchema(schema)
+        self.schema = self.schemaMapper.getOutputSchema()
         self.makeSubtask("multibandDeblend", schema=self.schema, peakSchema=self.peakSchema)
         self.outputSchema = afwTable.SourceCatalog(self.schema)
 
@@ -240,3 +261,14 @@ def run(self, coadds, filters, mergedDetections, idFactory):
         catalog, modelData = self.multibandDeblend.run(multiExposure, sources)
         retStruct = Struct(deblendedCatalog=catalog, scarletModelData=modelData)
         return retStruct
+
+    def _makeSourceCatalog(self, mergedDetections, idFactory):
+        # There may be gaps in the mergeDet catalog, which will cause the
+        # source ids to be inconsistent. So we update the id factory
+        # with the largest id already in the catalog.
+        maxId = np.max(mergedDetections["id"])
+        idFactory.notify(maxId)
+        table = afwTable.SourceTable.make(self.schema, idFactory)
+        sources = afwTable.SourceCatalog(table)
+        sources.extend(mergedDetections, self.schemaMapper)
+        return sources

python/lsst/pipe/tasks/multiBand.py

@@ -221,11 +221,19 @@ def run(self, exposure, idFactory, expId):
         return Struct(outputSources=sources, outputBackgrounds=backgrounds, outputExposure=exposure)
 
 
-class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections,
-                                           dimensions=("tract", "patch", "band", "skymap"),
-                                           defaultTemplates={"inputCoaddName": "deep",
-                                                             "outputCoaddName": "deep",
-                                                             "deblendedCatalog": "deblendedFlux"}):
+class MeasureMergedCoaddSourcesConnections(
+    PipelineTaskConnections,
+    dimensions=("tract", "patch", "band", "skymap"),
+    defaultTemplates={
+        "inputCoaddName": "deep",
+        "outputCoaddName": "deep",
+        "deblendedCatalog": "deblendedFlux",
+    },
+    deprecatedTemplates={
+        # TODO[DM-47797]: remove this deprecated connection template.
+        "deblendedCatalog": "Support for old deblender outputs will be removed after v29."
+    },
+):
     inputSchema = cT.InitInput(
         doc="Input schema for measure merged task produced by a deblender or detection task",
         name="{inputCoaddName}Coadd_deblendedFlux_schema",
@@ -236,13 +244,15 @@ class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections,
         name="{inputCoaddName}Coadd_meas_schema",
         storageClass="SourceCatalog"
     )
+    # TODO[DM-47797]: remove this deprecated connection.
     refCat = cT.PrerequisiteInput(
         doc="Reference catalog used to match measured sources against known sources",
         name="ref_cat",
         storageClass="SimpleCatalog",
         dimensions=("skypix",),
         deferLoad=True,
-        multiple=True
+        multiple=True,
+        deprecated="Reference matching in measureCoaddSources will be removed after v29.",
     )
     exposure = cT.Input(
         doc="Input coadd image",
@@ -284,6 +294,7 @@ class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections,
         multiple=True,
         deferLoad=True,
     )
+    # TODO[DM-47797]: remove this deprecated connection.
     inputCatalog = cT.Input(
         doc=("Name of the input catalog to use."
              "If the single band deblender was used this should be 'deblendedFlux."
@@ -293,6 +304,7 @@ class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections,
              "be 'mergeDet'"),
         name="{inputCoaddName}Coadd_{deblendedCatalog}",
         storageClass="SourceCatalog",
+        deprecated="Support for old deblender outputs will be removed after v29.",
         dimensions=("tract", "patch", "band", "skymap"),
     )
     scarletCatalog = cT.Input(
@@ -313,18 +325,22 @@ class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections,
         dimensions=("tract", "patch", "band", "skymap"),
         storageClass="SourceCatalog",
     )
+    # TODO[DM-47797]: remove this deprecated connection.
     matchResult = cT.Output(
         doc="Match catalog produced by configured matcher, optional on doMatchSources",
         name="{outputCoaddName}Coadd_measMatch",
         dimensions=("tract", "patch", "band", "skymap"),
         storageClass="Catalog",
+        deprecated="Reference matching in measureCoaddSources will be removed after v29.",
     )
+    # TODO[DM-47797]: remove this deprecated connection.
     denormMatches = cT.Output(
         doc="Denormalized Match catalog produced by configured matcher, optional on "
             "doWriteMatchesDenormalized",
         name="{outputCoaddName}Coadd_measMatchFull",
         dimensions=("tract", "patch", "band", "skymap"),
         storageClass="Catalog",
+        deprecated="Reference matching in measureCoaddSources will be removed after v29.",
     )
 
     def __init__(self, *, config=None):
@@ -339,6 +355,7 @@ def __init__(self, *, config=None):
                 del self.sourceTableHandles
             if not config.propagateFlags.finalized_source_flags:
                 del self.finalizedSourceTableHandles
+        # TODO[DM-47797]: only the 'if' block contents here should survive.
         if config.inputCatalog == "deblendedCatalog":
             del self.inputCatalog
             if not config.doAddFootprints:
@@ -347,6 +364,7 @@ def __init__(self, *, config=None):
             del self.deblendedCatalog
             del self.scarletModels
 
+        # TODO[DM-47797]: delete the conditionals below.
         if not config.doMatchSources:
             del self.refCat
             del self.matchResult
@@ -368,6 +386,8 @@ class MeasureMergedCoaddSourcesConfig(PipelineTaskConfig,
             "mergeDet": "The merged detections before deblending."
         },
         doc="The name of the input catalog.",
+        # TODO[DM-47797]: remove this config option and anything using it.
+        deprecated="Support for old deblender outputs will be removed after v29.",
     )
     doAddFootprints = Field(dtype=bool,
                             default=True,
@@ -389,13 +409,23 @@ class MeasureMergedCoaddSourcesConfig(PipelineTaskConfig,
         doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
     )
     propagateFlags = ConfigurableField(target=PropagateSourceFlagsTask, doc="Propagate source flags to coadd")
-    doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
-    match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
+    doMatchSources = Field(
+        dtype=bool,
+        default=False,
+        doc="Match sources to reference catalog?",
+        deprecated="Reference matching in measureCoaddSources will be removed after v29.",
+    )
+    match = ConfigurableField(
+        target=DirectMatchTask,
+        doc="Matching to reference catalog",
+        deprecated="Reference matching in measureCoaddSources will be removed after v29.",
+    )
     doWriteMatchesDenormalized = Field(
         dtype=bool,
         default=False,
         doc=("Write reference matches in denormalized format? "
              "This format uses more disk space, but is more convenient to read."),
+        deprecated="Reference matching in measureCoaddSources will be removed after v29.",
     )
     coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
     psfCache = Field(dtype=int, default=100, doc="Size of psfCache")
@@ -516,6 +546,7 @@ def __init__(self, schema=None, peakSchema=None, refObjLoader=None, initInputs=N
         self.algMetadata = PropertyList()
         self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
         self.makeSubtask("setPrimaryFlags", schema=self.schema)
+        # TODO[DM-47797]: remove match subtask
         if self.config.doMatchSources:
             self.makeSubtask("match", refObjLoader=refObjLoader)
         if self.config.doPropagateFlags:
@@ -531,6 +562,7 @@ def __init__(self, schema=None, peakSchema=None, refObjLoader=None, initInputs=N
     def runQuantum(self, butlerQC, inputRefs, outputRefs):
         inputs = butlerQC.get(inputRefs)
 
+        # TODO[DM-47797]: remove this block
         if self.config.doMatchSources:
             refObjLoader = ReferenceObjectLoader([ref.datasetRef.dataId for ref in inputRefs.refCat],
                                                  inputs.pop('refCat'),
@@ -678,6 +710,7 @@ def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None,
 
         results = Struct()
 
+        # TODO[DM-47797]: remove this block
         if self.config.doMatchSources:
             matchResult = self.match.run(sources, exposure.getInfo().getFilter().bandLabel)
             matches = afwTable.packMatches(matchResult.matches)