Fix processing.

pipelines/production/comCamRefitWcsUSDF.yaml

@@ -0,0 +1,31 @@
+description: RefitWCS pipeline for ComCam
+instrument: lsst.obs.lsst.LsstComCam
+tasks:
+  generateDonutFromRefitWcsTask:
+    class: lsst.ts.wep.task.generateDonutFromRefitWcsTask.GenerateDonutFromRefitWcsTask
+    config:
+      connections.astromRefCat: "gaia_dr3_20230707"
+      connections.photoRefCat: "the_monster_20240904"
+      connections.exposure: "postISRCCD"
+      connections.outputExposure: "postFitPostISRCCD"
+      connections.fitDonutCatalog: "donutTable"
+      connections.donutCatalog: "refitWcsDonutTable"
+      donutSelector.useCustomMagLimit: True
+      astromRefFilter: "phot_g_mean"
+      photoRefFilterPrefix: "monster_SynthLSST"
+      astromTask.referenceSelector.doMagLimit: False
+      astromTask.maxIter: 3
+      astromTask.matcher.maxOffsetPix: 2000
+      astromTask.matcher.maxRotationDeg: 3.0
+      failTask: True
+      # donutSelector.unblendedSeparation: 1
+  cutOutDonutsScienceSensorGroupTask:
+    class: lsst.ts.wep.task.cutOutDonutsScienceSensorTask.CutOutDonutsScienceSensorTask
+    config:
+      connections.exposures: "postFitPostISRCCD"
+      connections.donutCatalog: "refitWcsDonutTable"
+      python: |
+        from lsst.ts.wep.task.pairTask import GroupPairer
+        config.pairer.retarget(GroupPairer)
+      donutStampSize: 200
+      initialCutoutPadding: 40

pipelines/production/comCamRefitWcsUSDF_Danish.yaml

@@ -0,0 +1,50 @@
+description: Run WEP + DonutViz daily at USDF
+instrument: lsst.obs.lsst.LsstComCam
+imports:
+  - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml
+  - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml
+
+tasks:
+  calcZernikesTask:
+    class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask
+    config:
+      python: |
+        from lsst.ts.wep.task import EstimateZernikesDanishTask
+        config.estimateZernikes.retarget(EstimateZernikesDanishTask)
+      donutStampSelector.maxFracBadPixels: 2.0e-4
+      estimateZernikes.binning: 2
+      estimateZernikes.nollIndices:
+        [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 21, 22, 27, 28]
+      estimateZernikes.saveHistory: true
+      estimateZernikes.lstsqKwargs:
+        ftol: 1.0e-3
+        xtol: 1.0e-3
+        gtol: 1.0e-3
+      donutStampSelector.maxSelect: -1
+  aggregateDonutTablesGroupTask:
+    class: lsst.donut.viz.AggregateDonutTablesTask
+    config:
+        python: |
+            from lsst.ts.wep.task.pairTask import GroupPairer
+            config.pairer.retarget(GroupPairer)
+        connections.donutTables: "refitWcsDonutTable"
+
+# Define pipeline steps
+subsets:
+  step1b:
+    subset:
+      - calcZernikesTask
+    description: |
+      This step runs the Zernike calculation with danish.
+  step2:
+    subset:
+      - aggregateZernikeTablesTask
+      - aggregateDonutTablesGroupTask
+      - aggregateAOSVisitTableTask
+      - plotAOSTask
+      - aggregateDonutStampsTask
+      - plotDonutTask
+    description: |
+      AOS Donut visualization plotting tasks. This step generates plots
+      (including the pyramid residual and donut gallery) and
+      tables for the AOS visit.

pipelines/production/comCamRefitWcsUSDF_TIE.yaml

@@ -0,0 +1,47 @@
+description: Run WEP + DonutViz daily at USDF
+instrument: lsst.obs.lsst.LsstComCam
+imports:
+  - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml
+  - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml
+
+tasks:
+  calcZernikesTask:
+    class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask
+    config:
+      estimateZernikes.nollIndices: [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 18, 19, 20, 21, 22, 27, 28]
+      estimateZernikes.convergeTol: 10.0e-9
+      estimateZernikes.compGain: 0.75
+      estimateZernikes.compSequence: [4, 4, 6, 6, 13, 13, 13, 13]
+      estimateZernikes.maxIter: 50
+      estimateZernikes.requireConverge: True
+      estimateZernikes.saveHistory: True
+      estimateZernikes.maskKwargs: { "doMaskBlends": False }
+      donutStampSelector.maxFracBadPixels: 2.0e-4
+      donutStampSelector.maxSelect: -1
+  aggregateDonutTablesGroupTask:
+    class: lsst.donut.viz.AggregateDonutTablesTask
+    config:
+        python: |
+            from lsst.ts.wep.task.pairTask import GroupPairer
+            config.pairer.retarget(GroupPairer)
+        connections.donutTables: "refitWcsDonutTable"
+
+# Define pipeline steps
+subsets:
+  step1b:
+    subset:
+      - calcZernikesTask
+    description: |
+      This step runs the Zernike calculation with danish.
+  step2:
+    subset:
+      - aggregateZernikeTablesTask
+      - aggregateDonutTablesGroupTask
+      - aggregateAOSVisitTableTask
+      - plotAOSTask
+      - aggregateDonutStampsTask
+      - plotDonutTask
+    description: |
+      AOS Donut visualization plotting tasks. This step generates plots
+      (including the pyramid residual and donut gallery) and
+      tables for the AOS visit.

python/lsst/ts/wep/task/cutOutDonutsBase.py

@@ -695,9 +695,10 @@ def cutOutStamps(self, exposure, donutCatalog, defocalType, cameraName):
         stampsMetadata = PropertyList()
         stampsMetadata["VISIT"] = np.array([visitId] * catalogLength, dtype=int)
         # Save the donut flux as magnitude
-        if len(donutCatalog["source_flux"]) > 0:
+        fluxLabel = next((colName for colName in donutCatalog.columns if colName.endswith(f'{bandLabel}_flux')), None)
+        if fluxLabel is not None and len(donutCatalog[fluxLabel]) > 0:
             stampsMetadata["MAG"] = (
-                donutCatalog["source_flux"].value * u.nJy
+                donutCatalog[fluxLabel].value * u.nJy
             ).to_value(u.ABmag)
         else:
             stampsMetadata["MAG"] = np.array([])

python/lsst/ts/wep/task/generateDonutCatalogUtils.py

@@ -85,7 +85,11 @@ def runSelection(refObjLoader, detector, wcs, filterName, donutSelectorTask):
 
 
 def donutCatalogToAstropy(
-    donutCatalog=None, filterName=None, blendCentersX=None, blendCentersY=None
+    donutCatalog=None,
+    filterName=None,
+    blendCentersX=None,
+    blendCentersY=None,
+    sortFilterIdx=0,
 ):
     """
     Reformat afwCatalog into an astropy QTable sorted by flux with
@@ -98,19 +102,22 @@ def donutCatalogToAstropy(
         ReferenceObjectLoader search over the detector footprint.
         If None then it will return an empty QTable.
         (the default is None.)
-    filterName : `str` or `None`, optional
-        Name of camera filter. If donutCatalog is not None then
+    filterName : `list` or `None`, optional
+        Name of catalog flux filters. If donutCatalog is not None then
         this cannot be None. (the default is None.)
     blendCentersX : `list` or `None`, optional
-            X pixel position of centroids for blended objects. List
-            should be the same length as the donutCatalog. If
-            blendCentersY is not None then this cannot be None. (the default
-            is None.)
+        X pixel position of centroids for blended objects. List
+        should be the same length as the donutCatalog. If
+        blendCentersY is not None then this cannot be None. (the default
+        is None.)
     blendCentersY : `list` or `None`, optional
-            Y pixel position of centroids for blended objects. List
-            should be the same length as the donutCatalog. If
-            blendCentersX is not None then this cannot be None. (the default
-            is None.)
+        Y pixel position of centroids for blended objects. List
+        should be the same length as the donutCatalog. If
+        blendCentersX is not None then this cannot be None. (the default
+        is None.)
+    sortFilterIdx : int, optional
+        Index for which filter in filterName to sort the entire catalog
+        by brightness. (the default is 0.)
 
     Returns
     -------
@@ -144,12 +151,12 @@ def donutCatalogToAstropy(
         dec = donutCatalog["coord_dec"]
         centroidX = donutCatalog["centroid_x"]
         centroidY = donutCatalog["centroid_y"]
-        sourceFlux = donutCatalog[f"{filterName}_flux"]
+        sourceFlux = [donutCatalog[f"{fName}_flux"] for fName in filterName]
 
         if (blendCentersX is None) and (blendCentersY is None):
             blendCX = list()
             blendCY = list()
-            for idx in range(len(donutCatalog)):
+            for _ in range(len(donutCatalog)):
                 blendCX.append(list())
                 blendCY.append(list())
         elif isinstance(blendCentersX, list) and isinstance(blendCentersY, list):
@@ -178,16 +185,17 @@ def donutCatalogToAstropy(
             )
             raise ValueError(blendErrMsg)
 
-    flux_sort = np.argsort(sourceFlux)[::-1]
+    flux_sort = np.argsort(sourceFlux[sortFilterIdx])[::-1]
 
     fieldObjects = QTable()
     fieldObjects["coord_ra"] = ra * u.rad
     fieldObjects["coord_dec"] = dec * u.rad
     fieldObjects["centroid_x"] = centroidX
     fieldObjects["centroid_y"] = centroidY
-    fieldObjects["source_flux"] = sourceFlux * u.nJy
+    for idx in range(len(filterName)):
+        fieldObjects[f"{filterName[idx]}_flux"] = sourceFlux[idx] * u.nJy
 
-    fieldObjects.sort("source_flux", reverse=True)
+    fieldObjects.sort(f"{filterName[sortFilterIdx]}_flux", reverse=True)
 
     fieldObjects.meta["blend_centroid_x"] = [blendCX[idx] for idx in flux_sort]
     fieldObjects.meta["blend_centroid_y"] = [blendCY[idx] for idx in flux_sort]

python/lsst/ts/wep/task/generateDonutFromRefitWcsTask.py

@@ -134,10 +134,25 @@ class GenerateDonutFromRefitWcsTaskConfig(
         default="phot_g_mean",
     )
     photoRefFilter = pexConfig.Field(
-        doc="Set filter to use in Photometry catalog. If not set will try to use exposure filter.",
+        doc="Set filter to use in Photometry catalog. "
+        + "Cannot set both this and photoRefFilter. "
+        + "If neither is set then will just try to use the name of the exposure filter.",
         dtype=str,
         optional=True,
     )
+    photoRefFilterPrefix = pexConfig.Field(
+        doc="Set filter prefix to use. "
+        + "Will then try to use exposure band label with given catalog prefix. "
+        + "Cannot set both this and photoRefFilter. "
+        + "If neither is set then will just try to use the name of the exposure filter.",
+        dtype=str,
+        optional=True,
+    )
+    failTask = pexConfig.Field(
+        doc="Fail if error raised.",
+        dtype=bool,
+        default=False,
+    )
 
     # Took these defaults from atmospec/centroiding which I used
     # as a template for implementing WCS fitting in a task.
@@ -302,12 +317,14 @@ def run(
         # fitting successfully or not. This will
         # give us information on our donut catalog output.
         self.metadata["wcsFitSuccess"] = False
+        self.metadata["refCatalogSuccess"] = False
         try:
             astromResult = self.astromTask.run(
                 sourceCat=afwCat,
                 exposure=exposure,
             )
             scatter = astromResult.scatterOnSky.asArcseconds()
+            print(scatter, self.config.maxFitScatter)
             if scatter < self.config.maxFitScatter:
                 successfulFit = True
                 self.metadata["wcsFitSuccess"] = True
@@ -321,15 +338,17 @@ def run(
             # AttributeError: 'NoneType' object has no attribute 'asArcseconds'
             # when the result is a failure as the wcs is set to None on failure
             self.log.warning(f"Solving for WCS failed: {e}")
-            # this is set to None when the fit fails, so restore it
-            exposure.setWcs(originalWcs)
-            donutCatalog = fitDonutCatalog
-            self.log.warning(
-                "Returning original exposure and WCS \
-and direct detect catalog as output."
-            )
+            if self.config.failTask:
+                raise TaskError("Failing task due to wcs fit failure.")
+            else:
+                # this is set to None when the fit fails, so restore it
+                exposure.setWcs(originalWcs)
+                donutCatalog = fitDonutCatalog
+                self.log.warning(
+                    "Returning original exposure and WCS \
+    and direct detect catalog as output."
+                )
 
-        self.metadata["refCatalogSuccess"] = False
         if successfulFit:
             photoRefObjLoader = ReferenceObjectLoader(
                 dataIds=[ref.dataId for ref in photoRefCat],
@@ -351,26 +370,32 @@ def run(
                 )
 
             # Check that specified filter exists in catalogs
+            if (
+                self.config.photoRefFilter is not None
+                and self.config.photoRefFilterPrefix is not None
+            ):
+                raise ValueError(
+                    "photoRefFilter and photoRefFilterConfig cannot both be set."
+                )
             if self.config.photoRefFilter is not None:
-                if (
-                    f"{self.config.photoRefFilter}_flux"
-                    not in photoRefCat[0].get().schema
-                ):
-                    filterFailMsg = (
-                        "Photometric Reference Catalog does not contain "
-                        + f"photoRefFilter: {self.config.photoRefFilter}"
-                    )
-                    self.log.warning(filterFailMsg)
-                    donutCatalog = fitDonutCatalog
-                    self.log.warning(catCreateErrorMsg)
-                    return pipeBase.Struct(
-                        outputExposure=exposure, donutCatalog=donutCatalog
-                    )
-                else:
-                    photoRefObjLoader.config.anyFilterMapsToThis = (
-                        self.config.photoRefFilter
-                    )
-                    filterName = self.config.photoRefFilter
+                filterName = self.config.photoRefFilter
+            elif self.config.photoRefFilterPrefix is not None:
+                filterName = (
+                    f"{self.config.photoRefFilterPrefix}_{exposure.filter.bandLabel}"
+                )
+
+            # Test that given filter name exists in catalog.
+            if f"{filterName}_flux" not in photoRefCat[0].get().schema:
+                filterFailMsg = (
+                    "Photometric Reference Catalog does not contain "
+                    + f"photoRefFilter: {filterName}"
+                )
+                self.log.warning(filterFailMsg)
+                donutCatalog = fitDonutCatalog
+                self.log.warning(catCreateErrorMsg)
+                return pipeBase.Struct(
+                    outputExposure=exposure, donutCatalog=donutCatalog
+                )
 
             try:
                 # Match detector layout to reference catalog
@@ -386,8 +411,16 @@ def run(
                     donutSelectorTask,
                 )
 
+                filterName = [
+                    f"{self.config.photoRefFilterPrefix}_{band}"
+                    for band in ["u", "g", "r", "i", "z", "y"]
+                ]
                 donutCatalog = donutCatalogToAstropy(
-                    refSelection, filterName, blendCentersX, blendCentersY
+                    refSelection,
+                    filterName,
+                    blendCentersX,
+                    blendCentersY,
+                    sortFilterIdx=2,
                 )
                 self.metadata["refCatalogSuccess"] = True
             # Except RuntimeError caused when no reference catalog