Merge pull request #183 from lsst/tickets/DM-46991

DM-46991: switch to calibrateImage
lsst · Nov 26, 2024 · ef1aa0a · ef1aa0a
2 parents 869d593 + 65bac0b
commit ef1aa0a
Show file tree

Hide file tree

Showing 21 changed files with 156 additions and 164 deletions.
diff --git a/bps/clustering/DRP-recalibrated.yaml b/bps/clustering/DRP-recalibrated.yaml
@@ -12,7 +12,7 @@
 clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering
 cluster:
   visit_detector:
-    pipetasks: isr,inject_exposure,characterizeImage,calibrate,inject_visit,writePreSourceTable,transformPreSourceTable
+    pipetasks: isr,inject_exposure,calibrateImage,inject_visit,transformPreSourceTable
     dimensions: visit,detector
     equalDimensions: visit:exposure
 

diff --git a/bps/clustering/LSSTCam-imSim/DRP-DC2-clustering.yaml b/bps/clustering/LSSTCam-imSim/DRP-DC2-clustering.yaml
@@ -11,7 +11,7 @@
 clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering
 cluster:
   visit_detector:
-    pipetasks: isr,inject_exposure,characterizeImage,calibrate,inject_visit,writeSourceTable,transformSourceTable
+    pipetasks: isr,inject_exposure,calibrateImage,inject_visit,transformSourceTable
     dimensions: visit,detector
     equalDimensions: visit:exposure
 

diff --git a/pipelines/HSC/DRP-Prod.yaml b/pipelines/HSC/DRP-Prod.yaml
@@ -16,9 +16,7 @@ subsets:
   step1:
     subset:
       - isr
-      - characterizeImage
-      - calibrate
-      - writePreSourceTable
+      - calibrateImage
       - transformPreSourceTable
       - analyzeAmpOffsetMetadata
     description: |

diff --git a/pipelines/HSC/DRP-RC2.yaml b/pipelines/HSC/DRP-RC2.yaml
@@ -81,12 +81,10 @@ subsets:
   step1:
     subset:
       - isr
-      - characterizeImage
-      - calibrate
-      - writePreSourceTable
+      - calibrateImage
       - transformPreSourceTable
       - analyzeAmpOffsetMetadata
-      - analyzeCalibrateMetadata
+      - analyzeCalibrateImageMetadata
     description: |
       Per-detector tasks that can be run together to start the DRP pipeline.
 

diff --git a/pipelines/HSC/DRP-RC2_subset.yaml b/pipelines/HSC/DRP-RC2_subset.yaml
@@ -26,8 +26,7 @@ subsets:
   simpleSingleFrame:
     subset:
       - isr
-      - characterizeImage
-      - calibrate
+      - calibrateImage
     description: |
        A simplified single frame pipeline to make the quantum graph more readable.
   multiVisit:
@@ -39,9 +38,7 @@ subsets:
   singleFrame:
     subset:
       - isr
-      - characterizeImage
-      - calibrate
-      - writePreSourceTable
+      - calibrateImage
       - transformPreSourceTable
       - consolidatePreSourceTable
       - consolidateVisitSummary
@@ -81,9 +78,7 @@ subsets:
   nightlyStep1:
     subset:
       - isr
-      - characterizeImage
-      - calibrate
-      - writePreSourceTable
+      - calibrateImage
       - transformPreSourceTable
     description: |
       Per-detector tasks that can be run together to start the DRP pipeline.

diff --git a/pipelines/HSC/DRP-ci_hsc.yaml b/pipelines/HSC/DRP-ci_hsc.yaml
@@ -32,13 +32,13 @@ tasks:
       multibandDeblend.useCiLimits: true
       # Do not run the deblender on isolated sources
       multibandDeblend.processSingles: false
-  calibrate:
-    class: lsst.pipe.tasks.calibrate.CalibrateTask
+  calibrateImage:
+    class: lsst.pipe.tasks.calibrateImage.CalibrateImageTask
     # Significantly limit the number of sources to deblend, which saves time
     # in both deblending and measurement
     config:
       # Only run the deblender on a small subset of blended parents
-      deblend.useCiLimits: true
+      star_deblend.useCiLimits: true
   writeRecalibratedSourceTable:
     class: lsst.pipe.tasks.postprocess.WriteRecalibratedSourceTableTask
     config:

diff --git a/pipelines/LATISS/DRP.yaml b/pipelines/LATISS/DRP.yaml
@@ -119,9 +119,7 @@ subsets:
   step1:
     subset:
       - isr
-      - characterizeImage
-      - calibrate
-      - writePreSourceTable
+      - calibrateImage
       - transformPreSourceTable
     description: |
       Per-detector tasks that can be run together to start the DRP pipeline.

diff --git a/pipelines/LSSTCam-imSim/DRP-ci_imsim.yaml b/pipelines/LSSTCam-imSim/DRP-ci_imsim.yaml
@@ -18,13 +18,13 @@ tasks:
       multibandDeblend.useCiLimits: true
       # Do not run the deblender on isolated sources
       multibandDeblend.processSingles: false
-  calibrate:
-    class: lsst.pipe.tasks.calibrate.CalibrateTask
+  calibrateImage:
+    class: lsst.pipe.tasks.calibrateImage.CalibrateImageTask
     # Significantly limit the number of sources to deblend, which saves time
     # in both deblending and measurement
     config:
       # Only run the deblender on a small subset of blended parents
-      deblend.useCiLimits: true
+      star_deblend.useCiLimits: true
   catalogMatchVisit:
     class: lsst.analysis.tools.tasks.astrometricCatalogMatch.AstrometricCatalogMatchVisitTask
     config:

diff --git a/pipelines/LSSTCam/quickLook.yaml b/pipelines/LSSTCam/quickLook.yaml
@@ -22,47 +22,14 @@ tasks:
       doDefect: false
       doDark: false
       doFlat: false
-
-  # TO DO: Replace with calibrateImage in a way that we can use the
-  # preSources downstream.
-  characterizeImage:
-    class: lsst.pipe.tasks.characterizeImage.CharacterizeImageTask
-    config:
-      doApCorr: false
-      doDeblend: false
-  calibrate:
-    class: lsst.pipe.tasks.calibrate.CalibrateTask
-    config:
-      python: |
-          config.measurement.plugins.names = ['base_CircularApertureFlux',
-                                              'base_PsfFlux',
-                                              'base_LocalBackground',
-                                              'base_SdssCentroid',
-                                              'base_SdssShape',
-                                              'base_Variance',
-                                              'base_Jacobian',
-                                              'base_PixelFlags',
-                                              'base_GaussianFlux',
-                                              'base_SkyCoord',
-                                              'base_FPPosition',
-                                              'base_ClassificationSizeExtendedness',
-                                              'base_CompensatedTophatFlux',
-                                              ]
-          config.measurement.slots.shape = "base_SdssShape"
-          config.measurement.slots.psfShape = "base_SdssShape_psf"
-          config.measurement.plugins['base_PixelFlags'].masksFpAnywhere.remove('STREAK')
-          config.measurement.plugins['base_PixelFlags'].masksFpCenter.remove('STREAK')
-          config.doApCorr = False
-          config.doDeblend = False
-          config.astrometry.sourceSelector["science"].doRequirePrimary = False
-          config.astrometry.sourceSelector["science"].doIsolated = False
+  calibrateImage:
+    class: lsst.pipe.tasks.calibrateImage.CalibrateImageTask
       # TO DO: DM-41189 Get pipeline that will attach an atool here from lauren
 subsets:
   step1:
     subset:
       - isr
-      - characterizeImage
-      - calibrate
+      - calibrateImage
       # TO DO: configure and add writePreSourceTable, transformPreSourceTable
     description: |
       Per-detector tasks that can be run together

diff --git a/pipelines/LSSTComCamSim/DRP.yaml b/pipelines/LSSTComCamSim/DRP.yaml
@@ -7,9 +7,7 @@ subsets:
   step1:
     subset:
       - isr
-      - characterizeImage
-      - calibrate
-      - writePreSourceTable
+      - calibrateImage
       - transformPreSourceTable
     description: |
       Per-detector tasks that can be run together to start the DRP pipeline.

diff --git a/pipelines/LSSTComCamSim/nightly-validation.yaml b/pipelines/LSSTComCamSim/nightly-validation.yaml
@@ -15,15 +15,8 @@ tasks:
       connections.data: preSourceTable_visit
       connections.inputName: preSourceTable_visit
       connections.outputName: preSourceTableCore
-  calibrate:
-    class: lsst.pipe.tasks.calibrate.CalibrateTask
-    config:
-      python: |
-        from lsst.analysis.tools.tasks import CalexpSummaryAnalysisTask
-        config.createSummaryMetrics.retarget(CalexpSummaryAnalysisTask)
-        from lsst.analysis.tools.atools import CalexpSummaryMetrics
-        config.createSummaryMetrics.atools.calexpSummaryMetrics = CalexpSummaryMetrics
-      doCreateSummaryMetrics: true
+  calibrateImage:
+    class: lsst.pipe.tasks.calibrateImage.CalibrateImageTask
   updateVisitSummary:
     class: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask
     config:
@@ -36,10 +29,10 @@ subsets:
   step1:
     subset:
       - isr
-      - characterizeImage
-      - calibrate
-      - writePreSourceTable
+      - calibrateImage
       - transformPreSourceTable
+      - calexpSummary
+      - analyzeCalibrateImageMetadata
     description: |
       Detector level tasks. TO DO: requests include synthetic source injection,
       subtractImages, detectAndMeasureDiaSources.

diff --git a/pipelines/LSSTComCamSim/quickLook.yaml b/pipelines/LSSTComCamSim/quickLook.yaml
@@ -12,45 +12,15 @@ tasks:
       doBrighterFatter: false
   # TO DO: Replace with calibrateImage in a way that we can use the
   # preSources downstream.
-  characterizeImage:
-    class: lsst.pipe.tasks.characterizeImage.CharacterizeImageTask
-    config:
-      doApCorr: false
-      doDeblend: false
-  calibrate:
-    class: lsst.pipe.tasks.calibrate.CalibrateTask
-    config:
-      python: |
-          config.measurement.plugins.names = ['base_CircularApertureFlux',
-                                              'base_PsfFlux',
-                                              'base_LocalBackground',
-                                              'base_SdssCentroid',
-                                              'base_SdssShape',
-                                              'base_Variance',
-                                              'base_Jacobian',
-                                              'base_PixelFlags',
-                                              'base_GaussianFlux',
-                                              'base_SkyCoord',
-                                              'base_FPPosition',
-                                              'base_ClassificationSizeExtendedness',
-                                              'base_CompensatedTophatFlux',
-                                              ]
-          config.measurement.slots.shape = "base_SdssShape"
-          config.measurement.slots.psfShape = "base_SdssShape_psf"
-          config.measurement.plugins['base_PixelFlags'].masksFpAnywhere.remove('STREAK')
-          config.measurement.plugins['base_PixelFlags'].masksFpCenter.remove('STREAK')
-          config.doApCorr = False
-          config.doDeblend = False
-          config.astrometry.sourceSelector["science"].doRequirePrimary = False
-          config.astrometry.sourceSelector["science"].doIsolated = False
+  calibrateImage:
+    class: lsst.pipe.tasks.calibrateImage.CalibrateImageTask
       # TO DO: DM-41189 Get pipeline that will attach an atool here from lauren
 subsets:
   step1:
     subset:
       - isr
-      - characterizeImage
-      - calibrate
-      # TO DO: configure and add writePreSourceTable, transformPreSourceTable
+      - calibrateImage
+      # TO DO: configure and add transformPreSourceTable
     description: |
       Per-detector tasks that can be run together
   step2a:

diff --git a/pipelines/_ingredients/DECam/DRP.yaml b/pipelines/_ingredients/DECam/DRP.yaml
@@ -34,11 +34,12 @@ tasks:
     config:
       connections.crosstalkSources: overscanRaw
       doCrosstalk: true
-  calibrate:
-    class: lsst.pipe.tasks.calibrate.CalibrateTask
+  calibrateImage:
+    class: lsst.pipe.tasks.calibrateImage.CalibrateImageTask
+    # TODO: these should live in obs_decam, probably!
     config:
-      photoCal.match.referenceSelection.magLimit.fluxField: i_flux
-      photoCal.match.referenceSelection.magLimit.maximum: 22.0
+      photometry.match.referenceSelection.magLimit.fluxField: i_flux
+      photometry.match.referenceSelection.magLimit.maximum: 22.0
   updateVisitSummary:
     class: lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask
     config:
@@ -132,10 +133,8 @@ subsets:
   step1:
     subset:
       - isr
-      - characterizeImage
-      - calibrate
+      - calibrateImage
       - transformPreSourceTable
-      - writePreSourceTable
     description: |
       Per-detector tasks that can be run together to start the DRP pipeline.
 

diff --git a/pipelines/_ingredients/DRP-full.yaml b/pipelines/_ingredients/DRP-full.yaml
@@ -6,8 +6,6 @@ description:
   the 'pipelines' directory (or create a custom one) instead.
 imports:
   - location: $DRP_PIPE_DIR/pipelines/_ingredients/DRP-minimal-calibration.yaml
-    exclude:
-      - writeSourceTable  # cannot reconfigure because different dimensions.
 tasks:
   isr:
     class: lsst.ip.isr.IsrTask
@@ -16,6 +14,14 @@ tasks:
       # to compute them for analyzeAmpOffsetMetadata to report on as metrics.
       doAmpOffset: true
       ampOffset.doApplyAmpOffset: false
+  calibrateImage:
+    class: lsst.pipe.tasks.calibrateImage.CalibrateImageTask
+    config:
+
+      connections.stars: preSource
+      connections.exposure: calexp
+      connections.background: calexpBackground
+
   skyCorr:
     class: lsst.pipe.tasks.skyCorrection.SkyCorrectionTask
   isolatedStarAssociation:
@@ -34,15 +40,16 @@ tasks:
     class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask
   fgcmOutputProducts:
     class: lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask
-  writePreSourceTable:
-    class: lsst.pipe.tasks.postprocess.WriteSourceTableTask
-    config:
-      connections.outputCatalog: preSource
   transformPreSourceTable:
     class: lsst.pipe.tasks.postprocess.TransformSourceTableTask
     config:
+      functorFile: $PIPE_TASKS_DIR/schemas/PreSource.yaml
       connections.inputCatalog: preSource
       connections.outputCatalog: preSourceTable
+  transformSourceTable:
+    class: lsst.pipe.tasks.postprocess.TransformSourceTableTask
+    config:
+      functorFile: $PIPE_TASKS_DIR/schemas/Source.yaml
   consolidatePreSourceTable:
     class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask
     config:
@@ -91,7 +98,6 @@ subsets:
       Subset that includes all FGCM tasks.
   preSourceTable:
     subset:
-      - writePreSourceTable
       - transformPreSourceTable
       - consolidatePreSourceTable
     description: >