From d9a5dfec7aa6ab66ed6326bb6f4bc340d8e624fe Mon Sep 17 00:00:00 2001 From: John Parejko Date: Tue, 30 Jul 2024 16:33:51 -0700 Subject: [PATCH 01/11] Switch to calibrateImage outputs --- python/lsst/pipe/tasks/finalizeCharacterization.py | 6 +++--- python/lsst/pipe/tasks/makeWarp.py | 8 ++++---- python/lsst/pipe/tasks/make_direct_warp.py | 4 ++-- python/lsst/pipe/tasks/metrics.py | 4 ++-- python/lsst/pipe/tasks/multiBand.py | 2 +- python/lsst/pipe/tasks/postprocess.py | 10 +++++----- python/lsst/pipe/tasks/skyCorrection.py | 4 ++-- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/python/lsst/pipe/tasks/finalizeCharacterization.py b/python/lsst/pipe/tasks/finalizeCharacterization.py index 956251681..c15500cb2 100644 --- a/python/lsst/pipe/tasks/finalizeCharacterization.py +++ b/python/lsst/pipe/tasks/finalizeCharacterization.py @@ -48,12 +48,12 @@ class FinalizeCharacterizationConnections(pipeBase.PipelineTaskConnections, defaultTemplates={}): src_schema = pipeBase.connectionTypes.InitInput( doc='Input schema used for src catalogs.', - name='src_schema', + name='initial_stars_schema', storageClass='SourceCatalog', ) srcs = pipeBase.connectionTypes.Input( doc='Source catalogs for the visit', - name='src', + name='initial_stars_footprints_detector', storageClass='SourceCatalog', dimensions=('instrument', 'visit', 'detector'), deferLoad=True, @@ -61,7 +61,7 @@ class FinalizeCharacterizationConnections(pipeBase.PipelineTaskConnections, ) calexps = pipeBase.connectionTypes.Input( doc='Calexps for the visit', - name='calexp', + name='initial_pvi', storageClass='ExposureF', dimensions=('instrument', 'visit', 'detector'), deferLoad=True, diff --git a/python/lsst/pipe/tasks/makeWarp.py b/python/lsst/pipe/tasks/makeWarp.py index 64469a867..5fd15d7e7 100644 --- a/python/lsst/pipe/tasks/makeWarp.py +++ b/python/lsst/pipe/tasks/makeWarp.py @@ -50,21 +50,21 @@ class MakeWarpConnections(pipeBase.PipelineTaskConnections, "calexpType": ""}): calExpList = connectionTypes.Input( doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", - name="{calexpType}calexp", + name="{calexpType}initial_pvi", storageClass="ExposureF", dimensions=("instrument", "visit", "detector"), multiple=True, deferLoad=True, ) backgroundList = connectionTypes.Input( - doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", - name="calexpBackground", + doc="Input backgrounds to be added back into the exposure if bgSubtracted=False", + name="initial_pvi_background", storageClass="Background", dimensions=("instrument", "visit", "detector"), multiple=True, ) skyCorrList = connectionTypes.Input( - doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", + doc="Input Sky Correction to be subtracted from the exposure if doApplySkyCorr=True", name="skyCorr", storageClass="Background", dimensions=("instrument", "visit", "detector"), diff --git a/python/lsst/pipe/tasks/make_direct_warp.py b/python/lsst/pipe/tasks/make_direct_warp.py index 5bc35f2b4..cec1ea2cc 100644 --- a/python/lsst/pipe/tasks/make_direct_warp.py +++ b/python/lsst/pipe/tasks/make_direct_warp.py @@ -78,7 +78,7 @@ class MakeDirectWarpConnections( calexp_list = Input( doc="Input exposures to be interpolated and resampled onto a SkyMap " "projection/patch.", - name="{calexpType}calexp", + name="{calexpType}initial_pvi", storageClass="ExposureF", dimensions=("instrument", "visit", "detector"), multiple=True, @@ -87,7 +87,7 @@ class MakeDirectWarpConnections( background_revert_list = Input( doc="Background to be reverted (i.e., added back to the calexp). " "This connection is used only if doRevertOldBackground=False.", - name="calexpBackground", + name="initial_pvi_background", storageClass="Background", dimensions=("instrument", "visit", "detector"), multiple=True, diff --git a/python/lsst/pipe/tasks/metrics.py b/python/lsst/pipe/tasks/metrics.py index d6ef47a3c..9713b0dac 100644 --- a/python/lsst/pipe/tasks/metrics.py +++ b/python/lsst/pipe/tasks/metrics.py @@ -41,7 +41,7 @@ class NumberDeblendedSourcesMetricConnections( ): sources = connectionTypes.Input( doc="The catalog of science sources.", - name="src", + name="initial_stars_footprints_detector", storageClass="SourceCatalog", dimensions={"instrument", "visit", "detector"}, ) @@ -120,7 +120,7 @@ class NumberDeblendChildSourcesMetricConnections( ): sources = connectionTypes.Input( doc="The catalog of science sources.", - name="src", + name="initial_stars_footprints_detector", storageClass="SourceCatalog", dimensions={"instrument", "visit", "detector"}, ) diff --git a/python/lsst/pipe/tasks/multiBand.py b/python/lsst/pipe/tasks/multiBand.py index 33e4d729e..7ae9e591c 100644 --- a/python/lsst/pipe/tasks/multiBand.py +++ b/python/lsst/pipe/tasks/multiBand.py @@ -260,7 +260,7 @@ class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections, doc="Source catalogs for visits which overlap input tract, patch, band. Will be " "further filtered in the task for the purpose of propagating flags from image calibration " "and characterization to coadd objects. Only used in legacy PropagateVisitFlagsTask.", - name="src", + name="initial_stars_footprints_detector", dimensions=("instrument", "visit", "detector"), storageClass="SourceCatalog", multiple=True diff --git a/python/lsst/pipe/tasks/postprocess.py b/python/lsst/pipe/tasks/postprocess.py index 2418056fa..3d59823eb 100644 --- a/python/lsst/pipe/tasks/postprocess.py +++ b/python/lsst/pipe/tasks/postprocess.py @@ -194,14 +194,14 @@ class WriteSourceTableConnections(pipeBase.PipelineTaskConnections, catalog = connectionTypes.Input( doc="Input full-depth catalog of sources produced by CalibrateTask", - name="{catalogType}src", + name="{catalogType}initial_stars_footprints_detector", storageClass="SourceCatalog", dimensions=("instrument", "visit", "detector") ) outputCatalog = connectionTypes.Output( doc="Catalog of sources, `src` in DataFrame/Parquet format. The 'id' column is " "replaced with an index; all other columns are unchanged.", - name="{catalogType}source", + name="{catalogType}initial_stars_detector", storageClass="DataFrame", dimensions=("instrument", "visit", "detector") ) @@ -959,8 +959,8 @@ class TransformSourceTableConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector")): inputCatalog = connectionTypes.Input( - doc="Wide input catalog of sources produced by WriteSourceTableTask", - name="{catalogType}source", + doc="Wide input catalog of sources produced by WriteSourceTableTask or CalibrateImage.", + name="{catalogType}sources_detector", storageClass="DataFrame", dimensions=("instrument", "visit", "detector"), deferLoad=True @@ -996,7 +996,7 @@ class ConsolidateVisitSummaryConnections(pipeBase.PipelineTaskConnections, defaultTemplates={"calexpType": ""}): calexp = connectionTypes.Input( doc="Processed exposures used for metadata", - name="calexp", + name="initial_pvi", storageClass="ExposureF", dimensions=("instrument", "visit", "detector"), deferLoad=True, diff --git a/python/lsst/pipe/tasks/skyCorrection.py b/python/lsst/pipe/tasks/skyCorrection.py index df6f00f1c..0f7454b76 100644 --- a/python/lsst/pipe/tasks/skyCorrection.py +++ b/python/lsst/pipe/tasks/skyCorrection.py @@ -109,7 +109,7 @@ class SkyCorrectionConnections(PipelineTaskConnections, dimensions=("instrument" ) calExps = cT.Input( doc="Background-subtracted calibrated exposures.", - name="calexp", + name="initial_pvi", multiple=True, storageClass="ExposureF", dimensions=["instrument", "visit", "detector"], @@ -117,7 +117,7 @@ class SkyCorrectionConnections(PipelineTaskConnections, dimensions=("instrument" calBkgs = cT.Input( doc="Subtracted backgrounds for input calibrated exposures.", multiple=True, - name="calexpBackground", + name="initial_pvi_background", storageClass="Background", dimensions=["instrument", "visit", "detector"], ) From a20203cb838366e50e6d6b436182ee2acc6611fe Mon Sep 17 00:00:00 2001 From: John Parejko Date: Wed, 31 Jul 2024 13:08:36 -0700 Subject: [PATCH 02/11] Deprecation questions --- python/lsst/pipe/tasks/postprocess.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/lsst/pipe/tasks/postprocess.py b/python/lsst/pipe/tasks/postprocess.py index 3d59823eb..3be60d31e 100644 --- a/python/lsst/pipe/tasks/postprocess.py +++ b/python/lsst/pipe/tasks/postprocess.py @@ -188,6 +188,7 @@ def run(self, catalogs, tract, patch): return catalog +# TODO: should deprecate this? class WriteSourceTableConnections(pipeBase.PipelineTaskConnections, defaultTemplates={"catalogType": ""}, dimensions=("instrument", "visit", "detector")): @@ -207,11 +208,13 @@ class WriteSourceTableConnections(pipeBase.PipelineTaskConnections, ) +# TODO: should deprecate this! class WriteSourceTableConfig(pipeBase.PipelineTaskConfig, pipelineConnections=WriteSourceTableConnections): pass +# TODO: should deprecate this! class WriteSourceTableTask(pipeBase.PipelineTask): """Write source table to DataFrame Parquet format. """ @@ -281,6 +284,7 @@ class WriteRecalibratedSourceTableConfig(WriteSourceTableConfig, ) +# TODO: deprecate, since reprocessVisitImage does this more thoroughly? class WriteRecalibratedSourceTableTask(WriteSourceTableTask): """Write source table to DataFrame Parquet format. """ @@ -1122,6 +1126,7 @@ def _combineExposureMetadata(self, visit, dataRefs): class ConsolidateSourceTableConnections(pipeBase.PipelineTaskConnections, defaultTemplates={"catalogType": ""}, dimensions=("instrument", "visit")): + # TODO: Deprecate the dataframe connection? inputCatalogs = connectionTypes.Input( doc="Input per-detector Source Tables", name="{catalogType}sourceTable", From a939d4cefa67e5df5b4cc008c8cb9eee67e14e33 Mon Sep 17 00:00:00 2001 From: John Parejko Date: Wed, 31 Jul 2024 13:09:43 -0700 Subject: [PATCH 03/11] Add sdm standardization file for calibrateImage output --- .../initial_stars_detector_standardized.yaml | 196 ++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 schemas/initial_stars_detector_standardized.yaml diff --git a/schemas/initial_stars_detector_standardized.yaml b/schemas/initial_stars_detector_standardized.yaml new file mode 100644 index 000000000..6e83d5aa1 --- /dev/null +++ b/schemas/initial_stars_detector_standardized.yaml @@ -0,0 +1,196 @@ +# This file defines the mapping between the columns in a single visit+detector +# initial_stars table and their respective DPDD-style column names, as used by +# `lsst.pipe.tasks.postprocess.TransformSourceTableTask`. +# See the DPDD for more information about the output: https://lse-163.lsst.io +funcs: + sourceId: + functor: Column + args: id + coord_ra: + # reference position required by db. Not in DPDD + functor: CoordColumn + args: coord_ra + coord_dec: + # Reference position required by db. Not in DPDD + functor: CoordColumn + args: coord_dec + # objectId: not avaliable + # ssObjectId: not avaliable + parentSourceId: + functor: Column + args: parent + # htmId20: not avaliable + x: + functor: Column + args: slot_Centroid_x + y: + functor: Column + args: slot_Centroid_y + xErr: + functor: Column + args: slot_Centroid_xErr + yErr: + functor: Column + args: slot_Centroid_yErr + # x_y_Cov: not available + ra: + functor: RAColumn + dec: + functor: DecColumn + + # RFC-924: Temporarily keep a duplicate "decl" entry for backwards + # compatibility. To be removed after September 2023. + decl: + functor: DecColumn + raErr: + functor: RAErrColumn + decErr: + functor: DecErrColumn + ra_dec_Cov: + functor: RADecCovColumn + # One calibrated Calib flux is important: + calibFlux: + functor: Column + args: slot_CalibFlux_instFlux + calibFluxErr: + functor: Column + args: slot_CalibFlux_instFluxErr + ap12Flux: + functor: Column + args: base_CircularApertureFlux_12_0_instFlux + ap12FluxErr: + functor: Column + args: base_CircularApertureFlux_12_0_instFluxErr + ap12Flux_flag: + functor: Column + args: base_CircularApertureFlux_12_0_flag + ap17Flux: + functor: Column + args: base_CircularApertureFlux_17_0_instFlux + ap17FluxErr: + functor: Column + args: base_CircularApertureFlux_17_0_instFluxErr + ap17Flux_flag: + functor: Column + args: base_CircularApertureFlux_17_0_flag + psfFlux: + functor: Column + args: slot_PsfFlux_instFlux + psfFluxErr: + functor: Column + args: slot_PsfFlux_instFluxErr + ixx: + functor: Column + args: slot_Shape_xx + iyy: + functor: Column + args: slot_Shape_yy + ixy: + functor: Column + args: slot_Shape_xy + # DPDD should include Psf Shape + ixxPSF: + functor: Column + args: slot_PsfShape_xx + iyyPSF: + functor: Column + args: slot_PsfShape_yy + ixyPSF: + functor: Column + args: slot_PsfShape_xy + # apNann: Replaced by raw Aperture instFluxes in flags section below + # apMeanSb: Replaced by raw Aperture instFluxes in flags section below + # apMeanSbErr: Replaced by raw Aperture instFluxes in flags section below + + # DPDD does not include gaussianFluxes, however they are used for + # the single frame extendedness column which is used for QA. + gaussianFlux: + functor: Column + args: base_GaussianFlux_instFlux + gaussianFluxErr: + functor: Column + args: base_GaussianFlux_instFluxErr + extendedness: + functor: Column + args: base_ClassificationExtendedness_value + sizeExtendedness: + functor: Column + args: base_ClassificationSizeExtendedness_value +flags: + - base_ClassificationExtendedness_flag + - base_ClassificationSizeExtendedness_flag + - base_LocalBackground_instFlux # needed by isolatedStarAssociation + - base_LocalBackground_instFluxErr # needed by isolatedStarAssociation + - base_LocalBackground_flag # needed by isolatedStarAssociation + - base_NormalizedCompensatedTophatFlux_flag + - base_NormalizedCompensatedTophatFlux_instFlux + - base_NormalizedCompensatedTophatFlux_instFluxErr + - base_CircularApertureFlux_12_0_flag + - base_CircularApertureFlux_12_0_flag_apertureTruncated + - base_CircularApertureFlux_12_0_instFlux + - base_CircularApertureFlux_12_0_instFluxErr + - base_CircularApertureFlux_17_0_flag + - base_CircularApertureFlux_17_0_instFlux + - base_CircularApertureFlux_17_0_instFluxErr + - base_FootprintArea_value + - base_PixelFlags_flag_bad + - base_PixelFlags_flag_cr + - base_PixelFlags_flag_crCenter + - base_PixelFlags_flag_edge + - base_PixelFlags_flag_interpolated + - base_PixelFlags_flag_interpolatedCenter + - base_PixelFlags_flag_offimage + - base_PixelFlags_flag_saturated + - base_PixelFlags_flag_saturatedCenter + - base_PixelFlags_flag_suspect + - base_PixelFlags_flag_suspectCenter + - base_PsfFlux_apCorr + - base_PsfFlux_apCorrErr + - base_PsfFlux_area + - base_PsfFlux_flag + - base_PsfFlux_flag_apCorr + - base_PsfFlux_flag_edge + - base_PsfFlux_flag_noGoodPixels + - base_GaussianFlux_flag + - base_SdssCentroid_flag + - base_SdssCentroid_flag_almostNoSecondDerivative + - base_SdssCentroid_flag_badError + - base_SdssCentroid_flag_edge + - base_SdssCentroid_flag_noSecondDerivative + - base_SdssCentroid_flag_notAtMaximum + - base_SdssCentroid_flag_resetToPeak + - ext_shapeHSM_HsmPsfMoments_flag + - ext_shapeHSM_HsmPsfMoments_flag_no_pixels + - ext_shapeHSM_HsmPsfMoments_flag_not_contained + - ext_shapeHSM_HsmPsfMoments_flag_parent_source + - calib_astrometry_used + - calib_photometry_reserved + - calib_photometry_used + - calib_psf_candidate + - calib_psf_reserved + - calib_psf_used + - deblend_deblendedAsPsf + - deblend_hasStrayFlux + - deblend_masked + - deblend_nChild + - deblend_parentTooBig + - deblend_patchedTemplate + - deblend_rampedTemplate + - deblend_skipped + - deblend_tooManyPeaks + - sky_source + - detect_isPrimary + +flag_rename_rules: + # Taken from db-meas-forced + - ['base_Local', 'local'] + - ['base_PixelFlags_flag', 'pixelFlags'] + - ['base_SdssCentroid', 'centroid'] + - ['base_Psf', 'psf'] + - ['base_GaussianFlux', 'gaussianFlux'] + - ['base_CircularApertureFlux', 'apFlux'] + - ['base_NormalizedCompensatedTophatFlux', 'normCompTophatFlux'] + - ['ext_shapeHSM_Hsm', 'hsm'] + - ['base_', ''] + - ['slot_', ''] + From 1df49118bb78f876ffcfa569a13a9ee7f211b5b5 Mon Sep 17 00:00:00 2001 From: John Parejko Date: Thu, 15 Aug 2024 12:02:03 -0700 Subject: [PATCH 04/11] Use regular column functor for id field Now that reprocessVisitImage is directly writing an ArrowAstropy, it no longer has an index column, so we can just treat the id like a normal column. --- schemas/Source.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schemas/Source.yaml b/schemas/Source.yaml index f2a7dfb8f..9122c11e5 100644 --- a/schemas/Source.yaml +++ b/schemas/Source.yaml @@ -4,7 +4,8 @@ # See the DPDD for more information about the output: https://lse-163.lsst.io funcs: sourceId: - functor: Index + functor: Column + args: id coord_ra: # reference position required by db. Not in DPDD functor: CoordColumn From e3836dacd48250a8267a6b6ebbbb35a315cd3940 Mon Sep 17 00:00:00 2001 From: John Parejko Date: Fri, 16 Aug 2024 00:55:17 -0700 Subject: [PATCH 05/11] Remove STREAK flag from sdm catalogs They will get added back in once we start propagating them from the diffim. Remove STREAK from MeasureMergedCoaddSources. --- python/lsst/pipe/tasks/multiBand.py | 8 +++++--- schemas/ForcedSource.yaml | 5 +++-- schemas/Object.yaml | 5 +++-- schemas/Source.yaml | 5 +++-- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/python/lsst/pipe/tasks/multiBand.py b/python/lsst/pipe/tasks/multiBand.py index 7ae9e591c..e99de63fb 100644 --- a/python/lsst/pipe/tasks/multiBand.py +++ b/python/lsst/pipe/tasks/multiBand.py @@ -449,11 +449,13 @@ def setDefaults(self): 'base_LocalPhotoCalib', 'base_LocalWcs'] - # TODO: Remove STREAK in DM-44658, streak masking to happen only in ip_diffim + # TODO: Remove STREAK in DM-44658, streak masking to happen only in + # ip_diffim; if we can propagate the streak mask from diffim, we can + # still set flags with it here. self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE', - 'INEXACT_PSF', 'STREAK'] + 'INEXACT_PSF'] self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE', - 'INEXACT_PSF', 'STREAK'] + 'INEXACT_PSF'] def validate(self): super().validate() diff --git a/schemas/ForcedSource.yaml b/schemas/ForcedSource.yaml index 70c501ab7..5fae0fe01 100644 --- a/schemas/ForcedSource.yaml +++ b/schemas/ForcedSource.yaml @@ -95,8 +95,9 @@ calexpFlags: - base_PixelFlags_flag_saturatedCenter - base_PixelFlags_flag_crCenter - base_PixelFlags_flag_suspectCenter - - base_PixelFlags_flag_streak - - base_PixelFlags_flag_streakCenter + # Streak flags not yet propagated from difference imaging. + # - base_PixelFlags_flag_streak + # - base_PixelFlags_flag_streakCenter - base_InvalidPsf_flag flag_rename_rules: - ['base_PixelFlags_flag', 'pixelFlags'] diff --git a/schemas/Object.yaml b/schemas/Object.yaml index 8bac141fd..1165c2e69 100644 --- a/schemas/Object.yaml +++ b/schemas/Object.yaml @@ -769,8 +769,9 @@ flags: - base_PixelFlags_flag_sensor_edgeCenter - base_PixelFlags_flag_suspect - base_PixelFlags_flag_suspectCenter - - base_PixelFlags_flag_streak - - base_PixelFlags_flag_streakCenter + # Streak flags not yet propagated from difference imaging. + # - base_PixelFlags_flag_streak + # - base_PixelFlags_flag_streakCenter - base_ClassificationExtendedness_flag - base_ClassificationSizeExtendedness_flag - base_InvalidPsf_flag diff --git a/schemas/Source.yaml b/schemas/Source.yaml index 9122c11e5..6cc53d383 100644 --- a/schemas/Source.yaml +++ b/schemas/Source.yaml @@ -381,8 +381,9 @@ flags: - base_PixelFlags_flag_saturatedCenter - base_PixelFlags_flag_suspect - base_PixelFlags_flag_suspectCenter - - base_PixelFlags_flag_streak - - base_PixelFlags_flag_streakCenter + # Streak flags not yet propagated from difference imaging. + # - base_PixelFlags_flag_streak + # - base_PixelFlags_flag_streakCenter - base_PsfFlux_apCorr - base_PsfFlux_apCorrErr - base_PsfFlux_area From eba22aadd4f49d14b370c63a92196a80f80a22da Mon Sep 17 00:00:00 2001 From: John Parejko Date: Tue, 17 Sep 2024 13:39:29 -0700 Subject: [PATCH 06/11] Remove calib_detected from sdm output calib_detected is now meaningless, as the icSource table is no more. --- schemas/Source.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/schemas/Source.yaml b/schemas/Source.yaml index 6cc53d383..a860bf384 100644 --- a/schemas/Source.yaml +++ b/schemas/Source.yaml @@ -403,7 +403,6 @@ flags: - base_Variance_flag_emptyFootprint - base_Variance_value - calib_astrometry_used - - calib_detected - calib_photometry_reserved - calib_photometry_used - calib_psf_candidate From b2b2d9555f17c9c2bb6726091c4d7418b7278a15 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Fri, 4 Oct 2024 15:25:05 -0400 Subject: [PATCH 07/11] Remove initial PhotoCalib in finalizeCharacterization. This is needed for compatibility with outputs produced by CalibrateImageTask. --- .../pipe/tasks/finalizeCharacterization.py | 94 ++++++++++++++++++- tests/test_finalizeCharacterization.py | 4 +- 2 files changed, 93 insertions(+), 5 deletions(-) diff --git a/python/lsst/pipe/tasks/finalizeCharacterization.py b/python/lsst/pipe/tasks/finalizeCharacterization.py index c15500cb2..5ae15f077 100644 --- a/python/lsst/pipe/tasks/finalizeCharacterization.py +++ b/python/lsst/pipe/tasks/finalizeCharacterization.py @@ -26,6 +26,8 @@ 'FinalizeCharacterizationConfig', 'FinalizeCharacterizationTask'] +import logging + import numpy as np import esutil import pandas as pd @@ -43,6 +45,9 @@ from .reserveIsolatedStars import ReserveIsolatedStarsTask +_LOG = logging.getLogger(__name__) + + class FinalizeCharacterizationConnections(pipeBase.PipelineTaskConnections, dimensions=('instrument', 'visit',), defaultTemplates={}): @@ -85,6 +90,17 @@ class FinalizeCharacterizationConnections(pipeBase.PipelineTaskConnections, deferLoad=True, multiple=True, ) + initial_photo_calibs = pipeBase.connectionTypes.Input( + doc=("Initial photometric calibration that was already applied to " + "calexps, to be removed prior to measurement in order to recover " + "instrumental fluxes."), + name="initial_photoCalib_detector", + storageClass="PhotoCalib", + dimensions=("instrument", "visit", "detector"), + multiple=True, + deferLoad=True, + minimum=0, + ) finalized_psf_ap_corr_cat = pipeBase.connectionTypes.Output( doc=('Per-visit finalized psf models and aperture corrections. This ' 'catalog uses detector id for the id and are sorted for fast ' @@ -100,6 +116,28 @@ class FinalizeCharacterizationConnections(pipeBase.PipelineTaskConnections, dimensions=('instrument', 'visit'), ) + def adjustQuantum(self, inputs, outputs, label, data_id): + if self.config.remove_initial_photo_calib and not inputs["initial_photo_calibs"]: + _LOG.warning( + "Dropping %s quantum %s because initial photo calibs are needed and none were present " + "this may be an upstream partial-outputs error covering an entire visit (which is why this " + "is not an error), but it may mean that 'config.remove_initial_photo_calib' should be " + "False.", + label, + data_id, + ) + raise pipeBase.NoWorkFound("No initial photo calibs.") + elif not self.config.remove_initial_photo_calib and inputs["initial_photo_calibs"]: + _LOG.warning( + "For %s quantum %s, input collections have initial photo calib datasets but " + "'config.remove_initial_photo_calib=False'. This is either a very unusual collection " + "search path or (more likely) a bad configuration. Not that this config option should " + "be true when using images produced by CalibrateImageTask.", + label, + data_id, + ) + return super().adjustQuantum(inputs, outputs, label, data_id) + class FinalizeCharacterizationConfig(pipeBase.PipelineTaskConfig, pipelineConnections=FinalizeCharacterizationConnections): @@ -113,6 +151,12 @@ class FinalizeCharacterizationConfig(pipeBase.PipelineTaskConfig, dtype=str, default='sourceId', ) + remove_initial_photo_calib = pexConfig.Field( + doc=("Expect an initial photo calib input to be present, and use it ", + "to restore the image to instrumental units."), + dtype=bool, + default=True, + ) reserve_selection = pexConfig.ConfigurableField( target=ReserveIsolatedStarsTask, doc='Task to select reserved stars', @@ -269,6 +313,8 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): for handle in input_handle_dict['srcs']} calexp_dict_temp = {handle.dataId['detector']: handle for handle in input_handle_dict['calexps']} + initial_photo_calib_dict_temp = {handle.dataId['detector']: handle + for handle in input_handle_dict['initial_photo_calibs']} isolated_star_cat_dict_temp = {handle.dataId['tract']: handle for handle in input_handle_dict['isolated_star_cats']} isolated_star_source_dict_temp = {handle.dataId['tract']: handle @@ -279,6 +325,8 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): detector in sorted(src_dict_temp.keys())} calexp_dict = {detector: calexp_dict_temp[detector] for detector in sorted(calexp_dict_temp.keys())} + initial_photo_calib_dict = {detector: initial_photo_calib_dict_temp[detector] + for detector in sorted(initial_photo_calib_dict_temp.keys())} isolated_star_cat_dict = {tract: isolated_star_cat_dict_temp[tract] for tract in sorted(isolated_star_cat_dict_temp.keys())} isolated_star_source_dict = {tract: isolated_star_source_dict_temp[tract] for @@ -289,14 +337,24 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): isolated_star_cat_dict, isolated_star_source_dict, src_dict, - calexp_dict) + calexp_dict, + initial_photo_calib_dict) butlerQC.put(struct.psf_ap_corr_cat, outputRefs.finalized_psf_ap_corr_cat) butlerQC.put(pd.DataFrame(struct.output_table), outputRefs.finalized_src_table) - def run(self, visit, band, isolated_star_cat_dict, isolated_star_source_dict, src_dict, calexp_dict): + def run( + self, + visit, + band, + isolated_star_cat_dict, + isolated_star_source_dict, + src_dict, + calexp_dict, + initial_photo_calib_dict, + ): """ Run the FinalizeCharacterizationTask. @@ -314,6 +372,8 @@ def run(self, visit, band, isolated_star_cat_dict, isolated_star_source_dict, sr Per-detector dict of src catalog handles. calexp_dict : `dict` Per-detector dict of calibrated exposure handles. + initial_photo_calib_dict : `dict` + Per-detector dict of initial photometric calibration handles Returns ------- @@ -349,13 +409,18 @@ def run(self, visit, band, isolated_star_cat_dict, isolated_star_source_dict, sr for detector in src_dict: src = src_dict[detector].get() exposure = calexp_dict[detector].get() + if detector in initial_photo_calib_dict: + initial_photo_calib = initial_photo_calib_dict[detector].get() + else: + initial_photo_calib = None psf, ap_corr_map, measured_src = self.compute_psf_and_ap_corr_map( visit, detector, exposure, src, - isolated_source_table + isolated_source_table, + initial_photo_calib ) # And now we package it together... @@ -591,7 +656,15 @@ def concat_isolated_star_cats(self, band, isolated_star_cat_dict, isolated_star_ return isolated_table, isolated_source_table - def compute_psf_and_ap_corr_map(self, visit, detector, exposure, src, isolated_source_table): + def compute_psf_and_ap_corr_map( + self, + visit, + detector, + exposure, + src, + isolated_source_table, + initial_photo_calib, + ): """Compute psf model and aperture correction map for a single exposure. Parameters @@ -603,6 +676,8 @@ def compute_psf_and_ap_corr_map(self, visit, detector, exposure, src, isolated_s exposure : `lsst.afw.image.ExposureF` src : `lsst.afw.table.SourceCatalog` isolated_source_table : `np.ndarray` + initial_photo_calib : `lsst.afw.image.PhotoCalib` or `None` + Initial photometric calibration to remove from the image. Returns ------- @@ -613,6 +688,17 @@ def compute_psf_and_ap_corr_map(self, visit, detector, exposure, src, isolated_s measured_src : `lsst.afw.table.SourceCatalog` Updated source catalog with measurements, flags and aperture corrections. """ + if self.config.remove_initial_photo_calib: + if initial_photo_calib is None: + self.log.warning("No initial photo calib found for visit %d, detector %d", visit, detector) + return None, None, None + if not initial_photo_calib._isConstant: + # TODO DM-46720: remove this limitation and usage of private (why?!) property. + raise NotImplementedError( + "removeInitialPhotoCalib=True can only work when the initialPhotoCalib is constant." + ) + exposure.maskedImage /= initial_photo_calib.getCalibrationMean() + # Extract footprints from the input src catalog for noise replacement. footprints = SingleFrameMeasurementTask.getFootprintsFromCatalog(src) diff --git a/tests/test_finalizeCharacterization.py b/tests/test_finalizeCharacterization.py index 174b4383d..ca0c0be6a 100644 --- a/tests/test_finalizeCharacterization.py +++ b/tests/test_finalizeCharacterization.py @@ -56,6 +56,7 @@ class FinalizeCharacterizationTestCase(lsst.utils.tests.TestCase): """ def setUp(self): config = FinalizeCharacterizationConfig() + config.remove_initial_photo_calib = False self.finalizeCharacterizationTask = TestFinalizeCharacterizationTask( config=config, @@ -232,7 +233,8 @@ def test_compute_psf_and_ap_corr_map_no_sources(self): detector, exposure, src, - isolated_source_table + isolated_source_table, + initial_photo_calib=None, ) self.assertIn( "No good sources remain after cuts for visit {}, detector {}".format(visit, detector), From 901f93c7fb435f783b8c7332c1290cf98e63885b Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Tue, 8 Oct 2024 12:15:41 -0400 Subject: [PATCH 08/11] Clean up MakeWarpTask docs and variable names. Some doc parameter lists had gotten out of sync with the code, and it's always good to clarify whether you've got a DeferredDatasetHandle or a real object. --- python/lsst/pipe/tasks/makeWarp.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/python/lsst/pipe/tasks/makeWarp.py b/python/lsst/pipe/tasks/makeWarp.py index 5fd15d7e7..bf459eff9 100644 --- a/python/lsst/pipe/tasks/makeWarp.py +++ b/python/lsst/pipe/tasks/makeWarp.py @@ -225,7 +225,7 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): detector order (to ensure reproducibility). Then ensure all input lists are in the same sorted detector order. """ - detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList] + detectorOrder = [handle.datasetRef.dataId['detector'] for handle in inputRefs.calExpList] detectorOrder.sort() inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') @@ -299,9 +299,9 @@ def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwar (violates LSST algorithms group policy), but will be fixed up by interpolating after the coaddition. - calexpRefList : `list` - List of data references for calexps that (may) - overlap the patch of interest. + calExpList : `list` [ `lsst.afw.image.Exposure` ] + List of single-detector input images that (may) overlap the patch + of interest. skyInfo : `lsst.pipe.base.Struct` Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with geometric information about the patch. @@ -435,14 +435,16 @@ def _prepareCalibratedExposures(self, *, visitSummary, calExpList=[], wcsList=No to `None` are ignored. calExpList : `list` [`lsst.afw.image.Exposure` or `lsst.daf.butler.DeferredDatasetHandle`] - Sequence of calexps to be modified in place. + Sequence of single-epoch images (or deferred load handles for + images) to be modified in place. On return this always has images, + not handles. wcsList : `list` [`lsst.afw.geom.SkyWcs`] The WCSs of the calexps in ``calExpList``. These will be used to determine if the calexp should be used in the warp. The list is dynamically updated with the WCSs from the visitSummary. - backgroundList : `list` [`lsst.afw.math.backgroundList`], optional + backgroundList : `list` [`lsst.afw.math.BackgroundList`], optional Sequence of backgrounds to be added back in if bgSubtracted=False. - skyCorrList : `list` [`lsst.afw.math.backgroundList`], optional + skyCorrList : `list` [`lsst.afw.math.BackgroundList`], optional Sequence of background corrections to be subtracted if doApplySkyCorr=True. **kwargs @@ -606,7 +608,7 @@ def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey): if hasattr(refs[0], "dataId"): inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs] else: - inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs] + inputSortKeyOrder = [handle.datasetRef.dataId[dataIdKey] for handle in refs] if inputSortKeyOrder != outputSortKeyOrder: setattr(inputRefs, connectionName, reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder)) From ae0b327f2afaee7e6494ff6a3a278bf700ad97f6 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Tue, 8 Oct 2024 13:01:03 -0400 Subject: [PATCH 09/11] Apply calibration to warps after applying skyCorr backgrounds. Background modeling is done on the image without the calibration applied, and that's how it should be subtracted, too. This was a clear bug in the old code, but probably a very minor one, because the photometric calibration is close to constant, and we were previously only incorrectly multiplying the skyCorr background by the ratio of the photometric calibration to its average. --- python/lsst/pipe/tasks/makeWarp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/lsst/pipe/tasks/makeWarp.py b/python/lsst/pipe/tasks/makeWarp.py index bf459eff9..23ad61812 100644 --- a/python/lsst/pipe/tasks/makeWarp.py +++ b/python/lsst/pipe/tasks/makeWarp.py @@ -528,6 +528,10 @@ def _prepareCalibratedExposures(self, *, visitSummary, calExpList=[], wcsList=No ) continue + # Apply skycorr + if self.config.doApplySkyCorr: + calexp.maskedImage -= skyCorr.getImage() + # Calibrate the image. calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage, includeScaleUncertainty=includeCalibVar) @@ -536,10 +540,6 @@ def _prepareCalibratedExposures(self, *, visitSummary, calExpList=[], wcsList=No # RFC-545 is implemented. # exposure.setCalib(afwImage.Calib(1.0)) - # Apply skycorr - if self.config.doApplySkyCorr: - calexp.maskedImage -= skyCorr.getImage() - indices.append(index) calExpList[index] = calexp From 142eaf9f4c02cd19934582f2b958fc7b5169d2ee Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Wed, 9 Oct 2024 15:31:52 -0400 Subject: [PATCH 10/11] Drop errant commas in docstring concatenation. --- python/lsst/pipe/tasks/makeWarp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lsst/pipe/tasks/makeWarp.py b/python/lsst/pipe/tasks/makeWarp.py index 23ad61812..64e4aa7cd 100644 --- a/python/lsst/pipe/tasks/makeWarp.py +++ b/python/lsst/pipe/tasks/makeWarp.py @@ -77,14 +77,14 @@ class MakeWarpConnections(pipeBase.PipelineTaskConnections, dimensions=("skymap",), ) direct = connectionTypes.Output( - doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", + doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling " "calexps onto the skyMap patch geometry."), name="{coaddName}Coadd_directWarp", storageClass="ExposureF", dimensions=("tract", "patch", "skymap", "visit", "instrument"), ) psfMatched = connectionTypes.Output( - doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", + doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling " "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), name="{coaddName}Coadd_psfMatchedWarp", storageClass="ExposureF", From ddcd8c975caefedd94ad789d32cb59a89547d589 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Tue, 8 Oct 2024 13:26:04 -0400 Subject: [PATCH 11/11] Remove initial PhotoCalib in makeWarp. This is needed for compatibility with outputs produced by CalibrateImageTask. --- python/lsst/pipe/tasks/makeWarp.py | 76 ++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/python/lsst/pipe/tasks/makeWarp.py b/python/lsst/pipe/tasks/makeWarp.py index 64e4aa7cd..d867e09ad 100644 --- a/python/lsst/pipe/tasks/makeWarp.py +++ b/python/lsst/pipe/tasks/makeWarp.py @@ -96,6 +96,15 @@ class MakeWarpConnections(pipeBase.PipelineTaskConnections, storageClass="ExposureCatalog", dimensions=("instrument", "visit",), ) + initialPhotoCalibList = pipeBase.connectionTypes.Input( + doc=("Initial photometric calibration that was already applied to calExpList, " + "to be removed prior to applying the final photometric calibration."), + name="initial_photoCalib_detector", + storageClass="PhotoCalib", + dimensions=("instrument", "visit", "detector"), + multiple=True, + minimum=0, + ) def __init__(self, *, config=None): if config.bgSubtracted: @@ -107,6 +116,33 @@ def __init__(self, *, config=None): if not config.makePsfMatched: del self.psfMatched + def adjustQuantum(self, inputs, outputs, label, data_id): + # Instead of disabling the initial photo calibs connection when + # config.removeInitialPhotoCalib is False, check here (in QG + # generation) that the configuration is consistent with what's + # available. This gives us a chance at least warn if somebody + # runs CalibrateImageTask but doesn't configure MakeWarpTask to + # use it properly. + if self.config.removeInitialPhotoCalib and not inputs["initialPhotoCalibList"]: + log.warning( + "Dropping %s quantum %s because initial photo calibs are needed and none were present " + "this may be an upstream partial-outputs error covering an entire visit (which is why this " + "is not an error), but it may mean that 'config.removeInitialPhotoCalib' should be False.", + label, + data_id, + ) + raise pipeBase.NoWorkFound("No initial photo calibs.") + elif not self.config.removeInitialPhotoCalib and inputs["initialPhotoCalibList"]: + log.warning( + "For %s quantum %s, input collections have initial photo calib datasets but " + "'config.removeInitialPhotoCalib=False'. This is either a very unusual collection " + "search path or (more likely) a bad configuration. Not that this config option should " + "be true when using images produced by CalibrateImageTask.", + label, + data_id, + ) + return super().adjustQuantum(inputs, outputs, label, data_id) + class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass, pipelineConnections=MakeWarpConnections): @@ -164,6 +200,12 @@ class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass, default=False, doc="Apply sky correction?", ) + removeInitialPhotoCalib = pexConfig.Field( + doc=("Expect an initial photo calib input to be present, and use it to restore the image ", + "to instrumental units before applying the final photometric calibration."), + dtype=bool, + default=True, + ) idGenerator = DetectorVisitIdGeneratorConfig.make_field() def validate(self): @@ -425,7 +467,8 @@ def filterInputs(self, indices, inputs): return inputs def _prepareCalibratedExposures(self, *, visitSummary, calExpList=[], wcsList=None, - backgroundList=None, skyCorrList=None, **kwargs): + backgroundList=None, skyCorrList=None, initialPhotoCalibList=None, + **kwargs): """Calibrate and add backgrounds to input calExpList in place. Parameters @@ -447,6 +490,10 @@ def _prepareCalibratedExposures(self, *, visitSummary, calExpList=[], wcsList=No skyCorrList : `list` [`lsst.afw.math.BackgroundList`], optional Sequence of background corrections to be subtracted if doApplySkyCorr=True. + initialPhotoCalibList : `list [`lsst.afw.image.PhotoCalib`], optional + Initial photometric calibrations that were already applied to the + images in `calExpList` and must be removed before applying the + final photometric calibration. **kwargs Additional keyword arguments. @@ -459,13 +506,16 @@ def _prepareCalibratedExposures(self, *, visitSummary, calExpList=[], wcsList=No wcsList = len(calExpList)*[None] if wcsList is None else wcsList backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList + initialPhotoCalibList = ( + len(calExpList)*[None] if initialPhotoCalibList is None else initialPhotoCalibList + ) includeCalibVar = self.config.includeCalibVar indices = [] - for index, (calexp, background, skyCorr) in enumerate(zip(calExpList, - backgroundList, - skyCorrList)): + for index, (calexp, background, skyCorr, initialPhotoCalib) in enumerate( + zip(calExpList, backgroundList, skyCorrList, initialPhotoCalibList) + ): if isinstance(calexp, DeferredDatasetHandle): calexp = calexp.get() @@ -528,10 +578,26 @@ def _prepareCalibratedExposures(self, *, visitSummary, calExpList=[], wcsList=No ) continue - # Apply skycorr + # Apply skycorr. We assume it has the same units as the calExpList + # image. if self.config.doApplySkyCorr: calexp.maskedImage -= skyCorr.getImage() + # Remove initial photometric calibration, if we expect one. + if self.config.removeInitialPhotoCalib: + if initialPhotoCalib is None: + self.log.warning( + "Detector id %d for visit %d has no initialPhotoCalib and will " + "not be used in the warp", detectorId, row["visit"], + ) + continue + if not initialPhotoCalib._isConstant: + # TODO DM-46720: remove this limitation and usage of private (why?!) property. + raise NotImplementedError( + "removeInitialPhotoCalib=True can only work when the initialPhotoCalib is constant." + ) + calexp.maskedImage /= initialPhotoCalib.getCalibrationMean() + # Calibrate the image. calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage, includeScaleUncertainty=includeCalibVar)