diff --git a/python/lsst/pipe/tasks/finalizeCharacterization.py b/python/lsst/pipe/tasks/finalizeCharacterization.py index c15500cb2..f33f32cae 100644 --- a/python/lsst/pipe/tasks/finalizeCharacterization.py +++ b/python/lsst/pipe/tasks/finalizeCharacterization.py @@ -26,6 +26,8 @@ 'FinalizeCharacterizationConfig', 'FinalizeCharacterizationTask'] +import logging + import numpy as np import esutil import pandas as pd @@ -43,6 +45,9 @@ from .reserveIsolatedStars import ReserveIsolatedStarsTask +_LOG = logging.getLogger(__name__) + + class FinalizeCharacterizationConnections(pipeBase.PipelineTaskConnections, dimensions=('instrument', 'visit',), defaultTemplates={}): @@ -85,6 +90,17 @@ class FinalizeCharacterizationConnections(pipeBase.PipelineTaskConnections, deferLoad=True, multiple=True, ) + initial_photo_calibs = pipeBase.connectionTypes.Input( + doc=("Initial photometric calibration that was already applied to " + "calexps, to be removed prior to measurement in order to recover " + "instrumental fluxes."), + name="initial_photoCalib_detector", + storageClass="PhotoCalib", + dimensions=("instrument", "visit", "detector"), + multiple=True, + deferLoad=True, + minimum=0, + ) finalized_psf_ap_corr_cat = pipeBase.connectionTypes.Output( doc=('Per-visit finalized psf models and aperture corrections. This ' 'catalog uses detector id for the id and are sorted for fast ' @@ -100,6 +116,23 @@ class FinalizeCharacterizationConnections(pipeBase.PipelineTaskConnections, dimensions=('instrument', 'visit'), ) + def adjustQuantum(self, inputs, outputs, label, data_id): + if self.config.remove_initial_photo_calib and not inputs["initial_photo_calib"]: + _LOG.warning( + "Dropping %s quantum %s because initial photo calibs are needed and none were present " + "this may be an upstream partial-outputs error covering an entire visit (which is why this " + "is not an error), but it may mean that 'config.remove_initial_photo_calib' should be False." + ) + raise pipeBase.NoWorkFound("No initial photo calibs.") + elif not self.config.remove_initial_photo_calib and inputs["initial_photo_calib"]: + _LOG.warning( + "Input collections have initial photo calib datasets but " + "'config.remove_initial_photo_calib=False'. This is either a very unusual collection " + "search path or (more likely) a bad configuration. Not that this config option should " + "be true when using images produced by CalibrateImageTask." + ) + return super().adjustQuantum(inputs, outputs, label, data_id) + class FinalizeCharacterizationConfig(pipeBase.PipelineTaskConfig, pipelineConnections=FinalizeCharacterizationConnections): @@ -113,6 +146,12 @@ class FinalizeCharacterizationConfig(pipeBase.PipelineTaskConfig, dtype=str, default='sourceId', ) + remove_initial_photo_calib = pexConfig.Field( + doc=("Expect an initial photo calib input to be present, and use it ", + "to restore the image to instrumental units."), + dtype=bool, + default=True, + ) reserve_selection = pexConfig.ConfigurableField( target=ReserveIsolatedStarsTask, doc='Task to select reserved stars', @@ -269,6 +308,8 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): for handle in input_handle_dict['srcs']} calexp_dict_temp = {handle.dataId['detector']: handle for handle in input_handle_dict['calexps']} + initial_photo_calib_dict_temp = {handle.dataId['detector']: handle + for handle in input_handle_dict['initial_photo_calibs']} isolated_star_cat_dict_temp = {handle.dataId['tract']: handle for handle in input_handle_dict['isolated_star_cats']} isolated_star_source_dict_temp = {handle.dataId['tract']: handle @@ -279,6 +320,8 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): detector in sorted(src_dict_temp.keys())} calexp_dict = {detector: calexp_dict_temp[detector] for detector in sorted(calexp_dict_temp.keys())} + initial_photo_calib_dict = {detector: initial_photo_calib_dict_temp[detector] + for detector in sorted(initial_photo_calib_dict_temp.keys())} isolated_star_cat_dict = {tract: isolated_star_cat_dict_temp[tract] for tract in sorted(isolated_star_cat_dict_temp.keys())} isolated_star_source_dict = {tract: isolated_star_source_dict_temp[tract] for @@ -289,14 +332,24 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): isolated_star_cat_dict, isolated_star_source_dict, src_dict, - calexp_dict) + calexp_dict, + initial_photo_calib_dict) butlerQC.put(struct.psf_ap_corr_cat, outputRefs.finalized_psf_ap_corr_cat) butlerQC.put(pd.DataFrame(struct.output_table), outputRefs.finalized_src_table) - def run(self, visit, band, isolated_star_cat_dict, isolated_star_source_dict, src_dict, calexp_dict): + def run( + self, + visit, + band, + isolated_star_cat_dict, + isolated_star_source_dict, + src_dict, + calexp_dict, + initial_photo_calib_dict, + ): """ Run the FinalizeCharacterizationTask. @@ -314,6 +367,8 @@ def run(self, visit, band, isolated_star_cat_dict, isolated_star_source_dict, sr Per-detector dict of src catalog handles. calexp_dict : `dict` Per-detector dict of calibrated exposure handles. + initial_photo_calib_dict : `dict` + Per-detector dict of initial photometric calibration handles Returns ------- @@ -349,13 +404,18 @@ def run(self, visit, band, isolated_star_cat_dict, isolated_star_source_dict, sr for detector in src_dict: src = src_dict[detector].get() exposure = calexp_dict[detector].get() + if detector in initial_photo_calib_dict: + initial_photo_calib = initial_photo_calib_dict[detector].get() + else: + initial_photo_calib = None psf, ap_corr_map, measured_src = self.compute_psf_and_ap_corr_map( visit, detector, exposure, src, - isolated_source_table + isolated_source_table, + initial_photo_calib ) # And now we package it together... @@ -591,7 +651,15 @@ def concat_isolated_star_cats(self, band, isolated_star_cat_dict, isolated_star_ return isolated_table, isolated_source_table - def compute_psf_and_ap_corr_map(self, visit, detector, exposure, src, isolated_source_table): + def compute_psf_and_ap_corr_map( + self, + visit, + detector, + exposure, + src, + isolated_source_table, + initial_photo_calib, + ): """Compute psf model and aperture correction map for a single exposure. Parameters @@ -603,6 +671,8 @@ def compute_psf_and_ap_corr_map(self, visit, detector, exposure, src, isolated_s exposure : `lsst.afw.image.ExposureF` src : `lsst.afw.table.SourceCatalog` isolated_source_table : `np.ndarray` + initial_photo_calib : `lsst.afw.image.PhotoCalib` or `None` + Initial photometric calibration to remove from the image. Returns ------- @@ -613,6 +683,12 @@ def compute_psf_and_ap_corr_map(self, visit, detector, exposure, src, isolated_s measured_src : `lsst.afw.table.SourceCatalog` Updated source catalog with measurements, flags and aperture corrections. """ + if self.config.remove_initial_photo_calib: + if initial_photo_calib is None: + self.log.warning("No initial photo calib found for visit %d, detector %d", visit, detector) + return None, None, None + exposure.maskedImage /= initial_photo_calib.getCalibrationMean() + # Extract footprints from the input src catalog for noise replacement. footprints = SingleFrameMeasurementTask.getFootprintsFromCatalog(src) diff --git a/tests/test_finalizeCharacterization.py b/tests/test_finalizeCharacterization.py index 174b4383d..ca0c0be6a 100644 --- a/tests/test_finalizeCharacterization.py +++ b/tests/test_finalizeCharacterization.py @@ -56,6 +56,7 @@ class FinalizeCharacterizationTestCase(lsst.utils.tests.TestCase): """ def setUp(self): config = FinalizeCharacterizationConfig() + config.remove_initial_photo_calib = False self.finalizeCharacterizationTask = TestFinalizeCharacterizationTask( config=config, @@ -232,7 +233,8 @@ def test_compute_psf_and_ap_corr_map_no_sources(self): detector, exposure, src, - isolated_source_table + isolated_source_table, + initial_photo_calib=None, ) self.assertIn( "No good sources remain after cuts for visit {}, detector {}".format(visit, detector),