diff --git a/doc/versionHistory.rst b/doc/versionHistory.rst index 4667f307..2e440874 100644 --- a/doc/versionHistory.rst +++ b/doc/versionHistory.rst @@ -6,6 +6,17 @@ Version History ################## +.. _lsst.ts.wep-12.4.0: + +------------- +12.4.0 +------------- + +* Added a threshold on fraction-of-bad-pixels to DonutStampSelectorTask +* Modified DonutStampSelectorTaskConfig so that, by default, selections are run on fraction-of-bad-pixels and signal-to-noise ratio. +* Modified CalcZernikesTask so that DonutStampSelectorTask is run by default +* Fixed bug where DM mask bits weren't persisting in DonutStamp + .. _lsst.ts.wep-12.3.0: ------------- diff --git a/pipelines/production/comCamRapidAnalysisPipeline.yaml b/pipelines/production/comCamRapidAnalysisPipeline.yaml index f7ddb21b..7df10179 100644 --- a/pipelines/production/comCamRapidAnalysisPipeline.yaml +++ b/pipelines/production/comCamRapidAnalysisPipeline.yaml @@ -19,9 +19,9 @@ tasks: config: estimateZernikes.maxNollIndex: 28 estimateZernikes.saveHistory: False - estimateZernikes.maskKwargs: {'doMaskBlends': False} + estimateZernikes.maskKwargs: { "doMaskBlends": False } isr: - class: lsst.ip.isr.IsrTask + class: lsst.ip.isr.IsrTaskLSST config: # Although we don't have to apply the amp offset corrections, we do want # to compute them for analyzeAmpOffsetMetadata to report on as metrics. @@ -29,7 +29,10 @@ tasks: ampOffset.doApplyAmpOffset: false # Turn off slow steps in ISR doBrighterFatter: false - doCrosstalk: false + # Mask saturated pixels, + # but turn off quadratic crosstalk because it's currently broken + doSaturation: True + crosstalk.doQuadraticCrosstalkCorrection: False aggregateZernikeTablesTask: class: lsst.donut.viz.AggregateZernikeTablesTask aggregateDonutTablesTask: diff --git a/python/lsst/ts/wep/task/calcZernikesTask.py b/python/lsst/ts/wep/task/calcZernikesTask.py index 322840b4..9c80be27 100644 --- a/python/lsst/ts/wep/task/calcZernikesTask.py +++ b/python/lsst/ts/wep/task/calcZernikesTask.py @@ -105,10 +105,13 @@ class CalcZernikesTaskConfig( ), ) donutStampSelector = pexConfig.ConfigurableField( - target=DonutStampSelectorTask, doc="How to select donut stamps." + target=DonutStampSelectorTask, + doc="How to select donut stamps.", ) doDonutStampSelector = pexConfig.Field( - doc="Whether or not to run donut stamp selector.", dtype=bool, default=False + doc="Whether or not to run donut stamp selector.", + dtype=bool, + default=True, ) @@ -159,6 +162,8 @@ def initZkTable(self) -> QTable: ("extra_sn", " 0 + fracBadPixels.append(np.mean(badPixels)) + finalStamps.append(donutStamp) # Calculate the difference between original centroid and final centroid @@ -781,4 +797,8 @@ def cutOutStamps(self, exposure, donutCatalog, defocalType, cameraName): # Save the peak of the correlated image stampsMetadata["PEAK_HEIGHT"] = peakHeight + + # Save the fraction of bad pixels + stampsMetadata["FRAC_BAD_PIX"] = np.array(fracBadPixels).astype(float) + return DonutStamps(finalStamps, metadata=stampsMetadata, use_archive=True) diff --git a/python/lsst/ts/wep/task/donutStamp.py b/python/lsst/ts/wep/task/donutStamp.py index 352c9ff3..db4178c8 100644 --- a/python/lsst/ts/wep/task/donutStamp.py +++ b/python/lsst/ts/wep/task/donutStamp.py @@ -301,12 +301,7 @@ def makeMask( ): """Create the mask for the image. - Note the mask is returned in the original coordinate system of the info - that came from the butler (i.e. the DVCS, and the CWFSs are rotated - with respect to the science sensors). See sitcomtn-003.lsst.io for more - information. - - Also note that technically the image masks depend on the optical + Note that technically the image masks depend on the optical aberrations, but this function assumes the aberrations are zero. Parameters @@ -363,6 +358,18 @@ def makeMask( nRot = int(eulerZ // 90) stampMask = np.rot90(stampMask, -nRot) + # First make sure the mask doesn't already have donut/blend bits + # This is so if this function gets called multiple times, the donut + # and blend bits don't get re-added. + mask0 = self.stamp_im.mask.array.copy() + bit = self.stamp_im.mask.getMaskPlaneDict()["DONUT"] + mask0 &= ~(1 << bit) + bit = self.stamp_im.mask.getMaskPlaneDict()["BLEND"] + mask0 &= ~(1 << bit) + + # Add original mask to the new mask + stampMask += mask0 + # Save mask self.stamp_im.setMask(afwImage.Mask(stampMask.copy())) diff --git a/python/lsst/ts/wep/task/donutStampSelectorTask.py b/python/lsst/ts/wep/task/donutStampSelectorTask.py index 9c0cd743..89bfdea2 100644 --- a/python/lsst/ts/wep/task/donutStampSelectorTask.py +++ b/python/lsst/ts/wep/task/donutStampSelectorTask.py @@ -38,10 +38,16 @@ class DonutStampSelectorTaskConfig(pexConfig.Config): ) selectWithSignalToNoise = pexConfig.Field( dtype=bool, - default=False, - doc="Whether to use signal to noise ratio in deciding to use the donut." + default=True, + doc="Whether to use signal to noise ratio in deciding to use the donut. " + "By default the values from snLimitStar.yaml config file are used.", ) + selectWithFracBadPixels = pexConfig.Field( + dtype=bool, + default=True, + doc="Whether to use fraction of bad pixels in deciding to use the donut. " + + "Bad pixels correspond to mask values of 'SAT', 'BAD', 'NO_DATA'.", + ) useCustomSnLimit = pexConfig.Field( dtype=bool, default=False, @@ -62,13 +68,17 @@ class DonutStampSelectorTaskConfig(pexConfig.Config): default=3.5, doc=str("The entropy threshold to use (keep donuts only below the threshold)."), ) + maxFracBadPixels = pexConfig.Field( + dtype=float, + default=0.0, + doc=str("Maximum fraction of bad pixels in selected donuts."), + ) class DonutStampSelectorTask(pipeBase.Task): """ Donut Stamp Selector uses information about donut stamp calculated at - the stamp cutting out stage to select those that fulfill entropy - and/or signal-to-noise criteria. + the stamp cutting out stage to select those that specified criteria. """ ConfigClass = DonutStampSelectorTaskConfig @@ -97,9 +107,9 @@ def run(self, donutStamps): Boolean array of stamps that were selected, same length as donutStamps. - donutsQuality : `astropy.table.QTable` - A table with calculated signal to noise measure and entropy - value per donut, together with selection outcome for all - input donuts. + A table with calculated signal to noise measure, entropy + value per donut, and fraction of bad pixels, together with + selection outcome for all input donuts. """ result = self.selectStamps(donutStamps) @@ -109,7 +119,7 @@ def run(self, donutStamps): ) selectedStamps._refresh_metadata() # Need to copy a few other fields by hand - for k in ["SN", "ENTROPY", "VISIT"]: + for k in ["SN", "ENTROPY", "FRAC_BAD_PIX", "VISIT"]: if k in donutStamps.metadata: selectedStamps.metadata[k] = np.array( [ @@ -157,7 +167,6 @@ def selectStamps(self, donutStamps): value per donut, together with selection outcome for all input donuts. """ - # Which donuts to use for Zernike estimation # initiate these by selecting all donuts entropySelect = np.ones(len(donutStamps), dtype="bool") @@ -168,10 +177,8 @@ def selectStamps(self, donutStamps): if self.config.selectWithEntropy: entropySelect = entropyValue < self.config.maxEntropy else: - self.log.warning( - "No entropy cut. Checking if signal-to-noise \ -should be applied." - ) + self.log.warning("No entropy cut. Checking other conditions.") + # By default select all donuts, only overwritten # if selectWithSignalToNoise is True snSelect = np.ones(len(donutStamps), dtype="bool") @@ -194,12 +201,24 @@ def selectStamps(self, donutStamps): # Select using the given threshold snSelect = snThreshold < snValue else: - self.log.warning("No signal-to-noise selection applied.") - # AND condition : if both selectWithEntropy - # and selectWithSignalToNoise, then - # only donuts that pass with SN criterion as well - # as entropy criterion are selected - selected = entropySelect * snSelect + self.log.warning( + "No signal-to-noise selection applied. Checking other conditions" + ) + + # By default select all donuts, only overwritten + # if selectWithFracBadPixels is True + fracBadPixSelect = np.ones(len(donutStamps), dtype="bool") + + # collect fraction-of-bad-pixels information if available + if "FRAC_BAD_PIX" in list(donutStamps.metadata): + fracBadPix = np.asarray(donutStamps.metadata.getArray("FRAC_BAD_PIX")) + if self.config.selectWithFracBadPixels: + fracBadPixSelect = fracBadPix <= self.config.maxFracBadPixels + else: + self.log.warning("No fraction-of-bad-pixels cut.") + + # choose only donuts that satisfy all selected conditions + selected = entropySelect * snSelect * fracBadPixSelect # store information about which donuts were selected # use QTable even though no units at the moment in @@ -209,11 +228,21 @@ def selectStamps(self, donutStamps): data=[ snValue, entropyValue, + fracBadPix, snSelect, entropySelect, + fracBadPixSelect, selected, ], - names=["SN", "ENTROPY", "SN_SELECT", "ENTROPY_SELECT", "FINAL_SELECT"], + names=[ + "SN", + "ENTROPY", + "FRAC_BAD_PIX", + "SN_SELECT", + "ENTROPY_SELECT", + "FRAC_BAD_PIX_SELECT", + "FINAL_SELECT", + ], ) self.log.info("Selected %d/%d donut stamps", selected.sum(), len(donutStamps)) diff --git a/tests/task/test_calcZernikesTieTaskScienceSensor.py b/tests/task/test_calcZernikesTieTaskScienceSensor.py index 22a9f1e6..fa6575dc 100644 --- a/tests/task/test_calcZernikesTieTaskScienceSensor.py +++ b/tests/task/test_calcZernikesTieTaskScienceSensor.py @@ -110,7 +110,7 @@ def testValidateConfigs(self): self.assertEqual(type(self.task.combineZernikes), CombineZernikesMeanTask) self.assertEqual(type(self.task.donutStampSelector), DonutStampSelectorTask) - self.assertEqual(self.task.doDonutStampSelector, False) + self.assertEqual(self.task.doDonutStampSelector, True) def testEstimateZernikes(self): donutStampsExtra = self.butler.get( @@ -171,6 +171,8 @@ def testCalcZernikes(self): "extra_sn", "intra_entropy", "extra_entropy", + "intra_frac_bad_pix", + "extra_frac_bad_pix", ] self.assertLessEqual(set(desired_colnames), set(structNormal.zernikes.colnames)) @@ -204,6 +206,8 @@ def testCalcZernikes(self): "ENTROPY", "ENTROPY_SELECT", "SN_SELECT", + "FRAC_BAD_PIX", + "FRAC_BAD_PIX_SELECT", "FINAL_SELECT", "DEFOCAL_TYPE", ] diff --git a/tests/task/test_calcZernikesUnpairedTask.py b/tests/task/test_calcZernikesUnpairedTask.py index 9bd3989f..995d9945 100644 --- a/tests/task/test_calcZernikesUnpairedTask.py +++ b/tests/task/test_calcZernikesUnpairedTask.py @@ -130,7 +130,7 @@ def testWithAndWithoutPairs(self): # Check that results are similar diff = np.sqrt(np.sum((meanZk - pairedZk) ** 2)) - self.assertLess(diff, 0.16) + self.assertLess(diff, 0.17) def testTable(self): # Load data from butler @@ -219,6 +219,8 @@ def testTable(self): "ENTROPY", "ENTROPY_SELECT", "SN_SELECT", + "FRAC_BAD_PIX", + "FRAC_BAD_PIX_SELECT", "FINAL_SELECT", "DEFOCAL_TYPE", ] diff --git a/tests/task/test_cutOutDonutsBase.py b/tests/task/test_cutOutDonutsBase.py index 045eeea4..541f9594 100644 --- a/tests/task/test_cutOutDonutsBase.py +++ b/tests/task/test_cutOutDonutsBase.py @@ -422,6 +422,7 @@ def testCutOutStampsTaskRunNormal(self): "EFFECTIVE", "ENTROPY", "PEAK_HEIGHT", + "FRAC_BAD_PIX", "MJD", "BORESIGHT_ROT_ANGLE_RAD", "BORESIGHT_PAR_ANGLE_RAD", @@ -447,6 +448,7 @@ def testCutOutStampsTaskRunNormal(self): "EFFECTIVE", "ENTROPY", "PEAK_HEIGHT", + "FRAC_BAD_PIX", ]: self.assertEqual( len(donutStamps), len(donutStamps.metadata.getArray(measure)) @@ -497,6 +499,7 @@ def testCalculateSNWithBlends(self): # Add blend to mask stamp.wep_im.blendOffsets = [[-50, -60]] + stamp.makeMask(self.task.instConfigFile, self.task.opticalModel) sn_dict = self.task.calculateSN(stamp) for val in sn_dict.values(): self.assertFalse(np.isnan(val)) @@ -517,3 +520,19 @@ def testCalculateSNWithLargeMask(self): ) infoMsg += "of the image; reducing the amount of donut mask dilation to 99" self.assertEqual(infoMsg, cm.output[0]) + + def testBadPixelMaskDefinitions(self): + # Load test data + exposure, donutCatalog = self._getExpAndCatalog(DefocalType.Extra) + + # Flag donut pixels as bad + self.config.badPixelMaskDefinitions = ["DONUT"] + task = CutOutDonutsBaseTask(config=self.config, name="Flag donut pix as bad") + donutStamps = task.cutOutStamps( + exposure, donutCatalog, DefocalType.Extra, self.cameraName + ) + + # Check that all the stamps have "bad" pixels + # (because we flagged donut pixels as bad) + fracBadPix = np.asarray(donutStamps.metadata.getArray("FRAC_BAD_PIX")) + self.assertTrue(np.all(fracBadPix > 0)) diff --git a/tests/task/test_donutStampSelectorTask.py b/tests/task/test_donutStampSelectorTask.py index e786eb64..8585b5a5 100644 --- a/tests/task/test_donutStampSelectorTask.py +++ b/tests/task/test_donutStampSelectorTask.py @@ -95,46 +95,54 @@ def testValidateConfigs(self): # Test the default config values self.OrigTask = DonutStampSelectorTask(config=self.config, name="Orig Task") self.assertEqual(self.OrigTask.config.selectWithEntropy, False) - self.assertEqual(self.OrigTask.config.selectWithSignalToNoise, False) + self.assertEqual(self.OrigTask.config.selectWithSignalToNoise, True) + self.assertEqual(self.OrigTask.config.selectWithFracBadPixels, True) self.assertEqual(self.OrigTask.config.useCustomSnLimit, False) # Test changing configs self.config.selectWithEntropy = True - self.config.selectWithSignalToNoise = True + self.config.selectWithSignalToNoise = False + self.config.selectWithFracBadPixels = False self.config.minSignalToNoise = 999 self.config.maxEntropy = 4 + self.config.maxFracBadPixels = 0.2 self.ModifiedTask = DonutStampSelectorTask(config=self.config, name="Mod Task") self.assertEqual(self.ModifiedTask.config.selectWithEntropy, True) - self.assertEqual(self.ModifiedTask.config.selectWithSignalToNoise, True) + self.assertEqual(self.ModifiedTask.config.selectWithSignalToNoise, False) + self.assertEqual(self.ModifiedTask.config.selectWithFracBadPixels, False) self.assertEqual(self.ModifiedTask.config.minSignalToNoise, 999) self.assertEqual(self.ModifiedTask.config.maxEntropy, 4) + self.assertEqual(self.ModifiedTask.config.maxFracBadPixels, 0.2) def testSelectStamps(self): donutStampsIntra = self.butler.get( "donutStampsIntra", dataId=self.dataIdExtra, collections=[self.runName] ) - # test default: no donuts are excluded + # test defaults selection = self.task.selectStamps(donutStampsIntra) + donutsQuality = selection.donutsQuality - # by default, config.selectWithEntropy is False, + # by default, config.selectWithEntropy is False, # so we select all donuts - self.assertEqual(np.sum(selection.donutsQuality["ENTROPY_SELECT"]), 3) + self.assertEqual(np.sum(donutsQuality["ENTROPY_SELECT"]), 3) - # by default, config.selectWithSignalToNoise is False, - # so we select all donuts - self.assertEqual(np.sum(selection.donutsQuality["SN_SELECT"]), 3) + # by default, SNR selection happens and uses yaml config values + # so that all donuts here would get selected + self.assertEqual(np.sum(donutsQuality["SN_SELECT"]), 3) - # The final selection is the union of what was selected - # according to SN selection and entropy selection - self.assertEqual(np.sum(selection.donutsQuality["FINAL_SELECT"]), 3) + # by default, it thresholds on fraction-of-bad-pixels + # only one of these test donuts is selected + self.assertEqual(np.sum(donutsQuality["FRAC_BAD_PIX_SELECT"]), 1) - # Test that identical information is conveyed here - self.assertEqual(np.sum(selection.selected), 3) + # Test that overall selection also shows only one donut + self.assertEqual(np.sum(donutsQuality["FINAL_SELECT"]), 1) + self.assertEqual(np.sum(selection.selected), 1) # switch on selectWithEntropy, # set config.maxEntropy so that one donut is selected + self.config.selectWithFracBadPixels = False self.config.selectWithEntropy = True entropyThreshold = 2.85 self.config.maxEntropy = entropyThreshold @@ -151,18 +159,8 @@ def testSelectStamps(self): entropyThreshold, ) - # switch on selectWithSignalToNoise - self.config.selectWithSignalToNoise = True - task = DonutStampSelectorTask(config=self.config, name="SN Task") - selection = task.selectStamps(donutStampsIntra) - donutsQuality = selection.donutsQuality - - # by default we use the yaml config values so that - # all donuts here would get selected - self.assertEqual(np.sum(donutsQuality["SN_SELECT"]), 3) - - # test that if we use the custom threshold, - # some donuts won't get selected + # test custom SNR thresholds + self.config.selectWithEntropy = False self.config.useCustomSnLimit = True minSignalToNoise = 1658 self.config.minSignalToNoise = minSignalToNoise @@ -175,48 +173,28 @@ def testSelectStamps(self): for v in donutsQuality["SN"][donutsQuality["SN_SELECT"]]: self.assertLess(minSignalToNoise, v) + # finally turn all selections off and make sure everything is selected + self.config.selectWithEntropy = False + self.config.selectWithSignalToNoise = False + self.config.selectWithFracBadPixels = False + task = DonutStampSelectorTask(config=self.config, name="All off") + selection = task.selectStamps(donutStampsIntra) + self.assertEqual(np.sum(selection.donutsQuality["ENTROPY_SELECT"]), 3) + self.assertEqual(np.sum(selection.donutsQuality["SN_SELECT"]), 3) + self.assertEqual(np.sum(selection.donutsQuality["FRAC_BAD_PIX_SELECT"]), 3) + self.assertEqual(np.sum(selection.donutsQuality["FINAL_SELECT"]), 3) + def testTaskRun(self): donutStampsIntra = self.butler.get( "donutStampsIntra", dataId=self.dataIdExtra, collections=[self.runName] ) - # test default: no donuts are excluded + + # test defaults taskOut = self.task.run(donutStampsIntra) donutsQuality = taskOut.donutsQuality selected = taskOut.selected donutStampsSelect = taskOut.donutStampsSelect - # Test that the length of the donutStamps is as expected - self.assertEqual(len(donutStampsSelect), 3) - - # by default, config.selectWithEntropy is False, - # so we select all donuts - self.assertEqual(np.sum(donutsQuality["ENTROPY_SELECT"]), 3) - - # by default, config.selectWithSignalToNoise is False, - # so we select all donuts - self.assertEqual(np.sum(donutsQuality["SN_SELECT"]), 3) - - # The final selection is the union of what was selected - # according to SN selection and entropy selection - self.assertEqual(np.sum(donutsQuality["FINAL_SELECT"]), 3) - - # Test that identical information is conveyed here - self.assertEqual(np.sum(selected), 3) - - # switch on selectWithEntropy, - # set config.maxEntropy so that one donut is selected - self.config.selectWithEntropy = True - entropyThreshold = 2.85 - self.config.maxEntropy = entropyThreshold - - task = DonutStampSelectorTask(config=self.config, name="Entropy Task") - taskOut = task.run(donutStampsIntra) - donutsQuality = taskOut.donutsQuality - self.assertEqual(np.sum(donutsQuality["ENTROPY_SELECT"]), 1) - - # also test that the entropy of the selected donut - # is indeed below threshold - self.assertLess( - donutsQuality["ENTROPY"][donutsQuality["ENTROPY_SELECT"]], - entropyThreshold, - ) + # Test that final selection numbers match + self.assertEqual(len(donutStampsSelect), selected.sum()) + self.assertEqual(len(donutStampsSelect), donutsQuality["FINAL_SELECT"].sum())