From e1161deae310053f99e3e61559ea68ce0e45f958 Mon Sep 17 00:00:00 2001 From: John Franklin Crenshaw Date: Sat, 26 Oct 2024 15:41:06 -0700 Subject: [PATCH 01/16] Added cut on fraction-of-bad-pixels; new defaults for stamp selection during calcZernikes. --- doc/versionHistory.rst | 10 ++ python/lsst/ts/wep/task/calcZernikesTask.py | 7 +- python/lsst/ts/wep/task/cutOutDonutsBase.py | 11 ++ .../ts/wep/task/donutStampSelectorTask.py | 64 +++++++--- tests/task/test_cutOutDonutsBase.py | 2 + tests/task/test_donutStampSelectorTask.py | 109 ++++++++---------- 6 files changed, 123 insertions(+), 80 deletions(-) diff --git a/doc/versionHistory.rst b/doc/versionHistory.rst index 4667f307..779587c5 100644 --- a/doc/versionHistory.rst +++ b/doc/versionHistory.rst @@ -6,6 +6,16 @@ Version History ################## +.. _lsst.ts.wep-12.4.0: + +------------- +12.4.0 +------------- + +* Added a threshold on fraction-of-bad-pixels to DonutStampSelectorTask +* Modified DonutStampSelectorTaskConfig so that, by default, selections are run on fraction-of-bad-pixels and signal-to-noise ratio. +* Modified CalcZernikesTask so that DonutStampSelectorTask is run by default + .. _lsst.ts.wep-12.3.0: ------------- diff --git a/python/lsst/ts/wep/task/calcZernikesTask.py b/python/lsst/ts/wep/task/calcZernikesTask.py index 322840b4..ac470daa 100644 --- a/python/lsst/ts/wep/task/calcZernikesTask.py +++ b/python/lsst/ts/wep/task/calcZernikesTask.py @@ -105,10 +105,13 @@ class CalcZernikesTaskConfig( ), ) donutStampSelector = pexConfig.ConfigurableField( - target=DonutStampSelectorTask, doc="How to select donut stamps." + target=DonutStampSelectorTask, + doc="How to select donut stamps.", ) doDonutStampSelector = pexConfig.Field( - doc="Whether or not to run donut stamp selector.", dtype=bool, default=False + doc="Whether or not to run donut stamp selector.", + dtype=bool, + default=True, ) diff --git a/python/lsst/ts/wep/task/cutOutDonutsBase.py b/python/lsst/ts/wep/task/cutOutDonutsBase.py index 6bcfe129..495910c0 100644 --- a/python/lsst/ts/wep/task/cutOutDonutsBase.py +++ b/python/lsst/ts/wep/task/cutOutDonutsBase.py @@ -572,6 +572,9 @@ def cutOutStamps(self, exposure, donutCatalog, defocalType, cameraName): # Value of entropy stampsEntropy = list() + # Fraction of bad pixels + fracBadPixels = list() + for idx, donutRow in enumerate(donutCatalog): # Make an initial cutout larger than the actual final stamp # so that we can centroid to get the stamp centered exactly @@ -672,6 +675,10 @@ def cutOutStamps(self, exposure, donutCatalog, defocalType, cameraName): isEffective.append(eff) stampsEntropy.append(entro) + # Calculate fraction of bad pixels + bits = finalStamp.mask.getPlaneBitMask(("SAT", "BAD", "NO_DATA")) + fracBadPixels.append(np.mean(np.bitwise_and(finalStamp.mask.array, bits))) + finalStamps.append(donutStamp) # Calculate the difference between original centroid and final centroid @@ -781,4 +788,8 @@ def cutOutStamps(self, exposure, donutCatalog, defocalType, cameraName): # Save the peak of the correlated image stampsMetadata["PEAK_HEIGHT"] = peakHeight + + # Save the fraction of bad pixels + stampsMetadata["FRAC_BAD_PIX"] = np.array(fracBadPixels).astype(float) + return DonutStamps(finalStamps, metadata=stampsMetadata, use_archive=True) diff --git a/python/lsst/ts/wep/task/donutStampSelectorTask.py b/python/lsst/ts/wep/task/donutStampSelectorTask.py index 9c0cd743..8769651a 100644 --- a/python/lsst/ts/wep/task/donutStampSelectorTask.py +++ b/python/lsst/ts/wep/task/donutStampSelectorTask.py @@ -38,10 +38,16 @@ class DonutStampSelectorTaskConfig(pexConfig.Config): ) selectWithSignalToNoise = pexConfig.Field( dtype=bool, - default=False, - doc="Whether to use signal to noise ratio in deciding to use the donut." + default=True, + doc="Whether to use signal to noise ratio in deciding to use the donut. " + "By default the values from snLimitStar.yaml config file are used.", ) + selectWithFracBadPixels = pexConfig.Field( + dtype=bool, + default=True, + doc="Whether to use fraction of bad pixels in deciding to use the donut. " + + "Bad pixels correspond to mask values of 'SAT', 'BAD', 'NO_DATA'.", + ) useCustomSnLimit = pexConfig.Field( dtype=bool, default=False, @@ -62,13 +68,17 @@ class DonutStampSelectorTaskConfig(pexConfig.Config): default=3.5, doc=str("The entropy threshold to use (keep donuts only below the threshold)."), ) + maxFracBadPixels = pexConfig.Field( + dtype=float, + default=0.0, + doc=str("Maximum fraction of bad pixels in selected donuts."), + ) class DonutStampSelectorTask(pipeBase.Task): """ Donut Stamp Selector uses information about donut stamp calculated at - the stamp cutting out stage to select those that fulfill entropy - and/or signal-to-noise criteria. + the stamp cutting out stage to select those that specified criteria. """ ConfigClass = DonutStampSelectorTaskConfig @@ -97,9 +107,9 @@ def run(self, donutStamps): Boolean array of stamps that were selected, same length as donutStamps. - donutsQuality : `astropy.table.QTable` - A table with calculated signal to noise measure and entropy - value per donut, together with selection outcome for all - input donuts. + A table with calculated signal to noise measure, entropy + value per donut, and fraction of bad pixels, together with + selection outcome for all input donuts. """ result = self.selectStamps(donutStamps) @@ -109,7 +119,7 @@ def run(self, donutStamps): ) selectedStamps._refresh_metadata() # Need to copy a few other fields by hand - for k in ["SN", "ENTROPY", "VISIT"]: + for k in ["SN", "ENTROPY", "FRAC_BAD_PIX", "VISIT"]: if k in donutStamps.metadata: selectedStamps.metadata[k] = np.array( [ @@ -157,7 +167,6 @@ def selectStamps(self, donutStamps): value per donut, together with selection outcome for all input donuts. """ - # Which donuts to use for Zernike estimation # initiate these by selecting all donuts entropySelect = np.ones(len(donutStamps), dtype="bool") @@ -168,10 +177,8 @@ def selectStamps(self, donutStamps): if self.config.selectWithEntropy: entropySelect = entropyValue < self.config.maxEntropy else: - self.log.warning( - "No entropy cut. Checking if signal-to-noise \ -should be applied." - ) + self.log.warning("No entropy cut. Checking other conditions.") + # By default select all donuts, only overwritten # if selectWithSignalToNoise is True snSelect = np.ones(len(donutStamps), dtype="bool") @@ -194,12 +201,27 @@ def selectStamps(self, donutStamps): # Select using the given threshold snSelect = snThreshold < snValue else: - self.log.warning("No signal-to-noise selection applied.") + self.log.warning( + "No signal-to-noise selection applied. Checking other conditions" + ) + + # By default select all donuts, only overwritten + # if selectWithFracBadPixels is True + fracBadPixSelect = np.ones(len(donutStamps), dtype="bool") + + # collect fraction-of-bad-pixels information if available + if "FRAC_BAD_PIX" in list(donutStamps.metadata): + fracBadPix = np.asarray(donutStamps.metadata.getArray("FRAC_BAD_PIX")) + if self.config.selectWithFracBadPixels: + fracBadPixSelect = fracBadPix <= self.config.maxFracBadPixels + else: + self.log.warning("No fraction-of-bad-pixels cut.") + # AND condition : if both selectWithEntropy # and selectWithSignalToNoise, then # only donuts that pass with SN criterion as well # as entropy criterion are selected - selected = entropySelect * snSelect + selected = entropySelect * snSelect * fracBadPixSelect # store information about which donuts were selected # use QTable even though no units at the moment in @@ -209,11 +231,21 @@ def selectStamps(self, donutStamps): data=[ snValue, entropyValue, + fracBadPix, snSelect, entropySelect, + fracBadPixSelect, selected, ], - names=["SN", "ENTROPY", "SN_SELECT", "ENTROPY_SELECT", "FINAL_SELECT"], + names=[ + "SN", + "ENTROPY", + "FRAC_BAD_PIX", + "SN_SELECT", + "ENTROPY_SELECT", + "FRAC_BAD_PIX_SELECT", + "FINAL_SELECT", + ], ) self.log.info("Selected %d/%d donut stamps", selected.sum(), len(donutStamps)) diff --git a/tests/task/test_cutOutDonutsBase.py b/tests/task/test_cutOutDonutsBase.py index 045eeea4..42ba226d 100644 --- a/tests/task/test_cutOutDonutsBase.py +++ b/tests/task/test_cutOutDonutsBase.py @@ -422,6 +422,7 @@ def testCutOutStampsTaskRunNormal(self): "EFFECTIVE", "ENTROPY", "PEAK_HEIGHT", + "FRAC_BAD_PIX", "MJD", "BORESIGHT_ROT_ANGLE_RAD", "BORESIGHT_PAR_ANGLE_RAD", @@ -447,6 +448,7 @@ def testCutOutStampsTaskRunNormal(self): "EFFECTIVE", "ENTROPY", "PEAK_HEIGHT", + "FRAC_BAD_PIX", ]: self.assertEqual( len(donutStamps), len(donutStamps.metadata.getArray(measure)) diff --git a/tests/task/test_donutStampSelectorTask.py b/tests/task/test_donutStampSelectorTask.py index e786eb64..f5463013 100644 --- a/tests/task/test_donutStampSelectorTask.py +++ b/tests/task/test_donutStampSelectorTask.py @@ -95,42 +95,49 @@ def testValidateConfigs(self): # Test the default config values self.OrigTask = DonutStampSelectorTask(config=self.config, name="Orig Task") self.assertEqual(self.OrigTask.config.selectWithEntropy, False) - self.assertEqual(self.OrigTask.config.selectWithSignalToNoise, False) + self.assertEqual(self.OrigTask.config.selectWithSignalToNoise, True) + self.assertEqual(self.OrigTask.config.selectWithFracBadPixels, True) self.assertEqual(self.OrigTask.config.useCustomSnLimit, False) # Test changing configs self.config.selectWithEntropy = True - self.config.selectWithSignalToNoise = True + self.config.selectWithSignalToNoise = False + self.config.selectWithFracBadPixels = False self.config.minSignalToNoise = 999 self.config.maxEntropy = 4 + self.config.maxFracBadPixels = 0.2 self.ModifiedTask = DonutStampSelectorTask(config=self.config, name="Mod Task") self.assertEqual(self.ModifiedTask.config.selectWithEntropy, True) - self.assertEqual(self.ModifiedTask.config.selectWithSignalToNoise, True) + self.assertEqual(self.ModifiedTask.config.selectWithSignalToNoise, False) + self.assertEqual(self.ModifiedTask.config.selectWithFracBadPixels, False) self.assertEqual(self.ModifiedTask.config.minSignalToNoise, 999) self.assertEqual(self.ModifiedTask.config.maxEntropy, 4) + self.assertEqual(self.ModifiedTask.config.maxFracBadPixels, 0.2) def testSelectStamps(self): donutStampsIntra = self.butler.get( "donutStampsIntra", dataId=self.dataIdExtra, collections=[self.runName] ) - # test default: no donuts are excluded + # test defaults selection = self.task.selectStamps(donutStampsIntra) + donutsQuality = selection.donutsQuality - # by default, config.selectWithEntropy is False, + # by default, config.selectWithEntropy is False, # so we select all donuts - self.assertEqual(np.sum(selection.donutsQuality["ENTROPY_SELECT"]), 3) + self.assertEqual(np.sum(donutsQuality["ENTROPY_SELECT"]), 3) - # by default, config.selectWithSignalToNoise is False, - # so we select all donuts - self.assertEqual(np.sum(selection.donutsQuality["SN_SELECT"]), 3) + # by default, SNR selection happens and uses yaml config values + # so that all donuts here would get selected + self.assertEqual(np.sum(donutsQuality["SN_SELECT"]), 3) - # The final selection is the union of what was selected - # according to SN selection and entropy selection - self.assertEqual(np.sum(selection.donutsQuality["FINAL_SELECT"]), 3) + # by default, fraction-of-bad-pixels happens + # these donuts are all fine, so all are selected + self.assertEqual(np.sum(donutsQuality["FRAC_BAD_PIX_SELECT"]), 3) - # Test that identical information is conveyed here + # Test that overall selection also shows all three donuts + self.assertEqual(np.sum(donutsQuality["FINAL_SELECT"]), 3) self.assertEqual(np.sum(selection.selected), 3) # switch on selectWithEntropy, @@ -151,20 +158,10 @@ def testSelectStamps(self): entropyThreshold, ) - # switch on selectWithSignalToNoise - self.config.selectWithSignalToNoise = True - task = DonutStampSelectorTask(config=self.config, name="SN Task") - selection = task.selectStamps(donutStampsIntra) - donutsQuality = selection.donutsQuality - - # by default we use the yaml config values so that - # all donuts here would get selected - self.assertEqual(np.sum(donutsQuality["SN_SELECT"]), 3) - - # test that if we use the custom threshold, - # some donuts won't get selected + # test custom SNR thresholds + self.config.selectWithEntropy = False self.config.useCustomSnLimit = True - minSignalToNoise = 1658 + minSignalToNoise = 1585 self.config.minSignalToNoise = minSignalToNoise task = DonutStampSelectorTask(config=self.config, name="SN Task") selection = task.selectStamps(donutStampsIntra) @@ -175,48 +172,36 @@ def testSelectStamps(self): for v in donutsQuality["SN"][donutsQuality["SN_SELECT"]]: self.assertLess(minSignalToNoise, v) + # Make sure that stamps with bad pixels are cut + badPix = np.asarray(donutStampsIntra.metadata.getArray("FRAC_BAD_PIX")) + badPix[0] = 0.1 + donutStampsIntra.metadata.set("FRAC_BAD_PIX", badPix) + selection = self.task.selectStamps(donutStampsIntra) + donutsQuality = selection.donutsQuality + self.assertEqual(np.sum(donutsQuality["FRAC_BAD_PIX_SELECT"]), 2) + + # finally turn all selections off and make sure everything is selected + self.config.selectWithEntropy = False + self.config.selectWithSignalToNoise = False + self.config.selectWithFracBadPixels = False + task = DonutStampSelectorTask(config=self.config, name="All off") + selection = task.selectStamps(donutStampsIntra) + self.assertEqual(np.sum(selection.donutsQuality["ENTROPY_SELECT"]), 3) + self.assertEqual(np.sum(selection.donutsQuality["SN_SELECT"]), 3) + self.assertEqual(np.sum(selection.donutsQuality["FRAC_BAD_PIX_SELECT"]), 3) + self.assertEqual(np.sum(selection.donutsQuality["FINAL_SELECT"]), 3) + def testTaskRun(self): donutStampsIntra = self.butler.get( "donutStampsIntra", dataId=self.dataIdExtra, collections=[self.runName] ) - # test default: no donuts are excluded + + # test defaults taskOut = self.task.run(donutStampsIntra) donutsQuality = taskOut.donutsQuality selected = taskOut.selected donutStampsSelect = taskOut.donutStampsSelect - # Test that the length of the donutStamps is as expected - self.assertEqual(len(donutStampsSelect), 3) - - # by default, config.selectWithEntropy is False, - # so we select all donuts - self.assertEqual(np.sum(donutsQuality["ENTROPY_SELECT"]), 3) - - # by default, config.selectWithSignalToNoise is False, - # so we select all donuts - self.assertEqual(np.sum(donutsQuality["SN_SELECT"]), 3) - - # The final selection is the union of what was selected - # according to SN selection and entropy selection - self.assertEqual(np.sum(donutsQuality["FINAL_SELECT"]), 3) - - # Test that identical information is conveyed here - self.assertEqual(np.sum(selected), 3) - - # switch on selectWithEntropy, - # set config.maxEntropy so that one donut is selected - self.config.selectWithEntropy = True - entropyThreshold = 2.85 - self.config.maxEntropy = entropyThreshold - - task = DonutStampSelectorTask(config=self.config, name="Entropy Task") - taskOut = task.run(donutStampsIntra) - donutsQuality = taskOut.donutsQuality - self.assertEqual(np.sum(donutsQuality["ENTROPY_SELECT"]), 1) - - # also test that the entropy of the selected donut - # is indeed below threshold - self.assertLess( - donutsQuality["ENTROPY"][donutsQuality["ENTROPY_SELECT"]], - entropyThreshold, - ) + # Test that final selection numbers match + self.assertEqual(len(donutStampsSelect), selected.sum()) + self.assertEqual(len(donutStampsSelect), donutsQuality["FINAL_SELECT"].sum()) From f217fdbd3d8c3d380d7658025d1b830ded45d3d8 Mon Sep 17 00:00:00 2001 From: John Franklin Crenshaw Date: Sat, 26 Oct 2024 15:46:12 -0700 Subject: [PATCH 02/16] Linting. --- tests/task/test_donutStampSelectorTask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/task/test_donutStampSelectorTask.py b/tests/task/test_donutStampSelectorTask.py index f5463013..780bc37d 100644 --- a/tests/task/test_donutStampSelectorTask.py +++ b/tests/task/test_donutStampSelectorTask.py @@ -128,7 +128,7 @@ def testSelectStamps(self): # so we select all donuts self.assertEqual(np.sum(donutsQuality["ENTROPY_SELECT"]), 3) - # by default, SNR selection happens and uses yaml config values + # by default, SNR selection happens and uses yaml config values # so that all donuts here would get selected self.assertEqual(np.sum(donutsQuality["SN_SELECT"]), 3) From f0c3218962ebbb37277647bc9703bccc11a5a510 Mon Sep 17 00:00:00 2001 From: John Franklin Crenshaw Date: Sat, 26 Oct 2024 16:56:12 -0700 Subject: [PATCH 03/16] Addressing Chris's comments --- python/lsst/ts/wep/task/cutOutDonutsBase.py | 7 ++++++- .../lsst/ts/wep/task/donutStampSelectorTask.py | 5 +---- tests/task/test_cutOutDonutsBase.py | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/python/lsst/ts/wep/task/cutOutDonutsBase.py b/python/lsst/ts/wep/task/cutOutDonutsBase.py index 495910c0..abf04846 100644 --- a/python/lsst/ts/wep/task/cutOutDonutsBase.py +++ b/python/lsst/ts/wep/task/cutOutDonutsBase.py @@ -147,6 +147,11 @@ class CutOutDonutsBaseTaskConfig( dtype=int, default=3, ) + badPixelMaskDefinitions = pexConfig.ListField( + doc="List of mask values flagged as 'bad' for Zernike estimation.", + dtype=str, + default=["SAT", "BAD", "NO_DATA"], + ) class CutOutDonutsBaseTask(pipeBase.PipelineTask): @@ -676,7 +681,7 @@ def cutOutStamps(self, exposure, donutCatalog, defocalType, cameraName): stampsEntropy.append(entro) # Calculate fraction of bad pixels - bits = finalStamp.mask.getPlaneBitMask(("SAT", "BAD", "NO_DATA")) + bits = finalStamp.mask.getPlaneBitMask(self.config.badPixelMaskDefinitions) fracBadPixels.append(np.mean(np.bitwise_and(finalStamp.mask.array, bits))) finalStamps.append(donutStamp) diff --git a/python/lsst/ts/wep/task/donutStampSelectorTask.py b/python/lsst/ts/wep/task/donutStampSelectorTask.py index 8769651a..b3710d3b 100644 --- a/python/lsst/ts/wep/task/donutStampSelectorTask.py +++ b/python/lsst/ts/wep/task/donutStampSelectorTask.py @@ -217,10 +217,7 @@ def selectStamps(self, donutStamps): else: self.log.warning("No fraction-of-bad-pixels cut.") - # AND condition : if both selectWithEntropy - # and selectWithSignalToNoise, then - # only donuts that pass with SN criterion as well - # as entropy criterion are selected + # AND statement: choose only donuts that satisfy all selected conditions selected = entropySelect * snSelect * fracBadPixSelect # store information about which donuts were selected diff --git a/tests/task/test_cutOutDonutsBase.py b/tests/task/test_cutOutDonutsBase.py index 42ba226d..bc7c269b 100644 --- a/tests/task/test_cutOutDonutsBase.py +++ b/tests/task/test_cutOutDonutsBase.py @@ -519,3 +519,19 @@ def testCalculateSNWithLargeMask(self): ) infoMsg += "of the image; reducing the amount of donut mask dilation to 99" self.assertEqual(infoMsg, cm.output[0]) + + def testBadPixelMaskDefinitions(self): + # Load test data + exposure, donutCatalog = self._getExpAndCatalog(DefocalType.Extra) + + # Flag donut pixels as bad + self.config.badPixelMaskDefinitions=["DONUT"] + task = CutOutDonutsBaseTask(config=self.config, name="Flag donut pix as bad") + donutStamps = task.cutOutStamps( + exposure, donutCatalog, DefocalType.Extra, self.cameraName + ) + + # Check that all the stamps have "bad" pixels + # (because we flagged donut pixels as bad) + fracBadPix = np.asarray(donutStamps.metadata.getArray("FRAC_BAD_PIX")) + self.assertTrue(np.all(fracBadPix > 0)) \ No newline at end of file From 651d73da17ae7a80a957c11d85405f3c86ec64e7 Mon Sep 17 00:00:00 2001 From: John Franklin Crenshaw Date: Sat, 26 Oct 2024 17:00:40 -0700 Subject: [PATCH 04/16] Linting. --- tests/task/test_cutOutDonutsBase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/task/test_cutOutDonutsBase.py b/tests/task/test_cutOutDonutsBase.py index bc7c269b..1ffa1b30 100644 --- a/tests/task/test_cutOutDonutsBase.py +++ b/tests/task/test_cutOutDonutsBase.py @@ -525,7 +525,7 @@ def testBadPixelMaskDefinitions(self): exposure, donutCatalog = self._getExpAndCatalog(DefocalType.Extra) # Flag donut pixels as bad - self.config.badPixelMaskDefinitions=["DONUT"] + self.config.badPixelMaskDefinitions = ["DONUT"] task = CutOutDonutsBaseTask(config=self.config, name="Flag donut pix as bad") donutStamps = task.cutOutStamps( exposure, donutCatalog, DefocalType.Extra, self.cameraName @@ -534,4 +534,4 @@ def testBadPixelMaskDefinitions(self): # Check that all the stamps have "bad" pixels # (because we flagged donut pixels as bad) fracBadPix = np.asarray(donutStamps.metadata.getArray("FRAC_BAD_PIX")) - self.assertTrue(np.all(fracBadPix > 0)) \ No newline at end of file + self.assertTrue(np.all(fracBadPix > 0)) From 5b5e9794c6c6e9f3b313cf2d6ea0ada625365044 Mon Sep 17 00:00:00 2001 From: John Franklin Crenshaw Date: Sat, 26 Oct 2024 17:04:00 -0700 Subject: [PATCH 05/16] Linting. --- python/lsst/ts/wep/task/donutStampSelectorTask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lsst/ts/wep/task/donutStampSelectorTask.py b/python/lsst/ts/wep/task/donutStampSelectorTask.py index b3710d3b..89bfdea2 100644 --- a/python/lsst/ts/wep/task/donutStampSelectorTask.py +++ b/python/lsst/ts/wep/task/donutStampSelectorTask.py @@ -217,7 +217,7 @@ def selectStamps(self, donutStamps): else: self.log.warning("No fraction-of-bad-pixels cut.") - # AND statement: choose only donuts that satisfy all selected conditions + # choose only donuts that satisfy all selected conditions selected = entropySelect * snSelect * fracBadPixSelect # store information about which donuts were selected From 12d87279c3bd162b5664aa1777a2e178de40a1a1 Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Sat, 26 Oct 2024 17:48:35 -0700 Subject: [PATCH 06/16] Switch to new ISR task --- pipelines/production/comCamRapidAnalysisPipeline.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pipelines/production/comCamRapidAnalysisPipeline.yaml b/pipelines/production/comCamRapidAnalysisPipeline.yaml index f7ddb21b..4004b766 100644 --- a/pipelines/production/comCamRapidAnalysisPipeline.yaml +++ b/pipelines/production/comCamRapidAnalysisPipeline.yaml @@ -21,7 +21,7 @@ tasks: estimateZernikes.saveHistory: False estimateZernikes.maskKwargs: {'doMaskBlends': False} isr: - class: lsst.ip.isr.IsrTask + isr: lsst.ip.isr.IsrTaskLSST config: # Although we don't have to apply the amp offset corrections, we do want # to compute them for analyzeAmpOffsetMetadata to report on as metrics. @@ -29,7 +29,6 @@ tasks: ampOffset.doApplyAmpOffset: false # Turn off slow steps in ISR doBrighterFatter: false - doCrosstalk: false aggregateZernikeTablesTask: class: lsst.donut.viz.AggregateZernikeTablesTask aggregateDonutTablesTask: From 3b1759cd485473cc66ff9be7a6bb2047a2c18b15 Mon Sep 17 00:00:00 2001 From: John Franklin Crenshaw <41785729+jfcrenshaw@users.noreply.github.com> Date: Sat, 26 Oct 2024 20:30:53 -0700 Subject: [PATCH 07/16] Fix class keyword in ComCam RA pipeline --- pipelines/production/comCamRapidAnalysisPipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/production/comCamRapidAnalysisPipeline.yaml b/pipelines/production/comCamRapidAnalysisPipeline.yaml index 4004b766..769947a5 100644 --- a/pipelines/production/comCamRapidAnalysisPipeline.yaml +++ b/pipelines/production/comCamRapidAnalysisPipeline.yaml @@ -21,7 +21,7 @@ tasks: estimateZernikes.saveHistory: False estimateZernikes.maskKwargs: {'doMaskBlends': False} isr: - isr: lsst.ip.isr.IsrTaskLSST + class: lsst.ip.isr.IsrTaskLSST config: # Although we don't have to apply the amp offset corrections, we do want # to compute them for analyzeAmpOffsetMetadata to report on as metrics. From 148ab0759dd62ccf54d103f4a30cdfc80b6f668c Mon Sep 17 00:00:00 2001 From: John Franklin Crenshaw Date: Sat, 26 Oct 2024 21:50:18 -0700 Subject: [PATCH 08/16] Fixing a few tests. --- tests/task/test_calcZernikesTieTaskScienceSensor.py | 4 +++- tests/task/test_calcZernikesUnpairedTask.py | 2 ++ tests/task/test_donutStampSelectorTask.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/task/test_calcZernikesTieTaskScienceSensor.py b/tests/task/test_calcZernikesTieTaskScienceSensor.py index 22a9f1e6..1f2d0248 100644 --- a/tests/task/test_calcZernikesTieTaskScienceSensor.py +++ b/tests/task/test_calcZernikesTieTaskScienceSensor.py @@ -110,7 +110,7 @@ def testValidateConfigs(self): self.assertEqual(type(self.task.combineZernikes), CombineZernikesMeanTask) self.assertEqual(type(self.task.donutStampSelector), DonutStampSelectorTask) - self.assertEqual(self.task.doDonutStampSelector, False) + self.assertEqual(self.task.doDonutStampSelector, True) def testEstimateZernikes(self): donutStampsExtra = self.butler.get( @@ -204,6 +204,8 @@ def testCalcZernikes(self): "ENTROPY", "ENTROPY_SELECT", "SN_SELECT", + "FRAC_BAD_PIX", + "FRAC_BAD_PIX_SELECT", "FINAL_SELECT", "DEFOCAL_TYPE", ] diff --git a/tests/task/test_calcZernikesUnpairedTask.py b/tests/task/test_calcZernikesUnpairedTask.py index 9bd3989f..7b3d5c68 100644 --- a/tests/task/test_calcZernikesUnpairedTask.py +++ b/tests/task/test_calcZernikesUnpairedTask.py @@ -219,6 +219,8 @@ def testTable(self): "ENTROPY", "ENTROPY_SELECT", "SN_SELECT", + "FRAC_BAD_PIX", + "FRAC_BAD_PIX_SELECT", "FINAL_SELECT", "DEFOCAL_TYPE", ] diff --git a/tests/task/test_donutStampSelectorTask.py b/tests/task/test_donutStampSelectorTask.py index 780bc37d..659796d3 100644 --- a/tests/task/test_donutStampSelectorTask.py +++ b/tests/task/test_donutStampSelectorTask.py @@ -161,7 +161,7 @@ def testSelectStamps(self): # test custom SNR thresholds self.config.selectWithEntropy = False self.config.useCustomSnLimit = True - minSignalToNoise = 1585 + minSignalToNoise = 1658 self.config.minSignalToNoise = minSignalToNoise task = DonutStampSelectorTask(config=self.config, name="SN Task") selection = task.selectStamps(donutStampsIntra) From b36ca26c07503c5d1616c5836780404d644f52aa Mon Sep 17 00:00:00 2001 From: John Franklin Crenshaw Date: Sat, 26 Oct 2024 21:52:49 -0700 Subject: [PATCH 09/16] Adding frac_bad_pix to calcZernikes output. --- python/lsst/ts/wep/task/calcZernikesTask.py | 2 +- tests/task/test_calcZernikesTieTaskScienceSensor.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/python/lsst/ts/wep/task/calcZernikesTask.py b/python/lsst/ts/wep/task/calcZernikesTask.py index ac470daa..02fe879d 100644 --- a/python/lsst/ts/wep/task/calcZernikesTask.py +++ b/python/lsst/ts/wep/task/calcZernikesTask.py @@ -286,7 +286,7 @@ def createZkTable( * u.pixel ) ) - for key in ["MAG", "SN", "ENTROPY"]: + for key in ["MAG", "SN", "ENTROPY", "FRAC_BAD_PIX"]: for stamps, foc in [ (intraStamps, "intra"), (extraStamps, "extra"), diff --git a/tests/task/test_calcZernikesTieTaskScienceSensor.py b/tests/task/test_calcZernikesTieTaskScienceSensor.py index 1f2d0248..fa6575dc 100644 --- a/tests/task/test_calcZernikesTieTaskScienceSensor.py +++ b/tests/task/test_calcZernikesTieTaskScienceSensor.py @@ -171,6 +171,8 @@ def testCalcZernikes(self): "extra_sn", "intra_entropy", "extra_entropy", + "intra_frac_bad_pix", + "extra_frac_bad_pix", ] self.assertLessEqual(set(desired_colnames), set(structNormal.zernikes.colnames)) From bc080a18be242eed2fd36d252ec5ee3b6162a030 Mon Sep 17 00:00:00 2001 From: John Franklin Crenshaw Date: Sun, 27 Oct 2024 13:52:05 -0700 Subject: [PATCH 10/16] Fixing fracBadPix > 1 --- python/lsst/ts/wep/task/cutOutDonutsBase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/lsst/ts/wep/task/cutOutDonutsBase.py b/python/lsst/ts/wep/task/cutOutDonutsBase.py index abf04846..937e7e15 100644 --- a/python/lsst/ts/wep/task/cutOutDonutsBase.py +++ b/python/lsst/ts/wep/task/cutOutDonutsBase.py @@ -682,7 +682,8 @@ def cutOutStamps(self, exposure, donutCatalog, defocalType, cameraName): # Calculate fraction of bad pixels bits = finalStamp.mask.getPlaneBitMask(self.config.badPixelMaskDefinitions) - fracBadPixels.append(np.mean(np.bitwise_and(finalStamp.mask.array, bits))) + badPixels = np.bitwise_and(finalStamp.mask.array, bits) > 0 + fracBadPixels.append(np.mean(badPixels)) finalStamps.append(donutStamp) From 1b0cbea198f629cedec54109589cb9c452687a7b Mon Sep 17 00:00:00 2001 From: John Franklin Crenshaw Date: Sun, 27 Oct 2024 14:30:33 -0700 Subject: [PATCH 11/16] Fixed bug where original mask bits aren't persisted in stamps. --- python/lsst/ts/wep/task/cutOutDonutsBase.py | 5 ++++- python/lsst/ts/wep/task/donutStamp.py | 10 ++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/python/lsst/ts/wep/task/cutOutDonutsBase.py b/python/lsst/ts/wep/task/cutOutDonutsBase.py index 937e7e15..06417537 100644 --- a/python/lsst/ts/wep/task/cutOutDonutsBase.py +++ b/python/lsst/ts/wep/task/cutOutDonutsBase.py @@ -343,7 +343,6 @@ def calculateSN(self, stamp): A dictionary of calculated quantities """ - stamp.makeMask(self.instConfigFile, self.opticalModel) imageArray = stamp.stamp_im.image.array mask = stamp.stamp_im.mask varianceArray = stamp.stamp_im.variance.array @@ -437,6 +436,7 @@ def calculateSN(self, stamp): "ttl_noise_bkgnd_variance": ttlNoiseBkgndVariance, "ttl_noise_donut_variance": ttlNoiseDonutVariance, } + return snDict def filterBadRecentering(self, xShifts, yShifts): @@ -672,6 +672,9 @@ def cutOutStamps(self, exposure, donutCatalog, defocalType, cameraName): archive_element=linear_wcs, ) + # Create image mask + donutStamp.makeMask(self.instConfigFile, self.opticalModel) + # Calculate the S/N per stamp snQuant.append(self.calculateSN(donutStamp)) diff --git a/python/lsst/ts/wep/task/donutStamp.py b/python/lsst/ts/wep/task/donutStamp.py index 352c9ff3..9cd08295 100644 --- a/python/lsst/ts/wep/task/donutStamp.py +++ b/python/lsst/ts/wep/task/donutStamp.py @@ -301,12 +301,7 @@ def makeMask( ): """Create the mask for the image. - Note the mask is returned in the original coordinate system of the info - that came from the butler (i.e. the DVCS, and the CWFSs are rotated - with respect to the science sensors). See sitcomtn-003.lsst.io for more - information. - - Also note that technically the image masks depend on the optical + Note that technically the image masks depend on the optical aberrations, but this function assumes the aberrations are zero. Parameters @@ -363,6 +358,9 @@ def makeMask( nRot = int(eulerZ // 90) stampMask = np.rot90(stampMask, -nRot) + # Add back to the original mask + stampMask += self.stamp_im.mask.array + # Save mask self.stamp_im.setMask(afwImage.Mask(stampMask.copy())) From 5b11683f609959497efbe1088404a3cd178c54e7 Mon Sep 17 00:00:00 2001 From: John Franklin Crenshaw Date: Sun, 27 Oct 2024 14:51:57 -0700 Subject: [PATCH 12/16] Fixed tests. --- python/lsst/ts/wep/task/calcZernikesTask.py | 2 ++ python/lsst/ts/wep/task/cutOutDonutsBase.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/python/lsst/ts/wep/task/calcZernikesTask.py b/python/lsst/ts/wep/task/calcZernikesTask.py index 02fe879d..9c80be27 100644 --- a/python/lsst/ts/wep/task/calcZernikesTask.py +++ b/python/lsst/ts/wep/task/calcZernikesTask.py @@ -162,6 +162,8 @@ def initZkTable(self) -> QTable: ("extra_sn", " Date: Sun, 27 Oct 2024 16:34:21 -0700 Subject: [PATCH 13/16] More fixes --- .../comCamRapidAnalysisPipeline.yaml | 6 ++++- python/lsst/ts/wep/task/donutStamp.py | 13 +++++++++-- tests/task/test_cutOutDonutsBase.py | 1 + tests/task/test_donutStampSelectorTask.py | 23 +++++++------------ 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/pipelines/production/comCamRapidAnalysisPipeline.yaml b/pipelines/production/comCamRapidAnalysisPipeline.yaml index 769947a5..7df10179 100644 --- a/pipelines/production/comCamRapidAnalysisPipeline.yaml +++ b/pipelines/production/comCamRapidAnalysisPipeline.yaml @@ -19,7 +19,7 @@ tasks: config: estimateZernikes.maxNollIndex: 28 estimateZernikes.saveHistory: False - estimateZernikes.maskKwargs: {'doMaskBlends': False} + estimateZernikes.maskKwargs: { "doMaskBlends": False } isr: class: lsst.ip.isr.IsrTaskLSST config: @@ -29,6 +29,10 @@ tasks: ampOffset.doApplyAmpOffset: false # Turn off slow steps in ISR doBrighterFatter: false + # Mask saturated pixels, + # but turn off quadratic crosstalk because it's currently broken + doSaturation: True + crosstalk.doQuadraticCrosstalkCorrection: False aggregateZernikeTablesTask: class: lsst.donut.viz.AggregateZernikeTablesTask aggregateDonutTablesTask: diff --git a/python/lsst/ts/wep/task/donutStamp.py b/python/lsst/ts/wep/task/donutStamp.py index 9cd08295..db4178c8 100644 --- a/python/lsst/ts/wep/task/donutStamp.py +++ b/python/lsst/ts/wep/task/donutStamp.py @@ -358,8 +358,17 @@ def makeMask( nRot = int(eulerZ // 90) stampMask = np.rot90(stampMask, -nRot) - # Add back to the original mask - stampMask += self.stamp_im.mask.array + # First make sure the mask doesn't already have donut/blend bits + # This is so if this function gets called multiple times, the donut + # and blend bits don't get re-added. + mask0 = self.stamp_im.mask.array.copy() + bit = self.stamp_im.mask.getMaskPlaneDict()["DONUT"] + mask0 &= ~(1 << bit) + bit = self.stamp_im.mask.getMaskPlaneDict()["BLEND"] + mask0 &= ~(1 << bit) + + # Add original mask to the new mask + stampMask += mask0 # Save mask self.stamp_im.setMask(afwImage.Mask(stampMask.copy())) diff --git a/tests/task/test_cutOutDonutsBase.py b/tests/task/test_cutOutDonutsBase.py index 1ffa1b30..541f9594 100644 --- a/tests/task/test_cutOutDonutsBase.py +++ b/tests/task/test_cutOutDonutsBase.py @@ -499,6 +499,7 @@ def testCalculateSNWithBlends(self): # Add blend to mask stamp.wep_im.blendOffsets = [[-50, -60]] + stamp.makeMask(self.task.instConfigFile, self.task.opticalModel) sn_dict = self.task.calculateSN(stamp) for val in sn_dict.values(): self.assertFalse(np.isnan(val)) diff --git a/tests/task/test_donutStampSelectorTask.py b/tests/task/test_donutStampSelectorTask.py index 659796d3..6f0e0d19 100644 --- a/tests/task/test_donutStampSelectorTask.py +++ b/tests/task/test_donutStampSelectorTask.py @@ -123,7 +123,7 @@ def testSelectStamps(self): # test defaults selection = self.task.selectStamps(donutStampsIntra) donutsQuality = selection.donutsQuality - + # by default, config.selectWithEntropy is False, # so we select all donuts self.assertEqual(np.sum(donutsQuality["ENTROPY_SELECT"]), 3) @@ -132,16 +132,17 @@ def testSelectStamps(self): # so that all donuts here would get selected self.assertEqual(np.sum(donutsQuality["SN_SELECT"]), 3) - # by default, fraction-of-bad-pixels happens - # these donuts are all fine, so all are selected - self.assertEqual(np.sum(donutsQuality["FRAC_BAD_PIX_SELECT"]), 3) + # by default, it thresholds on fraction-of-bad-pixels + # only one of these test donuts is selected + self.assertEqual(np.sum(donutsQuality["FRAC_BAD_PIX_SELECT"]), 1) - # Test that overall selection also shows all three donuts - self.assertEqual(np.sum(donutsQuality["FINAL_SELECT"]), 3) - self.assertEqual(np.sum(selection.selected), 3) + # Test that overall selection also shows only one donut + self.assertEqual(np.sum(donutsQuality["FINAL_SELECT"]), 1) + self.assertEqual(np.sum(selection.selected), 1) # switch on selectWithEntropy, # set config.maxEntropy so that one donut is selected + self.config.selectWithFracBadPixels = False self.config.selectWithEntropy = True entropyThreshold = 2.85 self.config.maxEntropy = entropyThreshold @@ -172,14 +173,6 @@ def testSelectStamps(self): for v in donutsQuality["SN"][donutsQuality["SN_SELECT"]]: self.assertLess(minSignalToNoise, v) - # Make sure that stamps with bad pixels are cut - badPix = np.asarray(donutStampsIntra.metadata.getArray("FRAC_BAD_PIX")) - badPix[0] = 0.1 - donutStampsIntra.metadata.set("FRAC_BAD_PIX", badPix) - selection = self.task.selectStamps(donutStampsIntra) - donutsQuality = selection.donutsQuality - self.assertEqual(np.sum(donutsQuality["FRAC_BAD_PIX_SELECT"]), 2) - # finally turn all selections off and make sure everything is selected self.config.selectWithEntropy = False self.config.selectWithSignalToNoise = False From 51604e2ac256db7287f83d26e3ce58cefcee2412 Mon Sep 17 00:00:00 2001 From: John Franklin Crenshaw Date: Sun, 27 Oct 2024 16:41:56 -0700 Subject: [PATCH 14/16] Linting. --- tests/task/test_donutStampSelectorTask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/task/test_donutStampSelectorTask.py b/tests/task/test_donutStampSelectorTask.py index 6f0e0d19..8585b5a5 100644 --- a/tests/task/test_donutStampSelectorTask.py +++ b/tests/task/test_donutStampSelectorTask.py @@ -123,7 +123,7 @@ def testSelectStamps(self): # test defaults selection = self.task.selectStamps(donutStampsIntra) donutsQuality = selection.donutsQuality - + # by default, config.selectWithEntropy is False, # so we select all donuts self.assertEqual(np.sum(donutsQuality["ENTROPY_SELECT"]), 3) From c12c9f6c47cedb63c5cbdd71692e6ee686b145a5 Mon Sep 17 00:00:00 2001 From: John Franklin Crenshaw Date: Sun, 27 Oct 2024 16:43:23 -0700 Subject: [PATCH 15/16] More version hist. --- doc/versionHistory.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/versionHistory.rst b/doc/versionHistory.rst index 779587c5..2e440874 100644 --- a/doc/versionHistory.rst +++ b/doc/versionHistory.rst @@ -15,6 +15,7 @@ Version History * Added a threshold on fraction-of-bad-pixels to DonutStampSelectorTask * Modified DonutStampSelectorTaskConfig so that, by default, selections are run on fraction-of-bad-pixels and signal-to-noise ratio. * Modified CalcZernikesTask so that DonutStampSelectorTask is run by default +* Fixed bug where DM mask bits weren't persisting in DonutStamp .. _lsst.ts.wep-12.3.0: From bd72d365b7eabfba9f30f39bb8ae45dd7e99d8d7 Mon Sep 17 00:00:00 2001 From: John Franklin Crenshaw Date: Sun, 27 Oct 2024 17:26:50 -0700 Subject: [PATCH 16/16] Fixing test --- tests/task/test_calcZernikesUnpairedTask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/task/test_calcZernikesUnpairedTask.py b/tests/task/test_calcZernikesUnpairedTask.py index 7b3d5c68..995d9945 100644 --- a/tests/task/test_calcZernikesUnpairedTask.py +++ b/tests/task/test_calcZernikesUnpairedTask.py @@ -130,7 +130,7 @@ def testWithAndWithoutPairs(self): # Check that results are similar diff = np.sqrt(np.sum((meanZk - pairedZk) ** 2)) - self.assertLess(diff, 0.16) + self.assertLess(diff, 0.17) def testTable(self): # Load data from butler