tickets/DM-47163: Cut on fraction of bad pixels; run stamp selection during CalcZernikeTask by default

doc/versionHistory.rst

@@ -6,6 +6,16 @@
 Version History
 ##################
 
+.. _lsst.ts.wep-12.4.0:
+
+-------------
+12.4.0
+-------------
+
+* Added a threshold on fraction-of-bad-pixels to DonutStampSelectorTask
+* Modified DonutStampSelectorTaskConfig so that, by default, selections are run on fraction-of-bad-pixels and signal-to-noise ratio.
+* Modified CalcZernikesTask so that DonutStampSelectorTask is run by default
+
 .. _lsst.ts.wep-12.3.0:
 
 -------------

python/lsst/ts/wep/task/calcZernikesTask.py

@@ -105,10 +105,13 @@ class CalcZernikesTaskConfig(
         ),
     )
     donutStampSelector = pexConfig.ConfigurableField(
-        target=DonutStampSelectorTask, doc="How to select donut stamps."
+        target=DonutStampSelectorTask,
+        doc="How to select donut stamps.",
     )
     doDonutStampSelector = pexConfig.Field(
-        doc="Whether or not to run donut stamp selector.", dtype=bool, default=False
+        doc="Whether or not to run donut stamp selector.",
+        dtype=bool,
+        default=True,
     )
 
 

python/lsst/ts/wep/task/cutOutDonutsBase.py

@@ -572,6 +572,9 @@ def cutOutStamps(self, exposure, donutCatalog, defocalType, cameraName):
         # Value of entropy
         stampsEntropy = list()
 
+        # Fraction of bad pixels
+        fracBadPixels = list()
+
         for idx, donutRow in enumerate(donutCatalog):
             # Make an initial cutout larger than the actual final stamp
             # so that we can centroid to get the stamp centered exactly
@@ -672,6 +675,10 @@ def cutOutStamps(self, exposure, donutCatalog, defocalType, cameraName):
             isEffective.append(eff)
             stampsEntropy.append(entro)
 
+            # Calculate fraction of bad pixels
+            bits = finalStamp.mask.getPlaneBitMask(("SAT", "BAD", "NO_DATA"))
+            fracBadPixels.append(np.mean(np.bitwise_and(finalStamp.mask.array, bits)))
+
             finalStamps.append(donutStamp)
 
         # Calculate the difference between original centroid and final centroid
@@ -781,4 +788,8 @@ def cutOutStamps(self, exposure, donutCatalog, defocalType, cameraName):
 
         # Save the peak of the correlated image
         stampsMetadata["PEAK_HEIGHT"] = peakHeight
+
+        # Save the fraction of bad pixels
+        stampsMetadata["FRAC_BAD_PIX"] = np.array(fracBadPixels).astype(float)
+
         return DonutStamps(finalStamps, metadata=stampsMetadata, use_archive=True)

python/lsst/ts/wep/task/donutStampSelectorTask.py

@@ -38,10 +38,16 @@ class DonutStampSelectorTaskConfig(pexConfig.Config):
     )
     selectWithSignalToNoise = pexConfig.Field(
         dtype=bool,
-        default=False,
-        doc="Whether to use signal to noise ratio in deciding to use the donut."
+        default=True,
+        doc="Whether to use signal to noise ratio in deciding to use the donut. "
         + "By default the values from snLimitStar.yaml config file are used.",
     )
+    selectWithFracBadPixels = pexConfig.Field(
+        dtype=bool,
+        default=True,
+        doc="Whether to use fraction of bad pixels in deciding to use the donut. "
+        + "Bad pixels correspond to mask values of 'SAT', 'BAD', 'NO_DATA'.",
+    )
     useCustomSnLimit = pexConfig.Field(
         dtype=bool,
         default=False,
@@ -62,13 +68,17 @@ class DonutStampSelectorTaskConfig(pexConfig.Config):
         default=3.5,
         doc=str("The entropy threshold to use (keep donuts only below the threshold)."),
     )
+    maxFracBadPixels = pexConfig.Field(
+        dtype=float,
+        default=0.0,
+        doc=str("Maximum fraction of bad pixels in selected donuts."),
+    )
 
 
 class DonutStampSelectorTask(pipeBase.Task):
     """
     Donut Stamp Selector uses information about donut stamp calculated at
-    the stamp cutting out stage to select those that fulfill entropy
-    and/or signal-to-noise criteria.
+    the stamp cutting out stage to select those that specified criteria.
     """
 
     ConfigClass = DonutStampSelectorTaskConfig
@@ -97,9 +107,9 @@ def run(self, donutStamps):
                     Boolean array of stamps that were selected, same length as
                     donutStamps.
                 - donutsQuality : `astropy.table.QTable`
-                    A table with calculated signal to noise measure and entropy
-                    value per donut, together with selection outcome for all
-                    input donuts.
+                    A table with calculated signal to noise measure, entropy
+                    value per donut, and fraction of bad pixels, together with
+                    selection outcome for all input donuts.
 
         """
         result = self.selectStamps(donutStamps)
@@ -109,7 +119,7 @@ def run(self, donutStamps):
         )
         selectedStamps._refresh_metadata()
         # Need to copy a few other fields by hand
-        for k in ["SN", "ENTROPY", "VISIT"]:
+        for k in ["SN", "ENTROPY", "FRAC_BAD_PIX", "VISIT"]:
             if k in donutStamps.metadata:
                 selectedStamps.metadata[k] = np.array(
                     [
@@ -157,7 +167,6 @@ def selectStamps(self, donutStamps):
                     value per donut, together with selection outcome for all
                     input donuts.
         """
-
         # Which donuts to use for Zernike estimation
         # initiate these by selecting all donuts
         entropySelect = np.ones(len(donutStamps), dtype="bool")
@@ -168,10 +177,8 @@ def selectStamps(self, donutStamps):
             if self.config.selectWithEntropy:
                 entropySelect = entropyValue < self.config.maxEntropy
         else:
-            self.log.warning(
-                "No entropy cut. Checking if signal-to-noise \
-should be applied."
-            )
+            self.log.warning("No entropy cut. Checking other conditions.")
+
         # By default select all donuts,  only overwritten
         # if selectWithSignalToNoise is True
         snSelect = np.ones(len(donutStamps), dtype="bool")
@@ -194,12 +201,27 @@ def selectStamps(self, donutStamps):
                 # Select using the given threshold
                 snSelect = snThreshold < snValue
         else:
-            self.log.warning("No signal-to-noise selection applied.")
+            self.log.warning(
+                "No signal-to-noise selection applied. Checking other conditions"
+            )
+
+        # By default select all donuts,  only overwritten
+        # if selectWithFracBadPixels is True
+        fracBadPixSelect = np.ones(len(donutStamps), dtype="bool")
+
+        # collect fraction-of-bad-pixels information if available
+        if "FRAC_BAD_PIX" in list(donutStamps.metadata):
+            fracBadPix = np.asarray(donutStamps.metadata.getArray("FRAC_BAD_PIX"))
+            if self.config.selectWithFracBadPixels:
+                fracBadPixSelect = fracBadPix <= self.config.maxFracBadPixels
+        else:
+            self.log.warning("No fraction-of-bad-pixels cut.")
+
         # AND condition : if both selectWithEntropy
         # and selectWithSignalToNoise, then
         # only donuts that pass with SN criterion as well
         # as entropy criterion are selected
-        selected = entropySelect * snSelect
+        selected = entropySelect * snSelect * fracBadPixSelect
 
         # store information about which donuts were selected
         # use QTable even though no units at the moment in
@@ -209,11 +231,21 @@ def selectStamps(self, donutStamps):
             data=[
                 snValue,
                 entropyValue,
+                fracBadPix,
                 snSelect,
                 entropySelect,
+                fracBadPixSelect,
                 selected,
             ],
-            names=["SN", "ENTROPY", "SN_SELECT", "ENTROPY_SELECT", "FINAL_SELECT"],
+            names=[
+                "SN",
+                "ENTROPY",
+                "FRAC_BAD_PIX",
+                "SN_SELECT",
+                "ENTROPY_SELECT",
+                "FRAC_BAD_PIX_SELECT",
+                "FINAL_SELECT",
+            ],
         )
 
         self.log.info("Selected %d/%d donut stamps", selected.sum(), len(donutStamps))

tests/task/test_cutOutDonutsBase.py

@@ -422,6 +422,7 @@ def testCutOutStampsTaskRunNormal(self):
             "EFFECTIVE",
             "ENTROPY",
             "PEAK_HEIGHT",
+            "FRAC_BAD_PIX",
             "MJD",
             "BORESIGHT_ROT_ANGLE_RAD",
             "BORESIGHT_PAR_ANGLE_RAD",
@@ -447,6 +448,7 @@ def testCutOutStampsTaskRunNormal(self):
             "EFFECTIVE",
             "ENTROPY",
             "PEAK_HEIGHT",
+            "FRAC_BAD_PIX",
         ]:
             self.assertEqual(
                 len(donutStamps), len(donutStamps.metadata.getArray(measure))

tests/task/test_donutStampSelectorTask.py

@@ -95,42 +95,49 @@ def testValidateConfigs(self):
         # Test the default config values
         self.OrigTask = DonutStampSelectorTask(config=self.config, name="Orig Task")
         self.assertEqual(self.OrigTask.config.selectWithEntropy, False)
-        self.assertEqual(self.OrigTask.config.selectWithSignalToNoise, False)
+        self.assertEqual(self.OrigTask.config.selectWithSignalToNoise, True)
+        self.assertEqual(self.OrigTask.config.selectWithFracBadPixels, True)
         self.assertEqual(self.OrigTask.config.useCustomSnLimit, False)
 
         # Test changing configs
         self.config.selectWithEntropy = True
-        self.config.selectWithSignalToNoise = True
+        self.config.selectWithSignalToNoise = False
+        self.config.selectWithFracBadPixels = False
         self.config.minSignalToNoise = 999
         self.config.maxEntropy = 4
+        self.config.maxFracBadPixels = 0.2
         self.ModifiedTask = DonutStampSelectorTask(config=self.config, name="Mod Task")
 
         self.assertEqual(self.ModifiedTask.config.selectWithEntropy, True)
-        self.assertEqual(self.ModifiedTask.config.selectWithSignalToNoise, True)
+        self.assertEqual(self.ModifiedTask.config.selectWithSignalToNoise, False)
+        self.assertEqual(self.ModifiedTask.config.selectWithFracBadPixels, False)
         self.assertEqual(self.ModifiedTask.config.minSignalToNoise, 999)
         self.assertEqual(self.ModifiedTask.config.maxEntropy, 4)
+        self.assertEqual(self.ModifiedTask.config.maxFracBadPixels, 0.2)
 
     def testSelectStamps(self):
         donutStampsIntra = self.butler.get(
             "donutStampsIntra", dataId=self.dataIdExtra, collections=[self.runName]
         )
 
-        # test default: no donuts are excluded
+        # test defaults
         selection = self.task.selectStamps(donutStampsIntra)
+        donutsQuality = selection.donutsQuality
 
-        # by default,  config.selectWithEntropy is False,
+        # by default, config.selectWithEntropy is False,
         # so we select all donuts
-        self.assertEqual(np.sum(selection.donutsQuality["ENTROPY_SELECT"]), 3)
+        self.assertEqual(np.sum(donutsQuality["ENTROPY_SELECT"]), 3)
 
-        # by default, config.selectWithSignalToNoise is False,
-        # so we select all donuts
-        self.assertEqual(np.sum(selection.donutsQuality["SN_SELECT"]), 3)
+        # by default, SNR selection happens and uses yaml config values
+        # so that all donuts here would get selected
+        self.assertEqual(np.sum(donutsQuality["SN_SELECT"]), 3)
 
-        # The final selection is the union of what was selected
-        # according to SN selection and entropy selection
-        self.assertEqual(np.sum(selection.donutsQuality["FINAL_SELECT"]), 3)
+        # by default, fraction-of-bad-pixels happens
+        # these donuts are all fine, so all are selected
+        self.assertEqual(np.sum(donutsQuality["FRAC_BAD_PIX_SELECT"]), 3)
 
-        # Test that identical information is conveyed here
+        # Test that overall selection also shows all three donuts
+        self.assertEqual(np.sum(donutsQuality["FINAL_SELECT"]), 3)
         self.assertEqual(np.sum(selection.selected), 3)
 
         # switch on selectWithEntropy,
@@ -151,20 +158,10 @@ def testSelectStamps(self):
             entropyThreshold,
         )
 
-        # switch on selectWithSignalToNoise
-        self.config.selectWithSignalToNoise = True
-        task = DonutStampSelectorTask(config=self.config, name="SN Task")
-        selection = task.selectStamps(donutStampsIntra)
-        donutsQuality = selection.donutsQuality
-
-        # by default we use the yaml config values so that
-        # all donuts here would get selected
-        self.assertEqual(np.sum(donutsQuality["SN_SELECT"]), 3)
-
-        # test that if we use the custom threshold,
-        # some donuts won't get selected
+        # test custom SNR thresholds
+        self.config.selectWithEntropy = False
         self.config.useCustomSnLimit = True
-        minSignalToNoise = 1658
+        minSignalToNoise = 1585
         self.config.minSignalToNoise = minSignalToNoise
         task = DonutStampSelectorTask(config=self.config, name="SN Task")
         selection = task.selectStamps(donutStampsIntra)
@@ -175,48 +172,36 @@ def testSelectStamps(self):
         for v in donutsQuality["SN"][donutsQuality["SN_SELECT"]]:
             self.assertLess(minSignalToNoise, v)
 
+        # Make sure that stamps with bad pixels are cut
+        badPix = np.asarray(donutStampsIntra.metadata.getArray("FRAC_BAD_PIX"))
+        badPix[0] = 0.1
+        donutStampsIntra.metadata.set("FRAC_BAD_PIX", badPix)
+        selection = self.task.selectStamps(donutStampsIntra)
+        donutsQuality = selection.donutsQuality
+        self.assertEqual(np.sum(donutsQuality["FRAC_BAD_PIX_SELECT"]), 2)
+
+        # finally turn all selections off and make sure everything is selected
+        self.config.selectWithEntropy = False
+        self.config.selectWithSignalToNoise = False
+        self.config.selectWithFracBadPixels = False
+        task = DonutStampSelectorTask(config=self.config, name="All off")
+        selection = task.selectStamps(donutStampsIntra)
+        self.assertEqual(np.sum(selection.donutsQuality["ENTROPY_SELECT"]), 3)
+        self.assertEqual(np.sum(selection.donutsQuality["SN_SELECT"]), 3)
+        self.assertEqual(np.sum(selection.donutsQuality["FRAC_BAD_PIX_SELECT"]), 3)
+        self.assertEqual(np.sum(selection.donutsQuality["FINAL_SELECT"]), 3)
+
     def testTaskRun(self):
         donutStampsIntra = self.butler.get(
             "donutStampsIntra", dataId=self.dataIdExtra, collections=[self.runName]
         )
-        # test default: no donuts are excluded
+
+        # test defaults
         taskOut = self.task.run(donutStampsIntra)
         donutsQuality = taskOut.donutsQuality
         selected = taskOut.selected
         donutStampsSelect = taskOut.donutStampsSelect
 
-        # Test that the length of the donutStamps is as expected
-        self.assertEqual(len(donutStampsSelect), 3)
-
-        # by default,  config.selectWithEntropy is False,
-        # so we select all donuts
-        self.assertEqual(np.sum(donutsQuality["ENTROPY_SELECT"]), 3)
-
-        # by default, config.selectWithSignalToNoise is False,
-        # so we select all donuts
-        self.assertEqual(np.sum(donutsQuality["SN_SELECT"]), 3)
-
-        # The final selection is the union of what was selected
-        # according to SN selection and entropy selection
-        self.assertEqual(np.sum(donutsQuality["FINAL_SELECT"]), 3)
-
-        # Test that identical information is conveyed here
-        self.assertEqual(np.sum(selected), 3)
-
-        # switch on selectWithEntropy,
-        # set config.maxEntropy so that one donut is selected
-        self.config.selectWithEntropy = True
-        entropyThreshold = 2.85
-        self.config.maxEntropy = entropyThreshold
-
-        task = DonutStampSelectorTask(config=self.config, name="Entropy Task")
-        taskOut = task.run(donutStampsIntra)
-        donutsQuality = taskOut.donutsQuality
-        self.assertEqual(np.sum(donutsQuality["ENTROPY_SELECT"]), 1)
-
-        # also test that the entropy of the selected donut
-        # is indeed below threshold
-        self.assertLess(
-            donutsQuality["ENTROPY"][donutsQuality["ENTROPY_SELECT"]],
-            entropyThreshold,
-        )
+        # Test that final selection numbers match
+        self.assertEqual(len(donutStampsSelect), selected.sum())
+        self.assertEqual(len(donutStampsSelect), donutsQuality["FINAL_SELECT"].sum())