From a91519af750d5edcaad48bb732a16af35fcb7501 Mon Sep 17 00:00:00 2001 From: "J. Bryce Kalmbach" Date: Fri, 10 Jan 2025 17:03:54 -0800 Subject: [PATCH] Fix processing. --- pipelines/production/comCamRefitWcsUSDF.yaml | 31 +++++++ .../production/comCamRefitWcsUSDF_Danish.yaml | 50 ++++++++++ .../production/comCamRefitWcsUSDF_TIE.yaml | 47 ++++++++++ python/lsst/ts/wep/task/cutOutDonutsBase.py | 5 +- .../ts/wep/task/generateDonutCatalogUtils.py | 40 ++++---- .../wep/task/generateDonutFromRefitWcsTask.py | 91 +++++++++++++------ 6 files changed, 217 insertions(+), 47 deletions(-) create mode 100644 pipelines/production/comCamRefitWcsUSDF.yaml create mode 100644 pipelines/production/comCamRefitWcsUSDF_Danish.yaml create mode 100644 pipelines/production/comCamRefitWcsUSDF_TIE.yaml diff --git a/pipelines/production/comCamRefitWcsUSDF.yaml b/pipelines/production/comCamRefitWcsUSDF.yaml new file mode 100644 index 00000000..98797a3b --- /dev/null +++ b/pipelines/production/comCamRefitWcsUSDF.yaml @@ -0,0 +1,31 @@ +description: RefitWCS pipeline for ComCam +instrument: lsst.obs.lsst.LsstComCam +tasks: + generateDonutFromRefitWcsTask: + class: lsst.ts.wep.task.generateDonutFromRefitWcsTask.GenerateDonutFromRefitWcsTask + config: + connections.astromRefCat: "gaia_dr3_20230707" + connections.photoRefCat: "the_monster_20240904" + connections.exposure: "postISRCCD" + connections.outputExposure: "postFitPostISRCCD" + connections.fitDonutCatalog: "donutTable" + connections.donutCatalog: "refitWcsDonutTable" + donutSelector.useCustomMagLimit: True + astromRefFilter: "phot_g_mean" + photoRefFilterPrefix: "monster_SynthLSST" + astromTask.referenceSelector.doMagLimit: False + astromTask.maxIter: 3 + astromTask.matcher.maxOffsetPix: 2000 + astromTask.matcher.maxRotationDeg: 3.0 + failTask: True + # donutSelector.unblendedSeparation: 1 + cutOutDonutsScienceSensorGroupTask: + class: lsst.ts.wep.task.cutOutDonutsScienceSensorTask.CutOutDonutsScienceSensorTask + config: + connections.exposures: "postFitPostISRCCD" + connections.donutCatalog: "refitWcsDonutTable" + python: | + from lsst.ts.wep.task.pairTask import GroupPairer + config.pairer.retarget(GroupPairer) + donutStampSize: 200 + initialCutoutPadding: 40 diff --git a/pipelines/production/comCamRefitWcsUSDF_Danish.yaml b/pipelines/production/comCamRefitWcsUSDF_Danish.yaml new file mode 100644 index 00000000..1cca78d7 --- /dev/null +++ b/pipelines/production/comCamRefitWcsUSDF_Danish.yaml @@ -0,0 +1,50 @@ +description: Run WEP + DonutViz daily at USDF +instrument: lsst.obs.lsst.LsstComCam +imports: + - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml + - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml + +tasks: + calcZernikesTask: + class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask + config: + python: | + from lsst.ts.wep.task import EstimateZernikesDanishTask + config.estimateZernikes.retarget(EstimateZernikesDanishTask) + donutStampSelector.maxFracBadPixels: 2.0e-4 + estimateZernikes.binning: 2 + estimateZernikes.nollIndices: + [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 21, 22, 27, 28] + estimateZernikes.saveHistory: true + estimateZernikes.lstsqKwargs: + ftol: 1.0e-3 + xtol: 1.0e-3 + gtol: 1.0e-3 + donutStampSelector.maxSelect: -1 + aggregateDonutTablesGroupTask: + class: lsst.donut.viz.AggregateDonutTablesTask + config: + python: | + from lsst.ts.wep.task.pairTask import GroupPairer + config.pairer.retarget(GroupPairer) + connections.donutTables: "refitWcsDonutTable" + +# Define pipeline steps +subsets: + step1b: + subset: + - calcZernikesTask + description: | + This step runs the Zernike calculation with danish. + step2: + subset: + - aggregateZernikeTablesTask + - aggregateDonutTablesGroupTask + - aggregateAOSVisitTableTask + - plotAOSTask + - aggregateDonutStampsTask + - plotDonutTask + description: | + AOS Donut visualization plotting tasks. This step generates plots + (including the pyramid residual and donut gallery) and + tables for the AOS visit. diff --git a/pipelines/production/comCamRefitWcsUSDF_TIE.yaml b/pipelines/production/comCamRefitWcsUSDF_TIE.yaml new file mode 100644 index 00000000..f85abdb5 --- /dev/null +++ b/pipelines/production/comCamRefitWcsUSDF_TIE.yaml @@ -0,0 +1,47 @@ +description: Run WEP + DonutViz daily at USDF +instrument: lsst.obs.lsst.LsstComCam +imports: + - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml + - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml + +tasks: + calcZernikesTask: + class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask + config: + estimateZernikes.nollIndices: [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 18, 19, 20, 21, 22, 27, 28] + estimateZernikes.convergeTol: 10.0e-9 + estimateZernikes.compGain: 0.75 + estimateZernikes.compSequence: [4, 4, 6, 6, 13, 13, 13, 13] + estimateZernikes.maxIter: 50 + estimateZernikes.requireConverge: True + estimateZernikes.saveHistory: True + estimateZernikes.maskKwargs: { "doMaskBlends": False } + donutStampSelector.maxFracBadPixels: 2.0e-4 + donutStampSelector.maxSelect: -1 + aggregateDonutTablesGroupTask: + class: lsst.donut.viz.AggregateDonutTablesTask + config: + python: | + from lsst.ts.wep.task.pairTask import GroupPairer + config.pairer.retarget(GroupPairer) + connections.donutTables: "refitWcsDonutTable" + +# Define pipeline steps +subsets: + step1b: + subset: + - calcZernikesTask + description: | + This step runs the Zernike calculation with danish. + step2: + subset: + - aggregateZernikeTablesTask + - aggregateDonutTablesGroupTask + - aggregateAOSVisitTableTask + - plotAOSTask + - aggregateDonutStampsTask + - plotDonutTask + description: | + AOS Donut visualization plotting tasks. This step generates plots + (including the pyramid residual and donut gallery) and + tables for the AOS visit. diff --git a/python/lsst/ts/wep/task/cutOutDonutsBase.py b/python/lsst/ts/wep/task/cutOutDonutsBase.py index e835090e..f03c8f9c 100644 --- a/python/lsst/ts/wep/task/cutOutDonutsBase.py +++ b/python/lsst/ts/wep/task/cutOutDonutsBase.py @@ -695,9 +695,10 @@ def cutOutStamps(self, exposure, donutCatalog, defocalType, cameraName): stampsMetadata = PropertyList() stampsMetadata["VISIT"] = np.array([visitId] * catalogLength, dtype=int) # Save the donut flux as magnitude - if len(donutCatalog["source_flux"]) > 0: + fluxLabel = next((colName for colName in donutCatalog.columns if colName.endswith(f'{bandLabel}_flux')), None) + if fluxLabel is not None and len(donutCatalog[fluxLabel]) > 0: stampsMetadata["MAG"] = ( - donutCatalog["source_flux"].value * u.nJy + donutCatalog[fluxLabel].value * u.nJy ).to_value(u.ABmag) else: stampsMetadata["MAG"] = np.array([]) diff --git a/python/lsst/ts/wep/task/generateDonutCatalogUtils.py b/python/lsst/ts/wep/task/generateDonutCatalogUtils.py index 08a7f130..a830df27 100644 --- a/python/lsst/ts/wep/task/generateDonutCatalogUtils.py +++ b/python/lsst/ts/wep/task/generateDonutCatalogUtils.py @@ -85,7 +85,11 @@ def runSelection(refObjLoader, detector, wcs, filterName, donutSelectorTask): def donutCatalogToAstropy( - donutCatalog=None, filterName=None, blendCentersX=None, blendCentersY=None + donutCatalog=None, + filterName=None, + blendCentersX=None, + blendCentersY=None, + sortFilterIdx=0, ): """ Reformat afwCatalog into an astropy QTable sorted by flux with @@ -98,19 +102,22 @@ def donutCatalogToAstropy( ReferenceObjectLoader search over the detector footprint. If None then it will return an empty QTable. (the default is None.) - filterName : `str` or `None`, optional - Name of camera filter. If donutCatalog is not None then + filterName : `list` or `None`, optional + Name of catalog flux filters. If donutCatalog is not None then this cannot be None. (the default is None.) blendCentersX : `list` or `None`, optional - X pixel position of centroids for blended objects. List - should be the same length as the donutCatalog. If - blendCentersY is not None then this cannot be None. (the default - is None.) + X pixel position of centroids for blended objects. List + should be the same length as the donutCatalog. If + blendCentersY is not None then this cannot be None. (the default + is None.) blendCentersY : `list` or `None`, optional - Y pixel position of centroids for blended objects. List - should be the same length as the donutCatalog. If - blendCentersX is not None then this cannot be None. (the default - is None.) + Y pixel position of centroids for blended objects. List + should be the same length as the donutCatalog. If + blendCentersX is not None then this cannot be None. (the default + is None.) + sortFilterIdx : int, optional + Index for which filter in filterName to sort the entire catalog + by brightness. (the default is 0.) Returns ------- @@ -144,12 +151,12 @@ def donutCatalogToAstropy( dec = donutCatalog["coord_dec"] centroidX = donutCatalog["centroid_x"] centroidY = donutCatalog["centroid_y"] - sourceFlux = donutCatalog[f"{filterName}_flux"] + sourceFlux = [donutCatalog[f"{fName}_flux"] for fName in filterName] if (blendCentersX is None) and (blendCentersY is None): blendCX = list() blendCY = list() - for idx in range(len(donutCatalog)): + for _ in range(len(donutCatalog)): blendCX.append(list()) blendCY.append(list()) elif isinstance(blendCentersX, list) and isinstance(blendCentersY, list): @@ -178,16 +185,17 @@ def donutCatalogToAstropy( ) raise ValueError(blendErrMsg) - flux_sort = np.argsort(sourceFlux)[::-1] + flux_sort = np.argsort(sourceFlux[sortFilterIdx])[::-1] fieldObjects = QTable() fieldObjects["coord_ra"] = ra * u.rad fieldObjects["coord_dec"] = dec * u.rad fieldObjects["centroid_x"] = centroidX fieldObjects["centroid_y"] = centroidY - fieldObjects["source_flux"] = sourceFlux * u.nJy + for idx in range(len(filterName)): + fieldObjects[f"{filterName[idx]}_flux"] = sourceFlux[idx] * u.nJy - fieldObjects.sort("source_flux", reverse=True) + fieldObjects.sort(f"{filterName[sortFilterIdx]}_flux", reverse=True) fieldObjects.meta["blend_centroid_x"] = [blendCX[idx] for idx in flux_sort] fieldObjects.meta["blend_centroid_y"] = [blendCY[idx] for idx in flux_sort] diff --git a/python/lsst/ts/wep/task/generateDonutFromRefitWcsTask.py b/python/lsst/ts/wep/task/generateDonutFromRefitWcsTask.py index bec2f17e..f956a2bd 100644 --- a/python/lsst/ts/wep/task/generateDonutFromRefitWcsTask.py +++ b/python/lsst/ts/wep/task/generateDonutFromRefitWcsTask.py @@ -134,10 +134,25 @@ class GenerateDonutFromRefitWcsTaskConfig( default="phot_g_mean", ) photoRefFilter = pexConfig.Field( - doc="Set filter to use in Photometry catalog. If not set will try to use exposure filter.", + doc="Set filter to use in Photometry catalog. " + + "Cannot set both this and photoRefFilter. " + + "If neither is set then will just try to use the name of the exposure filter.", dtype=str, optional=True, ) + photoRefFilterPrefix = pexConfig.Field( + doc="Set filter prefix to use. " + + "Will then try to use exposure band label with given catalog prefix. " + + "Cannot set both this and photoRefFilter. " + + "If neither is set then will just try to use the name of the exposure filter.", + dtype=str, + optional=True, + ) + failTask = pexConfig.Field( + doc="Fail if error raised.", + dtype=bool, + default=False, + ) # Took these defaults from atmospec/centroiding which I used # as a template for implementing WCS fitting in a task. @@ -302,12 +317,14 @@ def run( # fitting successfully or not. This will # give us information on our donut catalog output. self.metadata["wcsFitSuccess"] = False + self.metadata["refCatalogSuccess"] = False try: astromResult = self.astromTask.run( sourceCat=afwCat, exposure=exposure, ) scatter = astromResult.scatterOnSky.asArcseconds() + print(scatter, self.config.maxFitScatter) if scatter < self.config.maxFitScatter: successfulFit = True self.metadata["wcsFitSuccess"] = True @@ -321,15 +338,17 @@ def run( # AttributeError: 'NoneType' object has no attribute 'asArcseconds' # when the result is a failure as the wcs is set to None on failure self.log.warning(f"Solving for WCS failed: {e}") - # this is set to None when the fit fails, so restore it - exposure.setWcs(originalWcs) - donutCatalog = fitDonutCatalog - self.log.warning( - "Returning original exposure and WCS \ -and direct detect catalog as output." - ) + if self.config.failTask: + raise TaskError("Failing task due to wcs fit failure.") + else: + # this is set to None when the fit fails, so restore it + exposure.setWcs(originalWcs) + donutCatalog = fitDonutCatalog + self.log.warning( + "Returning original exposure and WCS \ + and direct detect catalog as output." + ) - self.metadata["refCatalogSuccess"] = False if successfulFit: photoRefObjLoader = ReferenceObjectLoader( dataIds=[ref.dataId for ref in photoRefCat], @@ -351,26 +370,32 @@ def run( ) # Check that specified filter exists in catalogs + if ( + self.config.photoRefFilter is not None + and self.config.photoRefFilterPrefix is not None + ): + raise ValueError( + "photoRefFilter and photoRefFilterConfig cannot both be set." + ) if self.config.photoRefFilter is not None: - if ( - f"{self.config.photoRefFilter}_flux" - not in photoRefCat[0].get().schema - ): - filterFailMsg = ( - "Photometric Reference Catalog does not contain " - + f"photoRefFilter: {self.config.photoRefFilter}" - ) - self.log.warning(filterFailMsg) - donutCatalog = fitDonutCatalog - self.log.warning(catCreateErrorMsg) - return pipeBase.Struct( - outputExposure=exposure, donutCatalog=donutCatalog - ) - else: - photoRefObjLoader.config.anyFilterMapsToThis = ( - self.config.photoRefFilter - ) - filterName = self.config.photoRefFilter + filterName = self.config.photoRefFilter + elif self.config.photoRefFilterPrefix is not None: + filterName = ( + f"{self.config.photoRefFilterPrefix}_{exposure.filter.bandLabel}" + ) + + # Test that given filter name exists in catalog. + if f"{filterName}_flux" not in photoRefCat[0].get().schema: + filterFailMsg = ( + "Photometric Reference Catalog does not contain " + + f"photoRefFilter: {filterName}" + ) + self.log.warning(filterFailMsg) + donutCatalog = fitDonutCatalog + self.log.warning(catCreateErrorMsg) + return pipeBase.Struct( + outputExposure=exposure, donutCatalog=donutCatalog + ) try: # Match detector layout to reference catalog @@ -386,8 +411,16 @@ def run( donutSelectorTask, ) + filterName = [ + f"{self.config.photoRefFilterPrefix}_{band}" + for band in ["u", "g", "r", "i", "z", "y"] + ] donutCatalog = donutCatalogToAstropy( - refSelection, filterName, blendCentersX, blendCentersY + refSelection, + filterName, + blendCentersX, + blendCentersY, + sortFilterIdx=2, ) self.metadata["refCatalogSuccess"] = True # Except RuntimeError caused when no reference catalog