From 7eea7ffb58ce094597f7e97d539766e82c5db7f9 Mon Sep 17 00:00:00 2001 From: LR-inaf Date: Thu, 21 Nov 2024 20:44:55 +0000 Subject: [PATCH 1/3] add the plotPsfZernTask in the pipelines --- .../donutVizGroupPipeline-checkpoint.yaml | 42 ++ .../_ingredients/donutVizGroupPipeline.yaml | 5 + ...apidAnalysisDanishPipeline-checkpoint.yaml | 40 ++ ...omCamRapidAnalysisPipeline-checkpoint.yaml | 45 +++ .../comCamRapidAnalysisDanishPipeline.yaml | 1 + .../comCamRapidAnalysisPipeline.yaml | 1 + .../donutSourceSelectorTask-checkpoint.py | 373 ++++++++++++++++++ .../.ipynb_checkpoints/pairTask-checkpoint.py | 304 ++++++++++++++ 8 files changed, 811 insertions(+) create mode 100644 pipelines/_ingredients/.ipynb_checkpoints/donutVizGroupPipeline-checkpoint.yaml create mode 100644 pipelines/production/.ipynb_checkpoints/comCamRapidAnalysisDanishPipeline-checkpoint.yaml create mode 100644 pipelines/production/.ipynb_checkpoints/comCamRapidAnalysisPipeline-checkpoint.yaml create mode 100644 python/lsst/ts/wep/task/.ipynb_checkpoints/donutSourceSelectorTask-checkpoint.py create mode 100644 python/lsst/ts/wep/task/.ipynb_checkpoints/pairTask-checkpoint.py diff --git a/pipelines/_ingredients/.ipynb_checkpoints/donutVizGroupPipeline-checkpoint.yaml b/pipelines/_ingredients/.ipynb_checkpoints/donutVizGroupPipeline-checkpoint.yaml new file mode 100644 index 00000000..96d5c03d --- /dev/null +++ b/pipelines/_ingredients/.ipynb_checkpoints/donutVizGroupPipeline-checkpoint.yaml @@ -0,0 +1,42 @@ +description: donut viz pipeline tasks + +tasks: + aggregateZernikeTablesTask: + class: lsst.donut.viz.AggregateZernikeTablesTask + aggregateDonutTablesGroupTask: + class: lsst.donut.viz.AggregateDonutTablesTask + config: + python: | + from lsst.ts.wep.task.pairTask import GroupPairer + config.pairer.retarget(GroupPairer) + aggregateAOSVisitTableTask: + class: lsst.donut.viz.AggregateAOSVisitTableTask + plotAOSTask: + class: lsst.donut.viz.PlotAOSTask + config: + doRubinTVUpload: true + aggregateDonutStampsTask: + class: lsst.donut.viz.AggregateDonutStampsTask + plotDonutTask: + class: lsst.donut.viz.PlotDonutTask + config: + doRubinTVUpload: true + plotPsfZernTask: + class: lsst.donut.viz.PlotPsfZernTask + config: + doRubinTVUpload: true + +subsets: + donutVizGroups: + subset: + - aggregateZernikeTablesTask + - aggregateDonutTablesGroupTask + - aggregateAOSVisitTableTask + - plotAOSTask + - aggregateDonutStampsTask + - plotDonutTask + - plotPsfZernTask + description: | + AOS Donut visualization plotting tasks. This step generates plots + (including the pyramid residual and donut gallery) and + tables for the AOS visit. diff --git a/pipelines/_ingredients/donutVizGroupPipeline.yaml b/pipelines/_ingredients/donutVizGroupPipeline.yaml index 9abc23ec..96d5c03d 100644 --- a/pipelines/_ingredients/donutVizGroupPipeline.yaml +++ b/pipelines/_ingredients/donutVizGroupPipeline.yaml @@ -21,6 +21,10 @@ tasks: class: lsst.donut.viz.PlotDonutTask config: doRubinTVUpload: true + plotPsfZernTask: + class: lsst.donut.viz.PlotPsfZernTask + config: + doRubinTVUpload: true subsets: donutVizGroups: @@ -31,6 +35,7 @@ subsets: - plotAOSTask - aggregateDonutStampsTask - plotDonutTask + - plotPsfZernTask description: | AOS Donut visualization plotting tasks. This step generates plots (including the pyramid residual and donut gallery) and diff --git a/pipelines/production/.ipynb_checkpoints/comCamRapidAnalysisDanishPipeline-checkpoint.yaml b/pipelines/production/.ipynb_checkpoints/comCamRapidAnalysisDanishPipeline-checkpoint.yaml new file mode 100644 index 00000000..647411ab --- /dev/null +++ b/pipelines/production/.ipynb_checkpoints/comCamRapidAnalysisDanishPipeline-checkpoint.yaml @@ -0,0 +1,40 @@ +description: rapid analysis pipeline for ComCam w/ danish zernike estimation +instrument: lsst.obs.lsst.LsstComCam +imports: + - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml + - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml + +tasks: + calcZernikesTask: + class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask + config: + python: | + from lsst.ts.wep.task import EstimateZernikesDanishTask + config.estimateZernikes.retarget(EstimateZernikesDanishTask) + donutStampSelector.maxSelect: 2 + +# Define pipeline steps +subsets: + step1: + subset: + - isr + - generateDonutDirectDetectTask + - cutOutDonutsScienceSensorGroupTask + - calcZernikesTask + description: | + This step processes the input images with ISR, + finds and cuts out the donut stamps, + and estimates the Zernike coefficients from the donut pairs. + step2a: + subset: + - aggregateZernikeTablesTask + - aggregateDonutTablesGroupTask + - aggregateAOSVisitTableTask + - plotAOSTask + - aggregateDonutStampsTask + - plotDonutTask + - plotPsfZernTask + description: | + AOS Donut visualization plotting tasks. This step generates plots + (including the pyramid residual and donut gallery) and + tables for the AOS visit. diff --git a/pipelines/production/.ipynb_checkpoints/comCamRapidAnalysisPipeline-checkpoint.yaml b/pipelines/production/.ipynb_checkpoints/comCamRapidAnalysisPipeline-checkpoint.yaml new file mode 100644 index 00000000..803521d8 --- /dev/null +++ b/pipelines/production/.ipynb_checkpoints/comCamRapidAnalysisPipeline-checkpoint.yaml @@ -0,0 +1,45 @@ +description: rapid analysis pipeline for ComCam +instrument: lsst.obs.lsst.LsstComCam +imports: + - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml + - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml + +tasks: + calcZernikesTask: + class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask + config: + estimateZernikes.nollIndices: [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] + estimateZernikes.convergeTol: 10.0e-9 + estimateZernikes.compGain: 0.75 + estimateZernikes.compSequence: [4, 4, 6, 6, 13, 13, 13, 13] + estimateZernikes.maxIter: 50 + estimateZernikes.requireConverge: True + estimateZernikes.saveHistory: False + estimateZernikes.maskKwargs: { "doMaskBlends": False } + donutStampSelector.maxSelect: 5 + +# Define pipeline steps +subsets: + step1: + subset: + - isr + - generateDonutDirectDetectTask + - cutOutDonutsScienceSensorGroupTask + - calcZernikesTask + description: | + This step processes the input images with ISR, + finds and cuts out the donut stamps, + and estimates the Zernike coefficients from the donut pairs. + step2a: + subset: + - aggregateZernikeTablesTask + - aggregateDonutTablesGroupTask + - aggregateAOSVisitTableTask + - plotAOSTask + - aggregateDonutStampsTask + - plotDonutTask + - plotPsfZernTask + description: | + AOS Donut visualization plotting tasks. This step generates plots + (including the pyramid residual and donut gallery) and + tables for the AOS visit. diff --git a/pipelines/production/comCamRapidAnalysisDanishPipeline.yaml b/pipelines/production/comCamRapidAnalysisDanishPipeline.yaml index f630b66c..647411ab 100644 --- a/pipelines/production/comCamRapidAnalysisDanishPipeline.yaml +++ b/pipelines/production/comCamRapidAnalysisDanishPipeline.yaml @@ -33,6 +33,7 @@ subsets: - plotAOSTask - aggregateDonutStampsTask - plotDonutTask + - plotPsfZernTask description: | AOS Donut visualization plotting tasks. This step generates plots (including the pyramid residual and donut gallery) and diff --git a/pipelines/production/comCamRapidAnalysisPipeline.yaml b/pipelines/production/comCamRapidAnalysisPipeline.yaml index 537bb4e9..803521d8 100644 --- a/pipelines/production/comCamRapidAnalysisPipeline.yaml +++ b/pipelines/production/comCamRapidAnalysisPipeline.yaml @@ -38,6 +38,7 @@ subsets: - plotAOSTask - aggregateDonutStampsTask - plotDonutTask + - plotPsfZernTask description: | AOS Donut visualization plotting tasks. This step generates plots (including the pyramid residual and donut gallery) and diff --git a/python/lsst/ts/wep/task/.ipynb_checkpoints/donutSourceSelectorTask-checkpoint.py b/python/lsst/ts/wep/task/.ipynb_checkpoints/donutSourceSelectorTask-checkpoint.py new file mode 100644 index 00000000..9f135f1c --- /dev/null +++ b/python/lsst/ts/wep/task/.ipynb_checkpoints/donutSourceSelectorTask-checkpoint.py @@ -0,0 +1,373 @@ +# This file is part of ts_wep. +# +# Developed for the LSST Telescope and Site Systems. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +__all__ = ["DonutSourceSelectorTaskConfig", "DonutSourceSelectorTask"] + +import astropy.units as u +import lsst.geom +import lsst.pex.config as pexConfig +import lsst.pipe.base as pipeBase +import numpy as np +import pandas as pd +from lsst.afw.cameraGeom import FIELD_ANGLE, PIXELS +from lsst.meas.algorithms.sourceSelector import _getFieldFromCatalog +from lsst.ts.wep.utils import readConfigYaml +from lsst.utils.timer import timeMethod +from sklearn.neighbors import NearestNeighbors + + +class DonutSourceSelectorTaskConfig(pexConfig.Config): + xCoordField = pexConfig.Field( + dtype=str, default="centroid_x", doc="Name of x-coordinate column." + ) + yCoordField = pexConfig.Field( + dtype=str, default="centroid_y", doc="Name of y-coordinate column." + ) + useCustomMagLimit = pexConfig.Field( + dtype=bool, + default=False, + doc="Apply user-defined magnitude limit? If this is False then the code" + + " will default to use the magnitude values in policy:magLimitStar.yaml.", + ) + magMax = pexConfig.Field( + dtype=float, + default=99.0, + doc="Maximum magnitude for selection. Only used if useCustomMagLimit is True.", + ) + magMin = pexConfig.Field( + dtype=float, + default=-99.0, + doc="Minimum magnitude for selection. Only used if useCustomMagLimit is True.", + ) + # For information on where this default maxFieldDist comes from see details + # in ts_analysis_notebooks/aos/vignetting. + maxFieldDist = pexConfig.Field( + dtype=float, + default=1.808, + doc="Maximum distance from center of focal plane (in degrees).", + ) + unblendedSeparation = pexConfig.Field( + dtype=int, + default=160, + doc="Distance in pixels between two donut centers for them to be considered unblended. " + + "This setting and minBlendedSeparation will both be affected by the defocal distance.", + ) + minBlendedSeparation = pexConfig.Field( + dtype=int, + default=120, + doc="Minimum separation in pixels between blended donut centers. " + + "This setting and unblendedSeparation will both be affected by the defocal distance.", + ) + isolatedMagDiff = pexConfig.Field( + dtype=float, + default=2, + doc="Min. difference in magnitude for 'isolated' star.", + ) + sourceLimit = pexConfig.Field( + dtype=int, + default=-1, + doc="Maximum number of desired sources (default is -1 which will give all in catalog).", + ) + maxBlended = pexConfig.Field( + dtype=int, + default=0, + doc="Number of blended objects (defined by unblendedSeparation and isolatedMagDiff) " + + "allowed with a bright source.", + ) + + +class DonutSourceSelectorTask(pipeBase.Task): + """ + Donut Source Selector that uses a nearest neighbors radius + query to find all donuts within the pixel radius set in the + config. Then it goes from the brightest sources down to the faintest + picking donuts that are at least isolatedMagDiff brighter than any sources + with centers within 2 times the unblendedSeparation until reaching + numSources kept or going through the whole list. + """ + + ConfigClass = DonutSourceSelectorTaskConfig + _DefaultName = "donutSourceSelectorTask" + + def __init__(self, **kwargs): + pipeBase.Task.__init__(self, **kwargs) + + def run(self, sourceCat, detector, filterName): + """Select sources and return them. + + Parameters + ---------- + sourceCat : `lsst.afw.table.SourceCatalog` or `pandas.DataFrame` + or `astropy.table.Table` + Catalog of sources to select from. + detector : `lsst.afw.cameraGeom.Detector` + Detector object from the camera. + filterName : `str` + Name of camera filter. + + Returns + ------- + struct : `lsst.pipe.base.Struct` + The struct contains the following data: + - sourceCat : `lsst.afw.table.SourceCatalog` + or `pandas.DataFrame` or `astropy.table.Table` + The catalog of sources that were selected. + (may not be memory-contiguous) + - selected : `numpy.ndarray` of `bool` + Boolean array of sources that were selected, same length as + sourceCat. + + Raises + ------ + `RuntimeError` + Raised if ``sourceCat`` is not contiguous. + """ + if hasattr(sourceCat, "isContiguous"): + # Check for continuity on afwTable catalogs + if not sourceCat.isContiguous(): + raise RuntimeError( + "Input catalogs for source selection must be contiguous." + ) + + result = self.selectSources(sourceCat, detector, filterName) + + return pipeBase.Struct( + sourceCat=sourceCat[result.selected], + selected=result.selected, + blendCentersX=result.blendCentersX, + blendCentersY=result.blendCentersY, + ) + + @timeMethod + def selectSources(self, sourceCat, detector, filterName): + """ + Run the source selection algorithm and return the indices to keep + in the original catalog. + + Parameters + ---------- + sourceCat : `lsst.afw.table.SourceCatalog` or `pandas.DataFrame` + or `astropy.table.Table` + Catalog of sources to select from. + detector : `lsst.afw.cameraGeom.Detector` + Detector object from the camera. + filterName : `str` + Name of camera filter. + + Returns + ------- + struct : `lsst.pipe.base.Struct` + The struct contains the following data: + - selected : `numpy.ndarray` of `bool` + Boolean array of sources that were selected, same length as + sourceCat. + + Raises + ------ + `ValueError` + sourceLimit in config for task must be -1 or a positive integer. + """ + + bbox = detector.getBBox() + + selected = np.zeros(len(sourceCat), dtype=bool) + if len(selected) == 0: + return pipeBase.Struct( + selected=selected, + blendCentersX=None, + blendCentersY=None, + ) + + fluxField = f"{filterName}_flux" + flux = _getFieldFromCatalog(sourceCat, fluxField) + mag = (flux * u.nJy).to_value(u.ABmag) + minMagDiff = self.config.isolatedMagDiff + unblendedSeparation = self.config.unblendedSeparation + minBlendedSeparation = self.config.minBlendedSeparation + + # Use user defined inputs or ts_wep defaults + # depending on useCustomMagLimit. + if self.config.useCustomMagLimit: + magMin = self.config.magMin + magMax = self.config.magMax + else: + magPolicyDefaults = readConfigYaml("policy:magLimitStar.yaml") + defaultFilterKey = f"filter{filterName.upper()}" + magMax = magPolicyDefaults[defaultFilterKey]["high"] + magMin = magPolicyDefaults[defaultFilterKey]["low"] + + magSelected = np.ones(len(sourceCat), dtype=bool) + magSelected &= mag < (magMax + minMagDiff) + mag = mag[magSelected] + + xCoord = _getFieldFromCatalog(sourceCat[magSelected], self.config.xCoordField) + yCoord = _getFieldFromCatalog(sourceCat[magSelected], self.config.yCoordField) + + df = pd.DataFrame({"x": xCoord, "y": yCoord, "mag": mag}) + # Grab any donut centers within unblended distance. + xyNeigh = NearestNeighbors(radius=unblendedSeparation) + + # Get distance to center of field + fieldXY = detector.transform( + [lsst.geom.Point2D(xPix, yPix) for xPix, yPix in zip(xCoord, yCoord)], + PIXELS, + FIELD_ANGLE, + ) + fieldDist = [ + np.degrees(np.sqrt(fieldLoc[0] ** 2 + fieldLoc[1] ** 2)) + for fieldLoc in fieldXY + ] + df["fieldDist"] = fieldDist + + # Remove area too close to edge with new bounding box that allows + # only area at least distance for unblended separation from edges + trimmedBBox = bbox.erodedBy(unblendedSeparation) + + index = list() + magSortedDf = df.sort_values("mag") + groupIndices = magSortedDf.index.values + xyNeigh.fit(magSortedDf[["x", "y"]]) + radDist, radIdx = xyNeigh.radius_neighbors( + magSortedDf[["x", "y"]], sort_results=True + ) + + errMsg = str( + "config.sourceLimit must be a positive integer " + + "or turned off by setting it to '-1'" + ) + if not ((self.config.sourceLimit == -1) or (self.config.sourceLimit > 0)): + raise ValueError(errMsg) + + maxBlended = self.config.maxBlended + blendCentersX = [list() for _ in range(len(magSortedDf))] + blendCentersY = [list() for _ in range(len(magSortedDf))] + sourcesKept = 0 + # Go through catalog with nearest neighbor information + # and keep sources that match our configuration settings + srcOn = -1 + for nbrDist, idxList in zip(radDist, radIdx): + srcOn += 1 + # Move on if source is within unblendedSeparation + # of the edge of a given exposure + srcX = magSortedDf["x"].iloc[srcOn] + srcY = magSortedDf["y"].iloc[srcOn] + if trimmedBBox.contains(srcX, srcY) is False: + continue + + # If distance from field center is greater than + # maxFieldDist discard the source and move on + if magSortedDf["fieldDist"].iloc[srcOn] > self.config.maxFieldDist: + continue + + # If this source's magnitude is outside our bounds then discard + srcMag = magSortedDf["mag"].iloc[srcOn] + if (srcMag > magMax) | (srcMag < magMin): + continue + + # If there is no overlapping source keep + # the source and move on to next + if len(idxList) == 1: + index.append(groupIndices[srcOn]) + sourcesKept += 1 + # In this case there is at least one overlapping source + else: + # Measure magnitude differences with overlapping objects + magDiff = magSortedDf["mag"].iloc[idxList[1:]] - srcMag + magTooClose = magDiff.values < minMagDiff + + # Measure distances to overlapping objects + blendSeparations = nbrDist[1:] + blendTooClose = blendSeparations < minBlendedSeparation + + # If this is the fainter source of the overlaps move on + if np.min(magDiff) < 0.0: + continue + # If this source overlaps but is brighter than all its + # overlapping sources by minMagDiff then keep it + elif np.min(magDiff) >= minMagDiff: + index.append(groupIndices[srcOn]) + sourcesKept += 1 + # If the centers of any blended objects with a magnitude + # within minMagDiff of the source magnitude + # are closer than minBlendedSeparation move on + elif np.sum(blendTooClose & magTooClose) > 0: + continue + # If the number of overlapping sources with magnitudes close + # enough to count as blended is less than or equal to + # maxBlended then keep this source + elif len(magDiff) <= maxBlended: + index.append(groupIndices[srcOn]) + # Only include sources bright enough to count as + # blended based upon isolatedMagDiff. Otherwise + # masks for deblending will include footprints of + # all the faint sources that we don't care about + # when deblending. Add one to index because + # magDiff is all sources in magSortedDf after index=0. + blendMagIdx = np.where(magDiff < minMagDiff)[0] + 1 + blendCentersX[groupIndices[srcOn]] = ( + magSortedDf["x"].iloc[idxList[blendMagIdx]].values + ) + blendCentersY[groupIndices[srcOn]] = ( + magSortedDf["y"].iloc[idxList[blendMagIdx]].values + ) + sourcesKept += 1 + # Keep the source if it is blended with up to maxBlended + # number of sources. To check this we look at the maxBlended+1 + # source in the magDiff list and check that the object + # is at least minMagDiff brighter than this. Satisfying this + # criterion means it is blended with maxBlended + # or fewer sources. + elif np.partition(magDiff, maxBlended)[maxBlended] > minMagDiff: + index.append(groupIndices[srcOn]) + # Same process as above to make sure we only get + # the blend centers we care about + blendMagIdx = np.where(magDiff < minMagDiff)[0] + 1 + blendCentersX[groupIndices[srcOn]] = ( + magSortedDf["x"].iloc[idxList[blendMagIdx]].values + ) + blendCentersY[groupIndices[srcOn]] = ( + magSortedDf["y"].iloc[idxList[blendMagIdx]].values + ) + sourcesKept += 1 + else: + continue + + if (self.config.sourceLimit > 0) and ( + sourcesKept == self.config.sourceLimit + ): + break + + # magSelected is a boolean array so we can + # find indices with True by finding nonzero elements + magIndex = magSelected.nonzero()[0] + finalIndex = magIndex[index] + selected[finalIndex] = True + sortedIndex = np.sort(index) + selectedBlendCentersX = [blendCentersX[idx] for idx in sortedIndex] + selectedBlendCentersY = [blendCentersY[idx] for idx in sortedIndex] + + self.log.info("Selected %d/%d references", selected.sum(), len(sourceCat)) + + return pipeBase.Struct( + selected=selected, + blendCentersX=selectedBlendCentersX, + blendCentersY=selectedBlendCentersY, + ) diff --git a/python/lsst/ts/wep/task/.ipynb_checkpoints/pairTask-checkpoint.py b/python/lsst/ts/wep/task/.ipynb_checkpoints/pairTask-checkpoint.py new file mode 100644 index 00000000..5526b1a6 --- /dev/null +++ b/python/lsst/ts/wep/task/.ipynb_checkpoints/pairTask-checkpoint.py @@ -0,0 +1,304 @@ +# This file is part of ts_wep. +# +# Developed for the LSST Telescope and Site Systems. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +__all__ = [ + "ExposurePairerConfig", + "ExposurePairer", + "TablePairerConfig", + "TablePairer", + "GroupPairerConfig", + "GroupPairer", +] + +import typing + +import lsst.afw.image as afwImage +import lsst.pex.config as pexConfig +import lsst.pipe.base as pipeBase +import numpy as np +from astropy.coordinates import SkyCoord +from astropy.table import Table +from lsst.geom import radians + + +class IntraExtraIdxPair(typing.NamedTuple): + intra: int + extra: int + + +class ExposurePairerConfig(pexConfig.Config): + timeThreshold = pexConfig.Field[float]( + doc="Maximum time difference between paired intra- and extra-focal exposures (s)", + default=50, + ) + + pointingThreshold = pexConfig.Field[float]( + doc="Maximum pointing difference between paired intra- and extra-focal exposures (arcsec)", + default=60, + ) + + rotationThreshold = pexConfig.Field[float]( + doc="Maximum rotator angle difference between paired intra- and extra-focal exposures (deg)", + default=1.0, + ) + + ignoreThresholds = pexConfig.Field[bool]( + doc=( + "If True, ignore time, pointing, and rotation thresholds. Useful when grouping by " + "exposure.group" + ), + default=False, + ) + + doOverrideSeparation = pexConfig.Field[bool]( + doc="Whether to override expected intra-focal to focal separation", + default=False, + ) + + overrideSeparation = pexConfig.Field[float]( + doc="Expected extra-focal to focal separation (mm). Note that this is signed such that " + "a positive value means an extra-focal exposure has a greater focusZ value than an " + "intra-focal exposure.", + default=1.5, + ) + + groupingThreshold = pexConfig.Field[float]( + doc="Threshold for assigning visit to intra/extra/focal group as a fraction of the expected " + "intra-focal to focal separation.", + default=0.1, + ) + + forceUniquePairs = pexConfig.Field[bool]( + doc="If True, force each extra exposure to be paired with a unique intra exposure.", + default=True, + ) + + +class ExposurePairer(pipeBase.Task): + ConfigClass = ExposurePairerConfig + _DefaultName = "exposurePairer" + _needsPairTable = False + _needsGroupDimension = False + + def makeTables( + self, + visitInfos: typing.Dict[int, afwImage.VisitInfo], + ) -> typing.Dict[str, Table]: + """ + Make tables of intra/extra/focal/and pairs of exposures. + + Parameters + ---------- + visitInfos : dict + A dictionary of VisitInfo objects keyed by exposure. + + Returns + ------- + dict + A dictionary of astropy tables keyed by type. + """ + if self.config.doOverrideSeparation: + separation = self.config.overrideSeparation + else: + instrument = next(iter(visitInfos.values())).instrumentLabel + match instrument: + case "LSSTCam" | "LSSTComCam" | "LSSTComCamSim": + separation = 1.5 + case "LATISS": + separation = -0.8 + + # Partition intra/extra/focal groups by finding a common offset that + # maximizes the number of assigned visits. + # If we're handed a pair of visitInfos with a focusZ difference + # of ~1.5 mm, then we can't generally decide if that's an + # intra/focal pair or a focal/extra pair. Since the AOS pipeline + # is keyed on extrafocal dataIds, we'll assume we're always handed + # an extrafocal visitInfo in this case. + + table = Table( + names=["exposure", "ra", "dec", "mjd", "focusZ", "rtp"], + dtype=[int, float, float, float, float, float], + ) + for exposure, visitInfo in visitInfos.items(): + radec = visitInfo.boresightRaDec + ra = radec.getLongitude().asRadians() + dec = radec.getLatitude().asRadians() + mjd = visitInfo.date.toAstropy().mjd + rtp = ( + visitInfo.boresightParAngle + - visitInfo.boresightRotAngle + - (np.pi / 2 * radians) + ).asDegrees() + focusZ = visitInfo.focusZ + table.add_row([exposure, ra, dec, mjd, focusZ, rtp]) + table["radec"] = SkyCoord(table["ra"], table["dec"], unit="radian") + thresh = self.config.groupingThreshold * abs(separation) + + best_n_inliers = 0 + best_offset = None + for offset in np.unique(table["focusZ"]): + dz = table["focusZ"] - offset + n_extra = np.sum(np.abs(dz) < thresh) + n_focal = np.sum(np.abs(dz + separation) < thresh) + n_intra = np.sum(np.abs(dz + 2 * separation) < thresh) + if n_extra + n_focal + n_intra > best_n_inliers: + best_n_inliers = n_extra + n_focal + n_intra + best_offset = offset + + extraTable = table[np.abs(table["focusZ"] - best_offset) < thresh] + focalTable = table[np.abs(table["focusZ"] - best_offset + separation) < thresh] + intraTable = table[ + np.abs(table["focusZ"] - best_offset + 2 * separation) < thresh + ] + + # For each extra focal exposure, find the best intra focal exposure. + # Note that it's possible that the same intra exposure is paired with + # multiple extra exposures. + dtype = [ + ("extra", " typing.List[IntraExtraIdxPair]: + """ + Pair up the intra- and extra-focal exposures. + """ + tables = self.makeTables(visitInfos) + pairTable = tables["pairTable"] + out = [] + for row in pairTable: + out.append(IntraExtraIdxPair(row["intra"], row["extra"])) + return out + + +class TablePairerConfig(pexConfig.Config): + pass + + +class TablePairer(pipeBase.Task): + ConfigClass = TablePairerConfig + _DefaultName = "tablePairer" + _needsPairTable = True + _needsGroupDimension = False + + def run( + self, + visitInfos: typing.Dict[int, afwImage.VisitInfo], + pairTable: Table, + ) -> typing.List[IntraExtraIdxPair]: + """ + Pair up the intra- and extra-focal exposures. + """ + out = [] + for row in pairTable: + if row["intra"] in visitInfos and row["extra"] in visitInfos: + out.append(IntraExtraIdxPair(row["intra"], row["extra"])) + return out + + +class GroupPairerConfig(ExposurePairerConfig): + def setDefaults(self): + super().setDefaults() + self.ignoreThresholds = True + + def validate(self): + super().validate() + + if not self.ignoreThresholds: + raise pexConfig.FieldValidationError( + self.__class__.ignoreThresholds, + self, + "ignoreThresholds must be True for GroupPairer", + ) + + +class GroupPairer(ExposurePairer): + ConfigClass = GroupPairerConfig + _DefaultName = "groupPairer" + _needsGroupDimension = True + + def run( + self, + visitInfos: typing.Dict[int, afwImage.VisitInfo], + ) -> typing.List[IntraExtraIdxPair]: + # visitInfos should either be an intra/extra pair, or possibly an + # intra/extra/in-focus triplet. Catch that here. + if len(visitInfos) not in (2, 3): + self.log.warning("Expected 2 or 3 visits, but got %d.", len(visitInfos)) + self.log.warning("Received exposure ids:") + for vi in visitInfos.keys(): + self.log.warning(" %d", vi) + return [] + return super().run(visitInfos) From dfd82e79e4dcb27dcf197452e53eacb66b1a11c8 Mon Sep 17 00:00:00 2001 From: LR-inaf Date: Fri, 22 Nov 2024 13:40:38 +0000 Subject: [PATCH 2/3] versionHistory.rst updated --- .gitignore | 1 + .../versionHistory-checkpoint.rst | 1935 +++++++++++++++++ doc/versionHistory.rst | 10 +- 3 files changed, 1945 insertions(+), 1 deletion(-) create mode 100644 doc/.ipynb_checkpoints/versionHistory-checkpoint.rst diff --git a/.gitignore b/.gitignore index c5f08039..2055257d 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ pytest_session.txt # Existing Phosim Donut Templates policy/cwfs/donutTemplateData/phosimTemplates/ +.ruff.toml diff --git a/doc/.ipynb_checkpoints/versionHistory-checkpoint.rst b/doc/.ipynb_checkpoints/versionHistory-checkpoint.rst new file mode 100644 index 00000000..ceea4484 --- /dev/null +++ b/doc/.ipynb_checkpoints/versionHistory-checkpoint.rst @@ -0,0 +1,1935 @@ +.. py:currentmodule:: lsst.ts.wep + +.. _lsst.ts.wep-version_history: + +################## +Version History +################## + +.. _lsst.ts.wep-13.0.3: + +------------- +13.0.3 +------------- + +* Task plotPsfFromZern added in comCamRapidAnalysisPipeline and comCamRapidAnalysisDanishPipeline. + +.. _lsst.ts.wep-13.0.3: + +------------- +13.0.2 +------------- + +* Use _refresh_metadata in cutOutStamps function so DonutStamps have correct set of metadata when running cutOutDonuts tasks interactively. + +.. _lsst.ts.wep-13.0.1: + +------------- +13.0.1 +------------- + +* Reorganize pipelines and add daily processing and danish pipelines. + +.. _lsst.ts.wep-13.0.0: + +------------- +13.0.0 +------------- + +* enabled sparse Zernike estimation +* removed most jmax and return4Up configs in favor of nollIndices configs +* removed return4Up from estimator WfEstimator and WfAlgorithm +* added makeSparse and makeDense to Zernike utils + +.. _lsst.ts.wep-12.7.0: + +------------- +12.7.0 +------------- + +* Added requireConverge to TIE and defaulted to True in task +* Fixed bug with None types in EstimateZernikeTask metadata histories + +.. _lsst.ts.wep-12.6.1: + +------------- +12.6.2 +------------- + +* Update RA production pipeline to use group dimension in aggregate donut tables step. + +.. _lsst.ts.wep-12.6.1: + +------------- +12.6.1 +------------- + +* Added a unit test for specifying DonutStampSelector.config.maxSelect in a pipeline config yaml. + +.. _lsst.ts.wep-12.6.0: + +------------- +12.6.0 +------------- + +* Added maxSelect config to DonutStampSelector + +.. _lsst.ts.wep-12.5.0: + +------------- +12.5.0 +------------- + +* Enable CutOutDonutsScienceSensorTask to operate for a pair with same-sign focusZ. + +.. _lsst.ts.wep-12.4.2: + +------------- +12.4.2 +------------- + +* Increase stamp size in Rapid Analysis pipeline to avoid clipping donut edges. + +.. _lsst.ts.wep-12.4.1: + +------------- +12.4.1 +------------- + +* Fixed bug where CalcZernikesTask fails when the number of intra/extra stamps is not equal + +.. _lsst.ts.wep-12.4.0: + +------------- +12.4.0 +------------- + +* Added a threshold on fraction-of-bad-pixels to DonutStampSelectorTask +* Modified DonutStampSelectorTaskConfig so that, by default, selections are run on fraction-of-bad-pixels and signal-to-noise ratio. +* Modified CalcZernikesTask so that DonutStampSelectorTask is run by default +* Fixed bug where DM mask bits weren't persisting in DonutStamp + +.. _lsst.ts.wep-12.3.0: + +------------- +12.3.0 +------------- + +* Added CutOutDonutsUnpairedTask and CalcZernikesUnpairedTask + +.. _lsst.ts.wep-12.2.0: + +------------- +12.2.0 +------------- + +* Update pipelines to use zernikes table instead of separate raw, avg zernike arrays. +* Propogate visit info from donut table into donutStamps to avoid calling visitInfo from the butler. + +.. _lsst.ts.wep-12.1.0: + +------------- +12.1.0 +------------- + +* Change zernikes butler storage format to QTable. + +.. _lsst.ts.wep-12.0.0: + +------------- +12.0.0 +------------- + +* Change pandas.DataFrame outputs to Astropy Tables. + +.. _lsst.ts.wep-11.5.2: + +------------- +11.5.2 +------------- + +* Added a ComCamSim production pipeline for testing purposes. + +.. _lsst.ts.wep-11.5.1: + +------------- +11.5.1 +------------- + +* Fixed bug in donutSourceSelectorTask where the task set with maxBlended > 0 and sources with a number of overlapping donuts greater than maxBlended did not give correct blend centers in the final catalog. + +.. _lsst.ts.wep-11.5.0: + +------------- +11.5.0 +------------- + +* Add astropy table output to CalcZernikesTask. + +.. _lsst.ts.wep-11.4.2: + +------------- +11.4.2 +------------- + +* Add full comcam pipeline to pipelines folder including wep and donut_viz tasks. + +.. _lsst.ts.wep-11.4.1: + +------------- +11.4.1 +------------- + +* Fix treatment of binary dilation in calculateSN. +* Fix how calculateSN masks treat blended pixels. +* Make calculateSN formatting consistent with the rest of cutOutDonutsBaseTask. +* Add a test with a blended stamp for calculateSN. +* Make variance plane warning only appear once. +* Fix test values in test_donutStampSelectorTask due to changes to ISR in w_2024_38. + +.. _lsst.ts.wep-11.4.0: + +------------- +11.4.0 +------------- + +* Set default maxNollIndex to zk28 in estimateZernikesBase. + +.. _lsst.ts.wep-11.3.0: + +------------- +11.3.0 +------------- + +* Add option to bin donut stamps before estimating the wavefront. + +.. _lsst.ts.wep-11.2.0: + +------------- +11.2.0 +------------- + +* Change CalcZernikesTask output to be at least 2D for average as well as raw to make integration with MTAOS easier. + +.. _lsst.ts.wep-11.1.0: + +------------- +11.1.0 +------------- + +* Make maxRecenteringDistance cut more robust in cutOutDonutsBase by first subtracting median shift and then comparing shifts to maxRecenteringDistance. + +.. _lsst.ts.wep-11.0.0: + +------------- +11.0.0 +------------- + +* Add donut image quality checking. + +.. _lsst.ts.wep-10.6.0: + +------------- +10.6.0 +------------- + +* Update Image bandLabel setter to handle condition where the bandLabel is string but the string is not a valid BandLabel enumeration. + +.. _lsst.ts.wep-10.5.0: + +------------- +10.5.0 +------------- + +* Fix handling of empty exposures in generateDonutDirectDetect. + +.. _lsst.ts.wep-10.4.2: + +------------- +10.4.2 +------------- + +* Add pipelines directory to easily share pipeline templates. + +.. _lsst.ts.wep-10.4.1: + +------------- +10.4.1 +------------- + +* Add visit to donutStamps metadata. + +.. _lsst.ts.wep-10.4.0: + +------------- +10.4.0 +------------- + +* Added random field angles in lsst.ts.wep.utils.modelUtils.forwardModelPair +* Fixed two bugs related to the random number generator in lsst.ts.wep.utils.modelUtils.forwardModelPair +* Added tests for lsst.ts.wep.utils.modelUtils.forwardModelPair + +.. _lsst.ts.wep-10.3.0: + +------------- +10.3.0 +------------- + +* Added single-side-of-focus mode to the TIE. + +.. _lsst.ts.wep-10.2.0: + +------------- +10.2.0 +------------- + +* Add option to pair intra/extra focal exposures by group dimension. + +.. _lsst.ts.wep-10.1.1: + +------------- +10.1.1 +------------- + +* Separate recenterFlags in cutOutDonuts tasks metadata into recenterFlagsExtra and recenterFlagsIntra. + +.. _lsst.ts.wep-10.1.0: + +------------- +10.1.0 +------------- + +* Added lsst.ts.wep.utils.modelUtils.forwardModelPair to facilitate forward modeling donuts for testing and data exploration +* Added lsst.ts.wep.utils.plotUtils.plotTieConvergence to diagnose TIE convergence + +.. _lsst.ts.wep-10.0.0: + +------------- +10.0.0 +------------- + +* Removed Zernike units configuration from tasks so that tasks always return Zernikes in microns + +.. _lsst.ts.wep-9.9.0: + +------------- +9.9.0 +------------- + +* Add auto-dilation option to making blend masks in ImageMapper. +* Fixed bugs with blend offsets for extrafocal image masks. + +.. _lsst.ts.wep-9.8.1: + +------------- +9.8.1 +------------- + +* Fixed bug in convertMetadataToHistory that failed when array shape values were floats. + +.. _lsst.ts.wep-9.8.0: + +------------- +9.8.0 +------------- + +* Add maxRecenterDistance configuration option to cutOutDonutsBase. + +.. _lsst.ts.wep-9.7.0: + +------------- +9.7.0 +------------- + +* Change configuration options for GenerateDonutFromRefitWcsTask to specify filter for photometric catalog as well. + +.. _lsst.ts.wep-9.6.0: + +------------- +9.6.0 +------------- + +* Change CombineZernikesSigmaClipTask to use kwargs dict to set arguments in astropy.stats.sigma_clip. + +.. _lsst.ts.wep-9.5.8: + +------------- +9.5.8 +------------- + +* Update to use ts_jenkins_shared_library. + +.. _lsst.ts.wep-9.5.7: + +------------- +9.5.7 +------------- + +* Update default maxFieldDist in donutSourceSelectorTask.py after analysis in DM-42067 (see ts_analysis_notebooks/aos/vignetting). + +.. _lsst.ts.wep-9.5.6: + +------------- +9.5.6 +------------- + +* Move class diagrams to mermaid from plantUML. + +.. _lsst.ts.wep-9.5.5: + +------------- +9.5.5 +------------- + +* Correct indices used to calculate Zernike average. +* Update tests to discern whether flags and mean use the same indices. + +.. _lsst.ts.wep-9.5.4: + +------------- +9.5.4 +------------- + +* Fix blend centroid coordinates in donut stamp generation. + +.. _lsst.ts.wep-9.5.3: + +------------- +9.5.3 +------------- + +* Fixed bug where blended masks have sharp edges when using dilateBlends. + +.. _lsst.ts.wep-9.5.2: + +------------- +9.5.2 +------------- + +* Fix units in ExposurePairer and add tests. + +.. _lsst.ts.wep-9.5.1: + +------------- +9.5.1 +------------- + +* Fixed compatibility with Batoid 0.6.2 + +.. _lsst.ts.wep-9.5.0: + +------------- +9.5.0 +------------- + +* Add exposure pairing for full array mode. + +.. _lsst.ts.wep-9.4.0: + +------------- +9.4.0 +------------- + +* Added the Danish wavefront estimation algorithm. + +.. _lsst.ts.wep-9.3.1: + +------------- +9.3.1 +------------- + +* Added conditional sigma clipping for averaging Zernike coefficients. + +.. _lsst.ts.wep-9.3.0: + +------------- +9.3.0 +------------- + +* Added a separate instrument for full-array mode +* Updated the ComCam mask model to match the bug fixes in Batoid + +.. _lsst.ts.wep-9.2.1: + +------------- +9.2.1 +------------- + +* Added unit test directly comparing ``ImageMapper`` optical models to Batoid raytracing. + +.. _lsst.ts.wep-9.2.0: + +------------- +9.2.0 +------------- + +* Add ``LSSTComCamSim`` as allowed camera type. + +.. _lsst.ts.wep-9.1.1: + +------------- +9.1.1 +------------- + +* Fix latiss tests by using getpass, and updating Zk values + +.. _lsst.ts.wep-9.1.0: + +------------- +9.1.0 +------------- + +* Added ``jmin`` arguments to Zernike utility functions. +* Added ``jmin`` and ``jmax`` value checks to the Zernike utility functions. + +.. _lsst.ts.wep-9.0.0: + +------------- +9.0.0 +------------- + +This is a big backwards-incompatible refactor of WEP. The major changes are: + +* Split the ``cwfs`` modules into ``centroid``, and ``estimation``. +* Donut Images are now held by the ``Image`` class. This class is meant to hold information in the global camera coordinate system (CCS). +* A new ``Instrument`` class with new configurations in the ``policy/instruments`` directory. This class holds geometric information about the different telescopes and cameras, as well as interfaces with the Batoid models. +* The ``ImageMapper`` class maps ``Image`` objects between the image and pupil planes, and creates pupil and image masks. The "offAxis" model now uses a real-time band-dependent fit with Batoid. The "onAxis" and "paraxial" models work the same as before. +* The Zernike estimation classes have been generalized to allow different wavefront algorithm classes to plug into ``WfEstimator``. +* The TIE algorithm is implemented in ``estimation.TieAlgorithm``. +* There are new utilities in ``utils`` for fitting mask models and plotting mask models and the ``ImageMapper`` methods. +* ``Instrument`` configuration in tasks is now pulled from the default parameter files for each camera type. Overrides can be provided via the ``instConfigFile`` parameter. With the default instrument configurations, defocal offsets are pulled from the exposure metadata. If ``defocalOffset`` is explicitly set in the ``instConfigFile`` override, that defocal offset is used instead of the values from the exposure metadata. +* The ``donutTemplateSize`` config parameter has been removed from all the relevant tasks, as the new ``ImageMapper`` can predict the required template size. ``initialCutoutPadding`` provides padding beyond this predicted value. +* The ``multiplyMask`` and ``maskGrowthIter`` parameters have been removed from ``CutOutDonutsBase``. To mask blends during TIE fitting, instead use the ``maskKwargs`` parameter of the ``EstimateZernikesTieTask``. +* When estimating Zernikes, the maximum Noll index (jmax) is now a configurable parameter (``maxNollIndex`` in ``EstimateZernikesBaseConfig``). You can also toggle whether estimation starts from zero or from the telescope's instrinsic Zernikes. You can toggle whether the task returns the full optical path difference (OPD) or just the wavefront deviation (OPD - intrinsic Zernikes). You can toggle whether the returned Zernikes start with Noll index 4 (the previous standard), or with index 0 (matching the Galsim convention). You can also set the units of the returned Zernikes. +* The algorithm history can now be saved at the Task level using the ``saveHistory`` option in ``EstimateZernikesBaseConfig``. The history is saved in the task metadata in a json-compatible format. To convert the history back to the native format, use `utils.convertMetadataToHistory`. +* Changing from the native butler coordinate system (data visualization coordinate system with rotated wavefront sensors) to the WEP coordinate system (camera coordinate system with de-rotated wavefront sensors) now happens entirely in ``task.DonutStamp._setWepImage``. Furthermore, the ``defocal_distance`` saved in the stamp is now the detector offset (or equivalent detector offset) rather than the raw focusZ info. +* The AuxTel/LATISS unit tests have been fixed, and the LATISS Zernike calculation test has been explicitly switched to a regression test (rather than an accuracy test). +* Enum's now map to strings instead of integers. This natural Enum-string connection replaces the various utils that previously existed to map between Enums and strings. + +.. _lsst.ts.wep-8.3.1: + +------------- +8.3.1 +------------- + +* Update tests to be more robust to DM changes and fix failures after DM stack update to w_2024_08. +* Run black v24.2. + +.. _lsst.ts.wep-8.3.0: + +------------- +8.3.0 +------------- + +* Remove mask_comp and mask_pupil from DonutStamp since they don't persist and mask is already contained in MaskedImage stamp. + +.. _lsst.ts.wep-8.2.0: + +------------- +8.2.0 +------------- + +* Add background subtraction to cutOutDonutsBase. + +.. _lsst.ts.wep-8.1.1: + +------------- +8.1.1 +------------- + +* Replace calls to removed pipeBase.ButlerQuantumContext with pipeBase.QuantumContext. + +.. _lsst.ts.wep-8.1.0: + +------------- +8.1.0 +------------- + +* Remove Zemax Coordinate System (ZCS) conversions now that ts_ofc works exclusively in Camera Coordinate System (CCS). + +.. _lsst.ts.wep-8.0.4: + +------------- +8.0.4 +------------- + +* Update default config on GenerateDonutFromRefitWcsTask after updates in meas_astrom. + +.. _lsst.ts.wep-8.0.3: + +------------- +8.0.3 +------------- + +* Attach locally linear WCSs to DonutStamps. + +.. _lsst.ts.wep-8.0.2: + +------------- +8.0.2 +------------- + +* Adds support for MacOS. + +.. _lsst.ts.wep-8.0.1: + +------------- +8.0.1 +------------- + +* Add convertZernikesToPsfWidth to zernikeUtils. + +.. _lsst.ts.wep-8.0.0: + +------------- +8.0.0 +------------- + +* Save all DonutStamps with images aligned with focal plane science sensors. +* This version will break compatibility in the closed loop with Phosim and ts_phosim going forward. + + +.. _lsst.ts.wep-7.0.1: + +------------- +7.0.1 +------------- + +* Fix generateDonutDirectDetect when doDonutSelection is not run. + +.. _lsst.ts.wep-7.0.0: + +------------- +7.0.0 +------------- + +* Organize all utility functions inside the ``utils`` module. + +.. _lsst.ts.wep-6.4.12: + +------------- +6.4.12 +------------- + +* Update ts_pre_commit_config with ruff. + +.. _lsst.ts.wep-6.4.11: + +------------- +6.4.11 +------------- + +* Fix GenerateDonutFromRefitWcsTask adding coord_raErr, coord_decErr fields. + +.. _lsst.ts.wep-6.4.10: + +------------- +6.4.10 +------------- + +* Update calcZernikesLatissPipeline yaml with instrument-specific setup for generateDonutDirectDetectTask. + +.. _lsst.ts.wep-6.4.9: + +------------- +6.4.9 +------------- + +* Replacing lookUpCalibrations function to use the one in lsst.fgcmcal.utilities + +.. _lsst.ts.wep-6.4.8: + +------------- +6.4.8 +------------- + +* Add github actions to check version history was updated and linting. +* Fix black and flake8 violations. +* Fix Jenkinfile. + +.. _lsst.ts.wep-6.4.7: + +------------- +6.4.7 +------------- + +* Set default optical model for comCam to onAxis. + +.. _lsst.ts.wep-6.4.6: + +------------- +6.4.6 +------------- + +* Fix tests that failed due to changes in numpy testing methods and WCS output. + +.. _lsst.ts.wep-6.4.5: + +------------- +6.4.5 +------------- + +* Update setup files with pre-commit hooks, run black and isort. + +.. _lsst.ts.wep-6.4.4: + +------------- +6.4.4 +------------- + +* In ``utility``, update ``getFilterTypeFromBandLabel`` to return ``FilterType.REF`` if the ``bandLabel`` is not recognized. + +.. _lsst.ts.wep-6.4.3: + +------------- +6.4.3 +------------- + +* Fix error in Jenkinsfile that caused git-lfs to fail when running on develop branch. + +.. _lsst.ts.wep-6.4.2: + +------------- +6.4.2 +------------- + +* Move fits files to git-lfs. + +.. _lsst.ts.wep-6.4.1: + +------------- +6.4.1 +------------- + +* Add documentation explaining how to run the WEP pipeline on the USDF batch system. + +.. _lsst.ts.wep-6.4.0: + +------------- +6.4.0 +------------- + +* Create generateDonutCatalogUtils to store common methods. +* Update generateDonutCatalogOnlineTask to match output of other generateDonutCatalog...Tasks. + +.. _lsst.ts.wep-6.3.5: + +------------- +6.3.5 +------------- + +* Make sure output from empty catalogs match that expected from catalogs with sources in donutSourceSelectorTask. +* Add tests for run method in donutSourceSelectorTask. + +.. _lsst.ts.wep-6.3.4: + +------------- +6.3.4 +------------- + +* Patch refCatalogInterface to eliminate warnings from latest version of daf_butler. + +.. _lsst.ts.wep-6.3.3: + +------------- +6.3.3 +------------- + +* Change filter name in testData/gen3TestRepo camera fits files to comply with new obs_lsst convention. + +.. _lsst.ts.wep-6.3.2: + +------------- +6.3.2 +------------- + +* Change CWFS pipeline configuration files to have 1.5mm offset included and to handle this properly in CWFS version of tasks. + +.. _lsst.ts.wep-6.3.1: + +------------- +6.3.1 +------------- + +* Directly calculate dI/dz in Algorithm, without the intermediate dI. +* Save dI/dz and I0 in Algorithm history when debugLevel>=1. + +.. _lsst.ts.wep-6.3.0: + +------------- +6.3.0 +------------- + +* Add filterLabel property to CompensableImage. + +.. _lsst.ts.wep-6.2.0: + +------------- +6.2.0 +------------- + +* Add optional pipeline task to fit WCS from direct detect donut catalogs and generate new donut catalog from reference catalogs with fit WCS. + +.. _lsst.ts.wep-6.1.3: + +------------- +6.1.3 +------------- + +* Add license information to test_generateDonutDirectDetectTask. + +.. _lsst.ts.wep-6.1.2: + +------------- +6.1.2 +------------- + +* Import MaskedImage directly from afw.image to fix errors from change in w_2023_16. + +.. _lsst.ts.wep-6.1.1: + +------------- +6.1.1 +------------- + +* Fix blend_centroid_x and blend_centroid_y to only return donuts bright enough to count as blended when maxBlended is greater than the total number of donuts. + +.. _lsst.ts.wep-6.1.0: + +------------- +6.1.0 +------------- + +* Add optional ability to specify filter in GenerateDonutCatalogWcsTask. + +.. _lsst.ts.wep-6.0.2: + +------------- +6.0.2 +------------- + +* Fix assignment of blend_centroid_x and blend_centroid_y in donut catalogs. + +.. _lsst.ts.wep-6.0.1: + +------------- +6.0.1 +------------- + +* Redesign and enhance documentation to match style and detail of other TS repositories. + +.. _lsst.ts.wep-6.0.0: + +------------- +6.0.0 +------------- + +* Rename all modules to start with lowercase in order to align with DM convention. +* Add information into documentation about how this new change breaks repositories with data processed with older versions of ts_wep and how to fix it. + +.. _lsst.ts.wep-5.1.0: + +------------- +5.1.0 +------------- + +* Add bandpass information into DonutStamp objects. + +.. _lsst.ts.wep-5.0.1: + +------------- +5.0.1 +------------- + +* Run black v23.1.0. + +.. _lsst.ts.wep-5.0.0: + +------------- +5.0.0 +------------- + +* Remove deprecated keywords in cwfs/Tool.py and deprecated function in cwfs/CompensableImage.py. +* Remove deprecated EstimateZernikes Tasks. +* Remove deprecated pipelineConfig test files. + +.. _lsst.ts.wep-4.2.3: + +------------- +4.2.3 +------------- + +* Add transposeImages as optional config to CalcZernikesTask. + +.. _lsst.ts.wep-4.2.2: + +------------- +4.2.2 +------------- + +* Add flux sort into GenerateDonutDirectDetectTask to make it consistent with other catalog generation tasks. + +.. _lsst.ts.wep-4.2.1: + +------------- +4.2.1 +------------- + +* Use nan string instead of None so we can convert to float and use writeFits method in DonutStamps successfully and save in butler. + +.. _lsst.ts.wep-4.2.0: + +------------- +4.2.0 +------------- + +* Add DonutQuickMeasurementTask.py to incorporate quick donut detection and measurement using LSST Science Pipelines. +* Integrate DonutQuickMeasurementTask into GenerateDonutDirectDetectTask to speed up direct detection catalog generation. + +.. _lsst.ts.wep-4.1.0: + +------------- +4.1.0 +------------- + +* GenerateDonutCatalogWcsTask takes filter information from exposures automatically. +* DonutSourceSelectorTask uses policy/task/magLimitStar.yaml for default magnitude limits. + +.. _lsst.ts.wep-4.0.4: + +------------- +4.0.4 +------------- + +* Update calls to deprecated LoadIndexedReferenceObjectsTask to use LoadReferenceObjectsTask. + +.. _lsst.ts.wep-4.0.3: + +------------- +4.0.3 +------------- + +* Add blend_centroid_x and blend_centroid_y to GenerateDonutDirectDetectTask catalogs. + +.. _lsst.ts.wep-4.0.2: + +------------- +4.0.2 +------------- + +* Fix test_estimateZernikesCwfsTask call to ButlerQuantumContext. + +.. _lsst.ts.wep-4.0.1: + +------------- +4.0.1 +------------- + +* Remove Gen2 daf_persistence from UPS table. + +.. _lsst.ts.wep-4.0.0: + +------------- +4.0.0 +------------- + +* Add masked deblending to CompensableImage and pipeline tasks. +* Change how DonutSourceSelectorTask works by adding minBlendedSeparation parameter and changing DonutRadius to unblendedSeparation parameter. + +.. _lsst.ts.wep-3.2.0: + +------------- +3.2.0 +------------- + +* Port Latiss functionality from EstimateZernikesLatissTask into CutOutDonutsScienceSensorTask + CalcZernikesTask pipeline. +* Deprecate EstimateZernikes family of tasks. These tasks will no longer be updated and will be removed after January 2023. + +.. _lsst.ts.wep-3.1.5: + +------------- +3.1.5 +------------- + +* Throw exception when auxTel is trying to use offAxis model. + +.. _lsst.ts.wep-3.1.4: + +------------- +3.1.4 +------------- + +* Remove imageCoCenter step from Algorithm. +* Add DeprecationWarning that imageCoCenter function in CompensableImage will be removed after January 2023. + +.. _lsst.ts.wep-3.1.3: + +------------- +3.1.3 +------------- + +* Added default value to DonutStamp for DFC_DIST to allow the butler to read DonutStamp from repositories created with older versions of ts_wep. + +.. _lsst.ts.wep-3.1.2: + +------------- +3.1.2 +------------- + +* Update phosimOutput corner sensors test files. + +.. _lsst.ts.wep-3.1.1: + +------------- +3.1.1 +------------- + +* Fix tests pipeline yaml files updating the ISR setting to use 'MEDIAN' for overscan fit type. +* Remove obsolete _generateTestExposures. +* Fix `test_generateDonutDirectDetectTask.py` + +.. _lsst.ts.wep-3.1.0: + +------------- +3.1.0 +------------- + +* Added a history to the Algorithm class that stores intermediate products of the algorithm (see `Algorithm.getHistory()`). +* Fixed the algorithm so that it is once again symmetric with respect to I1 and I2. + This involved simplifying the way that mask and image orientation are handled for the extrafocal image (see below). +* Added the option to create masks in the orientation of the original images by setting `compensated=False` in `CompensableImage.makeMask()`. + +.. _lsst.ts.wep-3.0.1: + +------------- +3.0.1 +------------- + +* Fix ``test_generateDonutCatalogWcsTask.py`` to work with more recent versions of the DM stack. + +.. _lsst.ts.wep-3.0.0: + +------------- +3.0.0 +------------- + +* Refactor tasks to directly accept instrument parameters in their configuration. + +.. _lsst.ts.wep-2.7.0: + +------------- +2.7.0 +------------- + +* Remove dictionary defining allowable offsets in Instrument.py and replace with settable parameter. +* Allow Instrument.py to be configured directly from dictionary of instrument parameters in addition to policy file. + +.. _lsst.ts.wep-2.6.0: + +------------- +2.6.0 +------------- + +* Replace getters and setters in Instrument.py with properties to make more pythonic. +* Update Algorithm, CompensableImage and DonutTemplateModel with new Instrument.py design. + +.. _lsst.ts.wep-2.5.8: + +------------- +2.5.8 +------------- + +* Change focusZ in headers of repackaged phosim data to be in mm instead of microns after phosim_utils update. + +.. _lsst.ts.wep-2.5.7: + +------------- +2.5.7 +------------- + +* Add defocal distance into DonutStamp. + +.. _lsst.ts.wep-2.5.6: + +------------- +2.5.6 +------------- + +* Fix task input order in test_estimateZernikes... tests. + +.. _lsst.ts.wep-2.5.5: + +------------- +2.5.5 +------------- + +* Change default maxFieldDistance in DonutSourceSelectorTask.py to 1.813 degrees based upon results from DM-33180. +* Fix test in test_calcZernikesTaskScienceSensor to use correct intraFocal dataId. + +.. _lsst.ts.wep-2.5.4: + +------------- +2.5.4 +------------- + +* Update science sensor and LATISS tasks to get focusZ from exposure visitInfo instead of metadata after update in DM-35186. + +.. _lsst.ts.wep-2.5.3: + +------------- +2.5.3 +------------- + +* Update tests and gen3TestRepo to work with latest version of the stack (w_2022_28). + +.. _lsst.ts.wep-2.5.2: + +------------- +2.5.2 +------------- + +* Add ComCam to donutTemplateModel. +* Add error message to donutTemplateModel for AuxTel if not run with 'onAxis' optical model. + +.. _lsst.ts.wep-2.5.1: + +------------- +2.5.1 +------------- + +* Correct orientation of masks in pipeline tasks. + +.. _lsst.ts.wep-2.5.0: + +------------- +2.5.0 +------------- + +* Update names of cMask to mask_comp (padded), pMask to mask_pupil (non-padded) +* Correct output of getPaddedMask to mask_comp, getNonPaddedMask to mask_pupil + +.. _lsst.ts.wep-2.4.4: + +------------- +2.4.4 +------------- + +* Added documentation link to the README. + +.. _lsst.ts.wep-2.4.3: + +------------- +2.4.3 +------------- + +* Fix online documentation build errors. + +.. _lsst.ts.wep-2.4.2: + +------------- +2.4.2 +------------- + +* Remove matplotlib backend switching in PlotUtil.py + +.. _lsst.ts.wep-2.4.1: + +------------- +2.4.1 +------------- + +* Add information on Jupyter Notebooks in ts_analysis_notebooks to README. + +.. _lsst.ts.wep-2.4.0: + +------------- +2.4.0 +------------- + +* Add CutOutDonuts tasks and CalcZernikesTask to separate cutting out donut stamps and calculating Zernikes from donut stamps as separate tasks. + +.. _lsst.ts.wep-2.3.8: + +------------- +2.3.8 +------------- + +* Remove phosim_utils dependency. + +.. _lsst.ts.wep-2.3.7: + +------------- +2.3.7 +------------- + +* Optimize CWFS algorithms. + +.. _lsst.ts.wep-2.3.6: + +------------- +2.3.6 +------------- + +* Fix rotation of sensors in EstimateZernikesBase. + +.. _lsst.ts.wep-2.3.5: + +------------- +2.3.5 +------------- + +* Update scipy.ndimage namespace to fix deprecation warnings. +* Run black v22.3. + +.. _lsst.ts.wep-2.3.4: + +------------- +2.3.4 +------------- + +* Fix test for `EstimateZernikesLatissTask`, to run for any user with /repo/main/ access. + +.. _lsst.ts.wep-2.3.3: + +------------- +2.3.3 +------------- + +* Add donut location configuration setting to `DonutSourceSelectorTask`. + +.. _lsst.ts.wep-2.3.2: + +------------- +2.3.2 +------------- + +* Change `CombineZernikesSigmaClip` to use the more robust `mad_std` standard deviation algorithm. +* Add `maxZernClip` configuration parameter to `CombineZernikesSigmaClip`. +* Change `CombineZernikes` metadata to use integer flags. + +.. _lsst.ts.wep-2.3.1: + +------------- +2.3.1 +------------- + +* Rely on GalSim for Zernike and Cartesian polynomial evaluation. + +.. _lsst.ts.wep-2.3.0: + +------------- +2.3.0 +------------- + +* Add `EstimateZernikesLatissTask` to process auxTel data +* Add `GenerateDonutDirectDetectTask` to find donuts with template fitting +* Add choices for binary image creation in `DonutDetector` +* Add `getCamType` and `getDefocalDisInMm` to `Utility` +* Add donut template for auxTel in `DonutTemplateModel` + +.. _lsst.ts.wep-2.2.4: + +------------- +2.2.4 +------------- + +* Update Jenkinsfile to always pull the image before new builds and improve cleanup stages to make build more robust. + +.. _lsst.ts.wep-2.2.3: + +------------- +2.2.3 +------------- + +* Change `EstimateZernikesCwfsTask` to be able to accept only a single pair of wavefront sensors. +* Remove `runQuantum` function from `EstimateZernikesScienceSensorTask` since it does not add any functionality now that the task gets the camera from the butler. + +.. _lsst.ts.wep-2.2.2: + +------------- +2.2.2 +------------- + +* Update functions marked deprecated as of stack version `w_2022_06`. + +.. _lsst.ts.wep-2.2.1: + +------------- +2.2.1 +------------- + +* Distinguish AuxTel ZWO camera from LATISS + +.. _lsst.ts.wep-2.2.0: + +------------- +2.2.0 +------------- + +* Add CombineZernikes...Tasks that combine the Zernike coefficients from multiple donut pairs into a single set of coefficients. + +.. _lsst.ts.wep-2.1.4: + +------------- +2.1.4 +------------- + +* Remove `timeMethod` deprecation warnings and use static calibration camera. + +.. _lsst.ts.wep-2.1.3: + +------------- +2.1.3 +------------- + +* Fix maxBlended parameter in DonutSourceSelectorTask and improve tests to check this configuration setting. + +.. _lsst.ts.wep-2.1.2: + +------------- +2.1.2 +------------- + +* Make sure catalogs from GenerateDonutCatalog...Tasks have same columns. + +.. _lsst.ts.wep-2.1.1: + +------------- +2.1.1 +------------- + +* Get camera from the butler when running pipeline tasks. + +.. _lsst.ts.wep-2.1.0: + +------------- +2.1.0 +------------- + +* Refactor GenerateDonutCatalog*.py tasks. +* Update EstimateZernikes...Tasks after DonutCatalog refactor. + +.. _lsst.ts.wep-2.0.4: + +------------- +2.0.4 +------------- + +* Add DonutSourceSelectorTask to task module. + +.. _lsst.ts.wep-2.0.3: + +------------- +2.0.3 +------------- + +* Add RefCatalogInterface to task module. + +.. _lsst.ts.wep-2.0.2: + +------------- +2.0.2 +------------- + +* Patch to work with weekly `w_2022_2`: + * `loadSkyCircle` no longer returns centroid column, use `loadPixelBox` instead. + +.. _lsst.ts.wep-2.0.1: + +------------- +2.0.1 +------------- + +* Patch to work with latest weekly. +* Update Jenkinsfile for CI job: + * git command is no longer working after the latest update on our Jenkins server. + * update path to plantuml. + +.. _lsst.ts.wep-2.0.0: + +------------- +2.0.0 +------------- + +* Removed code not used in Gen3 Pipelines. + +.. _lsst.ts.wep-1.8.2: + +------------- +1.8.2 +------------- + +* Removed CreatePhosimDonutTemplates.py and moved to `ts_phosim`. + +.. _lsst.ts.wep-1.8.1: + +------------- +1.8.1 +------------- + +* Get sensor orientation and field position directly from camera through new DonutStamp objects instead of using SourceProcessor. +* Fix rotation of postage stamps sent to WFEsti. + +.. _lsst.ts.wep-1.8.0: + +------------- +1.8.0 +------------- + +* Refactored DonutStamp.py and added ability to recreate masks as afwImage.Mask objects. + +.. _lsst.ts.wep-1.7.10: + +------------- +1.7.10 +------------- + +* Save outputZernikes for pairs of wavefront detectors not just a single output for all detectors. + +.. _lsst.ts.wep-1.7.9: + +------------- +1.7.9 +------------- + +* Remove _shiftCenterWfs from Source Processor. + +.. _lsst.ts.wep-1.7.8: + +------------- +1.7.8 +------------- + +* Update stamp rotations to work with CWFS. + +.. _lsst.ts.wep-1.7.7: + +------------- +1.7.7 +------------- + +* Update focalplanelayout.txt with new Euler angle for SW0 sensors. + +.. _lsst.ts.wep-1.7.6: + +------------- +1.7.6 +------------- +* Update donutStamp with archive property. +* Add `LSSTCam/calib` to collections path in test Gen3 pipelines. + +.. _lsst.ts.wep-1.7.5: + +------------- +1.7.5 +------------- + +* Break generic pieces of GenerateDonutCatalogOnlineTask.py into GenerateDonutCatalogOnlineBase.py +* Add GenerateDonutCatalogWcsTask.py to calculate donut catalogs when WCS is available + +.. _lsst.ts.wep-1.7.4: + +------------- +1.7.4 +------------- + +* Remove old e-image corner wavefront sensor files. +* Add updated corner wavefront sensor test data. +* Add CWFS Zernikes code and tests. + +.. _lsst.ts.wep-1.7.3: + +------------- +1.7.3 +------------- + +* Break generic pieces of EstimateZernikesFamTask.py into EstimateZernikesBase.py + +.. _lsst.ts.wep-1.7.2: + +------------- +1.7.2 +------------- + +* Fix ``append`` and ``extend`` methods in ``DonutStamps.py``. +* Update tests in ``test_donutStamps.py`` to properly check ``append`` and ``extend`` methods. + +.. _lsst.ts.wep-1.7.1: + +------------- +1.7.1 +------------- + +* Update ``FOCUSZ`` parameter in test data. + +.. _lsst.ts.wep-1.7.0: + +------------- +1.7.0 +------------- + +* Replace ``WcsSol`` by DM's wcs code in ``GenerateDonutCatalogOnlineTask``. +* Fix intra/extra zernike selection. + +.. _lsst.ts.wep-1.6.9: + +------------- +1.6.9 +------------- + +* Add focusz as an argument to repackagePhosimImages in CreatePhosimDonutTemplates.py + +.. _lsst.ts.wep-1.6.8: + +------------- +1.6.8 +------------- + +* Return both raw and averaged Zernikes to Butler repository in EstimateZernikesFamTask.py. + +.. _lsst.ts.wep-1.6.7: + +------------- +1.6.7 +------------- + +* Fix flake error and update Jenkinsfile + +.. _lsst.ts.wep-1.6.6: + +------------- +1.6.6 +------------- + +* Remove 90 degree offset from WcsSol.py now that phosim headers are updated. + +.. _lsst.ts.wep-1.6.5: + +------------- +1.6.5 +------------- + +* Use `FOCUSZ` header information in EstimateZernikesFamTask.py. + +.. _lsst.ts.wep-1.6.4: + +------------- +1.6.4 +------------- + +* Add EstimateZernikesFamTask.py to calculate Zernike coefficients in full-array mode through a Gen 3 pipeline. + +.. _lsst.ts.wep-1.6.3: + +------------- +1.6.3 +------------- + +* Add DonutStamp and DonutStamps storage classes to hold postage stamps of donuts. + +.. _lsst.ts.wep-1.6.2: + +------------- +1.6.2 +------------- + +* Update ROTANG header in realComcam test files + +.. _lsst.ts.wep-1.6.1: + +------------- +1.6.1 +------------- + +* Update GenerateDonutCatalogOnlineTask.py to get instrument directly from pipeline configuration. +* Setup `ctrl_mpexec` package in Jenkinsfile so tests can run `pipetask` command. + +.. _lsst.ts.wep-1.6.0: + +------------- +1.6.0 +------------- + +* Create new task module +* Add GenerateDonutCatalogOnlineTask.py in task module +* Add `tests/testData/gen3TestRepo` as sample Gen 3 repo for testing + +.. _lsst.ts.wep-1.5.9: + +------------- +1.5.9 +------------- + +* Build and upload documentation as part of the CI job. +* Use develop-env image for the CI job, due to the need of java to build the documentation. +* Disable concurrent builds. +* Fix docstring in `SourceSelector.connect` method. + +.. _lsst.ts.wep-1.5.8: + +------------- +1.5.8 +------------- + +* Reformat the code by `black` v20.8b1. + +.. _lsst.ts.wep-1.5.7: + +------------- +1.5.7 +------------- + +* Update import of `DetectorType`. + +.. _lsst.ts.wep-1.5.6: + +------------- +1.5.6 +------------- + +* Reformat code with `black`. + +.. _lsst.ts.wep-1.5.5: + +------------- +1.5.5 +------------- + +* Add `DonutDetector` class. + +.. _lsst.ts.wep-1.5.4: + +------------- +1.5.4 +------------- + +* Update to using ``LsstCamMapper`` and new geometry, including ``focalplanelayout.txt`` + +.. _lsst.ts.wep-1.5.3: + +------------- +1.5.3 +------------- + +* Add ``DonutTemplatePhosim`` class. +* Add ``CreatePhosimDonutTemplates`` class and add ``bin.src/runCreatePhosimDonutTemplates.py`` + +.. _lsst.ts.wep-1.5.2: + +------------- +1.5.2 +------------- + +* Fix the ``ZernikeMaskedFit()`` when passing masked data + +.. _lsst.ts.wep-1.5.1: + +------------- +1.5.1 +------------- + +* Add donut template classes to make templates for ``CentroidConvolveTemplate``. +* Add ``DonutTemplateFactory``, ``DonutTemplateDefault``, and ``DonutTemplateModel``. + +.. _lsst.ts.wep-1.5.0: + +------------- +1.5.0 +------------- + +* Add ``CentroidConvolveTemplate`` as a new centroid finding method. + +.. _lsst.ts.wep-1.4.9: + +------------- +1.4.9 +------------- + +* Unify the line ending to LF. + +.. _lsst.ts.wep-1.4.8: + +------------- +1.4.8 +------------- + +* Remove the ``abbrevDectectorName()`` and ``expandDetectorName()``. +* Remove the unused arguments of ``epoch``, ``includeDistortion``, and ``mjd`` in WCS related functions. +* Fix the ``calcWfErr()`` for the **LsstCamMapper**. + +.. _lsst.ts.wep-1.4.7: + +------------- +1.4.7 +------------- + +* Remove ``sims`` and ``obs_lsstSim`` dependencies. +* Update WCS code to use ``obs_lsst``. + +.. _lsst.ts.wep-1.4.6: + +------------- +1.4.6 +------------- + +* Use the ``sims_w_2020_38``. + +.. _lsst.ts.wep-1.4.5: + +------------- +1.4.5 +------------- + +* Use the ``sims_w_2020_36``. +* Support the LSST full-array mode (FAM). Add the classes of **BaseCwfsTestCase** and **BaseBscTestCase**. +* Put the limits of star's magnitude into a configuration file. +* Remove the serialization functions in **FilterType** enum. + +.. _lsst.ts.wep-1.4.4: + +------------- +1.4.4 +------------- + +* Use the ``pybind11`` instead of ``cython``. +* Add the ``clang-format`` check to ``.githooks``. + +.. _lsst.ts.wep-1.4.3: + +------------- +1.4.3 +------------- + +* Reformat the code by ``black``. +* Add the ``black`` check to ``.githooks``. +* Ignore ``flake8`` check of E203 ans W503 for the ``black``. +* Use the ``sims_w_2020_21``. + +.. _lsst.ts.wep-1.4.2: + +------------- +1.4.2 +------------- + +* Improved handling of IO errors - catch more OS Errors instead of only file not exists. + +.. _lsst.ts.wep-1.4.1: + +------------- +1.4.1 +------------- + +* Add the function to recenter the donut image with the template. +* Add the instrument and test data of auxilirary telescope. + +.. _lsst.ts.wep-1.4.0: + +------------- +1.4.0 +------------- + +* Use the ``sims_w_2020_15``. +* Use the factory pattern for deblend module. + +.. _lsst.ts.wep-1.3.9: + +------------- +1.3.9 +------------- + +* Use the ``sims_w_2020_14``. + +.. _lsst.ts.wep-1.3.8: + +------------- +1.3.8 +------------- + +* Use the ``sims_w_2020_07``. + +.. _lsst.ts.wep-1.3.7: + +------------- +1.3.7 +------------- + +* Use the ``sims_w_2020_06``. +* Skip two tests in **test_butlerWrapper.py** and **test_camIsrWrapper.py** for the bugs in upstream. +* Feedback to DM team. + +.. _lsst.ts.wep-1.3.6: + +------------- +1.3.6 +------------- + +* Use the ``sims_w_2020_04``. + +.. _lsst.ts.wep-1.3.5: + +------------- +1.3.5 +------------- + +* Use the ``sims_w_2019_50``. + +.. _lsst.ts.wep-1.3.4: + +------------- +1.3.4 +------------- + +* Use the ``sims_w_2019_38``. + +.. _lsst.ts.wep-1.3.3: + +------------- +1.3.3 +------------- + +* Use the ``sims_w_2019_31``. +* Remove the ``conda`` package installation in **Jenkinsfile**. +* Update the permission of workspace after the unit test. + +.. _lsst.ts.wep-1.3.2: + +------------- +1.3.2 +------------- + +* Use the ``sims_w_2019_29``. +* Add the unit tests of ``cwfs`` module to check the outputs of cython related code. +* Move the ``plotImage()`` from **Tool.py** to **PlotUtil.py**. +* Install the ``ipython`` in **Jenkinsfile** to make the test environment to be consistent with the development. + +.. _lsst.ts.wep-1.3.1: + +------------- +1.3.1 +------------- + +* Use the factory pattern for centroid find algorithms. +* Move the **SensorWavefrontError** class of ``ts_ofc`` to here. + +.. _lsst.ts.wep-1.3.0: + +------------- +1.3.0 +------------- + +* Use ``sims_w_2019_24``. +* Support the eimage. +* Enable to update and save the setting file. + +.. _lsst.ts.wep-1.2.9: + +------------- +1.2.9 +------------- + +* Use ``sims_w_2019_22``. +* Adapt the new version of ``ip_isr`` that fixes the bug that can not do the ISR continuously. + +.. _lsst.ts.wep-1.2.8: + +------------- +1.2.8 +------------- + +* Use ``sims_w_2019_20``. + +.. _lsst.ts.wep-1.2.7: + +------------- +1.2.7 +------------- + +* Put the default BSC path and sky file path in default ``yaml`` file. +* Concrete **WEPCalculation** class will connect and disconnect the database at each query. +* Use ``sims_w_2019_18``. + +.. _lsst.ts.wep-1.2.6: + +------------- +1.2.6 +------------- + +* Utilize the interface classes to main telescope active optics system (MTAOS). +* Use ``sims_w_2019_17``. + +.. _lsst.ts.wep-1.2.5: + +------------- +1.2.5 +------------- + +* Support the ``documenteer``. + +.. _lsst.ts.wep-1.2.4: + +------------- +1.2.4 +------------- + +* Use the ``yaml`` format for configuration files of ``cwfs`` module. +* Use ``sims_w_2019_15``. + +.. _lsst.ts.wep-1.2.3: + +------------- +1.2.3 +------------- + +* Add the ``eups`` as the package manager. +* Use ``sims_w_2019_12``. + +.. _lsst.ts.wep-1.2.2: + +------------- +1.2.2 +------------- + +* Add the **RawExpData** class and update the related functions. + +.. _lsst.ts.wep-1.2.1: + +------------- +1.2.1 +------------- + +* Add the interface to **MTAOS** in ``ctrlIntf`` module. + +.. _lsst.ts.wep-1.1.1: + +------------- +1.1.1 +------------- + +* Updated to use the scientific pipeline of ``sims_w_2019_02``. +* Add the referece filter type. + +.. _lsst.ts.wep-1.1.0: + +------------- +1.1.0 +------------- + +* Updated the WEP to use the ``obs_lsst`` and scientific pipeline of ``sims_w_2018_47``. +* The ``phosim_utils`` is used to repackage the PhoSim output amplifer images to the format of multi-extention FITS. + +.. _lsst.ts.wep-1.0.1: + +------------- +1.0.1 +------------- + +* Updated the WEP to use the obs_lsst and scientific pipeline of ``sims_w_2018_47``. +* The phosim_utils is used to repackage the PhoSim output amplifer images to the format of multi-extention FITS. + +.. _lsst.ts.wep-1.0.0: + +------------- +1.0.0 +------------- + +* Finished the WEP in totally ideal condition with the scientific pipeline v.14. diff --git a/doc/versionHistory.rst b/doc/versionHistory.rst index 73fb4b1e..ceea4484 100644 --- a/doc/versionHistory.rst +++ b/doc/versionHistory.rst @@ -6,7 +6,15 @@ Version History ################## -.. _lsst.ts.wep-13.0.2: +.. _lsst.ts.wep-13.0.3: + +------------- +13.0.3 +------------- + +* Task plotPsfFromZern added in comCamRapidAnalysisPipeline and comCamRapidAnalysisDanishPipeline. + +.. _lsst.ts.wep-13.0.3: ------------- 13.0.2 From e9064ad9697a7b799b5a1a89b7743e8706471971 Mon Sep 17 00:00:00 2001 From: LR-inaf Date: Mon, 25 Nov 2024 12:29:37 +0000 Subject: [PATCH 3/3] checkpoints removed and fix .gitignore --- .gitignore | 1 - .../versionHistory-checkpoint.rst | 1935 ----------------- .../donutVizGroupPipeline-checkpoint.yaml | 42 - ...apidAnalysisDanishPipeline-checkpoint.yaml | 40 - ...omCamRapidAnalysisPipeline-checkpoint.yaml | 45 - .../donutSourceSelectorTask-checkpoint.py | 373 ---- .../.ipynb_checkpoints/pairTask-checkpoint.py | 304 --- 7 files changed, 2740 deletions(-) delete mode 100644 doc/.ipynb_checkpoints/versionHistory-checkpoint.rst delete mode 100644 pipelines/_ingredients/.ipynb_checkpoints/donutVizGroupPipeline-checkpoint.yaml delete mode 100644 pipelines/production/.ipynb_checkpoints/comCamRapidAnalysisDanishPipeline-checkpoint.yaml delete mode 100644 pipelines/production/.ipynb_checkpoints/comCamRapidAnalysisPipeline-checkpoint.yaml delete mode 100644 python/lsst/ts/wep/task/.ipynb_checkpoints/donutSourceSelectorTask-checkpoint.py delete mode 100644 python/lsst/ts/wep/task/.ipynb_checkpoints/pairTask-checkpoint.py diff --git a/.gitignore b/.gitignore index 2055257d..c5f08039 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,3 @@ pytest_session.txt # Existing Phosim Donut Templates policy/cwfs/donutTemplateData/phosimTemplates/ -.ruff.toml diff --git a/doc/.ipynb_checkpoints/versionHistory-checkpoint.rst b/doc/.ipynb_checkpoints/versionHistory-checkpoint.rst deleted file mode 100644 index ceea4484..00000000 --- a/doc/.ipynb_checkpoints/versionHistory-checkpoint.rst +++ /dev/null @@ -1,1935 +0,0 @@ -.. py:currentmodule:: lsst.ts.wep - -.. _lsst.ts.wep-version_history: - -################## -Version History -################## - -.. _lsst.ts.wep-13.0.3: - -------------- -13.0.3 -------------- - -* Task plotPsfFromZern added in comCamRapidAnalysisPipeline and comCamRapidAnalysisDanishPipeline. - -.. _lsst.ts.wep-13.0.3: - -------------- -13.0.2 -------------- - -* Use _refresh_metadata in cutOutStamps function so DonutStamps have correct set of metadata when running cutOutDonuts tasks interactively. - -.. _lsst.ts.wep-13.0.1: - -------------- -13.0.1 -------------- - -* Reorganize pipelines and add daily processing and danish pipelines. - -.. _lsst.ts.wep-13.0.0: - -------------- -13.0.0 -------------- - -* enabled sparse Zernike estimation -* removed most jmax and return4Up configs in favor of nollIndices configs -* removed return4Up from estimator WfEstimator and WfAlgorithm -* added makeSparse and makeDense to Zernike utils - -.. _lsst.ts.wep-12.7.0: - -------------- -12.7.0 -------------- - -* Added requireConverge to TIE and defaulted to True in task -* Fixed bug with None types in EstimateZernikeTask metadata histories - -.. _lsst.ts.wep-12.6.1: - -------------- -12.6.2 -------------- - -* Update RA production pipeline to use group dimension in aggregate donut tables step. - -.. _lsst.ts.wep-12.6.1: - -------------- -12.6.1 -------------- - -* Added a unit test for specifying DonutStampSelector.config.maxSelect in a pipeline config yaml. - -.. _lsst.ts.wep-12.6.0: - -------------- -12.6.0 -------------- - -* Added maxSelect config to DonutStampSelector - -.. _lsst.ts.wep-12.5.0: - -------------- -12.5.0 -------------- - -* Enable CutOutDonutsScienceSensorTask to operate for a pair with same-sign focusZ. - -.. _lsst.ts.wep-12.4.2: - -------------- -12.4.2 -------------- - -* Increase stamp size in Rapid Analysis pipeline to avoid clipping donut edges. - -.. _lsst.ts.wep-12.4.1: - -------------- -12.4.1 -------------- - -* Fixed bug where CalcZernikesTask fails when the number of intra/extra stamps is not equal - -.. _lsst.ts.wep-12.4.0: - -------------- -12.4.0 -------------- - -* Added a threshold on fraction-of-bad-pixels to DonutStampSelectorTask -* Modified DonutStampSelectorTaskConfig so that, by default, selections are run on fraction-of-bad-pixels and signal-to-noise ratio. -* Modified CalcZernikesTask so that DonutStampSelectorTask is run by default -* Fixed bug where DM mask bits weren't persisting in DonutStamp - -.. _lsst.ts.wep-12.3.0: - -------------- -12.3.0 -------------- - -* Added CutOutDonutsUnpairedTask and CalcZernikesUnpairedTask - -.. _lsst.ts.wep-12.2.0: - -------------- -12.2.0 -------------- - -* Update pipelines to use zernikes table instead of separate raw, avg zernike arrays. -* Propogate visit info from donut table into donutStamps to avoid calling visitInfo from the butler. - -.. _lsst.ts.wep-12.1.0: - -------------- -12.1.0 -------------- - -* Change zernikes butler storage format to QTable. - -.. _lsst.ts.wep-12.0.0: - -------------- -12.0.0 -------------- - -* Change pandas.DataFrame outputs to Astropy Tables. - -.. _lsst.ts.wep-11.5.2: - -------------- -11.5.2 -------------- - -* Added a ComCamSim production pipeline for testing purposes. - -.. _lsst.ts.wep-11.5.1: - -------------- -11.5.1 -------------- - -* Fixed bug in donutSourceSelectorTask where the task set with maxBlended > 0 and sources with a number of overlapping donuts greater than maxBlended did not give correct blend centers in the final catalog. - -.. _lsst.ts.wep-11.5.0: - -------------- -11.5.0 -------------- - -* Add astropy table output to CalcZernikesTask. - -.. _lsst.ts.wep-11.4.2: - -------------- -11.4.2 -------------- - -* Add full comcam pipeline to pipelines folder including wep and donut_viz tasks. - -.. _lsst.ts.wep-11.4.1: - -------------- -11.4.1 -------------- - -* Fix treatment of binary dilation in calculateSN. -* Fix how calculateSN masks treat blended pixels. -* Make calculateSN formatting consistent with the rest of cutOutDonutsBaseTask. -* Add a test with a blended stamp for calculateSN. -* Make variance plane warning only appear once. -* Fix test values in test_donutStampSelectorTask due to changes to ISR in w_2024_38. - -.. _lsst.ts.wep-11.4.0: - -------------- -11.4.0 -------------- - -* Set default maxNollIndex to zk28 in estimateZernikesBase. - -.. _lsst.ts.wep-11.3.0: - -------------- -11.3.0 -------------- - -* Add option to bin donut stamps before estimating the wavefront. - -.. _lsst.ts.wep-11.2.0: - -------------- -11.2.0 -------------- - -* Change CalcZernikesTask output to be at least 2D for average as well as raw to make integration with MTAOS easier. - -.. _lsst.ts.wep-11.1.0: - -------------- -11.1.0 -------------- - -* Make maxRecenteringDistance cut more robust in cutOutDonutsBase by first subtracting median shift and then comparing shifts to maxRecenteringDistance. - -.. _lsst.ts.wep-11.0.0: - -------------- -11.0.0 -------------- - -* Add donut image quality checking. - -.. _lsst.ts.wep-10.6.0: - -------------- -10.6.0 -------------- - -* Update Image bandLabel setter to handle condition where the bandLabel is string but the string is not a valid BandLabel enumeration. - -.. _lsst.ts.wep-10.5.0: - -------------- -10.5.0 -------------- - -* Fix handling of empty exposures in generateDonutDirectDetect. - -.. _lsst.ts.wep-10.4.2: - -------------- -10.4.2 -------------- - -* Add pipelines directory to easily share pipeline templates. - -.. _lsst.ts.wep-10.4.1: - -------------- -10.4.1 -------------- - -* Add visit to donutStamps metadata. - -.. _lsst.ts.wep-10.4.0: - -------------- -10.4.0 -------------- - -* Added random field angles in lsst.ts.wep.utils.modelUtils.forwardModelPair -* Fixed two bugs related to the random number generator in lsst.ts.wep.utils.modelUtils.forwardModelPair -* Added tests for lsst.ts.wep.utils.modelUtils.forwardModelPair - -.. _lsst.ts.wep-10.3.0: - -------------- -10.3.0 -------------- - -* Added single-side-of-focus mode to the TIE. - -.. _lsst.ts.wep-10.2.0: - -------------- -10.2.0 -------------- - -* Add option to pair intra/extra focal exposures by group dimension. - -.. _lsst.ts.wep-10.1.1: - -------------- -10.1.1 -------------- - -* Separate recenterFlags in cutOutDonuts tasks metadata into recenterFlagsExtra and recenterFlagsIntra. - -.. _lsst.ts.wep-10.1.0: - -------------- -10.1.0 -------------- - -* Added lsst.ts.wep.utils.modelUtils.forwardModelPair to facilitate forward modeling donuts for testing and data exploration -* Added lsst.ts.wep.utils.plotUtils.plotTieConvergence to diagnose TIE convergence - -.. _lsst.ts.wep-10.0.0: - -------------- -10.0.0 -------------- - -* Removed Zernike units configuration from tasks so that tasks always return Zernikes in microns - -.. _lsst.ts.wep-9.9.0: - -------------- -9.9.0 -------------- - -* Add auto-dilation option to making blend masks in ImageMapper. -* Fixed bugs with blend offsets for extrafocal image masks. - -.. _lsst.ts.wep-9.8.1: - -------------- -9.8.1 -------------- - -* Fixed bug in convertMetadataToHistory that failed when array shape values were floats. - -.. _lsst.ts.wep-9.8.0: - -------------- -9.8.0 -------------- - -* Add maxRecenterDistance configuration option to cutOutDonutsBase. - -.. _lsst.ts.wep-9.7.0: - -------------- -9.7.0 -------------- - -* Change configuration options for GenerateDonutFromRefitWcsTask to specify filter for photometric catalog as well. - -.. _lsst.ts.wep-9.6.0: - -------------- -9.6.0 -------------- - -* Change CombineZernikesSigmaClipTask to use kwargs dict to set arguments in astropy.stats.sigma_clip. - -.. _lsst.ts.wep-9.5.8: - -------------- -9.5.8 -------------- - -* Update to use ts_jenkins_shared_library. - -.. _lsst.ts.wep-9.5.7: - -------------- -9.5.7 -------------- - -* Update default maxFieldDist in donutSourceSelectorTask.py after analysis in DM-42067 (see ts_analysis_notebooks/aos/vignetting). - -.. _lsst.ts.wep-9.5.6: - -------------- -9.5.6 -------------- - -* Move class diagrams to mermaid from plantUML. - -.. _lsst.ts.wep-9.5.5: - -------------- -9.5.5 -------------- - -* Correct indices used to calculate Zernike average. -* Update tests to discern whether flags and mean use the same indices. - -.. _lsst.ts.wep-9.5.4: - -------------- -9.5.4 -------------- - -* Fix blend centroid coordinates in donut stamp generation. - -.. _lsst.ts.wep-9.5.3: - -------------- -9.5.3 -------------- - -* Fixed bug where blended masks have sharp edges when using dilateBlends. - -.. _lsst.ts.wep-9.5.2: - -------------- -9.5.2 -------------- - -* Fix units in ExposurePairer and add tests. - -.. _lsst.ts.wep-9.5.1: - -------------- -9.5.1 -------------- - -* Fixed compatibility with Batoid 0.6.2 - -.. _lsst.ts.wep-9.5.0: - -------------- -9.5.0 -------------- - -* Add exposure pairing for full array mode. - -.. _lsst.ts.wep-9.4.0: - -------------- -9.4.0 -------------- - -* Added the Danish wavefront estimation algorithm. - -.. _lsst.ts.wep-9.3.1: - -------------- -9.3.1 -------------- - -* Added conditional sigma clipping for averaging Zernike coefficients. - -.. _lsst.ts.wep-9.3.0: - -------------- -9.3.0 -------------- - -* Added a separate instrument for full-array mode -* Updated the ComCam mask model to match the bug fixes in Batoid - -.. _lsst.ts.wep-9.2.1: - -------------- -9.2.1 -------------- - -* Added unit test directly comparing ``ImageMapper`` optical models to Batoid raytracing. - -.. _lsst.ts.wep-9.2.0: - -------------- -9.2.0 -------------- - -* Add ``LSSTComCamSim`` as allowed camera type. - -.. _lsst.ts.wep-9.1.1: - -------------- -9.1.1 -------------- - -* Fix latiss tests by using getpass, and updating Zk values - -.. _lsst.ts.wep-9.1.0: - -------------- -9.1.0 -------------- - -* Added ``jmin`` arguments to Zernike utility functions. -* Added ``jmin`` and ``jmax`` value checks to the Zernike utility functions. - -.. _lsst.ts.wep-9.0.0: - -------------- -9.0.0 -------------- - -This is a big backwards-incompatible refactor of WEP. The major changes are: - -* Split the ``cwfs`` modules into ``centroid``, and ``estimation``. -* Donut Images are now held by the ``Image`` class. This class is meant to hold information in the global camera coordinate system (CCS). -* A new ``Instrument`` class with new configurations in the ``policy/instruments`` directory. This class holds geometric information about the different telescopes and cameras, as well as interfaces with the Batoid models. -* The ``ImageMapper`` class maps ``Image`` objects between the image and pupil planes, and creates pupil and image masks. The "offAxis" model now uses a real-time band-dependent fit with Batoid. The "onAxis" and "paraxial" models work the same as before. -* The Zernike estimation classes have been generalized to allow different wavefront algorithm classes to plug into ``WfEstimator``. -* The TIE algorithm is implemented in ``estimation.TieAlgorithm``. -* There are new utilities in ``utils`` for fitting mask models and plotting mask models and the ``ImageMapper`` methods. -* ``Instrument`` configuration in tasks is now pulled from the default parameter files for each camera type. Overrides can be provided via the ``instConfigFile`` parameter. With the default instrument configurations, defocal offsets are pulled from the exposure metadata. If ``defocalOffset`` is explicitly set in the ``instConfigFile`` override, that defocal offset is used instead of the values from the exposure metadata. -* The ``donutTemplateSize`` config parameter has been removed from all the relevant tasks, as the new ``ImageMapper`` can predict the required template size. ``initialCutoutPadding`` provides padding beyond this predicted value. -* The ``multiplyMask`` and ``maskGrowthIter`` parameters have been removed from ``CutOutDonutsBase``. To mask blends during TIE fitting, instead use the ``maskKwargs`` parameter of the ``EstimateZernikesTieTask``. -* When estimating Zernikes, the maximum Noll index (jmax) is now a configurable parameter (``maxNollIndex`` in ``EstimateZernikesBaseConfig``). You can also toggle whether estimation starts from zero or from the telescope's instrinsic Zernikes. You can toggle whether the task returns the full optical path difference (OPD) or just the wavefront deviation (OPD - intrinsic Zernikes). You can toggle whether the returned Zernikes start with Noll index 4 (the previous standard), or with index 0 (matching the Galsim convention). You can also set the units of the returned Zernikes. -* The algorithm history can now be saved at the Task level using the ``saveHistory`` option in ``EstimateZernikesBaseConfig``. The history is saved in the task metadata in a json-compatible format. To convert the history back to the native format, use `utils.convertMetadataToHistory`. -* Changing from the native butler coordinate system (data visualization coordinate system with rotated wavefront sensors) to the WEP coordinate system (camera coordinate system with de-rotated wavefront sensors) now happens entirely in ``task.DonutStamp._setWepImage``. Furthermore, the ``defocal_distance`` saved in the stamp is now the detector offset (or equivalent detector offset) rather than the raw focusZ info. -* The AuxTel/LATISS unit tests have been fixed, and the LATISS Zernike calculation test has been explicitly switched to a regression test (rather than an accuracy test). -* Enum's now map to strings instead of integers. This natural Enum-string connection replaces the various utils that previously existed to map between Enums and strings. - -.. _lsst.ts.wep-8.3.1: - -------------- -8.3.1 -------------- - -* Update tests to be more robust to DM changes and fix failures after DM stack update to w_2024_08. -* Run black v24.2. - -.. _lsst.ts.wep-8.3.0: - -------------- -8.3.0 -------------- - -* Remove mask_comp and mask_pupil from DonutStamp since they don't persist and mask is already contained in MaskedImage stamp. - -.. _lsst.ts.wep-8.2.0: - -------------- -8.2.0 -------------- - -* Add background subtraction to cutOutDonutsBase. - -.. _lsst.ts.wep-8.1.1: - -------------- -8.1.1 -------------- - -* Replace calls to removed pipeBase.ButlerQuantumContext with pipeBase.QuantumContext. - -.. _lsst.ts.wep-8.1.0: - -------------- -8.1.0 -------------- - -* Remove Zemax Coordinate System (ZCS) conversions now that ts_ofc works exclusively in Camera Coordinate System (CCS). - -.. _lsst.ts.wep-8.0.4: - -------------- -8.0.4 -------------- - -* Update default config on GenerateDonutFromRefitWcsTask after updates in meas_astrom. - -.. _lsst.ts.wep-8.0.3: - -------------- -8.0.3 -------------- - -* Attach locally linear WCSs to DonutStamps. - -.. _lsst.ts.wep-8.0.2: - -------------- -8.0.2 -------------- - -* Adds support for MacOS. - -.. _lsst.ts.wep-8.0.1: - -------------- -8.0.1 -------------- - -* Add convertZernikesToPsfWidth to zernikeUtils. - -.. _lsst.ts.wep-8.0.0: - -------------- -8.0.0 -------------- - -* Save all DonutStamps with images aligned with focal plane science sensors. -* This version will break compatibility in the closed loop with Phosim and ts_phosim going forward. - - -.. _lsst.ts.wep-7.0.1: - -------------- -7.0.1 -------------- - -* Fix generateDonutDirectDetect when doDonutSelection is not run. - -.. _lsst.ts.wep-7.0.0: - -------------- -7.0.0 -------------- - -* Organize all utility functions inside the ``utils`` module. - -.. _lsst.ts.wep-6.4.12: - -------------- -6.4.12 -------------- - -* Update ts_pre_commit_config with ruff. - -.. _lsst.ts.wep-6.4.11: - -------------- -6.4.11 -------------- - -* Fix GenerateDonutFromRefitWcsTask adding coord_raErr, coord_decErr fields. - -.. _lsst.ts.wep-6.4.10: - -------------- -6.4.10 -------------- - -* Update calcZernikesLatissPipeline yaml with instrument-specific setup for generateDonutDirectDetectTask. - -.. _lsst.ts.wep-6.4.9: - -------------- -6.4.9 -------------- - -* Replacing lookUpCalibrations function to use the one in lsst.fgcmcal.utilities - -.. _lsst.ts.wep-6.4.8: - -------------- -6.4.8 -------------- - -* Add github actions to check version history was updated and linting. -* Fix black and flake8 violations. -* Fix Jenkinfile. - -.. _lsst.ts.wep-6.4.7: - -------------- -6.4.7 -------------- - -* Set default optical model for comCam to onAxis. - -.. _lsst.ts.wep-6.4.6: - -------------- -6.4.6 -------------- - -* Fix tests that failed due to changes in numpy testing methods and WCS output. - -.. _lsst.ts.wep-6.4.5: - -------------- -6.4.5 -------------- - -* Update setup files with pre-commit hooks, run black and isort. - -.. _lsst.ts.wep-6.4.4: - -------------- -6.4.4 -------------- - -* In ``utility``, update ``getFilterTypeFromBandLabel`` to return ``FilterType.REF`` if the ``bandLabel`` is not recognized. - -.. _lsst.ts.wep-6.4.3: - -------------- -6.4.3 -------------- - -* Fix error in Jenkinsfile that caused git-lfs to fail when running on develop branch. - -.. _lsst.ts.wep-6.4.2: - -------------- -6.4.2 -------------- - -* Move fits files to git-lfs. - -.. _lsst.ts.wep-6.4.1: - -------------- -6.4.1 -------------- - -* Add documentation explaining how to run the WEP pipeline on the USDF batch system. - -.. _lsst.ts.wep-6.4.0: - -------------- -6.4.0 -------------- - -* Create generateDonutCatalogUtils to store common methods. -* Update generateDonutCatalogOnlineTask to match output of other generateDonutCatalog...Tasks. - -.. _lsst.ts.wep-6.3.5: - -------------- -6.3.5 -------------- - -* Make sure output from empty catalogs match that expected from catalogs with sources in donutSourceSelectorTask. -* Add tests for run method in donutSourceSelectorTask. - -.. _lsst.ts.wep-6.3.4: - -------------- -6.3.4 -------------- - -* Patch refCatalogInterface to eliminate warnings from latest version of daf_butler. - -.. _lsst.ts.wep-6.3.3: - -------------- -6.3.3 -------------- - -* Change filter name in testData/gen3TestRepo camera fits files to comply with new obs_lsst convention. - -.. _lsst.ts.wep-6.3.2: - -------------- -6.3.2 -------------- - -* Change CWFS pipeline configuration files to have 1.5mm offset included and to handle this properly in CWFS version of tasks. - -.. _lsst.ts.wep-6.3.1: - -------------- -6.3.1 -------------- - -* Directly calculate dI/dz in Algorithm, without the intermediate dI. -* Save dI/dz and I0 in Algorithm history when debugLevel>=1. - -.. _lsst.ts.wep-6.3.0: - -------------- -6.3.0 -------------- - -* Add filterLabel property to CompensableImage. - -.. _lsst.ts.wep-6.2.0: - -------------- -6.2.0 -------------- - -* Add optional pipeline task to fit WCS from direct detect donut catalogs and generate new donut catalog from reference catalogs with fit WCS. - -.. _lsst.ts.wep-6.1.3: - -------------- -6.1.3 -------------- - -* Add license information to test_generateDonutDirectDetectTask. - -.. _lsst.ts.wep-6.1.2: - -------------- -6.1.2 -------------- - -* Import MaskedImage directly from afw.image to fix errors from change in w_2023_16. - -.. _lsst.ts.wep-6.1.1: - -------------- -6.1.1 -------------- - -* Fix blend_centroid_x and blend_centroid_y to only return donuts bright enough to count as blended when maxBlended is greater than the total number of donuts. - -.. _lsst.ts.wep-6.1.0: - -------------- -6.1.0 -------------- - -* Add optional ability to specify filter in GenerateDonutCatalogWcsTask. - -.. _lsst.ts.wep-6.0.2: - -------------- -6.0.2 -------------- - -* Fix assignment of blend_centroid_x and blend_centroid_y in donut catalogs. - -.. _lsst.ts.wep-6.0.1: - -------------- -6.0.1 -------------- - -* Redesign and enhance documentation to match style and detail of other TS repositories. - -.. _lsst.ts.wep-6.0.0: - -------------- -6.0.0 -------------- - -* Rename all modules to start with lowercase in order to align with DM convention. -* Add information into documentation about how this new change breaks repositories with data processed with older versions of ts_wep and how to fix it. - -.. _lsst.ts.wep-5.1.0: - -------------- -5.1.0 -------------- - -* Add bandpass information into DonutStamp objects. - -.. _lsst.ts.wep-5.0.1: - -------------- -5.0.1 -------------- - -* Run black v23.1.0. - -.. _lsst.ts.wep-5.0.0: - -------------- -5.0.0 -------------- - -* Remove deprecated keywords in cwfs/Tool.py and deprecated function in cwfs/CompensableImage.py. -* Remove deprecated EstimateZernikes Tasks. -* Remove deprecated pipelineConfig test files. - -.. _lsst.ts.wep-4.2.3: - -------------- -4.2.3 -------------- - -* Add transposeImages as optional config to CalcZernikesTask. - -.. _lsst.ts.wep-4.2.2: - -------------- -4.2.2 -------------- - -* Add flux sort into GenerateDonutDirectDetectTask to make it consistent with other catalog generation tasks. - -.. _lsst.ts.wep-4.2.1: - -------------- -4.2.1 -------------- - -* Use nan string instead of None so we can convert to float and use writeFits method in DonutStamps successfully and save in butler. - -.. _lsst.ts.wep-4.2.0: - -------------- -4.2.0 -------------- - -* Add DonutQuickMeasurementTask.py to incorporate quick donut detection and measurement using LSST Science Pipelines. -* Integrate DonutQuickMeasurementTask into GenerateDonutDirectDetectTask to speed up direct detection catalog generation. - -.. _lsst.ts.wep-4.1.0: - -------------- -4.1.0 -------------- - -* GenerateDonutCatalogWcsTask takes filter information from exposures automatically. -* DonutSourceSelectorTask uses policy/task/magLimitStar.yaml for default magnitude limits. - -.. _lsst.ts.wep-4.0.4: - -------------- -4.0.4 -------------- - -* Update calls to deprecated LoadIndexedReferenceObjectsTask to use LoadReferenceObjectsTask. - -.. _lsst.ts.wep-4.0.3: - -------------- -4.0.3 -------------- - -* Add blend_centroid_x and blend_centroid_y to GenerateDonutDirectDetectTask catalogs. - -.. _lsst.ts.wep-4.0.2: - -------------- -4.0.2 -------------- - -* Fix test_estimateZernikesCwfsTask call to ButlerQuantumContext. - -.. _lsst.ts.wep-4.0.1: - -------------- -4.0.1 -------------- - -* Remove Gen2 daf_persistence from UPS table. - -.. _lsst.ts.wep-4.0.0: - -------------- -4.0.0 -------------- - -* Add masked deblending to CompensableImage and pipeline tasks. -* Change how DonutSourceSelectorTask works by adding minBlendedSeparation parameter and changing DonutRadius to unblendedSeparation parameter. - -.. _lsst.ts.wep-3.2.0: - -------------- -3.2.0 -------------- - -* Port Latiss functionality from EstimateZernikesLatissTask into CutOutDonutsScienceSensorTask + CalcZernikesTask pipeline. -* Deprecate EstimateZernikes family of tasks. These tasks will no longer be updated and will be removed after January 2023. - -.. _lsst.ts.wep-3.1.5: - -------------- -3.1.5 -------------- - -* Throw exception when auxTel is trying to use offAxis model. - -.. _lsst.ts.wep-3.1.4: - -------------- -3.1.4 -------------- - -* Remove imageCoCenter step from Algorithm. -* Add DeprecationWarning that imageCoCenter function in CompensableImage will be removed after January 2023. - -.. _lsst.ts.wep-3.1.3: - -------------- -3.1.3 -------------- - -* Added default value to DonutStamp for DFC_DIST to allow the butler to read DonutStamp from repositories created with older versions of ts_wep. - -.. _lsst.ts.wep-3.1.2: - -------------- -3.1.2 -------------- - -* Update phosimOutput corner sensors test files. - -.. _lsst.ts.wep-3.1.1: - -------------- -3.1.1 -------------- - -* Fix tests pipeline yaml files updating the ISR setting to use 'MEDIAN' for overscan fit type. -* Remove obsolete _generateTestExposures. -* Fix `test_generateDonutDirectDetectTask.py` - -.. _lsst.ts.wep-3.1.0: - -------------- -3.1.0 -------------- - -* Added a history to the Algorithm class that stores intermediate products of the algorithm (see `Algorithm.getHistory()`). -* Fixed the algorithm so that it is once again symmetric with respect to I1 and I2. - This involved simplifying the way that mask and image orientation are handled for the extrafocal image (see below). -* Added the option to create masks in the orientation of the original images by setting `compensated=False` in `CompensableImage.makeMask()`. - -.. _lsst.ts.wep-3.0.1: - -------------- -3.0.1 -------------- - -* Fix ``test_generateDonutCatalogWcsTask.py`` to work with more recent versions of the DM stack. - -.. _lsst.ts.wep-3.0.0: - -------------- -3.0.0 -------------- - -* Refactor tasks to directly accept instrument parameters in their configuration. - -.. _lsst.ts.wep-2.7.0: - -------------- -2.7.0 -------------- - -* Remove dictionary defining allowable offsets in Instrument.py and replace with settable parameter. -* Allow Instrument.py to be configured directly from dictionary of instrument parameters in addition to policy file. - -.. _lsst.ts.wep-2.6.0: - -------------- -2.6.0 -------------- - -* Replace getters and setters in Instrument.py with properties to make more pythonic. -* Update Algorithm, CompensableImage and DonutTemplateModel with new Instrument.py design. - -.. _lsst.ts.wep-2.5.8: - -------------- -2.5.8 -------------- - -* Change focusZ in headers of repackaged phosim data to be in mm instead of microns after phosim_utils update. - -.. _lsst.ts.wep-2.5.7: - -------------- -2.5.7 -------------- - -* Add defocal distance into DonutStamp. - -.. _lsst.ts.wep-2.5.6: - -------------- -2.5.6 -------------- - -* Fix task input order in test_estimateZernikes... tests. - -.. _lsst.ts.wep-2.5.5: - -------------- -2.5.5 -------------- - -* Change default maxFieldDistance in DonutSourceSelectorTask.py to 1.813 degrees based upon results from DM-33180. -* Fix test in test_calcZernikesTaskScienceSensor to use correct intraFocal dataId. - -.. _lsst.ts.wep-2.5.4: - -------------- -2.5.4 -------------- - -* Update science sensor and LATISS tasks to get focusZ from exposure visitInfo instead of metadata after update in DM-35186. - -.. _lsst.ts.wep-2.5.3: - -------------- -2.5.3 -------------- - -* Update tests and gen3TestRepo to work with latest version of the stack (w_2022_28). - -.. _lsst.ts.wep-2.5.2: - -------------- -2.5.2 -------------- - -* Add ComCam to donutTemplateModel. -* Add error message to donutTemplateModel for AuxTel if not run with 'onAxis' optical model. - -.. _lsst.ts.wep-2.5.1: - -------------- -2.5.1 -------------- - -* Correct orientation of masks in pipeline tasks. - -.. _lsst.ts.wep-2.5.0: - -------------- -2.5.0 -------------- - -* Update names of cMask to mask_comp (padded), pMask to mask_pupil (non-padded) -* Correct output of getPaddedMask to mask_comp, getNonPaddedMask to mask_pupil - -.. _lsst.ts.wep-2.4.4: - -------------- -2.4.4 -------------- - -* Added documentation link to the README. - -.. _lsst.ts.wep-2.4.3: - -------------- -2.4.3 -------------- - -* Fix online documentation build errors. - -.. _lsst.ts.wep-2.4.2: - -------------- -2.4.2 -------------- - -* Remove matplotlib backend switching in PlotUtil.py - -.. _lsst.ts.wep-2.4.1: - -------------- -2.4.1 -------------- - -* Add information on Jupyter Notebooks in ts_analysis_notebooks to README. - -.. _lsst.ts.wep-2.4.0: - -------------- -2.4.0 -------------- - -* Add CutOutDonuts tasks and CalcZernikesTask to separate cutting out donut stamps and calculating Zernikes from donut stamps as separate tasks. - -.. _lsst.ts.wep-2.3.8: - -------------- -2.3.8 -------------- - -* Remove phosim_utils dependency. - -.. _lsst.ts.wep-2.3.7: - -------------- -2.3.7 -------------- - -* Optimize CWFS algorithms. - -.. _lsst.ts.wep-2.3.6: - -------------- -2.3.6 -------------- - -* Fix rotation of sensors in EstimateZernikesBase. - -.. _lsst.ts.wep-2.3.5: - -------------- -2.3.5 -------------- - -* Update scipy.ndimage namespace to fix deprecation warnings. -* Run black v22.3. - -.. _lsst.ts.wep-2.3.4: - -------------- -2.3.4 -------------- - -* Fix test for `EstimateZernikesLatissTask`, to run for any user with /repo/main/ access. - -.. _lsst.ts.wep-2.3.3: - -------------- -2.3.3 -------------- - -* Add donut location configuration setting to `DonutSourceSelectorTask`. - -.. _lsst.ts.wep-2.3.2: - -------------- -2.3.2 -------------- - -* Change `CombineZernikesSigmaClip` to use the more robust `mad_std` standard deviation algorithm. -* Add `maxZernClip` configuration parameter to `CombineZernikesSigmaClip`. -* Change `CombineZernikes` metadata to use integer flags. - -.. _lsst.ts.wep-2.3.1: - -------------- -2.3.1 -------------- - -* Rely on GalSim for Zernike and Cartesian polynomial evaluation. - -.. _lsst.ts.wep-2.3.0: - -------------- -2.3.0 -------------- - -* Add `EstimateZernikesLatissTask` to process auxTel data -* Add `GenerateDonutDirectDetectTask` to find donuts with template fitting -* Add choices for binary image creation in `DonutDetector` -* Add `getCamType` and `getDefocalDisInMm` to `Utility` -* Add donut template for auxTel in `DonutTemplateModel` - -.. _lsst.ts.wep-2.2.4: - -------------- -2.2.4 -------------- - -* Update Jenkinsfile to always pull the image before new builds and improve cleanup stages to make build more robust. - -.. _lsst.ts.wep-2.2.3: - -------------- -2.2.3 -------------- - -* Change `EstimateZernikesCwfsTask` to be able to accept only a single pair of wavefront sensors. -* Remove `runQuantum` function from `EstimateZernikesScienceSensorTask` since it does not add any functionality now that the task gets the camera from the butler. - -.. _lsst.ts.wep-2.2.2: - -------------- -2.2.2 -------------- - -* Update functions marked deprecated as of stack version `w_2022_06`. - -.. _lsst.ts.wep-2.2.1: - -------------- -2.2.1 -------------- - -* Distinguish AuxTel ZWO camera from LATISS - -.. _lsst.ts.wep-2.2.0: - -------------- -2.2.0 -------------- - -* Add CombineZernikes...Tasks that combine the Zernike coefficients from multiple donut pairs into a single set of coefficients. - -.. _lsst.ts.wep-2.1.4: - -------------- -2.1.4 -------------- - -* Remove `timeMethod` deprecation warnings and use static calibration camera. - -.. _lsst.ts.wep-2.1.3: - -------------- -2.1.3 -------------- - -* Fix maxBlended parameter in DonutSourceSelectorTask and improve tests to check this configuration setting. - -.. _lsst.ts.wep-2.1.2: - -------------- -2.1.2 -------------- - -* Make sure catalogs from GenerateDonutCatalog...Tasks have same columns. - -.. _lsst.ts.wep-2.1.1: - -------------- -2.1.1 -------------- - -* Get camera from the butler when running pipeline tasks. - -.. _lsst.ts.wep-2.1.0: - -------------- -2.1.0 -------------- - -* Refactor GenerateDonutCatalog*.py tasks. -* Update EstimateZernikes...Tasks after DonutCatalog refactor. - -.. _lsst.ts.wep-2.0.4: - -------------- -2.0.4 -------------- - -* Add DonutSourceSelectorTask to task module. - -.. _lsst.ts.wep-2.0.3: - -------------- -2.0.3 -------------- - -* Add RefCatalogInterface to task module. - -.. _lsst.ts.wep-2.0.2: - -------------- -2.0.2 -------------- - -* Patch to work with weekly `w_2022_2`: - * `loadSkyCircle` no longer returns centroid column, use `loadPixelBox` instead. - -.. _lsst.ts.wep-2.0.1: - -------------- -2.0.1 -------------- - -* Patch to work with latest weekly. -* Update Jenkinsfile for CI job: - * git command is no longer working after the latest update on our Jenkins server. - * update path to plantuml. - -.. _lsst.ts.wep-2.0.0: - -------------- -2.0.0 -------------- - -* Removed code not used in Gen3 Pipelines. - -.. _lsst.ts.wep-1.8.2: - -------------- -1.8.2 -------------- - -* Removed CreatePhosimDonutTemplates.py and moved to `ts_phosim`. - -.. _lsst.ts.wep-1.8.1: - -------------- -1.8.1 -------------- - -* Get sensor orientation and field position directly from camera through new DonutStamp objects instead of using SourceProcessor. -* Fix rotation of postage stamps sent to WFEsti. - -.. _lsst.ts.wep-1.8.0: - -------------- -1.8.0 -------------- - -* Refactored DonutStamp.py and added ability to recreate masks as afwImage.Mask objects. - -.. _lsst.ts.wep-1.7.10: - -------------- -1.7.10 -------------- - -* Save outputZernikes for pairs of wavefront detectors not just a single output for all detectors. - -.. _lsst.ts.wep-1.7.9: - -------------- -1.7.9 -------------- - -* Remove _shiftCenterWfs from Source Processor. - -.. _lsst.ts.wep-1.7.8: - -------------- -1.7.8 -------------- - -* Update stamp rotations to work with CWFS. - -.. _lsst.ts.wep-1.7.7: - -------------- -1.7.7 -------------- - -* Update focalplanelayout.txt with new Euler angle for SW0 sensors. - -.. _lsst.ts.wep-1.7.6: - -------------- -1.7.6 -------------- -* Update donutStamp with archive property. -* Add `LSSTCam/calib` to collections path in test Gen3 pipelines. - -.. _lsst.ts.wep-1.7.5: - -------------- -1.7.5 -------------- - -* Break generic pieces of GenerateDonutCatalogOnlineTask.py into GenerateDonutCatalogOnlineBase.py -* Add GenerateDonutCatalogWcsTask.py to calculate donut catalogs when WCS is available - -.. _lsst.ts.wep-1.7.4: - -------------- -1.7.4 -------------- - -* Remove old e-image corner wavefront sensor files. -* Add updated corner wavefront sensor test data. -* Add CWFS Zernikes code and tests. - -.. _lsst.ts.wep-1.7.3: - -------------- -1.7.3 -------------- - -* Break generic pieces of EstimateZernikesFamTask.py into EstimateZernikesBase.py - -.. _lsst.ts.wep-1.7.2: - -------------- -1.7.2 -------------- - -* Fix ``append`` and ``extend`` methods in ``DonutStamps.py``. -* Update tests in ``test_donutStamps.py`` to properly check ``append`` and ``extend`` methods. - -.. _lsst.ts.wep-1.7.1: - -------------- -1.7.1 -------------- - -* Update ``FOCUSZ`` parameter in test data. - -.. _lsst.ts.wep-1.7.0: - -------------- -1.7.0 -------------- - -* Replace ``WcsSol`` by DM's wcs code in ``GenerateDonutCatalogOnlineTask``. -* Fix intra/extra zernike selection. - -.. _lsst.ts.wep-1.6.9: - -------------- -1.6.9 -------------- - -* Add focusz as an argument to repackagePhosimImages in CreatePhosimDonutTemplates.py - -.. _lsst.ts.wep-1.6.8: - -------------- -1.6.8 -------------- - -* Return both raw and averaged Zernikes to Butler repository in EstimateZernikesFamTask.py. - -.. _lsst.ts.wep-1.6.7: - -------------- -1.6.7 -------------- - -* Fix flake error and update Jenkinsfile - -.. _lsst.ts.wep-1.6.6: - -------------- -1.6.6 -------------- - -* Remove 90 degree offset from WcsSol.py now that phosim headers are updated. - -.. _lsst.ts.wep-1.6.5: - -------------- -1.6.5 -------------- - -* Use `FOCUSZ` header information in EstimateZernikesFamTask.py. - -.. _lsst.ts.wep-1.6.4: - -------------- -1.6.4 -------------- - -* Add EstimateZernikesFamTask.py to calculate Zernike coefficients in full-array mode through a Gen 3 pipeline. - -.. _lsst.ts.wep-1.6.3: - -------------- -1.6.3 -------------- - -* Add DonutStamp and DonutStamps storage classes to hold postage stamps of donuts. - -.. _lsst.ts.wep-1.6.2: - -------------- -1.6.2 -------------- - -* Update ROTANG header in realComcam test files - -.. _lsst.ts.wep-1.6.1: - -------------- -1.6.1 -------------- - -* Update GenerateDonutCatalogOnlineTask.py to get instrument directly from pipeline configuration. -* Setup `ctrl_mpexec` package in Jenkinsfile so tests can run `pipetask` command. - -.. _lsst.ts.wep-1.6.0: - -------------- -1.6.0 -------------- - -* Create new task module -* Add GenerateDonutCatalogOnlineTask.py in task module -* Add `tests/testData/gen3TestRepo` as sample Gen 3 repo for testing - -.. _lsst.ts.wep-1.5.9: - -------------- -1.5.9 -------------- - -* Build and upload documentation as part of the CI job. -* Use develop-env image for the CI job, due to the need of java to build the documentation. -* Disable concurrent builds. -* Fix docstring in `SourceSelector.connect` method. - -.. _lsst.ts.wep-1.5.8: - -------------- -1.5.8 -------------- - -* Reformat the code by `black` v20.8b1. - -.. _lsst.ts.wep-1.5.7: - -------------- -1.5.7 -------------- - -* Update import of `DetectorType`. - -.. _lsst.ts.wep-1.5.6: - -------------- -1.5.6 -------------- - -* Reformat code with `black`. - -.. _lsst.ts.wep-1.5.5: - -------------- -1.5.5 -------------- - -* Add `DonutDetector` class. - -.. _lsst.ts.wep-1.5.4: - -------------- -1.5.4 -------------- - -* Update to using ``LsstCamMapper`` and new geometry, including ``focalplanelayout.txt`` - -.. _lsst.ts.wep-1.5.3: - -------------- -1.5.3 -------------- - -* Add ``DonutTemplatePhosim`` class. -* Add ``CreatePhosimDonutTemplates`` class and add ``bin.src/runCreatePhosimDonutTemplates.py`` - -.. _lsst.ts.wep-1.5.2: - -------------- -1.5.2 -------------- - -* Fix the ``ZernikeMaskedFit()`` when passing masked data - -.. _lsst.ts.wep-1.5.1: - -------------- -1.5.1 -------------- - -* Add donut template classes to make templates for ``CentroidConvolveTemplate``. -* Add ``DonutTemplateFactory``, ``DonutTemplateDefault``, and ``DonutTemplateModel``. - -.. _lsst.ts.wep-1.5.0: - -------------- -1.5.0 -------------- - -* Add ``CentroidConvolveTemplate`` as a new centroid finding method. - -.. _lsst.ts.wep-1.4.9: - -------------- -1.4.9 -------------- - -* Unify the line ending to LF. - -.. _lsst.ts.wep-1.4.8: - -------------- -1.4.8 -------------- - -* Remove the ``abbrevDectectorName()`` and ``expandDetectorName()``. -* Remove the unused arguments of ``epoch``, ``includeDistortion``, and ``mjd`` in WCS related functions. -* Fix the ``calcWfErr()`` for the **LsstCamMapper**. - -.. _lsst.ts.wep-1.4.7: - -------------- -1.4.7 -------------- - -* Remove ``sims`` and ``obs_lsstSim`` dependencies. -* Update WCS code to use ``obs_lsst``. - -.. _lsst.ts.wep-1.4.6: - -------------- -1.4.6 -------------- - -* Use the ``sims_w_2020_38``. - -.. _lsst.ts.wep-1.4.5: - -------------- -1.4.5 -------------- - -* Use the ``sims_w_2020_36``. -* Support the LSST full-array mode (FAM). Add the classes of **BaseCwfsTestCase** and **BaseBscTestCase**. -* Put the limits of star's magnitude into a configuration file. -* Remove the serialization functions in **FilterType** enum. - -.. _lsst.ts.wep-1.4.4: - -------------- -1.4.4 -------------- - -* Use the ``pybind11`` instead of ``cython``. -* Add the ``clang-format`` check to ``.githooks``. - -.. _lsst.ts.wep-1.4.3: - -------------- -1.4.3 -------------- - -* Reformat the code by ``black``. -* Add the ``black`` check to ``.githooks``. -* Ignore ``flake8`` check of E203 ans W503 for the ``black``. -* Use the ``sims_w_2020_21``. - -.. _lsst.ts.wep-1.4.2: - -------------- -1.4.2 -------------- - -* Improved handling of IO errors - catch more OS Errors instead of only file not exists. - -.. _lsst.ts.wep-1.4.1: - -------------- -1.4.1 -------------- - -* Add the function to recenter the donut image with the template. -* Add the instrument and test data of auxilirary telescope. - -.. _lsst.ts.wep-1.4.0: - -------------- -1.4.0 -------------- - -* Use the ``sims_w_2020_15``. -* Use the factory pattern for deblend module. - -.. _lsst.ts.wep-1.3.9: - -------------- -1.3.9 -------------- - -* Use the ``sims_w_2020_14``. - -.. _lsst.ts.wep-1.3.8: - -------------- -1.3.8 -------------- - -* Use the ``sims_w_2020_07``. - -.. _lsst.ts.wep-1.3.7: - -------------- -1.3.7 -------------- - -* Use the ``sims_w_2020_06``. -* Skip two tests in **test_butlerWrapper.py** and **test_camIsrWrapper.py** for the bugs in upstream. -* Feedback to DM team. - -.. _lsst.ts.wep-1.3.6: - -------------- -1.3.6 -------------- - -* Use the ``sims_w_2020_04``. - -.. _lsst.ts.wep-1.3.5: - -------------- -1.3.5 -------------- - -* Use the ``sims_w_2019_50``. - -.. _lsst.ts.wep-1.3.4: - -------------- -1.3.4 -------------- - -* Use the ``sims_w_2019_38``. - -.. _lsst.ts.wep-1.3.3: - -------------- -1.3.3 -------------- - -* Use the ``sims_w_2019_31``. -* Remove the ``conda`` package installation in **Jenkinsfile**. -* Update the permission of workspace after the unit test. - -.. _lsst.ts.wep-1.3.2: - -------------- -1.3.2 -------------- - -* Use the ``sims_w_2019_29``. -* Add the unit tests of ``cwfs`` module to check the outputs of cython related code. -* Move the ``plotImage()`` from **Tool.py** to **PlotUtil.py**. -* Install the ``ipython`` in **Jenkinsfile** to make the test environment to be consistent with the development. - -.. _lsst.ts.wep-1.3.1: - -------------- -1.3.1 -------------- - -* Use the factory pattern for centroid find algorithms. -* Move the **SensorWavefrontError** class of ``ts_ofc`` to here. - -.. _lsst.ts.wep-1.3.0: - -------------- -1.3.0 -------------- - -* Use ``sims_w_2019_24``. -* Support the eimage. -* Enable to update and save the setting file. - -.. _lsst.ts.wep-1.2.9: - -------------- -1.2.9 -------------- - -* Use ``sims_w_2019_22``. -* Adapt the new version of ``ip_isr`` that fixes the bug that can not do the ISR continuously. - -.. _lsst.ts.wep-1.2.8: - -------------- -1.2.8 -------------- - -* Use ``sims_w_2019_20``. - -.. _lsst.ts.wep-1.2.7: - -------------- -1.2.7 -------------- - -* Put the default BSC path and sky file path in default ``yaml`` file. -* Concrete **WEPCalculation** class will connect and disconnect the database at each query. -* Use ``sims_w_2019_18``. - -.. _lsst.ts.wep-1.2.6: - -------------- -1.2.6 -------------- - -* Utilize the interface classes to main telescope active optics system (MTAOS). -* Use ``sims_w_2019_17``. - -.. _lsst.ts.wep-1.2.5: - -------------- -1.2.5 -------------- - -* Support the ``documenteer``. - -.. _lsst.ts.wep-1.2.4: - -------------- -1.2.4 -------------- - -* Use the ``yaml`` format for configuration files of ``cwfs`` module. -* Use ``sims_w_2019_15``. - -.. _lsst.ts.wep-1.2.3: - -------------- -1.2.3 -------------- - -* Add the ``eups`` as the package manager. -* Use ``sims_w_2019_12``. - -.. _lsst.ts.wep-1.2.2: - -------------- -1.2.2 -------------- - -* Add the **RawExpData** class and update the related functions. - -.. _lsst.ts.wep-1.2.1: - -------------- -1.2.1 -------------- - -* Add the interface to **MTAOS** in ``ctrlIntf`` module. - -.. _lsst.ts.wep-1.1.1: - -------------- -1.1.1 -------------- - -* Updated to use the scientific pipeline of ``sims_w_2019_02``. -* Add the referece filter type. - -.. _lsst.ts.wep-1.1.0: - -------------- -1.1.0 -------------- - -* Updated the WEP to use the ``obs_lsst`` and scientific pipeline of ``sims_w_2018_47``. -* The ``phosim_utils`` is used to repackage the PhoSim output amplifer images to the format of multi-extention FITS. - -.. _lsst.ts.wep-1.0.1: - -------------- -1.0.1 -------------- - -* Updated the WEP to use the obs_lsst and scientific pipeline of ``sims_w_2018_47``. -* The phosim_utils is used to repackage the PhoSim output amplifer images to the format of multi-extention FITS. - -.. _lsst.ts.wep-1.0.0: - -------------- -1.0.0 -------------- - -* Finished the WEP in totally ideal condition with the scientific pipeline v.14. diff --git a/pipelines/_ingredients/.ipynb_checkpoints/donutVizGroupPipeline-checkpoint.yaml b/pipelines/_ingredients/.ipynb_checkpoints/donutVizGroupPipeline-checkpoint.yaml deleted file mode 100644 index 96d5c03d..00000000 --- a/pipelines/_ingredients/.ipynb_checkpoints/donutVizGroupPipeline-checkpoint.yaml +++ /dev/null @@ -1,42 +0,0 @@ -description: donut viz pipeline tasks - -tasks: - aggregateZernikeTablesTask: - class: lsst.donut.viz.AggregateZernikeTablesTask - aggregateDonutTablesGroupTask: - class: lsst.donut.viz.AggregateDonutTablesTask - config: - python: | - from lsst.ts.wep.task.pairTask import GroupPairer - config.pairer.retarget(GroupPairer) - aggregateAOSVisitTableTask: - class: lsst.donut.viz.AggregateAOSVisitTableTask - plotAOSTask: - class: lsst.donut.viz.PlotAOSTask - config: - doRubinTVUpload: true - aggregateDonutStampsTask: - class: lsst.donut.viz.AggregateDonutStampsTask - plotDonutTask: - class: lsst.donut.viz.PlotDonutTask - config: - doRubinTVUpload: true - plotPsfZernTask: - class: lsst.donut.viz.PlotPsfZernTask - config: - doRubinTVUpload: true - -subsets: - donutVizGroups: - subset: - - aggregateZernikeTablesTask - - aggregateDonutTablesGroupTask - - aggregateAOSVisitTableTask - - plotAOSTask - - aggregateDonutStampsTask - - plotDonutTask - - plotPsfZernTask - description: | - AOS Donut visualization plotting tasks. This step generates plots - (including the pyramid residual and donut gallery) and - tables for the AOS visit. diff --git a/pipelines/production/.ipynb_checkpoints/comCamRapidAnalysisDanishPipeline-checkpoint.yaml b/pipelines/production/.ipynb_checkpoints/comCamRapidAnalysisDanishPipeline-checkpoint.yaml deleted file mode 100644 index 647411ab..00000000 --- a/pipelines/production/.ipynb_checkpoints/comCamRapidAnalysisDanishPipeline-checkpoint.yaml +++ /dev/null @@ -1,40 +0,0 @@ -description: rapid analysis pipeline for ComCam w/ danish zernike estimation -instrument: lsst.obs.lsst.LsstComCam -imports: - - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml - - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml - -tasks: - calcZernikesTask: - class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask - config: - python: | - from lsst.ts.wep.task import EstimateZernikesDanishTask - config.estimateZernikes.retarget(EstimateZernikesDanishTask) - donutStampSelector.maxSelect: 2 - -# Define pipeline steps -subsets: - step1: - subset: - - isr - - generateDonutDirectDetectTask - - cutOutDonutsScienceSensorGroupTask - - calcZernikesTask - description: | - This step processes the input images with ISR, - finds and cuts out the donut stamps, - and estimates the Zernike coefficients from the donut pairs. - step2a: - subset: - - aggregateZernikeTablesTask - - aggregateDonutTablesGroupTask - - aggregateAOSVisitTableTask - - plotAOSTask - - aggregateDonutStampsTask - - plotDonutTask - - plotPsfZernTask - description: | - AOS Donut visualization plotting tasks. This step generates plots - (including the pyramid residual and donut gallery) and - tables for the AOS visit. diff --git a/pipelines/production/.ipynb_checkpoints/comCamRapidAnalysisPipeline-checkpoint.yaml b/pipelines/production/.ipynb_checkpoints/comCamRapidAnalysisPipeline-checkpoint.yaml deleted file mode 100644 index 803521d8..00000000 --- a/pipelines/production/.ipynb_checkpoints/comCamRapidAnalysisPipeline-checkpoint.yaml +++ /dev/null @@ -1,45 +0,0 @@ -description: rapid analysis pipeline for ComCam -instrument: lsst.obs.lsst.LsstComCam -imports: - - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml - - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml - -tasks: - calcZernikesTask: - class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask - config: - estimateZernikes.nollIndices: [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] - estimateZernikes.convergeTol: 10.0e-9 - estimateZernikes.compGain: 0.75 - estimateZernikes.compSequence: [4, 4, 6, 6, 13, 13, 13, 13] - estimateZernikes.maxIter: 50 - estimateZernikes.requireConverge: True - estimateZernikes.saveHistory: False - estimateZernikes.maskKwargs: { "doMaskBlends": False } - donutStampSelector.maxSelect: 5 - -# Define pipeline steps -subsets: - step1: - subset: - - isr - - generateDonutDirectDetectTask - - cutOutDonutsScienceSensorGroupTask - - calcZernikesTask - description: | - This step processes the input images with ISR, - finds and cuts out the donut stamps, - and estimates the Zernike coefficients from the donut pairs. - step2a: - subset: - - aggregateZernikeTablesTask - - aggregateDonutTablesGroupTask - - aggregateAOSVisitTableTask - - plotAOSTask - - aggregateDonutStampsTask - - plotDonutTask - - plotPsfZernTask - description: | - AOS Donut visualization plotting tasks. This step generates plots - (including the pyramid residual and donut gallery) and - tables for the AOS visit. diff --git a/python/lsst/ts/wep/task/.ipynb_checkpoints/donutSourceSelectorTask-checkpoint.py b/python/lsst/ts/wep/task/.ipynb_checkpoints/donutSourceSelectorTask-checkpoint.py deleted file mode 100644 index 9f135f1c..00000000 --- a/python/lsst/ts/wep/task/.ipynb_checkpoints/donutSourceSelectorTask-checkpoint.py +++ /dev/null @@ -1,373 +0,0 @@ -# This file is part of ts_wep. -# -# Developed for the LSST Telescope and Site Systems. -# This product includes software developed by the LSST Project -# (https://www.lsst.org). -# See the COPYRIGHT file at the top-level directory of this distribution -# for details of code ownership. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -__all__ = ["DonutSourceSelectorTaskConfig", "DonutSourceSelectorTask"] - -import astropy.units as u -import lsst.geom -import lsst.pex.config as pexConfig -import lsst.pipe.base as pipeBase -import numpy as np -import pandas as pd -from lsst.afw.cameraGeom import FIELD_ANGLE, PIXELS -from lsst.meas.algorithms.sourceSelector import _getFieldFromCatalog -from lsst.ts.wep.utils import readConfigYaml -from lsst.utils.timer import timeMethod -from sklearn.neighbors import NearestNeighbors - - -class DonutSourceSelectorTaskConfig(pexConfig.Config): - xCoordField = pexConfig.Field( - dtype=str, default="centroid_x", doc="Name of x-coordinate column." - ) - yCoordField = pexConfig.Field( - dtype=str, default="centroid_y", doc="Name of y-coordinate column." - ) - useCustomMagLimit = pexConfig.Field( - dtype=bool, - default=False, - doc="Apply user-defined magnitude limit? If this is False then the code" - + " will default to use the magnitude values in policy:magLimitStar.yaml.", - ) - magMax = pexConfig.Field( - dtype=float, - default=99.0, - doc="Maximum magnitude for selection. Only used if useCustomMagLimit is True.", - ) - magMin = pexConfig.Field( - dtype=float, - default=-99.0, - doc="Minimum magnitude for selection. Only used if useCustomMagLimit is True.", - ) - # For information on where this default maxFieldDist comes from see details - # in ts_analysis_notebooks/aos/vignetting. - maxFieldDist = pexConfig.Field( - dtype=float, - default=1.808, - doc="Maximum distance from center of focal plane (in degrees).", - ) - unblendedSeparation = pexConfig.Field( - dtype=int, - default=160, - doc="Distance in pixels between two donut centers for them to be considered unblended. " - + "This setting and minBlendedSeparation will both be affected by the defocal distance.", - ) - minBlendedSeparation = pexConfig.Field( - dtype=int, - default=120, - doc="Minimum separation in pixels between blended donut centers. " - + "This setting and unblendedSeparation will both be affected by the defocal distance.", - ) - isolatedMagDiff = pexConfig.Field( - dtype=float, - default=2, - doc="Min. difference in magnitude for 'isolated' star.", - ) - sourceLimit = pexConfig.Field( - dtype=int, - default=-1, - doc="Maximum number of desired sources (default is -1 which will give all in catalog).", - ) - maxBlended = pexConfig.Field( - dtype=int, - default=0, - doc="Number of blended objects (defined by unblendedSeparation and isolatedMagDiff) " - + "allowed with a bright source.", - ) - - -class DonutSourceSelectorTask(pipeBase.Task): - """ - Donut Source Selector that uses a nearest neighbors radius - query to find all donuts within the pixel radius set in the - config. Then it goes from the brightest sources down to the faintest - picking donuts that are at least isolatedMagDiff brighter than any sources - with centers within 2 times the unblendedSeparation until reaching - numSources kept or going through the whole list. - """ - - ConfigClass = DonutSourceSelectorTaskConfig - _DefaultName = "donutSourceSelectorTask" - - def __init__(self, **kwargs): - pipeBase.Task.__init__(self, **kwargs) - - def run(self, sourceCat, detector, filterName): - """Select sources and return them. - - Parameters - ---------- - sourceCat : `lsst.afw.table.SourceCatalog` or `pandas.DataFrame` - or `astropy.table.Table` - Catalog of sources to select from. - detector : `lsst.afw.cameraGeom.Detector` - Detector object from the camera. - filterName : `str` - Name of camera filter. - - Returns - ------- - struct : `lsst.pipe.base.Struct` - The struct contains the following data: - - sourceCat : `lsst.afw.table.SourceCatalog` - or `pandas.DataFrame` or `astropy.table.Table` - The catalog of sources that were selected. - (may not be memory-contiguous) - - selected : `numpy.ndarray` of `bool` - Boolean array of sources that were selected, same length as - sourceCat. - - Raises - ------ - `RuntimeError` - Raised if ``sourceCat`` is not contiguous. - """ - if hasattr(sourceCat, "isContiguous"): - # Check for continuity on afwTable catalogs - if not sourceCat.isContiguous(): - raise RuntimeError( - "Input catalogs for source selection must be contiguous." - ) - - result = self.selectSources(sourceCat, detector, filterName) - - return pipeBase.Struct( - sourceCat=sourceCat[result.selected], - selected=result.selected, - blendCentersX=result.blendCentersX, - blendCentersY=result.blendCentersY, - ) - - @timeMethod - def selectSources(self, sourceCat, detector, filterName): - """ - Run the source selection algorithm and return the indices to keep - in the original catalog. - - Parameters - ---------- - sourceCat : `lsst.afw.table.SourceCatalog` or `pandas.DataFrame` - or `astropy.table.Table` - Catalog of sources to select from. - detector : `lsst.afw.cameraGeom.Detector` - Detector object from the camera. - filterName : `str` - Name of camera filter. - - Returns - ------- - struct : `lsst.pipe.base.Struct` - The struct contains the following data: - - selected : `numpy.ndarray` of `bool` - Boolean array of sources that were selected, same length as - sourceCat. - - Raises - ------ - `ValueError` - sourceLimit in config for task must be -1 or a positive integer. - """ - - bbox = detector.getBBox() - - selected = np.zeros(len(sourceCat), dtype=bool) - if len(selected) == 0: - return pipeBase.Struct( - selected=selected, - blendCentersX=None, - blendCentersY=None, - ) - - fluxField = f"{filterName}_flux" - flux = _getFieldFromCatalog(sourceCat, fluxField) - mag = (flux * u.nJy).to_value(u.ABmag) - minMagDiff = self.config.isolatedMagDiff - unblendedSeparation = self.config.unblendedSeparation - minBlendedSeparation = self.config.minBlendedSeparation - - # Use user defined inputs or ts_wep defaults - # depending on useCustomMagLimit. - if self.config.useCustomMagLimit: - magMin = self.config.magMin - magMax = self.config.magMax - else: - magPolicyDefaults = readConfigYaml("policy:magLimitStar.yaml") - defaultFilterKey = f"filter{filterName.upper()}" - magMax = magPolicyDefaults[defaultFilterKey]["high"] - magMin = magPolicyDefaults[defaultFilterKey]["low"] - - magSelected = np.ones(len(sourceCat), dtype=bool) - magSelected &= mag < (magMax + minMagDiff) - mag = mag[magSelected] - - xCoord = _getFieldFromCatalog(sourceCat[magSelected], self.config.xCoordField) - yCoord = _getFieldFromCatalog(sourceCat[magSelected], self.config.yCoordField) - - df = pd.DataFrame({"x": xCoord, "y": yCoord, "mag": mag}) - # Grab any donut centers within unblended distance. - xyNeigh = NearestNeighbors(radius=unblendedSeparation) - - # Get distance to center of field - fieldXY = detector.transform( - [lsst.geom.Point2D(xPix, yPix) for xPix, yPix in zip(xCoord, yCoord)], - PIXELS, - FIELD_ANGLE, - ) - fieldDist = [ - np.degrees(np.sqrt(fieldLoc[0] ** 2 + fieldLoc[1] ** 2)) - for fieldLoc in fieldXY - ] - df["fieldDist"] = fieldDist - - # Remove area too close to edge with new bounding box that allows - # only area at least distance for unblended separation from edges - trimmedBBox = bbox.erodedBy(unblendedSeparation) - - index = list() - magSortedDf = df.sort_values("mag") - groupIndices = magSortedDf.index.values - xyNeigh.fit(magSortedDf[["x", "y"]]) - radDist, radIdx = xyNeigh.radius_neighbors( - magSortedDf[["x", "y"]], sort_results=True - ) - - errMsg = str( - "config.sourceLimit must be a positive integer " - + "or turned off by setting it to '-1'" - ) - if not ((self.config.sourceLimit == -1) or (self.config.sourceLimit > 0)): - raise ValueError(errMsg) - - maxBlended = self.config.maxBlended - blendCentersX = [list() for _ in range(len(magSortedDf))] - blendCentersY = [list() for _ in range(len(magSortedDf))] - sourcesKept = 0 - # Go through catalog with nearest neighbor information - # and keep sources that match our configuration settings - srcOn = -1 - for nbrDist, idxList in zip(radDist, radIdx): - srcOn += 1 - # Move on if source is within unblendedSeparation - # of the edge of a given exposure - srcX = magSortedDf["x"].iloc[srcOn] - srcY = magSortedDf["y"].iloc[srcOn] - if trimmedBBox.contains(srcX, srcY) is False: - continue - - # If distance from field center is greater than - # maxFieldDist discard the source and move on - if magSortedDf["fieldDist"].iloc[srcOn] > self.config.maxFieldDist: - continue - - # If this source's magnitude is outside our bounds then discard - srcMag = magSortedDf["mag"].iloc[srcOn] - if (srcMag > magMax) | (srcMag < magMin): - continue - - # If there is no overlapping source keep - # the source and move on to next - if len(idxList) == 1: - index.append(groupIndices[srcOn]) - sourcesKept += 1 - # In this case there is at least one overlapping source - else: - # Measure magnitude differences with overlapping objects - magDiff = magSortedDf["mag"].iloc[idxList[1:]] - srcMag - magTooClose = magDiff.values < minMagDiff - - # Measure distances to overlapping objects - blendSeparations = nbrDist[1:] - blendTooClose = blendSeparations < minBlendedSeparation - - # If this is the fainter source of the overlaps move on - if np.min(magDiff) < 0.0: - continue - # If this source overlaps but is brighter than all its - # overlapping sources by minMagDiff then keep it - elif np.min(magDiff) >= minMagDiff: - index.append(groupIndices[srcOn]) - sourcesKept += 1 - # If the centers of any blended objects with a magnitude - # within minMagDiff of the source magnitude - # are closer than minBlendedSeparation move on - elif np.sum(blendTooClose & magTooClose) > 0: - continue - # If the number of overlapping sources with magnitudes close - # enough to count as blended is less than or equal to - # maxBlended then keep this source - elif len(magDiff) <= maxBlended: - index.append(groupIndices[srcOn]) - # Only include sources bright enough to count as - # blended based upon isolatedMagDiff. Otherwise - # masks for deblending will include footprints of - # all the faint sources that we don't care about - # when deblending. Add one to index because - # magDiff is all sources in magSortedDf after index=0. - blendMagIdx = np.where(magDiff < minMagDiff)[0] + 1 - blendCentersX[groupIndices[srcOn]] = ( - magSortedDf["x"].iloc[idxList[blendMagIdx]].values - ) - blendCentersY[groupIndices[srcOn]] = ( - magSortedDf["y"].iloc[idxList[blendMagIdx]].values - ) - sourcesKept += 1 - # Keep the source if it is blended with up to maxBlended - # number of sources. To check this we look at the maxBlended+1 - # source in the magDiff list and check that the object - # is at least minMagDiff brighter than this. Satisfying this - # criterion means it is blended with maxBlended - # or fewer sources. - elif np.partition(magDiff, maxBlended)[maxBlended] > minMagDiff: - index.append(groupIndices[srcOn]) - # Same process as above to make sure we only get - # the blend centers we care about - blendMagIdx = np.where(magDiff < minMagDiff)[0] + 1 - blendCentersX[groupIndices[srcOn]] = ( - magSortedDf["x"].iloc[idxList[blendMagIdx]].values - ) - blendCentersY[groupIndices[srcOn]] = ( - magSortedDf["y"].iloc[idxList[blendMagIdx]].values - ) - sourcesKept += 1 - else: - continue - - if (self.config.sourceLimit > 0) and ( - sourcesKept == self.config.sourceLimit - ): - break - - # magSelected is a boolean array so we can - # find indices with True by finding nonzero elements - magIndex = magSelected.nonzero()[0] - finalIndex = magIndex[index] - selected[finalIndex] = True - sortedIndex = np.sort(index) - selectedBlendCentersX = [blendCentersX[idx] for idx in sortedIndex] - selectedBlendCentersY = [blendCentersY[idx] for idx in sortedIndex] - - self.log.info("Selected %d/%d references", selected.sum(), len(sourceCat)) - - return pipeBase.Struct( - selected=selected, - blendCentersX=selectedBlendCentersX, - blendCentersY=selectedBlendCentersY, - ) diff --git a/python/lsst/ts/wep/task/.ipynb_checkpoints/pairTask-checkpoint.py b/python/lsst/ts/wep/task/.ipynb_checkpoints/pairTask-checkpoint.py deleted file mode 100644 index 5526b1a6..00000000 --- a/python/lsst/ts/wep/task/.ipynb_checkpoints/pairTask-checkpoint.py +++ /dev/null @@ -1,304 +0,0 @@ -# This file is part of ts_wep. -# -# Developed for the LSST Telescope and Site Systems. -# This product includes software developed by the LSST Project -# (https://www.lsst.org). -# See the COPYRIGHT file at the top-level directory of this distribution -# for details of code ownership. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -__all__ = [ - "ExposurePairerConfig", - "ExposurePairer", - "TablePairerConfig", - "TablePairer", - "GroupPairerConfig", - "GroupPairer", -] - -import typing - -import lsst.afw.image as afwImage -import lsst.pex.config as pexConfig -import lsst.pipe.base as pipeBase -import numpy as np -from astropy.coordinates import SkyCoord -from astropy.table import Table -from lsst.geom import radians - - -class IntraExtraIdxPair(typing.NamedTuple): - intra: int - extra: int - - -class ExposurePairerConfig(pexConfig.Config): - timeThreshold = pexConfig.Field[float]( - doc="Maximum time difference between paired intra- and extra-focal exposures (s)", - default=50, - ) - - pointingThreshold = pexConfig.Field[float]( - doc="Maximum pointing difference between paired intra- and extra-focal exposures (arcsec)", - default=60, - ) - - rotationThreshold = pexConfig.Field[float]( - doc="Maximum rotator angle difference between paired intra- and extra-focal exposures (deg)", - default=1.0, - ) - - ignoreThresholds = pexConfig.Field[bool]( - doc=( - "If True, ignore time, pointing, and rotation thresholds. Useful when grouping by " - "exposure.group" - ), - default=False, - ) - - doOverrideSeparation = pexConfig.Field[bool]( - doc="Whether to override expected intra-focal to focal separation", - default=False, - ) - - overrideSeparation = pexConfig.Field[float]( - doc="Expected extra-focal to focal separation (mm). Note that this is signed such that " - "a positive value means an extra-focal exposure has a greater focusZ value than an " - "intra-focal exposure.", - default=1.5, - ) - - groupingThreshold = pexConfig.Field[float]( - doc="Threshold for assigning visit to intra/extra/focal group as a fraction of the expected " - "intra-focal to focal separation.", - default=0.1, - ) - - forceUniquePairs = pexConfig.Field[bool]( - doc="If True, force each extra exposure to be paired with a unique intra exposure.", - default=True, - ) - - -class ExposurePairer(pipeBase.Task): - ConfigClass = ExposurePairerConfig - _DefaultName = "exposurePairer" - _needsPairTable = False - _needsGroupDimension = False - - def makeTables( - self, - visitInfos: typing.Dict[int, afwImage.VisitInfo], - ) -> typing.Dict[str, Table]: - """ - Make tables of intra/extra/focal/and pairs of exposures. - - Parameters - ---------- - visitInfos : dict - A dictionary of VisitInfo objects keyed by exposure. - - Returns - ------- - dict - A dictionary of astropy tables keyed by type. - """ - if self.config.doOverrideSeparation: - separation = self.config.overrideSeparation - else: - instrument = next(iter(visitInfos.values())).instrumentLabel - match instrument: - case "LSSTCam" | "LSSTComCam" | "LSSTComCamSim": - separation = 1.5 - case "LATISS": - separation = -0.8 - - # Partition intra/extra/focal groups by finding a common offset that - # maximizes the number of assigned visits. - # If we're handed a pair of visitInfos with a focusZ difference - # of ~1.5 mm, then we can't generally decide if that's an - # intra/focal pair or a focal/extra pair. Since the AOS pipeline - # is keyed on extrafocal dataIds, we'll assume we're always handed - # an extrafocal visitInfo in this case. - - table = Table( - names=["exposure", "ra", "dec", "mjd", "focusZ", "rtp"], - dtype=[int, float, float, float, float, float], - ) - for exposure, visitInfo in visitInfos.items(): - radec = visitInfo.boresightRaDec - ra = radec.getLongitude().asRadians() - dec = radec.getLatitude().asRadians() - mjd = visitInfo.date.toAstropy().mjd - rtp = ( - visitInfo.boresightParAngle - - visitInfo.boresightRotAngle - - (np.pi / 2 * radians) - ).asDegrees() - focusZ = visitInfo.focusZ - table.add_row([exposure, ra, dec, mjd, focusZ, rtp]) - table["radec"] = SkyCoord(table["ra"], table["dec"], unit="radian") - thresh = self.config.groupingThreshold * abs(separation) - - best_n_inliers = 0 - best_offset = None - for offset in np.unique(table["focusZ"]): - dz = table["focusZ"] - offset - n_extra = np.sum(np.abs(dz) < thresh) - n_focal = np.sum(np.abs(dz + separation) < thresh) - n_intra = np.sum(np.abs(dz + 2 * separation) < thresh) - if n_extra + n_focal + n_intra > best_n_inliers: - best_n_inliers = n_extra + n_focal + n_intra - best_offset = offset - - extraTable = table[np.abs(table["focusZ"] - best_offset) < thresh] - focalTable = table[np.abs(table["focusZ"] - best_offset + separation) < thresh] - intraTable = table[ - np.abs(table["focusZ"] - best_offset + 2 * separation) < thresh - ] - - # For each extra focal exposure, find the best intra focal exposure. - # Note that it's possible that the same intra exposure is paired with - # multiple extra exposures. - dtype = [ - ("extra", " typing.List[IntraExtraIdxPair]: - """ - Pair up the intra- and extra-focal exposures. - """ - tables = self.makeTables(visitInfos) - pairTable = tables["pairTable"] - out = [] - for row in pairTable: - out.append(IntraExtraIdxPair(row["intra"], row["extra"])) - return out - - -class TablePairerConfig(pexConfig.Config): - pass - - -class TablePairer(pipeBase.Task): - ConfigClass = TablePairerConfig - _DefaultName = "tablePairer" - _needsPairTable = True - _needsGroupDimension = False - - def run( - self, - visitInfos: typing.Dict[int, afwImage.VisitInfo], - pairTable: Table, - ) -> typing.List[IntraExtraIdxPair]: - """ - Pair up the intra- and extra-focal exposures. - """ - out = [] - for row in pairTable: - if row["intra"] in visitInfos and row["extra"] in visitInfos: - out.append(IntraExtraIdxPair(row["intra"], row["extra"])) - return out - - -class GroupPairerConfig(ExposurePairerConfig): - def setDefaults(self): - super().setDefaults() - self.ignoreThresholds = True - - def validate(self): - super().validate() - - if not self.ignoreThresholds: - raise pexConfig.FieldValidationError( - self.__class__.ignoreThresholds, - self, - "ignoreThresholds must be True for GroupPairer", - ) - - -class GroupPairer(ExposurePairer): - ConfigClass = GroupPairerConfig - _DefaultName = "groupPairer" - _needsGroupDimension = True - - def run( - self, - visitInfos: typing.Dict[int, afwImage.VisitInfo], - ) -> typing.List[IntraExtraIdxPair]: - # visitInfos should either be an intra/extra pair, or possibly an - # intra/extra/in-focus triplet. Catch that here. - if len(visitInfos) not in (2, 3): - self.log.warning("Expected 2 or 3 visits, but got %d.", len(visitInfos)) - self.log.warning("Received exposure ids:") - for vi in visitInfos.keys(): - self.log.warning(" %d", vi) - return [] - return super().run(visitInfos)