DM-43340: Add exposure pairing for full array mode

.gitignore

@@ -11,7 +11,7 @@ config.log
 
 # Built by sconsUtils
 version.py
-bin/
+/bin/
 
 # Pytest
 tests/.tests

Jenkinsfile

@@ -75,7 +75,7 @@ pipeline {
                         git lfs fetch --all || echo git lfs fetch FAILED
                         git lfs checkout || echo git lfs checkout FAILED
                         setup -k -r .
-                        scons python
+                        scons lib python shebang
                     """
                 }
             }

bin.src/generatePairTable.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+
+# This file is part of ts_wep.
+#
+# Developed for the LSST Telescope and Site Systems.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from lsst.ts.wep.bin.generatePairTable import main
+
+if __name__ == "__main__":
+    main()

bin.src/ingestPairTable.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+
+# This file is part of ts_wep.
+#
+# Developed for the LSST Telescope and Site Systems.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from lsst.ts.wep.bin.ingestPairTable import main
+
+if __name__ == "__main__":
+    main()

doc/developer-guide/developer-guide.rst

@@ -148,6 +148,8 @@ This module has the tasks to run WEP as a pipeline with Gen 3 LSST DM middleware
 * **GenerateDonutFromRefitWcsTaskConnections**: Connections setup for GenerateDonutFromRefitWcsTask.
 * **GenerateDonutFromRefitWcsTaskConfig**: Configuration setup for GenerateDonutFromRefitWcsTask.
 * **GenerateDonutCatalogUtils**: Common utility functions for the GenerateDonutCatalog...Tasks.
+* **ExposurePairer**: Subtask to pair intra- and extra-focal exposures heuristically.
+* **TablePairer**: Subtask to pair intra- and extra-focal exposures manually from a table.  Use generatePairTable.py script as a useful starting point.
 
 .. _WEP_modules_wep_utils:
 
@@ -160,7 +162,7 @@ This module contains utility functions that are used elsewhere in WEP.
 * **ioUtils**: Functions for reading and writing files.
 * **maskUtils**: Functions for generating a mask model for an instrument.
 * **taskUtils**: Functions for running command line tasks from a python script.
-* **zernikeUtils**: Functions for evaluating and fitting Zernike polynomials. 
+* **zernikeUtils**: Functions for evaluating and fitting Zernike polynomials.
 * **plotUtils**: Functions for plotting results.
 * **miscUtils**: Miscellaneous utility functions.
 

doc/uml/taskClass.uml

@@ -29,6 +29,7 @@ CutOutDonutsScienceSensorTaskConfig *-- CutOutDonutsBaseTaskConnections
 CutOutDonutsScienceSensorTask *-- CutOutDonutsScienceSensorTaskConfig
 CutOutDonutsBaseTaskConnections <|-- CutOutDonutsScienceSensorTaskConnections
 CutOutDonutsScienceSensorTaskConfig *-- CutOutDonutsScienceSensorTaskConnections
+CutOutDonutsScienceSensorTaskConfig *-- ExposurePairer
 CutOutDonutsScienceSensorTask ..> DonutStamps
 CutOutDonutsBaseTaskConfig *-- CutOutDonutsBaseTaskConnections
 CutOutDonutsBaseTask *-- CutOutDonutsBaseTaskConfig
@@ -42,4 +43,6 @@ GenerateDonutFromRefitWcsTaskConfig *-- GenerateDonutFromRefitWcsTaskConnections
 GenerateDonutCatalogWcsTaskConfig <|-- GenerateDonutFromRefitWcsTaskConfig
 GenerateDonutFromRefitWcsTask *-- GenerateDonutFromRefitWcsTaskConfig
 GenerateDonutCatalogWcsTask <|-- GenerateDonutFromRefitWcsTask
+ExposurePairer *-- ExposurePairerConfig
+TablePairer *-- TablePairerConfig
 @enduml

doc/versionHistory.rst

@@ -6,13 +6,20 @@
 Version History
 ##################
 
+.. _lsst.ts.wep-9.5.0:
+-------------
+9.5.0
+-------------
+
+* Add exposure pairing for full array mode.
+
 .. _lsst.ts.wep-9.4.0:
 -------------
 9.4.0
 -------------
 
 * Added the Danish wavefront estimation algorithm.
-  
+
 .. _lsst.ts.wep-9.3.1:
 
 -------------

python/lsst/ts/wep/bin/generatePairTable.py

@@ -0,0 +1,158 @@
+# This file is part of ts_wep.
+#
+# Developed for the LSST Telescope and Site Systems.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from argparse import ArgumentParser
+
+from lsst.daf.butler import Butler
+from lsst.ts.wep.task.pairTask import ExposurePairer, ExposurePairerConfig
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument(
+        "-b",
+        "--butler-config",
+        help="Location of the butler/registry config file.",
+        required=True,
+        metavar="TEXT",
+    )
+    parser.add_argument(
+        "-c", "--collection", help="Collection name.", required=True, metavar="NAME"
+    )
+    parser.add_argument(
+        "-o",
+        type=str,
+        help="Output filename.",
+        required=True,
+        metavar="FILENAME",
+    )
+    parser.add_argument(
+        "--instrument",
+        type=str,
+        help="Name of the instrument.",
+        required=True,
+        metavar="INST",
+    )
+    parser.add_argument(
+        "-d",
+        "--data-query",
+        type=str,
+        help="User data selection expression.",
+        required=True,
+        metavar="QUERY",
+    )
+    parser.add_argument(
+        "--timeThreshold",
+        type=float,
+        help="Maximum time difference between paired intra- and extra-focal exposures (s)",
+        default=300.0,
+        metavar="SEC",
+    )
+    parser.add_argument(
+        "--pointingThreshold",
+        type=float,
+        help="Maximum pointing difference between paired intra- and extra-focal exposures (arcsec)",
+        default=60.0,
+        metavar="ARCSEC",
+    )
+    parser.add_argument(
+        "--rotationThreshold",
+        type=float,
+        help="Maximum rotator angle difference between paired intra- and extra-focal exposures (deg)",
+        default=1.0,
+        metavar="DEG",
+    )
+    parser.add_argument(
+        "--overrideSeparation",
+        type=float,
+        help="Expected intra-focal to focal separation (mm)",
+        default=None,
+        metavar="MM",
+    )
+    parser.add_argument(
+        "--groupingThreshold",
+        type=float,
+        help="Threshold for assigning visit to intra/extra/focal group as a fraction of the expected"
+        " intra-focal to focal separation.",
+        default=0.1,
+        metavar="FRAC",
+    )
+    parser.add_argument(
+        "--forceUniquePairs",
+        action="store_true",
+        help="If True, force each extra exposure to be paired with a unique intra exposure.",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Enable verbose output.",
+    )
+
+    args = parser.parse_args()
+
+    butler = Butler(args.butler_config, collections=args.collection)
+
+    doOverrideSeparation = False
+    overrideSeparation = 1.5
+    if args.overrideSeparation is not None:
+        overrideSeparation = args.overrideSeparation
+        doOverrideSeparation = True
+
+    config = ExposurePairerConfig(
+        timeThreshold=args.timeThreshold,
+        pointingThreshold=args.pointingThreshold,
+        rotationThreshold=args.rotationThreshold,
+        doOverrideSeparation=doOverrideSeparation,
+        overrideSeparation=overrideSeparation,
+        groupingThreshold=args.groupingThreshold,
+        forceUniquePairs=args.forceUniquePairs,
+    )
+    pairer = ExposurePairer(config=config)
+
+    visitInfos = {
+        v.dataId["exposure"]: butler.get("raw.visitInfo", dataId=v.dataId)
+        for v in butler.registry.queryDatasets("raw", where=args.data_query)
+    }
+
+    tables = pairer.makeTables(visitInfos)
+
+    print()
+    print("Found pairs")
+    tables["pairTable"].pprint_all()
+    print()
+    print("Writing pairs to file: ", args.o)
+    tables["pairTable"].write(args.o, format="ascii.ecsv", overwrite=True)
+
+    if args.verbose:
+        if len(tables["unusedExtraTable"]) > 0:
+            print()
+            print("Unused extra exposures")
+            tables["unusedExtraTable"].pprint_all()
+        if len(tables["unusedIntraTable"]) > 0:
+            print()
+            print("Unused intra exposures")
+            tables["unusedIntraTable"].pprint_all()
+        if len(tables["focalTable"]) > 0:
+            print()
+            print("Focal exposures")
+            tables["focalTable"].pprint_all()

python/lsst/ts/wep/bin/ingestPairTable.py

@@ -0,0 +1,97 @@
+# This file is part of ts_wep.
+#
+# Developed for the LSST Telescope and Site Systems.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+import logging
+import time
+from argparse import ArgumentParser
+
+from astropy.table import Table
+from lsst.daf.butler import Butler, CollectionType, DatasetType, DimensionUniverse
+
+
+def main():
+    tz = time.strftime("%z")
+    logging.basicConfig(
+        format="%(levelname)s %(asctime)s.%(msecs)03d" + tz + " - %(message)s",
+        datefmt="%Y-%m-%dT%H:%M:%S",
+    )
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.DEBUG)
+
+    parser = ArgumentParser()
+    parser.add_argument(
+        "-b",
+        "--butler-config",
+        help="Location of the butler/registry config file.",
+        required=True,
+        metavar="TEXT",
+    )
+    parser.add_argument(
+        "-o",
+        "--output-collection",
+        type=str,
+        help="Name of the output collection to ingest the injection catalog into.",
+        required=True,
+        metavar="COLL",
+    )
+    parser.add_argument(
+        "--instrument",
+        type=str,
+        help="Name of the instrument.",
+        required=True,
+        metavar="INST",
+    )
+    parser.add_argument(
+        "pairs",
+        type=str,
+        help="Path to the pair table to ingest.",
+    )
+    args = parser.parse_args()
+
+    logger.info(f"Reading pair table from {args.pairs}")
+    table = Table.read(args.pairs)
+
+    logger.info(f"Writing pair table to butler collection {args.output_collection}")
+    butler = Butler(args.butler_config, writeable=True)
+
+    # Register if needed
+    logger.info("Registering donutVisitPairTable dataset type")
+    donutVisitPairTableDatasetType = DatasetType(
+        "donutVisitPairTable",
+        tuple(["instrument"]),
+        "AstropyTable",
+        universe=DimensionUniverse(),
+    )
+    butler.registry.registerDatasetType(donutVisitPairTableDatasetType)
+    logger.info("Registering donutVisitPairTable collection")
+    butler.registry.registerCollection(
+        name=args.output_collection, type=CollectionType.RUN
+    )
+
+    logger.info("Ingesting pair table")
+    butler.put(
+        table,
+        donutVisitPairTableDatasetType,
+        dict(instrument=args.instrument),
+        run=args.output_collection,
+    )
+    logger.info("Done")

python/lsst/ts/wep/task/__init__.py

@@ -17,4 +17,5 @@
 from .generateDonutCatalogWcsTask import *
 from .generateDonutDirectDetectTask import *
 from .generateDonutFromRefitWcsTask import *
+from .pairTask import *
 from .refCatalogInterface import *