From b0c6f8c7e2a4fc45602e2b4c24d8f702bfaa85e2 Mon Sep 17 00:00:00 2001 From: hlin730 Date: Mon, 4 Dec 2023 14:58:45 -0600 Subject: [PATCH 1/3] Add difference imaging steps Add difference imaging tasks in step4, step5, and step6. Also modify step2bcde (move transformSourceTable, consolidateSourceTable, writeRecalibratedSourceTable from step4 to here) and step3b (add selectGoodSeeingVisits and templateGen). --- pipelines/LATISS/DRP.yaml | 55 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/pipelines/LATISS/DRP.yaml b/pipelines/LATISS/DRP.yaml index 4c1dedb5..603cccdd 100644 --- a/pipelines/LATISS/DRP.yaml +++ b/pipelines/LATISS/DRP.yaml @@ -135,7 +135,10 @@ subsets: - fgcmBuildFromIsolatedStars - fgcmFitCycle - fgcmOutputProducts + - transformSourceTable + - consolidateSourceTable - updateVisitSummary + - writeRecalibratedSourceTable - makeCcdVisitTable - makeVisitTable description: | @@ -178,6 +181,8 @@ subsets: step3b: subset: - consolidateObjectTable + - selectGoodSeeingVisits + - templateGen description: | Tract-level tasks. Allowed data query constraints: skymap, tract. @@ -212,9 +217,13 @@ subsets: step4: subset: - - writeRecalibratedSourceTable - - transformSourceTable - - consolidateSourceTable + - forcedPhotCcd + - forcedPhotDiffim + - getTemplate + - subtractImages + - detectAndMeasureDiaSources + - transformDiaSourceCat + - writeForcedSourceTable description: | Tasks that be run together, but only after the 'step1', 'step2', and 'step3' subsets. @@ -223,3 +232,43 @@ subsets: These detector-level tasks should not be run with 'tract' or 'patch' as part of the data ID expression if all reference catalogs are desired, and to ensure that consolidateSourceTable contains complete visits. + + step5: + subset: + - drpAssociation + - drpDiaCalculation + - forcedPhotCcdOnDiaObjects + - forcedPhotDiffOnDiaObjects + - transformForcedSourceTable + - consolidateForcedSourceTable + - consolidateAssocDiaSourceTable + - consolidateFullDiaObjectTable + - writeForcedSourceOnDiaObjectTable + - transformForcedSourceOnDiaObjectTable + - consolidateForcedSourceOnDiaObjectTable + description: | + Tasks that can be run together, but only after the 'step1', 'step2', + 'step3', and 'step4' subsets. + + This step includes patch-level aggregation Tasks. These should be run + with explicit 'tract' constraints in the data query, otherwise quanta + will be created for jobs with only partial visit coverage. + 'consolidateForcedSourceTable' is a tract-level task that aggregates + patches and should be rerun if any of the patches fail. + + step6: + subset: + - consolidateDiaSourceTable + description: | + Tasks that can be run together, but only after the 'step1', 'step2', + 'step3', 'step4', and 'step5' subsets. + + This step includes visit-level aggregation tasks. Running without tract + or patch in the data query is recommended, otherwise the outputs of + consolidateDiaSourceTable will not contain complete visits. + + This subset is separate from step4 to signal to operators to pause to + assess unexpected image differencing failures before these aggregation + steps. Otherwise, if run in the same quantum graph, aggregated data + products (e.g. diaObjects) would not be created if one or more of the + expected inputs is missing. From 681f25c6237ea7667c00dcf1521b2e6bce9eba0f Mon Sep 17 00:00:00 2001 From: hlin730 Date: Wed, 6 Dec 2023 15:21:05 -0600 Subject: [PATCH 2/3] Move tasks for consistency with HSC DRP-RC2.yaml After review, move some tasks for consistency with HSC DRP-RC2.yaml: selectGoodSeeingVisits and templateGen to step3a, writeRecalibratedSourceTable and transformSourceTable to step4, consolidateSourceTable to step5. --- pipelines/LATISS/DRP.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pipelines/LATISS/DRP.yaml b/pipelines/LATISS/DRP.yaml index 603cccdd..ac3a5643 100644 --- a/pipelines/LATISS/DRP.yaml +++ b/pipelines/LATISS/DRP.yaml @@ -135,10 +135,7 @@ subsets: - fgcmBuildFromIsolatedStars - fgcmFitCycle - fgcmOutputProducts - - transformSourceTable - - consolidateSourceTable - updateVisitSummary - - writeRecalibratedSourceTable - makeCcdVisitTable - makeVisitTable description: | @@ -167,6 +164,8 @@ subsets: - forcedPhotCoadd - writeObjectTable - transformObjectTable + - selectGoodSeeingVisits + - templateGen description: | Tract-level coaddition tasks. Allowed data query constraints: instrument, skymap, tract. @@ -181,8 +180,6 @@ subsets: step3b: subset: - consolidateObjectTable - - selectGoodSeeingVisits - - templateGen description: | Tract-level tasks. Allowed data query constraints: skymap, tract. @@ -224,6 +221,8 @@ subsets: - detectAndMeasureDiaSources - transformDiaSourceCat - writeForcedSourceTable + - writeRecalibratedSourceTable + - transformSourceTable description: | Tasks that be run together, but only after the 'step1', 'step2', and 'step3' subsets. @@ -250,15 +249,14 @@ subsets: Tasks that can be run together, but only after the 'step1', 'step2', 'step3', and 'step4' subsets. - This step includes patch-level aggregation Tasks. These should be run - with explicit 'tract' constraints in the data query, otherwise quanta - will be created for jobs with only partial visit coverage. + This step includes patch-level aggregation Tasks. 'consolidateForcedSourceTable' is a tract-level task that aggregates patches and should be rerun if any of the patches fail. step6: subset: - consolidateDiaSourceTable + - consolidateSourceTable description: | Tasks that can be run together, but only after the 'step1', 'step2', 'step3', 'step4', and 'step5' subsets. From 395a38f1d8b40bdcbf984267020cbf6b4fda545d Mon Sep 17 00:00:00 2001 From: hlin730 Date: Wed, 6 Dec 2023 16:36:02 -0600 Subject: [PATCH 3/3] fix trailing space linting error Fix trailing space linting error in line 252. --- pipelines/LATISS/DRP.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/LATISS/DRP.yaml b/pipelines/LATISS/DRP.yaml index ac3a5643..a9da7c12 100644 --- a/pipelines/LATISS/DRP.yaml +++ b/pipelines/LATISS/DRP.yaml @@ -249,7 +249,7 @@ subsets: Tasks that can be run together, but only after the 'step1', 'step2', 'step3', and 'step4' subsets. - This step includes patch-level aggregation Tasks. + This step includes patch-level aggregation Tasks. 'consolidateForcedSourceTable' is a tract-level task that aggregates patches and should be rerun if any of the patches fail.