From 561b52832e1811403d45aad796c6c7eee5640b58 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Tue, 17 Dec 2024 15:26:01 -0500 Subject: [PATCH 1/2] Remove analysis_drp. --- bps/clustering/DRP-recalibrated.yaml | 17 -------------- .../LSSTCam-imSim/DRP-DC2-clustering.yaml | 17 -------------- pipelines/DECam/DRP-Merian.yaml | 1 - pipelines/HSC/QaTractTables.yaml | 22 ------------------- pipelines/HSC/analysis_drp_plots_tier2_3.yaml | 4 ---- pipelines/LATISS/DRP.yaml | 21 +----------------- pipelines/LSSTCam-imSim/DRP-ci_imsim.yaml | 1 - pipelines/LSSTCam-imSim/DRP-test-med-1.yaml | 1 - pipelines/_ingredients/HSC/DRP.yaml | 1 - tests/test_pipelines.py | 2 -- ups/drp_pipe.table | 1 - 11 files changed, 1 insertion(+), 87 deletions(-) delete mode 100644 pipelines/HSC/QaTractTables.yaml delete mode 100644 pipelines/HSC/analysis_drp_plots_tier2_3.yaml diff --git a/bps/clustering/DRP-recalibrated.yaml b/bps/clustering/DRP-recalibrated.yaml index 21f0564b..aa41069a 100644 --- a/bps/clustering/DRP-recalibrated.yaml +++ b/bps/clustering/DRP-recalibrated.yaml @@ -1,6 +1,5 @@ # This is a prescription for quantum clustering with BPS, suitable for any # concrete pipeline based on the "DRP-recalibrated" pipeline. -# It also includes clustering for analysis_drp plots. # # Use it by adding: # @@ -16,22 +15,6 @@ cluster: dimensions: visit,detector equalDimensions: visit:exposure - visit_plots: - pipetasks: plot_CircAp12_sub_PS_meas_sky_stars_visit,plot_CircAp12_sub_PS_meas_sky_gals_visit,plot_CircAp12_sub_PS_meas_calib_psf_used_sky_stars_visit,plot_CircAp12_sub_PS_meas_calib_psf_used_sky_gals_visit,plot_CircAp12_sub_PS_calib_psf_used_sky_unknown_visit,plot_PSFluxSN_meas_sky_all_visit,plot_CircAp12_sub_PS_meas_calib_psf_used_scatter_visit,plot_CircAp12_sub_PS_all_scatter_visit,plot_CircAp25_sub_PS_all_scatter_visit,plot_CircAp12_sub_PS_gals_scatter_visit,plot_CircAp25_sub_PS_gals_scatter_visit,plot_CircAp12_sub_PS_meas_scatter_visit,plot_CircAp25_sub_PS_meas_scatter_visit,plot_skyObject_sky_visit,plot_astromRefCat_sky_visit_dRA,plot_astromRefCat_sky_visit_dDec,plot_astromRefCat_scatter_visit_dRA,plot_astromRefCat_scatter_visit_dDec - dimensions: visit - - visit_plots_ellip: - pipetasks: plot_e1_scatter_visit,plot_e1_sky_visit,plot_e2_scatter_visit,plot_e2_sky_visit,plot_shapeSize_scatter_visit,plot_shapeSize_sky_visit,plot_e1PSF_scatter_visit,plot_e1PSF_sky_visit,plot_e2PSF_sky_visit,plot_e2PSF_scatter_visit,plot_ellipResids_quiver_visit,plot_shapeSizePSF_scatter_visit,plot_shapeSizePSF_sky_visit,plot_shapeSizeDiff_scatter_visit,plot_shapeSizeFractionalDiff_scatter_visit,plot_E1Diff_scatter_visit,plot_E2Diff_scatter_visit,plot_E1Diff_sky_visit,plot_E2Diff_sky_visit,plot_ShapeDiff_sky_visit,plot_RhoStatistics_visit - dimensions: skymap,visit - - coadd_plots: - pipetasks: plot_CircAp12_sub_PSF_meas_sky_stars,plot_CircAp12_sub_PSF_meas_sky_gals,plot_CircAp12_sub_PSF_meas_calib_psf_used_sky_stars,plot_CircAp12_sub_PSF_meas_calib_psf_used_sky_gals,plot_CircAp12_sub_PSF_meas_calib_psf_used_sky_unknown,plot_PSFluxSN_meas_sky_all,plot_Kron_sub_PSFmag_meas_sky_galaxies,plot_Kron_sub_PSFmag_meas_sky_stars,plot_CModel_sub_PSFmag_meas_sky_galaxies,plot_CModel_sub_PSFmag_meas_sky_stars,plot_CircAp12_sub_PSF_meas_calib_psf_used_stars_scatter,plot_CircAp12_sub_PSF_meas_stars_scatter,plot_CircAp12_sub_PSF_meas_calib_psf_used_gals_scatter,plot_CircAp12_sub_PSF_meas_gals_scatter,plot_ri_gr_cmodel,plot_ri_gr_psf,plot_astromRefCat_sky_tract_dRA,plot_astromRefCat_sky_tract_dDec,plot_astromRefCat_scatter_tract_dRA,plot_astromRefCat_scatter_tract_dDec - dimensions: skymap,tract - - coadd_plots_ellip: - pipetasks: plot_E1Diff_scatter,plot_E2Diff_scatter,plot_shapeSizeFractionalDiff_scatter,plot_E1Diff_sky,plot_E2Diff_sky,plot_shapeSizeFractionalDiff_sky,plot_E1Diff_magDiff_scatter,plot_E2Diff_magDiff_scatter,plot_shapeSizeFractionalDiff_magDiff_scatter,plot_shapeSizeDiff_scatter,plot_shapeSizeDiff_magDiff_scatter,plot_shapeSizeDiff_sky,plot_ellipResids_quiver,plot_e1_scatter,plot_e2_scatter,plot_shapeSize_scatter,plot_e1PSF_scatter,plot_e2PSF_scatter,plot_shapeSizePSF_scatter,plot_RhoStatistics - dimensions: skymap,tract - coadd: pipetasks: assembleCoadd,inject_coadd,templateGen,detection dimensions: tract,patch,band diff --git a/bps/clustering/LSSTCam-imSim/DRP-DC2-clustering.yaml b/bps/clustering/LSSTCam-imSim/DRP-DC2-clustering.yaml index 9e417202..465117c5 100644 --- a/bps/clustering/LSSTCam-imSim/DRP-DC2-clustering.yaml +++ b/bps/clustering/LSSTCam-imSim/DRP-DC2-clustering.yaml @@ -1,5 +1,4 @@ # This is a prescription for quantum clustering with BPS with DC2 -# It also includes clustering for analysis_drp plots. # # Use it by adding: # @@ -15,22 +14,6 @@ cluster: dimensions: visit,detector equalDimensions: visit:exposure - visit_plots: - pipetasks: plot_CircAp12_sub_PS_meas_sky_stars_visit,plot_CircAp12_sub_PS_meas_sky_gals_visit,plot_CircAp12_sub_PS_meas_calib_psf_used_sky_stars_visit,plot_CircAp12_sub_PS_meas_calib_psf_used_sky_gals_visit,plot_CircAp12_sub_PS_calib_psf_used_sky_unknown_visit,plot_PSFluxSN_meas_sky_all_visit,plot_CircAp12_sub_PS_meas_calib_psf_used_scatter_visit,plot_CircAp12_sub_PS_all_scatter_visit,plot_CircAp25_sub_PS_all_scatter_visit,plot_CircAp12_sub_PS_gals_scatter_visit,plot_CircAp25_sub_PS_gals_scatter_visit,plot_CircAp12_sub_PS_meas_scatter_visit,plot_CircAp25_sub_PS_meas_scatter_visit,plot_skyObject_sky_visit,plot_astromRefCat_sky_visit_dRA,plot_astromRefCat_sky_visit_dDec,plot_astromRefCat_scatter_visit_dRA,plot_astromRefCat_scatter_visit_dDec - dimensions: visit - - visit_plots_ellip: - pipetasks: plot_e1_scatter_visit,plot_e1_sky_visit,plot_e2_scatter_visit,plot_e2_sky_visit,plot_shapeSize_scatter_visit,plot_shapeSize_sky_visit,plot_e1PSF_scatter_visit,plot_e1PSF_sky_visit,plot_e2PSF_sky_visit,plot_e2PSF_scatter_visit,plot_ellipResids_quiver_visit,plot_shapeSizePSF_scatter_visit,plot_shapeSizePSF_sky_visit,plot_shapeSizeDiff_scatter_visit,plot_shapeSizeFractionalDiff_scatter_visit,plot_E1Diff_scatter_visit,plot_E2Diff_scatter_visit,plot_E1Diff_sky_visit,plot_E2Diff_sky_visit,plot_ShapeDiff_sky_visit,plot_RhoStatistics_visit - dimensions: skymap,visit - - coadd_plots: - pipetasks: plot_CircAp12_sub_PSF_meas_sky_stars,plot_CircAp12_sub_PSF_meas_sky_gals,plot_CircAp12_sub_PSF_meas_calib_psf_used_sky_stars,plot_CircAp12_sub_PSF_meas_calib_psf_used_sky_gals,plot_CircAp12_sub_PSF_meas_calib_psf_used_sky_unknown,plot_PSFluxSN_meas_sky_all,plot_Kron_sub_PSFmag_meas_sky_galaxies,plot_Kron_sub_PSFmag_meas_sky_stars,plot_CModel_sub_PSFmag_meas_sky_galaxies,plot_CModel_sub_PSFmag_meas_sky_stars,plot_CircAp12_sub_PSF_meas_calib_psf_used_stars_scatter,plot_CircAp12_sub_PSF_meas_stars_scatter,plot_CircAp12_sub_PSF_meas_calib_psf_used_gals_scatter,plot_CircAp12_sub_PSF_meas_gals_scatter,plot_ri_gr_cmodel,plot_ri_gr_psf,plot_astromRefCat_sky_tract_dRA,plot_astromRefCat_sky_tract_dDec,plot_astromRefCat_scatter_tract_dRA,plot_astromRefCat_scatter_tract_dDec - dimensions: skymap,tract - - coadd_plots_ellip: - pipetasks: plot_E1Diff_scatter,plot_E2Diff_scatter,plot_shapeSizeFractionalDiff_scatter,plot_E1Diff_sky,plot_E2Diff_sky,plot_shapeSizeFractionalDiff_sky,plot_E1Diff_magDiff_scatter,plot_E2Diff_magDiff_scatter,plot_shapeSizeFractionalDiff_magDiff_scatter,plot_shapeSizeDiff_scatter,plot_shapeSizeDiff_magDiff_scatter,plot_shapeSizeDiff_sky,plot_ellipResids_quiver,plot_e1_scatter,plot_e2_scatter,plot_shapeSize_scatter,plot_e1PSF_scatter,plot_e2PSF_scatter,plot_shapeSizePSF_scatter,plot_RhoStatistics - dimensions: skymap,tract - coadd: pipetasks: assembleCoadd,inject_coadd,templateGen,detection dimensions: tract,patch,band diff --git a/pipelines/DECam/DRP-Merian.yaml b/pipelines/DECam/DRP-Merian.yaml index 77e221e1..77e3456d 100644 --- a/pipelines/DECam/DRP-Merian.yaml +++ b/pipelines/DECam/DRP-Merian.yaml @@ -9,4 +9,3 @@ imports: - location: $DRP_PIPE_DIR/pipelines/_ingredients/DECam/DRP.yaml exclude: - isrForCrosstalkSources - - location: $ANALYSIS_DRP_DIR/pipelines/analysis_drp_plots.yaml diff --git a/pipelines/HSC/QaTractTables.yaml b/pipelines/HSC/QaTractTables.yaml deleted file mode 100644 index d3e072a4..00000000 --- a/pipelines/HSC/QaTractTables.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# TODO: remove this pipeline once DM-28475 lands (as filterMap config will no -# longer be necessary) -description: Make collated tract-level QA tables for use in plotting pipelines -parameters: - filterMap: - { - "g": "HSC-G", - "r": "HSC-R", - "i": "HSC-I", - "z": "HSC-Z", - "y": "HSC-Y", - "N921": "NB0921", - } -tasks: - makeForcedQaTractTables: - class: lsst.analysis.drp.makeQaTractTables.MakeForcedQaTractTablesTask - config: - filterMap: parameters.filterMap - makeUnforcedQaTractTables: - class: lsst.analysis.drp.makeQaTractTables.MakeUnforcedQaTractTablesTask - config: - filterMap: parameters.filterMap diff --git a/pipelines/HSC/analysis_drp_plots_tier2_3.yaml b/pipelines/HSC/analysis_drp_plots_tier2_3.yaml deleted file mode 100644 index 126bfefb..00000000 --- a/pipelines/HSC/analysis_drp_plots_tier2_3.yaml +++ /dev/null @@ -1,4 +0,0 @@ -description: The tier 2 & 3 analysis_drp plots pipeline specialized for the HSC RC2 dataset. -instrument: lsst.obs.subaru.HyperSuprimeCam -imports: - - location: $ANALYSIS_DRP_DIR/pipelines/coaddQAPlots_tier2_3.yaml diff --git a/pipelines/LATISS/DRP.yaml b/pipelines/LATISS/DRP.yaml index 6b8c45c4..8c2b8660 100644 --- a/pipelines/LATISS/DRP.yaml +++ b/pipelines/LATISS/DRP.yaml @@ -8,18 +8,6 @@ imports: - location: $ANALYSIS_TOOLS_DIR/pipelines/exposureQualityCore.yaml exclude: - analyzeObjectTableCore - # The following are from analysis_drp - # TODO: remove once they have all been ported to analysis_tools (DM-39081) - - location: $ANALYSIS_DRP_DIR/pipelines/analysis_drp_plots.yaml - include: - - plot_ri_gr_psf - - plot_ri_gr_cmodel - - plot_wFit_PSF - - plot_xFit_PSF - - location: $ANALYSIS_DRP_DIR/pipelines/coaddQAPlots_tier2_3.yaml - include: - - plot_ri_gr_contour_psf - - plot_ri_gr_contour_cmodel tasks: isr: lsst.ip.isr.IsrTaskLSST @@ -221,16 +209,9 @@ subsets: - validateObjectTableCore - analyzeMatchedVisitCore - analyzeAmpOffsetMetadata - # The following are from analysis_drp - # TODO: remove once they have all been ported to analysis_tools (DM-39081) - - plot_ri_gr_psf - - plot_ri_gr_cmodel - - plot_wFit_PSF - - plot_xFit_PSF description: | Healsparse maps and "core" (most important), coadd-level plots/metrics - from analysis_tools (plus a few from analysis_drp to be deprecated once - replicated in analysis_tools). + from analysis_tools. Needs to be run after 'step3b'. Must be be run with only instrument and skymap dataquery constraints if analyzeObjectTableSurveyCore is included, diff --git a/pipelines/LSSTCam-imSim/DRP-ci_imsim.yaml b/pipelines/LSSTCam-imSim/DRP-ci_imsim.yaml index 67f0f885..49df13e7 100644 --- a/pipelines/LSSTCam-imSim/DRP-ci_imsim.yaml +++ b/pipelines/LSSTCam-imSim/DRP-ci_imsim.yaml @@ -2,7 +2,6 @@ description: DRP specialized for ci_imsim test package. instrument: lsst.obs.lsst.LsstCamImSim imports: - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTCam-imSim/DRP.yaml - - $ANALYSIS_DRP_DIR/pipelines/LSSTCam-imSim/analysis_drp_plots.yaml - $ANALYSIS_TOOLS_DIR/pipelines/diaTractQualityCore.yaml - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityExtended.yaml - $MEAS_EXTENSIONS_MULTIPROFIT_DIR/pipelines/multiprofit_match_dc2.yaml diff --git a/pipelines/LSSTCam-imSim/DRP-test-med-1.yaml b/pipelines/LSSTCam-imSim/DRP-test-med-1.yaml index 69e920b0..0c04c53d 100644 --- a/pipelines/LSSTCam-imSim/DRP-test-med-1.yaml +++ b/pipelines/LSSTCam-imSim/DRP-test-med-1.yaml @@ -2,4 +2,3 @@ description: DRP specialized for the ImSim-DC2 test-med-1 dataset instrument: lsst.obs.lsst.LsstCamImSim imports: - $DRP_PIPE_DIR/pipelines/_ingredients/LSSTCam-imSim/DRP.yaml - - $ANALYSIS_DRP_DIR/pipelines/LSSTCam-imSim/analysis_drp_plots.yaml diff --git a/pipelines/_ingredients/HSC/DRP.yaml b/pipelines/_ingredients/HSC/DRP.yaml index 9f80929d..a68bcf39 100644 --- a/pipelines/_ingredients/HSC/DRP.yaml +++ b/pipelines/_ingredients/HSC/DRP.yaml @@ -2,7 +2,6 @@ description: The DRP pipeline specialized for Subaru's HyperSuprime-Cam instrument: lsst.obs.subaru.HyperSuprimeCam imports: - $DRP_PIPE_DIR/pipelines/_ingredients/DRP-full.yaml - - $ANALYSIS_DRP_DIR/pipelines/analysis_drp_plots.yaml - $ANALYSIS_TOOLS_DIR/pipelines/coaddColumnValidate.yaml - $ANALYSIS_TOOLS_DIR/pipelines/coaddQualityCore.yaml - $ANALYSIS_TOOLS_DIR/pipelines/exposureQualityCore.yaml diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index ccc89b1a..fe9f4542 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -359,8 +359,6 @@ def test_hsc_drp_ci_hsc(self): "goodSeeingVisits", "mergedForcedSource", "mergedForcedSourceOnDiaObject", - "objectTable_tract_astrometryRefCat_match", - "sourceTable_visit_astrometryRefCat_match", }, ) tester.run(butler, self) diff --git a/ups/drp_pipe.table b/ups/drp_pipe.table index 6b778bc6..6f9abfa9 100644 --- a/ups/drp_pipe.table +++ b/ups/drp_pipe.table @@ -18,7 +18,6 @@ setupRequired(meas_transiNet) # dependency until all of its plots and metrics have analysis_tools # replacements in order to make it easier to run as a one-off. setupRequired(faro) -setupRequired(analysis_drp) setupRequired(analysis_tools) setupRequired(drp_tasks) setupRequired(ap_association) From 7c8906558afd2fb143b8db6f350044f006d25b91 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Wed, 18 Dec 2024 20:45:38 -0500 Subject: [PATCH 2/2] Fix capitalization of 'true' in YAML. --- pipelines/_ingredients/DRP-minimal-calibration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/_ingredients/DRP-minimal-calibration.yaml b/pipelines/_ingredients/DRP-minimal-calibration.yaml index 5474f769..dee1d222 100644 --- a/pipelines/_ingredients/DRP-minimal-calibration.yaml +++ b/pipelines/_ingredients/DRP-minimal-calibration.yaml @@ -165,7 +165,7 @@ tasks: transformDiaSourceCat: class: lsst.ap.association.TransformDiaSourceCatalogTask config: - doIncludeReliability: True + doIncludeReliability: true connections.coaddName: goodSeeing connections.diaSourceSchema: goodSeeingDiff_diaSrc_schema connections.diaSourceCat: goodSeeingDiff_diaSrc