Merge pull request #319 from LSSTDESC/use-qp

Switch to using QP for n(z) information throughout

.github/workflows/ci.yml

@@ -42,7 +42,7 @@ jobs:
     runs-on: ubuntu-latest
 
     container:
-      image: ghcr.io/lsstdesc/txpipe:v0.7
+      image: ghcr.io/lsstdesc/txpipe-dev
 
     steps:
     - name: Checkout repository
@@ -61,7 +61,7 @@ jobs:
     needs: Download_Data
 
     container:
-      image: ghcr.io/lsstdesc/txpipe:v0.7
+      image: ghcr.io/lsstdesc/txpipe-dev
 
     steps:
     - name: Checkout repository
@@ -99,7 +99,7 @@ jobs:
     needs: Download_Data
 
     container:
-      image: ghcr.io/lsstdesc/txpipe:v0.7
+      image: ghcr.io/lsstdesc/txpipe-dev
 
     steps:
     - name: Checkout repository
@@ -146,7 +146,7 @@ jobs:
       needs: Download_Data
 
       container:
-        image: ghcr.io/lsstdesc/txpipe:v0.7
+        image: ghcr.io/lsstdesc/txpipe-dev
 
       steps:
       - name: Checkout repository
@@ -185,7 +185,7 @@ jobs:
     needs: Download_Data
 
     container:
-      image: ghcr.io/lsstdesc/txpipe:v0.7
+      image: ghcr.io/lsstdesc/txpipe-dev
 
     steps:
     - name: Checkout repository
@@ -224,7 +224,7 @@ jobs:
     needs: Download_Data
 
     container:
-      image: ghcr.io/lsstdesc/txpipe:v0.7
+      image: ghcr.io/lsstdesc/txpipe-dev
 
     steps:
     - name: Checkout repository
@@ -260,7 +260,7 @@ jobs:
     runs-on: ubuntu-latest
 
     container:
-      image: ghcr.io/lsstdesc/txpipe:v0.7
+      image: ghcr.io/lsstdesc/txpipe-dev
 
     steps:
     - name: Checkout repository
@@ -272,7 +272,6 @@ jobs:
         ceci --dry-run examples/2.2i/pipeline.yml
         ceci --dry-run examples/buzzard/pipeline.yml
         ceci --dry-run examples/cosmodc2/pipeline.yml
-        ceci --dry-run examples/dr1b/pipeline.yml
         ceci --dry-run examples/skysim/pipeline.yml
 
 

docs/make-stages.py

@@ -129,7 +129,7 @@
         - TXTwoPointPlotsFourier
         - TXConvergenceMapPlots
         - TXMapPlots
-        - TXPhotozPlots
+        - TXPhotozPlot
 
 
 Diagnostics:

docs/src/stages/Plots.rst

@@ -3,7 +3,7 @@ Plots
 
 These stages make plots out TXPipe output data
 
-* :py:class:`~txpipe.photoz_stack.TXPhotozPlots` - Make n(z) plots of source and lens galaxies
+* :py:class:`~txpipe.photoz_stack.TXPhotozPlot` - Make n(z) plots of source and lens galaxies
 
 * :py:class:`~txpipe.map_plots.TXMapPlots` - Make plots of all the available maps
 
@@ -15,7 +15,7 @@ These stages make plots out TXPipe output data
 
 
 
-.. autoclass:: txpipe.photoz_stack.TXPhotozPlots
+.. autoclass:: txpipe.photoz_stack.TXPhotozPlot
    :members:
 
 

environment.yml

@@ -29,7 +29,7 @@ dependencies:
   - treecorr=4.3.*
   - pip
   - pip:
-    - ceci==1.13
+    - ceci==1.14
     - git+https://github.com/jlvdb/hyperbolic@b88b107a291fa16c2006cf971ce610248d58e94c
     - git+https://github.com/joezuntz/dask-mpi@31e35759cfb03780a12ad6ffe2b82279b0fab442
     - git+https://github.com/LSSTDESC/[email protected]

examples/2.2i/config.yml

@@ -29,15 +29,44 @@ PZPDFMLZ:
     zmax: 3.0
     chunk_rows: 100000
 
-TXPhotozStack:
-    chunk_rows: 100000
 
-# Mock version of stacking:
-TXLensTrueNumberDensity:
-    nz: 301
-    zmax: 3.0
-    chunk_rows: 100000
-
+TXPhotozPlotLens:
+    name: TXPhotozPlotLens
+    aliases:
+        photoz_stack: lens_photoz_stack
+        nz_plot: source_nz
+
+TXPhotozPlotSource:
+    name: TXPhotozPlotSource
+    aliases:
+        photoz_stack: shear_photoz_stack
+        nz_plot: lens_nz
+
+
+TXTruePhotozStackSource:
+    name: TXTruePhotozStackSource
+    aliases:
+        tomography_catalog: shear_tomography_catalog
+        catalog: shear_catalog
+        weights_catalog: shear_catalog
+        photoz_stack: shear_photoz_stack
+    weight_col: metacal/weight
+    redshift_group: metacal
+    zmax: 2.0
+    nz: 201
+
+TXTruePhotozStackLens:
+    name: TXTruePhotozStackLens
+    aliases:
+        tomography_catalog: lens_tomography_catalog
+        catalog: photometry_catalog
+        weights_catalog: dummy
+        photoz_stack: lens_photoz_stack
+    redshift_group: photometry
+    zmax: 2.0
+    nz: 201
+
+
 TXSourceTrueNumberDensity:
     nz: 301
     zmax: 3.0
@@ -147,3 +176,4 @@ TXTruthLensSelector:
     i_lo_cut: 17.5
     i_hi_cut: 21.9
     r_i_cut: 2.0
+

examples/2.2i/pipeline.yml

@@ -36,16 +36,22 @@ stages:
       nprocess: 8
     - name: TXTruthLensSelector
       nprocess: 32
-    - name: TXLensTrueNumberDensity
+    - name: TXTruePhotozStackSource
+      classname: TXTruePhotozStack
       nprocess: 32
       nodes: 2
-    - name: TXSourceTrueNumberDensity
+    - name: TXTruePhotozStackLens
+      classname: TXTruePhotozStack
       nprocess: 32
       nodes: 2
     - name: TXTwoPointTheoryReal
       nprocess: 1
       threads_per_process: 8
-    - name: TXPhotozPlots
+    - name: TXPhotozPlotSource
+      classname: TXPhotozPlot
+      nprocess: 8
+    - name: TXPhotozPlotLens
+      classname: TXPhotozPlot
       nprocess: 8
     - name: TXSourceMaps
       nprocess: 16
@@ -111,6 +117,7 @@ inputs:
     calibration_table: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/2.2i-inputs/sample_cosmodc2_w10year_errors.dat
     star_catalog: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/2.2i_dr6/star_catalog.hdf5
     exposures: /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/2.2i-inputs/exposures.hdf5
+    dummy: none
 
 resume: True
 log_dir: data/2.2i_dr6/logs

examples/2.2i/pipeline_1tract.yml

@@ -31,8 +31,10 @@ stages:
     - name: TXTruthLensSelector
     - name: TXLensCatalogSplitter
     - name: TXPhotozSourceStack
+      classname: TXPhotozStack
     - name: TXPhotozLensStack
-    - name: TXPhotozPlots
+      classname: TXPhotozStack
+    - name: TXPhotozPlot
     - name: TXSourceMaps
     - name: TXLensMaps
     - name: TXAuxiliarySourceMaps

examples/cosmodc2/config.yml

@@ -23,14 +23,29 @@ TXIngestRedmagic:
     lens_zbin_edges: [0.15, 0.3, 0.45, 0.6, 0.75, 0.9]
 
 TXSourceTrueNumberDensity:
+    name: TXSourceTrueNumberDensity
     nz: 601
     zmax: 3.0
     chunk_rows: 100000
+    aliases:
+        tomography_catalog: shear_tomography_catalog
+        catalog: shear_catalog
+        weights_catalog: shear_catalog
+        photoz_stack: shear_photoz_stack
+    weight_col: metacal/weight
+    redshift_group: metacal
 
 TXLensTrueNumberDensity:
+    name: TXLensTrueNumberDensity
     nz: 601
     zmax: 3.0
     chunk_rows: 100000
+    aliases:
+        tomography_catalog: lens_tomography_catalog
+        catalog: photometry_catalog
+        weights_catalog: none
+        photoz_stack: lens_photoz_stack
+    redshift_group: photometry
 
 TXLensMaps:
     chunk_rows:  100000
@@ -184,3 +199,15 @@ TXTruthLensSelector:
     i_lo_cut: 17.5
     i_hi_cut: 21.9
     r_i_cut: 2.0
+
+TXPhotozPlotLens:
+    name: TXPhotozPlotLens
+    aliases:
+        photoz_stack: lens_photoz_stack
+        nz_plot: lens_nz
+
+TXPhotozPlotSource:
+    name: TXPhotozPlotSource
+    aliases:
+        photoz_stack: shear_photoz_stack
+        nz_plot: source_nz

examples/cosmodc2/pipeline-20deg2.yml

@@ -21,7 +21,7 @@ stages:
       nprocess: 32
     - name: TXLensTrueNumberDensity
     - name: TXSourceTrueNumberDensity
-    - name: TXPhotozPlots
+    - name: TXPhotozPlot
     - name: TXShearCalibration
       nprocess: 32
     - name: TXTruthLensCatalogSplitter

examples/cosmodc2/pipeline.yml

@@ -19,8 +19,13 @@ stages:
     - name: TXTruthLensSelector
       nprocess: 32
     - name: TXLensTrueNumberDensity
+      classname: TXTruePhotozStack
     - name: TXSourceTrueNumberDensity
-    - name: TXPhotozPlots
+      classname: TXTruePhotozStack
+    - name: TXPhotozPlotSource
+      classname: TXPhotozPlot
+    - name: TXPhotozPlotLens
+      classname: TXPhotozPlot
     - name: TXShearCalibration
     - name: TXTruthLensCatalogSplitter
     - name: TXTwoPointFourier
@@ -83,7 +88,7 @@ inputs:
     photoz_trained_model:  /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/cosmoDC2_trees_i25.3.npy
     fiducial_cosmology: data/fiducial_cosmology.yml
     calibration_table:  /global/projecta/projectdirs/lsst/groups/WL/users/zuntz/data/cosmoDC2-1.1.4_oneyear_unit_response/sample_cosmodc2_w10year_errors.dat
-
+    none: none
 
 resume: True
 log_dir: data/cosmodc2/logs

examples/cosmodc2/pipeline_redmagic.yml

@@ -25,7 +25,7 @@ stages:
       nprocess: 5
       nodes: 1
     - name: TXSourceTrueNumberDensity
-    - name: TXPhotozPlots
+    - name: TXPhotozPlot
     - name: TXJackknifeCenters
     - name: TXMapPlots
     - name: TXTracerMetadata

examples/cosmodc2/pipeline_redmagic_full.yml

@@ -30,7 +30,7 @@ stages:
       nprocess: 5
       nodes: 1
     - name: TXSourceTrueNumberDensity
-    - name: TXPhotozPlots
+    - name: TXPhotozPlot
     - name: TXJackknifeCenters
     - name: TXMapPlots
     - name: TXTracerMetadata

examples/cosmodc2/pipeline_redmagic_no_shape_noise.yml

@@ -25,7 +25,7 @@ stages:
       nprocess: 5
       nodes: 1
     - name: TXSourceTrueNumberDensity
-    - name: TXPhotozPlots
+    - name: TXPhotozPlot
     - name: TXJackknifeCenters
     - name: TXMapPlots
     - name: TXTracerMetadata

examples/cosmodc2/pipeline_redmagic_no_shape_noise_full.yml

@@ -23,7 +23,7 @@ stages:
       nprocess: 64
       nodes: 2
     - name: TXSourceTrueNumberDensity
-    - name: TXPhotozPlots
+    - name: TXPhotozPlot
     - name: TXJackknifeCenters
     - name: TXMapPlots
     - name: TXTracerMetadata

examples/desy1/config.yml

@@ -16,7 +16,7 @@ global:
 # TXShearCalibration
 # TXTruthLensCatalogSplitter
 # TXStarCatalogSplitter     
-# TXPhotozPlots 
+# TXPhotozPlot 
 # TXSourceMaps
 # TXLensMaps
 # TXAuxiliarySourceMaps  
@@ -64,7 +64,7 @@ TXSourceSelectorMetacal:
 TXShearCalibration: {}
 TXTruthLensCatalogSplitter: {}
 TXStarCatalogSplitter: {} 
-TXPhotozPlots: {}
+TXPhotozPlot: {}
 
 TXSourceMaps:
     sparse: True  # Generate sparse maps - faster if using small areas

examples/desy1/desy1_input_pz.yml

@@ -15,7 +15,7 @@ modules: txpipe
 stages:
     - name: TXSelector
       nprocess: 1
-    - name: TXPhotozPlots
+    - name: TXPhotozPlot
     - name: TXDiagnosticMaps
       nprocess: 1
     - name: TXRandomCat

examples/desy1/pipeline.yml

@@ -6,7 +6,7 @@ stages:
     - name: TXShearCalibration   # Calibrate and split the source sample tomographically
     - name: TXTruthLensCatalogSplitter
     - name: TXStarCatalogSplitter  # Split the star catalog into separate bins (psf/non-psf)    
-    - name: TXPhotozPlots 
+    - name: TXPhotozPlot 
     - name: TXSourceMaps
       #nprocess: 6
       #threads_per_process: 1

examples/dp0.2/pipeline.yml

@@ -53,7 +53,7 @@ stages:
     - name: TXDensityMaps
     - name: TXRealGaussianCovariance
       threads_per_process: 32
-    - name: TXPhotozPlots
+    - name: TXPhotozPlot
 
 # Where to put outputs
 output_dir: data/dp0.2/outputs