patch diagnostic.py and ci.yml

.github/workflows/ci.yml

@@ -273,6 +273,7 @@ jobs:
         ceci --dry-run examples/buzzard/pipeline.yml
         ceci --dry-run examples/cosmodc2/pipeline.yml
         ceci --dry-run examples/skysim/pipeline.yml
+        ceci --dry-run examples/desy3/pipeline_Bmodes.yml
 
 
   Test_Auto_Installer:

examples/desy3/config_Bmodes.yml

@@ -7,7 +7,7 @@ global:
     chunk_rows: 100000
     # These mapping options are also read by a range of stages
     pixelization: healpix
-    nside: 128
+    nside: 512
     sparse: True  # Generate sparse maps - faster if using small areas
 
 TXGCRTwoCatalogInput:
@@ -193,9 +193,10 @@ TXTwoPoint:
     flip_g2: True  # use true when using metacal shears
     min_sep: 2.5
     max_sep: 60.0
-    nbins: 10
+    nbins: 1000
     verbose: 0
     subtract_mean_shear: True
+    var_method: "jackknife"
 
 TXGammaTBrightStars: {}
 
@@ -398,3 +399,15 @@ Inform_NZDirLens:
     aliases:
         input: spectroscopic_catalog
         model: lens_direct_calibration_model
+
+TXpureB:
+    method    : 'namaster'                 # 'namaster' or 'hybrideb'
+    Nell      : 20                         # Number of ell bins
+    lmin      : 100                        # ell min 
+    lmax      : 2000                       # ell max
+    lspacing  : 'log'                      # ell spacing for binning 
+    bin_file  :                            # Load predfined bin edges instead        
+    theta_min : 2.5                        # (only for hybrideb) theta min in arcmin 
+    theta_max : 250                        # (only for hybrideb) theta max in arcmin 
+    Ntheta    : 1000                       # (only for hybrideb) Number of theta bins 
+    Nsims     : 3                         # (only for hybrideb) Number of Gaussian draws to use to estimate errors 

examples/desy3/pipeline_Bmodes.yml

@@ -36,6 +36,10 @@ stages:
     - name: TXLensMaps              # make source lens and n_gal maps
     - name: TXDensityMaps           # turn mask and ngal maps into overdensity maps
     - name: TXTwoPointFourier       # Compute power spectra C_ell
+    - name: TXTwoPoint              # Compute real-space 2-point correlations
+      threads_per_process: 128
+    - name: TXpureB                 # Compute pureBB
+
     #- name: TXStarCatalogSplitter  # Split the star catalog into separate bins (psf/non-psf)
     #- name: TXSourceMaps           # make source g1 and g2 maps
     #- name: TXLensMaps             # make source lens and n_gal maps
@@ -47,14 +51,14 @@ stages:
     #- name: TXDensityMaps        # turn mask and ngal maps into overdensity maps
     #- name: TXTracerMetadata     # collate metadata
     #- name: TXRandomCat          # generate lens bin random catalogs
-    #- name: TXJackknifeCenters   # Split the area into jackknife regions
+    - name: TXJackknifeCenters   # Split the area into jackknife regions
     #- name: TXTwoPoint           # Compute real-space 2-point correlations
     #  threads_per_process: 2
     #- name: TXBlinding           # Blind the data following Muir et al
     #  threads_per_process: 2
     #- name: TXTwoPointTheoryReal # compute theory using CCL to save in sacc file and plot later
     #- name: TXTwoPointPlotsTheory      # Make plots of 2pt correlations
-    #- name: TXSourceDiagnosticPlots    # Make a suite of diagnostic plots
+    - name: TXSourceDiagnosticPlots    # Make a suite of diagnostic plots
     #- name: TXLensDiagnosticPlots    # Make a suite of diagnostic plots
     #- name: TXGammaTFieldCenters # Compute and plot gamma_t around center points
     #  threads_per_process: 2
@@ -64,10 +68,10 @@ stages:
     #  threads_per_process: 2
     #- name: TXGalaxyStarDensity  # Compute and plot the star-galaxy density cross-correlation
     #- name: TXGalaxyStarShear    # Compute and plot the star-galaxy shear cross-correlation
-    - name: TXPhotozPlotSource        # Plot the bin n(z)
-      classname: TXPhotozPlot
-    - name: TXPhotozPlotLens        # Plot the bin n(z)
-      classname: TXPhotozPlot
+    #- name: TXPhotozPlotSource        # Plot the bin n(z)
+    #  classname: TXPhotozPlot
+    #- name: TXPhotozPlotLens        # Plot the bin n(z)
+    #  classname: TXPhotozPlot
     #- name: TXConvergenceMaps    # Make convergence kappa maps from g1, g2 maps
     #- name: TXConvergenceMapPlots # Plot the convergence map
     #- name: TXMapCorrelations    # plot the correlations between systematics and data
@@ -98,7 +102,7 @@ python_paths:
     - submodules/WLMassMap/python/desc/
 
 # Where to put outputs
-output_dir: data/desy3a/outputs
+output_dir: data/des-y3/outputs
 
 # How to run the pipeline: mini, parsl, or cwl
 launcher:
@@ -108,7 +112,7 @@ launcher:
 # Where to run the pipeline: cori-interactive, cori-batch, or local
 site:
     name: local
-    max_threads: 2
+    max_threads: 128
 
 # configuration settings
 config: examples/desy3/config_Bmodes.yml
@@ -118,10 +122,15 @@ inputs:
     # See README for paths to download these files
     shear_catalog     : /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y3/shear_catalog_desy3_unmasked_withfakez_v2.h5
     star_catalog      : /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y3/DES_psf_y3_catalog.hdf5
+    shear_photoz_stack: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y3//shear_photoz_stack.hdf5
+    random_cats       :  /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y3/randoms_desy3_RM.hdf5
+    binned_random_catalog: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y3/binned_randoms_desy3_RM.hdf5
+    binned_random_catalog_sub: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y3/binned_randoms_desy3_RM.hdf5
+    #patch_centers        : /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y3/binned_randoms_desy3_RM.hdf5
+
     #lens_tomography_catalog: /global/cfs/cdirs/lsst/groups/WL/users/yomori/repo/aaa/TXPipe/data/desy3a/outputs/lens_tomography_catalog_unweighted.hdf5 ##############<----- manually add this line
     # This file comes with the code
     fiducial_cosmology: data/fiducial_cosmology.yml
-    shear_photoz_stack: data/example/outputs/shear_photoz_stack_manual.hdf5
     photometry_catalog: data/example/inputs/photometry_catalog.hdf5
     calibration_table : data/example/inputs/sample_cosmodc2_w10year_errors.dat
     exposures         : data/example/inputs/exposures.hdf5
@@ -147,9 +156,9 @@ inputs:
 # if interrupted
 resume: True
 # where to put output logs for individual stages
-log_dir: data/desy3a/logs
+log_dir: data/des-y3/logs
 # where to put an overall parsl pipeline log
-pipeline_log: data/desy3a/log.txt
+pipeline_log: data/des-y3/log.txt
 
 
 
@@ -179,4 +188,4 @@ pipeline_log: data/desy3a/log.txt
 #    f['qp/meta/pdf_name']    = np.array([b'hist'], dtype='|S4')
 #    f['qp/meta/pdf_version'] = [0]
 #    f['qp/data/pdfs']        = (np.c_[nz1,nz2,nz3,nz4]).T
-
+

txpipe/diagnostics.py

@@ -300,8 +300,11 @@ def plot_psf_shear(self):
         fig.close()
 
         f = self.open_output("g_psf_g_out")
-        data   =[mu1,mu2,mean11,mean12,mean21,mean22,std11,std12,std21,std22,line11,line12,line21,line22]
-        f.write(''.join([str(i) + '\n' for i in  data]))
+        data = np.c_[mu1,mu2,mean11,mean12,mean21,mean22,std11,std12,std21,std22,line11,line12,line21,line22]
+        f.write('#[0]mu1 [1]mu2 [2]mean11 [3]mean12 [4]mean21 [5]mean22 [6]std11 [7]std12 [8]std21 [9]std22 [10]line11 [11]line12 [12]line21 [13]line22\n')
+        for row in data:
+            line = ' '.join(map(str, row)) + '\n'
+            f.write(line)
         f.close()
 
     def plot_psf_size_shear(self):
@@ -372,9 +375,14 @@ def plot_psf_size_shear(self):
         fig.close()
 
         f = self.open_output("g_psf_T_out")
-        data   =[mu,mean1,mean2,std1,std2,line1,line2]
-        f.write(''.join([str(i) + '\n' for i in  data]))
+        f.write('#[0]mu [1]mean1 [2]mean2 [3]std1 [4]std2 [5]line1 [6]line2\n')
+        data = np.c_[mu,mean1,mean2,std1,std2,line1,line2]
+        for row in data:
+            line = ' '.join(map(str, row)) + '\n'
+            f.write(line)
         f.close()
+
+
 
 
     def plot_snr_shear(self):
@@ -445,11 +453,17 @@ def plot_snr_shear(self):
         plt.tight_layout()
         fig.close()
 
+
         f = self.open_output("g_snr_out")
-        data   =[mu,mean1,mean2,std1,std2,line1,line2]
-        f.write(''.join([str(i) + '\n' for i in  data]))
+        f.write('#[0]mu [1]mean1 [2]mean2 [3]std1 [4]std2 [5]line1 [6]line2\n')
+        data = np.c_[mu,mean1,mean2,std1,std2,line1,line2]
+        for row in data:
+            line = ' '.join(map(str, row)) + '\n'
+            f.write(line)
         f.close()
-
+
+
+
     def plot_size_shear(self):
         # mean shear in bins of galaxy size
         print("Making mean shear galaxy size plot")
@@ -519,10 +533,15 @@ def plot_size_shear(self):
         plt.tight_layout()
         fig.close()
 
+
         f = self.open_output("g_T_out")
-        data   =[mu,mean1,mean2,std1,std2,line1,line2]
-        f.write(''.join([str(i) + '\n' for i in  data]))
+        f.write('#[0]mu [1]mean1 [2]mean2 [3]std1 [4]std2 [5]line1 [6]line2\n')
+        data = np.c_[mu,mean1,mean2,std1,std2,line1,line2]
+        for row in data:
+            line = ' '.join(map(str, row)) + '\n'
+            f.write(line)
         f.close()
+
 
     def plot_mag_shear(self):
         # mean shear in bins of magnitude