Merge branch 'master' into ricci-clusters

.github/workflows/ci.yml

@@ -37,7 +37,7 @@ jobs:
     runs-on: ubuntu-latest
 
     container:
-      image: joezuntz/txpipe
+      image: ghcr.io/lsstdesc/txpipe:v0.7
 
     steps:
     - name: Checkout repository
@@ -56,7 +56,7 @@ jobs:
     needs: Download_Data
 
     container:
-      image: joezuntz/txpipe
+      image: ghcr.io/lsstdesc/txpipe:v0.7
 
     steps:
     - name: Checkout repository
@@ -97,7 +97,7 @@ jobs:
     needs: Download_Data
 
     container:
-      image: joezuntz/txpipe
+      image: ghcr.io/lsstdesc/txpipe:v0.7
 
     steps:
     - name: Checkout repository
@@ -127,6 +127,15 @@ jobs:
         ceci examples/metadetect/pipeline.yml
         test -f data/example/outputs_metadetect/shear_xi_plus.png
 
+    - name: Run randoms pipeline
+      run: |
+        ceci examples/randoms/pipeline_randomsonly.yml
+        test -f data/example/outputs_randoms/random_cats.hdf5
+
+    - name: Run notebook
+      run: |
+        jupyter nbconvert --to notebook --execute "notebooks/Welcome to TXPipe.ipynb"
+
     - name: Show logs
       if: ${{ always() }}
       run: |
@@ -139,7 +148,7 @@ jobs:
     needs: Download_Data
 
     container:
-      image: joezuntz/txpipe
+      image: ghcr.io/lsstdesc/txpipe:v0.7
 
     steps:
     - name: Checkout repository
@@ -181,7 +190,7 @@ jobs:
     needs: Download_Data
 
     container:
-      image: joezuntz/txpipe
+      image: ghcr.io/lsstdesc/txpipe:v0.7
 
     steps:
     - name: Checkout repository
@@ -220,7 +229,7 @@ jobs:
     runs-on: ubuntu-latest
 
     container:
-      image: joezuntz/txpipe
+      image: ghcr.io/lsstdesc/txpipe:v0.7
 
     steps:
     - name: Checkout repository

.github/workflows/installer.yml

@@ -17,7 +17,7 @@ jobs:
           - os: ubuntu-latest
             INSTALL_DEPS: sudo apt-get update && sudo apt-get -y install wget
           - os: macos-latest
-            INSTALL_DEPS: brew update && brew install wget
+            INSTALL_DEPS: brew update-reset && brew install wget
     steps:
     - name: Checkout repository
       uses: actions/checkout@v2

.gitmodules

@@ -5,6 +5,3 @@
 [submodule "WLMassMap"]
 	path = submodules/WLMassMap
 	url = https://github.com/LSSTDESC/WLMassMap
-[submodule "submodules/TJPCov"]
-	path = submodules/TJPCov
-	url = https://github.com/LSSTDESC/TJPCov

.readthedocs.yaml

@@ -0,0 +1,16 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+sphinx:
+  configuration: docs/src/conf.py
+
+python:
+  install:
+  - requirements: docs/minimal-requirements.txt

README.md

@@ -49,6 +49,37 @@ ceci examples/metadetect/pipeline.yml
 You can find the outputs in the directory `data/example/outputs_metadetect`
 
 
+Running the example notebook on jupyter at NERSC
+-------------------------------------------------
+
+Follow these steps to run the jupyter notebook at nersc.
+
+1. In a terminal on NERSC, run these commands to prepare and download code and data
+
+```bash
+# Set up the TXPipe kernel
+/global/cfs/cdirs/lsst/groups/WL/users/zuntz/setup-txpipe-jupyter.sh
+
+# Get TXPipe
+cd $SCRATCH
+git clone --recurse-submodules https://github.com/LSSTDESC/TXPipe
+cd TXPipe
+# While this is still in a branch, you also have to do this:
+git checkout example-notebook
+
+# Get data
+curl -O https://portal.nersc.gov/cfs/lsst/txpipe/data/example.tar.gz
+tar -zxvf example.tar.gz
+```
+
+2. Go to https://jupyter.nersc.gov, log in, and select "Perlmutter shared node"
+
+3. Navigate to this notebook in your cloned directory's `notebooks` subdirectory (via the $PSCRATCH link) and open it.
+
+4. Click the button on the top right of the notebook that usually says something "NERSC Python" and select the "TXPipe" kernel in the pop-up.
+
+5. You can now run the notebook
+
 
 Learning more
 -------------

bin/delete_downstream.py

@@ -14,21 +14,25 @@
 import collections
 import os
 
-# start from a config file and a stage to delete
-config = yaml.safe_load(open(sys.argv[1]))
+config_file = sys.argv[1]
 stage_to_delete = sys.argv[2]
 
 # get the stages we need
-stage_names = [s['name'] for s in config['stages']]
-pipeline = ceci.Pipeline(config['stages'], None)
-stages = [ceci.PipelineStage.get_stage(stage_name) for stage_name in stage_names]
+pipe_config = ceci.Pipeline.build_config(config_file, ["resume=False"], dry_run=True)
+
+with ceci.main.prepare_for_pipeline(pipe_config):
+    pipeline = ceci.Pipeline.create(pipe_config)
+
+stage_names = [s['name'] for s in pipe_config['stages']]
+stages = pipeline.stages
 
 # build the mapping tag => stages depending on that tag
 dependencies = collections.defaultdict(list)
 for stage in stages:
-    for tag in stage.input_tags():
+    for tag in stage.find_inputs(pipeline.pipeline_files):
         dependencies[tag].append(stage)
 
+
 # initialize with deletng one stage and the tags it makes
 tags_to_delete = ceci.PipelineStage.get_stage(stage_to_delete).output_tags()
 stages_to_delete = {stage}
@@ -51,5 +55,5 @@
 
 # now at the end we delete all tags output by stage to delete
 for s in stages_to_delete:
-    for f in pipeline.find_outputs(s, config).values():
+    for f in s.find_outputs(pipe_config['output_dir']).values():
         print(f"rm -f {f}")

bin/install.sh

@@ -22,23 +22,10 @@ else
     CHIPSET=x86_64
 fi
 
-if [ "$CHIPSET" = "aarch64" ]
-then
-    echo "Sorry - TXPipe does not yet install on non-x86 systems like M1 macs"
-    exit 1
-fi
-
-if [ "$CHIPSET" = "arm64" ]
-then
-    echo "Sorry - TXPipe does not yet install on non-x86 systems like M1 macs"
-    exit 1
-fi
-
-
+export SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL=True
 
 # URL to download
-URL="https://github.com/conda-forge/miniforge/releases/download/4.11.0-4/Mambaforge-4.11.0-4-${OS}-${CHIPSET}.sh"
-
+URL="https://github.com/conda-forge/miniforge/releases/download/23.1.0-3/Mambaforge-23.1.0-3-${OS}-${CHIPSET}.sh"
 # Download and run the conda installer Miniforge conda installer
 echo "Downloading conda installer from $URL"
 wget -O Mambaforge3.sh $URL
@@ -47,9 +34,16 @@ chmod +x Mambaforge3.sh
 source ./conda/bin/activate
 
 # conda-installable stuff
-mamba install -c conda-forge -y --file conda.txt
-# everything else
-pip install -r requirements.txt || ( sed 's/git+https/git+git/' requirements.txt > requirements2.txt && pip install -r requirements2.txt )
+mamba env update  --file environment.yml
+
+
+if [[ "$CHIPSET" = "arm64" || "$CHIPSET" = "aarch64" ]]
+then
+    echo "Pymaster cannot be correctly conda- or pip-installed on Apple Silicon yet, so we are skipping it."
+    echo "The twopoint fourier and some covariance stage(s) will not work"
+else
+    mamba install -c conda-forge namaster
+fi
 
 echo ""
 echo "Installation successful!"

conda.txt

@@ -1,19 +1,27 @@
-scipy
-matplotlib
+# Additional non-conda dependencies are listed in requirements.txt
+astropy
 camb
+cosmosis
+dask
+dm-tree
+firecrown>=1.4.0
+fitsio
+h5py=*=mpi_mpich_*
+healpix
 healpy
-psutil
+healsparse
+jax
+matplotlib
+mpi4py
+mpich
+namaster
 numpy
-scikit-learn
-fitsio
 pandas
-astropy
+psutil
 pyccl
-mpi4py
-treecorr
-namaster
-dask
-mpich
-h5py=*=mpi_mpich_*
-cosmosis-standalone
-tables-io
+sacc
+scikit-learn
+scipy
+tables-io-full
+threadpoolctl
+treecorr>=4.2.2

environment.yml

@@ -0,0 +1,38 @@
+channels:
+  - conda-forge
+dependencies:
+  - astropy=5.2.*
+  - camb=1.4.*
+  - cosmosis=2.4.1
+  - dask=2023.4.1
+  - dm-tree=0.1.7
+  - firecrown=1.4.0
+  - fitsio=1.1.8
+  - h5py=3.8.*=mpi_mpich_*
+  - healpy=1.16.*
+  - healsparse=1.6.*
+  - jax=0.4.8
+  - jupyter=1.0.*
+  - matplotlib=3.7.*
+  - mpi4py=3.1.*
+  - mpich=4.1.*
+  - numpy=1.24.*
+  - pandas=1.5.*
+  - psutil=5.9.*
+  - pyccl=2.6.*
+  - python=3.10.*
+  - scikit-learn=1.2.*
+  - scipy=1.10.*
+  - tables-io-full=0.8.*
+  - threadpoolctl=3.1.*
+  - treecorr=4.3.*
+  - pip
+  - pip:
+    - ceci==1.13
+    - git+https://github.com/jlvdb/hyperbolic@b88b107a291fa16c2006cf971ce610248d58e94c
+    - git+https://github.com/joezuntz/dask-mpi@31e35759cfb03780a12ad6ffe2b82279b0fab442
+    - healpix==2023.1.13
+    - parallel-statistics==0.13
+    - pz-rail[dev]==0.99.1
+    - sacc==0.8
+    - tjpcov==0.3.0

examples/2.2i/pipeline.yml

@@ -7,7 +7,7 @@ launcher:
 # These site options tell the launcher to use shifter
 site:
     name: cori-interactive
-    image: joezuntz/txpipe
+    image: ghcr.io/lsstdesc/txpipe
     volume: ${PWD}:/opt/txpipe
 
 # modules and packages to import that have pipeline
@@ -18,7 +18,6 @@ modules: txpipe
 # and any other code we need.
 python_paths:
     - submodules/WLMassMap/python/desc/
-    - submodules/TJPCov
     - submodules/FlexZPipe
 
 stages:

examples/2.2i/pipeline_1tract.yml

@@ -7,7 +7,7 @@ launcher:
 # These site options tell the launcher to use shifter
 site:
     name: cori-interactive
-    image: joezuntz/txpipe
+    image: ghcr.io/lsstdesc/txpipe
     volume: ${PWD}:/opt/txpipe
 
 # modules and packages to import that have pipeline
@@ -18,7 +18,6 @@ modules: txpipe
 # and any other code we need.
 python_paths:
     - submodules/WLMassMap/python/desc/
-    - submodules/TJPCov
     - submodules/FlexZPipe
 
 stages:

examples/buzzard/pipeline.yml

@@ -4,7 +4,7 @@ launcher:
 
 site:
     name: cori-interactive
-    image: joezuntz/txpipe
+    image: ghcr.io/lsstdesc/txpipe
 
 modules: txpipe
 

examples/cosmodc2/pipeline.yml

@@ -4,13 +4,12 @@ launcher:
 
 site:
     name: cori-interactive
-    image: joezuntz/txpipe
+    image: ghcr.io/lsstdesc/txpipe
 
 modules: txpipe
 
 python_paths:
     - submodules/WLMassMap/python/desc/
-    - submodules/TJPCov
     - submodules/FlexZPipe
 
 stages:

examples/cosmodc2/pipeline_redmagic.yml

@@ -4,13 +4,12 @@ launcher:
 
 site:
     name: cori-interactive
-    image: joezuntz/txpipe
+    image: ghcr.io/lsstdesc/txpipe
 
 modules: txpipe
 
 python_paths:
     - submodules/WLMassMap/python/desc/
-    - submodules/TJPCov
     - submodules/FlexZPipe
 
 stages:

examples/cosmodc2/pipeline_redmagic_full.yml

@@ -4,13 +4,12 @@ launcher:
 
 site:
     name: cori-interactive
-    image: joezuntz/txpipe
+    image: ghcr.io/lsstdesc/txpipe
 
 modules: txpipe
 
 python_paths:
     - submodules/WLMassMap/python/desc/
-    - submodules/TJPCov
     - submodules/FlexZPipe
 
 stages: