merge wi24 into master (#45)

* build docker only on push master

* build docker on this branch

* test on smaller dockerfile

* programming env

* update dockerfile for building specific module

* update yml for multi-tag

* update yml and dockerfile

* fix dockerfile

* rename yaml to yml

* add unminimize to dockerfile

* Wi24 popgen (#41)

* Create popgen.yml

* add popgen code to create env

* add irkernel

* update dockerfile

---------

Co-authored-by: Arya Massarat <[email protected]>
Co-authored-by: kohan <[email protected]>

* add missing dependencies

* fix multiqc version

* add missing r packages

* update and add r packages

---------

Co-authored-by: Daniella Vo <[email protected]>
Co-authored-by: Arya Massarat <[email protected]>

.github/workflows/docker-image.yml

@@ -8,13 +8,13 @@ name: Docker
 
 on:
   push:
-    branches: [ "master", "*" ]
+    branches: [ "master", "wi24" ]
     # Publish semver tags as releases.
     tags: [ 'v*.*.*' ]
     paths-ignore:
       - 'README.md'
   pull_request:
-    branches: [ "*" ]
+    branches: [ "master", "wi24" ]
     # Publish semver tags as releases.
     tags: [ 'v*.*.*' ]
     paths-ignore:
@@ -25,11 +25,34 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  build-and-push-image:
+  # adapted from https://docs.github.com/en/actions/learn-github-actions/expressions#example-returning-a-json-object
+  metadata:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.generate-matrix.outputs.matrix }} # contains a list of the module names encoded as json
+      modulenames: ${{ steps.generate-matrix.outputs.modulenames }} # contains a simple, space-separated list of the module names
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: generate matrix
+        id: generate-matrix
+        # adapted from https://stackoverflow.com/a/74829694
+        run: |
+          modulenames="$(ls *.yml | sed 's/.yml//' | paste -s -d ' ')"
+          echo "modulenames=$modulenames" >> $GITHUB_OUTPUT
+          # echo $modulenames | tr ' ' '\n' | sed 's/^/type=gha,scope=builder-/' | paste -s
+          matrix=$(echo $modulenames | jq -R 'split(" ") | {include: map({modulename: .})}')
+          echo "matrix=$(echo $matrix | jq -c)" >> $GITHUB_OUTPUT
+
+  build-and-push-images:
     runs-on: ubuntu-latest
     permissions:
       contents: read
       packages: write
+    needs: [metadata]
+    strategy:
+      matrix: ${{ fromJson(needs.metadata.outputs.matrix) }}
 
     steps:
       - name: Checkout repository
@@ -50,15 +73,20 @@ jobs:
         uses: docker/metadata-action@v4
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
       - name: prune
         run: docker system prune -af
 
       - name: Build and push Docker image
+        id: builder
         uses: docker/build-push-action@v3
         with:
           context: .
+          file: Dockerfile
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.modulename }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
+          build-args: |
+            "modulename=${{ matrix.modulename }}"

Dockerfile

@@ -2,6 +2,8 @@ FROM ucsdets/datahub-base-notebook:2023.1-stable
 
 USER root
 
+ARG modulename
+
 RUN sed -i 's:^path-exclude=/usr/share/man:#path-exclude=/usr/share/man:' \
     /etc/dpkg/dpkg.cfg.d/excludes
 
@@ -25,46 +27,10 @@ RUN apt-get update && apt-get install -y \
 RUN conda config --set channel_priority strict && \
     mamba install -y -n base -c conda-forge --override-channels bash_kernel nb_conda_kernels
 
-COPY programming-R.yaml /tmp
-RUN mamba env create --file /tmp/programming-R.yaml && \
-    mamba clean -afy
-
-COPY chipseq.yml /tmp
-RUN mamba env create --file /tmp/chipseq.yml && \
-    mamba clean -afy
-
-COPY gwas.yml /tmp
-RUN mamba env create --file /tmp/gwas.yml && \
-    mamba clean -afy
-
-COPY stats.yml /tmp
-RUN mamba env create --file /tmp/stats.yml && \
-    mamba clean -afy
-
-COPY scrna-seq.yml /tmp
-RUN mamba env create --file /tmp/scrna-seq.yml && \
-    mamba clean -afy
-
-COPY imgproc.yml /tmp
-RUN mamba env create --file /tmp/imgproc.yml && \
-    mamba clean -afy
-
-COPY rna-seq.yaml /tmp
-RUN mamba env create --file /tmp/rna-seq.yaml && \
-    mamba clean -afy
-
-COPY spatial-tx.yml /tmp
-RUN mamba env create --file /tmp/spatial-tx.yml && \
-    mamba clean -afy
-
-COPY variant_calling.yml /tmp
-RUN mamba env create --file /tmp/variant_calling.yml && \
-    mamba clean -afy
-
-COPY networks.yml /tmp
-RUN mamba env create --file /tmp/networks.yml && \
+COPY ${modulename}.yml /tmp
+RUN mamba env create --file /tmp/${modulename}.yml && \
     mamba clean -afy
 
 RUN yes | unminimize || echo "done"
 
-USER $NB_USER
+USER $NB_USER

popgen.yml

@@ -0,0 +1,7 @@
+name: popgen
+channels:
+- conda-forge
+- nodefaults
+dependencies:
+- conda-forge::r-irkernel==1.3.2
+- conda-forge::r-data.table==1.14.8

rna-seq.yml

@@ -0,0 +1,28 @@
+name: rna-seq
+channels:
+- conda-forge
+- bioconda
+- nodefaults
+dependencies:
+- bioconda::fastqc==0.11.9
+- bioconda::star==2.7.10b
+- bioconda::samtools==1.16.1
+- bioconda::subread==2.0.3
+- bioconda::multiqc==1.17
+- conda-forge::ipykernel==6.29.0
+- conda-forge::r-irkernel==1.3.2
+- conda-forge::r-ggplot2==3.4.4
+- conda-forge::r-gplots==3.1.3
+- conda-forge::r-data.table==1.14.10
+- conda-forge::r-readr==2.1.5
+- bioconda::bioconductor-limma==3.58.1
+- bioconda::bioconductor-glimma==2.12.0
+- bioconda::bioconductor-deseq2==1.42.0
+- conda-forge::r-rcolorbrewer==1.1_3
+- bioconda::bioconductor-apeglm==1.24.0
+- bioconda::bioconductor-affycoretools==1.74.0
+- bioconda::bioconductor-biomart==2.58.0
+- bioconda::bioconductor-edger==4.0.2
+- bioconda::bioconductor-tximport==1.30.0
+- bioconda::bioconductor-tximportdata==1.30.0
+- bioconda::bioconductor-tximeta==1.20.1