From 82cf9d523b90ad6f7f2fc68943650be3d74c8b61 Mon Sep 17 00:00:00 2001 From: kohan Date: Mon, 4 Dec 2023 18:55:24 -0800 Subject: [PATCH 01/15] build docker only on push master --- .github/workflows/docker-image.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index e00007f..438bdd5 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -8,13 +8,7 @@ name: Docker on: push: - branches: [ "master", "*" ] - # Publish semver tags as releases. - tags: [ 'v*.*.*' ] - paths-ignore: - - 'README.md' - pull_request: - branches: [ "*" ] + branches: [ "master" ] # Publish semver tags as releases. tags: [ 'v*.*.*' ] paths-ignore: From 9eabb3b70a350e243ccadfcc8bc023b020791bb4 Mon Sep 17 00:00:00 2001 From: kohan Date: Mon, 4 Dec 2023 18:56:06 -0800 Subject: [PATCH 02/15] build docker on this branch --- .github/workflows/docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 438bdd5..2cb59b0 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -8,7 +8,7 @@ name: Docker on: push: - branches: [ "master" ] + branches: [ "master", "wi24" ] # Publish semver tags as releases. tags: [ 'v*.*.*' ] paths-ignore: From c88ac003c4ea283e8d6a3f6f6db43a0efb28c00e Mon Sep 17 00:00:00 2001 From: kohan Date: Mon, 4 Dec 2023 19:29:17 -0800 Subject: [PATCH 03/15] test on smaller dockerfile --- Dockerfile | 49 ------------------------------------------------- 1 file changed, 49 deletions(-) diff --git a/Dockerfile b/Dockerfile index f80b1ea..a777a49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,55 +16,6 @@ RUN apt-get update && apt-get install -y \ apt-utils=2.0.9 \ gdebi-core=0.9.5.7+nmu3 \ dpkg-sig=0.13.1+nmu4 \ - manpages=5.05-1 \ - man-db=2.9.1-1 \ - manpages-posix=2013a-2 \ - tree=1.8.0-1 \ && rm -rf /var/lib/apt/lists/* -RUN conda config --set channel_priority strict && \ - mamba install -y -n base -c conda-forge --override-channels bash_kernel nb_conda_kernels - -COPY programming-R.yaml /tmp -RUN mamba env create --file /tmp/programming-R.yaml && \ - mamba clean -afy - -COPY chipseq.yml /tmp -RUN mamba env create --file /tmp/chipseq.yml && \ - mamba clean -afy - -COPY gwas.yml /tmp -RUN mamba env create --file /tmp/gwas.yml && \ - mamba clean -afy - -COPY stats.yml /tmp -RUN mamba env create --file /tmp/stats.yml && \ - mamba clean -afy - -COPY scrna-seq.yml /tmp -RUN mamba env create --file /tmp/scrna-seq.yml && \ - mamba clean -afy - -COPY imgproc.yml /tmp -RUN mamba env create --file /tmp/imgproc.yml && \ - mamba clean -afy - -COPY rna-seq.yaml /tmp -RUN mamba env create --file /tmp/rna-seq.yaml && \ - mamba clean -afy - -COPY spatial-tx.yml /tmp -RUN mamba env create --file /tmp/spatial-tx.yml && \ - mamba clean -afy - -COPY variant_calling.yml /tmp -RUN mamba env create --file /tmp/variant_calling.yml && \ - mamba clean -afy - -COPY networks.yml /tmp -RUN mamba env create --file /tmp/networks.yml && \ - mamba clean -afy - -RUN yes | unminimize || echo "done" - USER $NB_USER From 75acf87f6498a6b5593caadeaf5ff72b11a2e268 Mon Sep 17 00:00:00 2001 From: kohan Date: Mon, 4 Dec 2023 19:53:11 -0800 Subject: [PATCH 04/15] programming env --- Dockerfile | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a777a49..6b26107 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,19 @@ RUN apt-get update && apt-get install -y \ apt-utils=2.0.9 \ gdebi-core=0.9.5.7+nmu3 \ dpkg-sig=0.13.1+nmu4 \ + manpages=5.05-1 \ + man-db=2.9.1-1 \ + manpages-posix=2013a-2 \ + tree=1.8.0-1 \ && rm -rf /var/lib/apt/lists/* -USER $NB_USER +RUN conda config --set channel_priority strict && \ + mamba install -y -n base -c conda-forge --override-channels bash_kernel nb_conda_kernels + +COPY programming-R.yaml /tmp +RUN mamba env create --file /tmp/programming-R.yaml && \ + mamba clean -afy + +RUN yes | unminimize || echo "done" + +USER $NB_USER \ No newline at end of file From 5b30372970489d82595c57ab025da5c52d0cd94a Mon Sep 17 00:00:00 2001 From: kohan Date: Tue, 5 Dec 2023 15:44:54 -0800 Subject: [PATCH 05/15] update dockerfile for building specific module --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6b26107..e756c96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,8 @@ FROM ucsdets/datahub-base-notebook:2023.1-stable USER root +ARG modulename + RUN sed -i 's:^path-exclude=/usr/share/man:#path-exclude=/usr/share/man:' \ /etc/dpkg/dpkg.cfg.d/excludes @@ -25,8 +27,8 @@ RUN apt-get update && apt-get install -y \ RUN conda config --set channel_priority strict && \ mamba install -y -n base -c conda-forge --override-channels bash_kernel nb_conda_kernels -COPY programming-R.yaml /tmp -RUN mamba env create --file /tmp/programming-R.yaml && \ +COPY ${modulename}.yml /tmp +RUN mamba env create --file /tmp/${modulename}.yaml && \ mamba clean -afy RUN yes | unminimize || echo "done" From b446fe6fff31b2cf6c2f958b6170c690df21031b Mon Sep 17 00:00:00 2001 From: kohan Date: Tue, 5 Dec 2023 15:51:23 -0800 Subject: [PATCH 06/15] update yml for multi-tag --- .github/workflows/docker-image.yml | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 2cb59b0..dd9dac2 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -19,11 +19,33 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: + metadata: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.generate-matrix.outputs.matrix }} # contains a list of the module names encoded as json + modulenames: ${{ steps.generate-matrix.outputs.modulenames }} # contains a simple, space-separated list of the module names + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: generate matrix + id: generate-matrix + # adapted from https://stackoverflow.com/a/74829694 + run: | + modulenames="$(ls *.yml | sed 's/.yml//' | paste -s -d ' ')" + echo "modulenames=$modulenames" >> $GITHUB_OUTPUT + # echo $modulenames | tr ' ' '\n' | sed 's/^/type=gha,scope=builder-/' | paste -s + matrix=$(echo $modulenames | jq -R 'split(" ") | {include: map({modulename: .})}') + echo "matrix=$(echo $matrix | jq -c)" >> $GITHUB_OUTPUT + build-and-push-image: runs-on: ubuntu-latest permissions: contents: read packages: write + needs: [metadata] + strategy: + matrix: ${{ fromJson(needs.metadata.outputs.matrix) }} steps: - name: Checkout repository @@ -44,15 +66,21 @@ jobs: uses: docker/metadata-action@v4 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: prune run: docker system prune -af - name: Build and push Docker image + id: builder uses: docker/build-push-action@v3 with: context: . + file: Dockerfile push: true - tags: ${{ steps.meta.outputs.tags }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.modulename }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max + target: builder + build-args: | + "modulename=${{ matrix.modulename }}" \ No newline at end of file From e5676f359c1401efffe82f24fa4a3da61514425a Mon Sep 17 00:00:00 2001 From: kohan Date: Tue, 5 Dec 2023 15:58:51 -0800 Subject: [PATCH 07/15] update yml and dockerfile --- .github/workflows/docker-image.yml | 11 +++++++-- Dockerfile | 36 +++++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index dd9dac2..8c1ad2a 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -13,12 +13,19 @@ on: tags: [ 'v*.*.*' ] paths-ignore: - 'README.md' + pull_request: + branches: [ "*" ] + # Publish semver tags as releases. + tags: [ 'v*.*.*' ] + paths-ignore: + - 'README.md' env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} jobs: + # adapted from https://docs.github.com/en/actions/learn-github-actions/expressions#example-returning-a-json-object metadata: runs-on: ubuntu-latest outputs: @@ -37,8 +44,8 @@ jobs: # echo $modulenames | tr ' ' '\n' | sed 's/^/type=gha,scope=builder-/' | paste -s matrix=$(echo $modulenames | jq -R 'split(" ") | {include: map({modulename: .})}') echo "matrix=$(echo $matrix | jq -c)" >> $GITHUB_OUTPUT - - build-and-push-image: + + build-and-push-images: runs-on: ubuntu-latest permissions: contents: read diff --git a/Dockerfile b/Dockerfile index e756c96..26f49e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,35 @@ -FROM ucsdets/datahub-base-notebook:2023.1-stable - -USER root +# BUILDER CONTAINER +FROM condaforge/mambaforge:4.14.0-0 as builder ARG modulename +COPY ${modulename}.yml /docker/environment.yml + +RUN . /opt/conda/etc/profile.d/conda.sh && \ + mamba create --name lock && \ + conda activate lock && \ + mamba env list && \ + mamba install --yes pip conda-lock>=1.2.2 setuptools wheel && \ + conda-lock lock \ + --platform linux-64 \ + --file /docker/environment.yml \ + --kind lock \ + --lockfile /docker/conda-lock.yml + +RUN . /opt/conda/etc/profile.d/conda.sh && \ + conda activate lock && \ + conda-lock install \ + --mamba \ + --copy \ + --prefix /opt/env \ + /docker/conda-lock.yml + + +# PRODUCTION (AKA PRIMARY) CONTAINER +FROM ucsdets/datahub-base-notebook:2023.1-stable as primary + +USER root + RUN sed -i 's:^path-exclude=/usr/share/man:#path-exclude=/usr/share/man:' \ /etc/dpkg/dpkg.cfg.d/excludes @@ -27,9 +53,7 @@ RUN apt-get update && apt-get install -y \ RUN conda config --set channel_priority strict && \ mamba install -y -n base -c conda-forge --override-channels bash_kernel nb_conda_kernels -COPY ${modulename}.yml /tmp -RUN mamba env create --file /tmp/${modulename}.yaml && \ - mamba clean -afy +#KEEP THIS COMMENT LINE - it is replaced dynamically within github actions to install each of the modules RUN yes | unminimize || echo "done" From f86d766b2d565ab33a03440b79370190a671e38c Mon Sep 17 00:00:00 2001 From: kohan Date: Tue, 5 Dec 2023 16:25:40 -0800 Subject: [PATCH 08/15] fix dockerfile --- .github/workflows/docker-image.yml | 3 +-- Dockerfile | 40 +++++------------------------- 2 files changed, 7 insertions(+), 36 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 8c1ad2a..be0dce1 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -8,7 +8,7 @@ name: Docker on: push: - branches: [ "master", "wi24" ] + branches: [ "master", "*" ] # Publish semver tags as releases. tags: [ 'v*.*.*' ] paths-ignore: @@ -88,6 +88,5 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - target: builder build-args: | "modulename=${{ matrix.modulename }}" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 26f49e4..39cae79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,35 +1,9 @@ -# BUILDER CONTAINER -FROM condaforge/mambaforge:4.14.0-0 as builder - -ARG modulename - -COPY ${modulename}.yml /docker/environment.yml - -RUN . /opt/conda/etc/profile.d/conda.sh && \ - mamba create --name lock && \ - conda activate lock && \ - mamba env list && \ - mamba install --yes pip conda-lock>=1.2.2 setuptools wheel && \ - conda-lock lock \ - --platform linux-64 \ - --file /docker/environment.yml \ - --kind lock \ - --lockfile /docker/conda-lock.yml - -RUN . /opt/conda/etc/profile.d/conda.sh && \ - conda activate lock && \ - conda-lock install \ - --mamba \ - --copy \ - --prefix /opt/env \ - /docker/conda-lock.yml - - -# PRODUCTION (AKA PRIMARY) CONTAINER -FROM ucsdets/datahub-base-notebook:2023.1-stable as primary +FROM ucsdets/datahub-base-notebook:2023.1-stable USER root +ARG modulename + RUN sed -i 's:^path-exclude=/usr/share/man:#path-exclude=/usr/share/man:' \ /etc/dpkg/dpkg.cfg.d/excludes @@ -53,8 +27,6 @@ RUN apt-get update && apt-get install -y \ RUN conda config --set channel_priority strict && \ mamba install -y -n base -c conda-forge --override-channels bash_kernel nb_conda_kernels -#KEEP THIS COMMENT LINE - it is replaced dynamically within github actions to install each of the modules - -RUN yes | unminimize || echo "done" - -USER $NB_USER \ No newline at end of file +COPY ${modulename}.yml /tmp +RUN mamba env create --file /tmp/${modulename}.yml && \ + mamba clean -afy \ No newline at end of file From f044e1525d22a93d48474bc12df2b15caa907c6c Mon Sep 17 00:00:00 2001 From: kohan Date: Tue, 5 Dec 2023 16:47:52 -0800 Subject: [PATCH 09/15] rename yaml to yml --- .github/workflows/docker-image.yml | 4 ++-- programming-R.yaml => programming-R.yml | 0 rna-seq.yaml => rna-seq.yml | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename programming-R.yaml => programming-R.yml (100%) rename rna-seq.yaml => rna-seq.yml (100%) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index be0dce1..d8ef0fa 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -8,13 +8,13 @@ name: Docker on: push: - branches: [ "master", "*" ] + branches: [ "master", "wi24" ] # Publish semver tags as releases. tags: [ 'v*.*.*' ] paths-ignore: - 'README.md' pull_request: - branches: [ "*" ] + branches: [ "master", "wi24" ] # Publish semver tags as releases. tags: [ 'v*.*.*' ] paths-ignore: diff --git a/programming-R.yaml b/programming-R.yml similarity index 100% rename from programming-R.yaml rename to programming-R.yml diff --git a/rna-seq.yaml b/rna-seq.yml similarity index 100% rename from rna-seq.yaml rename to rna-seq.yml From b6c75fe21dcc79a8ec69fea0dcdeba403c3835ab Mon Sep 17 00:00:00 2001 From: kohan Date: Tue, 5 Dec 2023 16:55:57 -0800 Subject: [PATCH 10/15] add unminimize to dockerfile --- Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 39cae79..1551ca1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,4 +29,8 @@ RUN conda config --set channel_priority strict && \ COPY ${modulename}.yml /tmp RUN mamba env create --file /tmp/${modulename}.yml && \ - mamba clean -afy \ No newline at end of file + mamba clean -afy + +RUN yes | unminimize || echo "done" + +USER $NB_USER \ No newline at end of file From cf888071fe243256bcdcb94465c61335676b2c1d Mon Sep 17 00:00:00 2001 From: Daniella Vo <35377884+daniellavo@users.noreply.github.com> Date: Mon, 11 Dec 2023 14:30:54 -0800 Subject: [PATCH 11/15] Wi24 popgen (#41) * Create popgen.yml * add popgen code to create env * add irkernel * update dockerfile --------- Co-authored-by: Arya Massarat <23412689+aryarm@users.noreply.github.com> Co-authored-by: kohan --- popgen.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 popgen.yml diff --git a/popgen.yml b/popgen.yml new file mode 100644 index 0000000..7bb2e11 --- /dev/null +++ b/popgen.yml @@ -0,0 +1,7 @@ +name: popgen +channels: +- conda-forge +- nodefaults +dependencies: +- conda-forge::r-irkernel==1.3.2 +- conda-forge::r-data.table==1.14.8 From fd7434ddd1f27993601a2e8bcfb581d37bdc990f Mon Sep 17 00:00:00 2001 From: Daniella Vo <35377884+daniellavo@users.noreply.github.com> Date: Fri, 19 Jan 2024 12:55:33 -0800 Subject: [PATCH 12/15] add missing dependencies --- rna-seq.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rna-seq.yml b/rna-seq.yml index 89946c5..cebf196 100644 --- a/rna-seq.yml +++ b/rna-seq.yml @@ -4,6 +4,12 @@ channels: - bioconda - nodefaults dependencies: +- bioconda::fastqc==0.11.9 +- bioconda::star==2.7.10b +- bioconda::samtools==1.16.1 +- bioconda::subread==2.0.3 +- bioconda::multiqc==1.14 +- conda-forge::ipykernel==6.22.0 - conda-forge::r-irkernel==1.3.1 - conda-forge::r-ggplot2==3.4.0 - conda-forge::r-gplots==3.1.3 From 3d56bcffe1ff8ed3aa8400f0e2350bf029894499 Mon Sep 17 00:00:00 2001 From: Daniella Vo <35377884+daniellavo@users.noreply.github.com> Date: Fri, 19 Jan 2024 14:28:24 -0800 Subject: [PATCH 13/15] fix multiqc version --- rna-seq.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rna-seq.yml b/rna-seq.yml index cebf196..cd4988a 100644 --- a/rna-seq.yml +++ b/rna-seq.yml @@ -8,7 +8,7 @@ dependencies: - bioconda::star==2.7.10b - bioconda::samtools==1.16.1 - bioconda::subread==2.0.3 -- bioconda::multiqc==1.14 +- bioconda::multiqc==1.17 - conda-forge::ipykernel==6.22.0 - conda-forge::r-irkernel==1.3.1 - conda-forge::r-ggplot2==3.4.0 From 809b1cfd34f11f1eea20940360e6962f4c85a1cc Mon Sep 17 00:00:00 2001 From: Daniella Vo <35377884+daniellavo@users.noreply.github.com> Date: Mon, 22 Jan 2024 09:00:56 -0800 Subject: [PATCH 14/15] add missing r packages --- rna-seq.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rna-seq.yml b/rna-seq.yml index cd4988a..eef89ab 100644 --- a/rna-seq.yml +++ b/rna-seq.yml @@ -14,6 +14,7 @@ dependencies: - conda-forge::r-ggplot2==3.4.0 - conda-forge::r-gplots==3.1.3 - conda-forge::r-data.table==1.14.6 +- conda-forge::r-readr==2.1.5 - bioconda::bioconductor-limma==3.54.0 - bioconda::bioconductor-glimma==2.8.0 - bioconda::bioconductor-deseq2==1.38.0 @@ -22,3 +23,6 @@ dependencies: - bioconda::bioconductor-affycoretools==1.70.0 - bioconda::bioconductor-biomart==2.54.0 - bioconda::bioconductor-edger==3.40.0 +- bioconda::bioconductor-tximport==1.30.0 +- bioconda::bioconductor-tximportdata==1.30.0 +- bioconda::bioconductor-tximeta==1.20.1 From b9b9f4e6ddd45df8254c0d4e4c7183365e198952 Mon Sep 17 00:00:00 2001 From: Daniella Vo <35377884+daniellavo@users.noreply.github.com> Date: Mon, 22 Jan 2024 10:10:11 -0800 Subject: [PATCH 15/15] update and add r packages --- rna-seq.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/rna-seq.yml b/rna-seq.yml index eef89ab..ce057d9 100644 --- a/rna-seq.yml +++ b/rna-seq.yml @@ -9,20 +9,20 @@ dependencies: - bioconda::samtools==1.16.1 - bioconda::subread==2.0.3 - bioconda::multiqc==1.17 -- conda-forge::ipykernel==6.22.0 -- conda-forge::r-irkernel==1.3.1 -- conda-forge::r-ggplot2==3.4.0 +- conda-forge::ipykernel==6.29.0 +- conda-forge::r-irkernel==1.3.2 +- conda-forge::r-ggplot2==3.4.4 - conda-forge::r-gplots==3.1.3 -- conda-forge::r-data.table==1.14.6 +- conda-forge::r-data.table==1.14.10 - conda-forge::r-readr==2.1.5 -- bioconda::bioconductor-limma==3.54.0 -- bioconda::bioconductor-glimma==2.8.0 -- bioconda::bioconductor-deseq2==1.38.0 +- bioconda::bioconductor-limma==3.58.1 +- bioconda::bioconductor-glimma==2.12.0 +- bioconda::bioconductor-deseq2==1.42.0 - conda-forge::r-rcolorbrewer==1.1_3 -- bioconda::bioconductor-apeglm==1.20.0 -- bioconda::bioconductor-affycoretools==1.70.0 -- bioconda::bioconductor-biomart==2.54.0 -- bioconda::bioconductor-edger==3.40.0 +- bioconda::bioconductor-apeglm==1.24.0 +- bioconda::bioconductor-affycoretools==1.74.0 +- bioconda::bioconductor-biomart==2.58.0 +- bioconda::bioconductor-edger==4.0.2 - bioconda::bioconductor-tximport==1.30.0 - bioconda::bioconductor-tximportdata==1.30.0 - bioconda::bioconductor-tximeta==1.20.1