Merge pull request #456 from sunbeam-labs/455-restructure-containeriz…

…ation 455 restructure containerization
sunbeam-labs · Mar 4, 2024 · f4a7a5c · f4a7a5c
2 parents 7192399 + 1ca5c60
commit f4a7a5c
Show file tree

Hide file tree

Showing 25 changed files with 507 additions and 156 deletions.
diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml
@@ -12,6 +12,9 @@ jobs:
 
         steps:
           - uses: actions/checkout@v4
+
+          - id: get_version
+            uses: battila7/get-version-action@v2
 
           - name: Log in to Docker Hub
             uses: docker/login-action@v3
@@ -23,13 +26,158 @@ jobs:
             id: meta
             uses: docker/metadata-action@v5
             with:
-              images: ctbushman/sunbeam
+              images: sunbeamlabs/sunbeam
+
+          - name: Build test
+            uses: docker/build-push-action@v5
+            with:
+              context: .
+              file: ./Dockerfile
+              load: true
+              tags: sunbeamlabs/sunbeam:test
+
+          - name: Test
+            run: |
+              docker run --rm sunbeamlabs/sunbeam:test pytest tests/
+            
+          - name: Build slim test
+            uses: docker/build-push-action@v5
+            with:
+              context: .
+              file: ./slim.Dockerfile
+              load: true
+              tags: sunbeamlabs/sunbeam:test-slim
+
+          - name: Test slim
+            run: |
+              docker run --rm sunbeamlabs/sunbeam:test-slim pytest tests/
           
           - name: Build and push Docker image
             uses: docker/build-push-action@v5
             with:
               context: .
               file: ./Dockerfile
               push: true
-              tags: ${{ steps.meta.outputs.tags }}
-              labels: ${{ steps.meta.outputs.labels }}
+              tags: sunbeamlabs/sunbeam:latest, sunbeamlabs/sunbeam:${{ steps.get_version.outputs.version-without-v }}
+              labels: ${{ steps.meta.outputs.labels }}
+
+          - name: Build and push slim Docker image
+            uses: docker/build-push-action@v5
+            with:
+              context: .
+              file: ./slim.Dockerfile
+              push: true
+              tags: sunbeamlabs/sunbeam:${{ steps.get_version.outputs.version-without-v }}-slim
+              labels: ${{ steps.meta.outputs.labels }}
+
+    push-cutadapt-dockerhub:
+      name: Push cutadapt env to Dockerhub
+      runs-on: ubuntu-latest
+
+      steps:
+        - uses: actions/checkout@v4
+
+        - name: Log in to Docker Hub
+          uses: docker/login-action@v3
+          with:
+            username: ${{ secrets.DOCKER_USERNAME }}
+            password: ${{ secrets.DOCKER_PASSWORD }}
+
+        - name: Extract metadata (tags, labels) for Docker
+          id: meta
+          uses: docker/metadata-action@v5
+          with:
+            images: sunbeamlabs/cutadapt
+
+        - name: Build and push Docker image
+          uses: docker/build-push-action@v5
+          with:
+            context: .
+            file: workflow/envs/cutadapt.Dockerfile
+            push: true
+            tags: ${{ steps.meta.outputs.tags }}
+            labels: ${{ steps.meta.outputs.labels }}
+
+    push-komplexity-dockerhub:
+      name: Push komplexity env to Dockerhub
+      runs-on: ubuntu-latest
+
+      steps:
+        - uses: actions/checkout@v4
+
+        - name: Log in to Docker Hub
+          uses: docker/login-action@v3
+          with:
+            username: ${{ secrets.DOCKER_USERNAME }}
+            password: ${{ secrets.DOCKER_PASSWORD }}
+
+        - name: Extract metadata (tags, labels) for Docker
+          id: meta
+          uses: docker/metadata-action@v5
+          with:
+            images: sunbeamlabs/komplexity
+
+        - name: Build and push Docker image
+          uses: docker/build-push-action@v5
+          with:
+            context: .
+            file: workflow/envs/komplexity.Dockerfile
+            push: true
+            tags: ${{ steps.meta.outputs.tags }}
+            labels: ${{ steps.meta.outputs.labels }}
+
+    push-qc-dockerhub:
+      name: Push qc env to Dockerhub
+      runs-on: ubuntu-latest
+
+      steps:
+        - uses: actions/checkout@v4
+
+        - name: Log in to Docker Hub
+          uses: docker/login-action@v3
+          with:
+            username: ${{ secrets.DOCKER_USERNAME }}
+            password: ${{ secrets.DOCKER_PASSWORD }}
+
+        - name: Extract metadata (tags, labels) for Docker
+          id: meta
+          uses: docker/metadata-action@v5
+          with:
+            images: sunbeamlabs/qc
+
+        - name: Build and push Docker image
+          uses: docker/build-push-action@v5
+          with:
+            context: .
+            file: workflow/envs/qc.Dockerfile
+            push: true
+            tags: ${{ steps.meta.outputs.tags }}
+            labels: ${{ steps.meta.outputs.labels }}
+
+    push-reports-dockerhub:
+      name: Push reports env to Dockerhub
+      runs-on: ubuntu-latest
+
+      steps:
+        - uses: actions/checkout@v4
+
+        - name: Log in to Docker Hub
+          uses: docker/login-action@v3
+          with:
+            username: ${{ secrets.DOCKER_USERNAME }}
+            password: ${{ secrets.DOCKER_PASSWORD }}
+
+        - name: Extract metadata (tags, labels) for Docker
+          id: meta
+          uses: docker/metadata-action@v5
+          with:
+            images: sunbeamlabs/reports
+
+        - name: Build and push Docker image
+          uses: docker/build-push-action@v5
+          with:
+            context: .
+            file: workflow/envs/reports.Dockerfile
+            push: true
+            tags: ${{ steps.meta.outputs.tags }}
+            labels: ${{ steps.meta.outputs.labels }}
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -4,9 +4,6 @@ on:
     pull_request:
       branches:
         - main
-    push:
-      branches:
-        - main
 
 jobs:
     run-tests:

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -0,0 +1,12 @@
+name: Tests
+
+on:
+    push:
+      branches:
+        - main
+
+jobs:
+    run-tests:
+        uses: ./.github/workflows/tests.yml
+        secrets: inherit
+
diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,8 @@ config.yaml
 config.yml
 configs/config*.yml
 .snakemake
+.conda/
+.condarc
 \#*
 output
 __pycache__
@@ -22,4 +24,5 @@ build
 projects/
 .vscode/
 .cache/
-.java/
+.java/
+sunbeam.tar.gz
diff --git a/Dockerfile b/Dockerfile
@@ -1,66 +1,44 @@
 FROM condaforge/mambaforge:latest
-LABEL io.github.snakemake.containerized="true"
-LABEL io.github.snakemake.conda_env_hash="12452d6af82ca1066b553b1905c85b62405942009ebafb5e897d30b84646e799"
 
-# Step 1: Retrieve conda environments
+# Setup
+WORKDIR /home/sunbeam
 
-# Conda environment:
-#   source: workflow/envs/cutadapt.yml
-#   prefix: /conda-envs/08b0272f8c744b8bb162030774cf9917
-#   channels:
-#     - bioconda
-#   dependencies:
-#     - cutadapt
-#     #- python =3.12.0
-#   name: cutadapt
-RUN mkdir -p /conda-envs/08b0272f8c744b8bb162030774cf9917
-COPY workflow/envs/cutadapt.yml /conda-envs/08b0272f8c744b8bb162030774cf9917/environment.yaml
+RUN mkdir -p etc/
+COPY etc/* etc/
 
-# Conda environment:
-#   source: workflow/envs/komplexity.yml
-#   prefix: /conda-envs/cc72134ebb57f052a6ac7d5a084f9bf6
-#   channels:
-#     - eclarke
-#     - conda-forge
-#   dependencies:
-#     - komplexity
-#     - python =3.12.0
-#   name: komplexity
-RUN mkdir -p /conda-envs/cc72134ebb57f052a6ac7d5a084f9bf6
-COPY workflow/envs/komplexity.yml /conda-envs/cc72134ebb57f052a6ac7d5a084f9bf6/environment.yaml
+RUN mkdir -p extensions/
+COPY extensions/.placeholder extensions/
 
-# Conda environment:
-#   source: workflow/envs/qc.yml
-#   prefix: /conda-envs/2e2d3aeac664927deed435e9fde5bf30
-#   channels:
-#     - bioconda
-#     - conda-forge
-#   dependencies:
-#     - bwa
-#     - fastqc
-#     - trimmomatic
-#     - python =3.12.0
-#   name: qc
-RUN mkdir -p /conda-envs/2e2d3aeac664927deed435e9fde5bf30
-COPY workflow/envs/qc.yml /conda-envs/2e2d3aeac664927deed435e9fde5bf30/environment.yaml
+RUN mkdir -p src/sunbeamlib/
+COPY src/sunbeamlib/* src/sunbeamlib/
 
-# Conda environment:
-#   source: workflow/envs/reports.yml
-#   prefix: /conda-envs/60172346a36951e9f4497f0c766b8d74
-#   channels:
-#     - conda-forge
-#   dependencies:
-#     - numpy
-#     - pandas
-#     - python =3.12.0
-#   name: reports
-RUN mkdir -p /conda-envs/60172346a36951e9f4497f0c766b8d74
-COPY workflow/envs/reports.yml /conda-envs/60172346a36951e9f4497f0c766b8d74/environment.yaml
+COPY tests/ tests/
 
-# Step 2: Generate conda environments
+COPY workflow/ workflow/
 
-RUN mamba env create --prefix /conda-envs/08b0272f8c744b8bb162030774cf9917 --file /conda-envs/08b0272f8c744b8bb162030774cf9917/environment.yaml && \
-    mamba env create --prefix /conda-envs/cc72134ebb57f052a6ac7d5a084f9bf6 --file /conda-envs/cc72134ebb57f052a6ac7d5a084f9bf6/environment.yaml && \
-    mamba env create --prefix /conda-envs/2e2d3aeac664927deed435e9fde5bf30 --file /conda-envs/2e2d3aeac664927deed435e9fde5bf30/environment.yaml && \
-    mamba env create --prefix /conda-envs/60172346a36951e9f4497f0c766b8d74 --file /conda-envs/60172346a36951e9f4497f0c766b8d74/environment.yaml && \
-    mamba clean --all -y
+COPY environment.yml install.sh MANIFEST.in pyproject.toml pytest.ini README.md ./
+
+# Install sunbeam
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y git
+RUN ./install.sh -e sunbeam -v
+
+ENV PATH="/opt/conda/envs/sunbeam/bin/:${PATH}"
+ENV SUNBEAM_DIR="/home/sunbeam"
+ENV SUNBEAM_VER="4.4.0"
+ENV SUNBEAM_MIN_MEM_MB="8000"
+ENV SUNBEAM_MIN_RUNTIME="60"
+
+# Install conda environments
+RUN mkdir -p projects
+RUN sunbeam init --data_fp tests/data/reads projects/init
+RUN sunbeam run --profile projects/init --conda-create-envs-only --mamba
+RUN rm -r projects
+
+# "Activate" the environment
+SHELL ["conda", "run", "-n", "sunbeam", "/bin/bash", "-c"]
+#RUN bash /opt/conda/envs/sunbeam/etc/conda/activate.d/env_vars.sh
+
+# Run
+CMD "bash"
diff --git a/README.md b/README.md
@@ -4,7 +4,7 @@
 
 [![Tests](https://github.com/sunbeam-labs/sunbeam/actions/workflows/pr.yml/badge.svg)](https://github.com/sunbeam-labs/sunbeam/actions/workflows/pr.yml)
 [![Documentation Status](https://readthedocs.org/projects/sunbeam/badge/?version=stable)](https://sunbeam.readthedocs.io/en/stable/?badge=stable)
-[![DockerHub](https://img.shields.io/docker/pulls/ctbushman/sunbeam)](https://hub.docker.com/repository/docker/ctbushman/sunbeam/)
+[![DockerHub](https://img.shields.io/docker/pulls/sunbeamlabs/sunbeam)](https://hub.docker.com/repository/docker/sunbeamlabs/sunbeam/)
 [![DOI:10.1186/s40168-019-0658-x](https://img.shields.io/badge/Published%20in-Microbiome-1abc9c.svg)](https://doi.org/10.1186/s40168-019-0658-x)
 
 Sunbeam is a pipeline written in [snakemake](http://snakemake.readthedocs.io) that simplifies and automates many of the steps in metagenomic sequencing analysis. It uses [conda](http://conda.io) to manage dependencies, so it doesn't have pre-existing dependencies or admin privileges, and can be deployed on most Linux workstations and clusters. Sunbeam was designed to be modular and extensible, allowing anyone to build off the core functionality. To read more, check out [our paper in Microbiome](https://microbiomejournal.biomedcentral.com/articles/10.1186/s40168-019-0658-x).