Merge branch 'bacpop-186-v9-db-support' of https://github.com/bacpop/…

…beebop_py into bacpop-202-fallback-to-refs

.github/workflows/build_and_push.yml

@@ -0,0 +1,26 @@
+name: build and push docker images
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - "*"
+env:
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Login to GHCR (GitHub Packages)
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # add --with-dev to below commands to build & push the dev image
+      - name: Build docker image
+        run: ./docker/build 
+      - name: Push docker image
+        run: ./docker/push 

README.md

@@ -112,3 +112,19 @@ TESTING=True poetry run pytest
 
 - There is a .drawio graph in the `diagrams` folder illustrating the process of running a analysis. This includes
 all the files created and how they are used in each job. You can open and view the diagram at [draw.io](https://draw.io).
+
+## Use/Deploy specific version of PopPUNK
+
+To use a specific version, commit or branch of PopPUNK in a beebop_py deployment, you can update `POPPUNK_VERSION` in `common`.
+
+The new dev images built with `/docker/build --with-dev` will have a *-dev* postfix.
+
+### Local Development
+
+You can build the image with `/docker/build --with-dev`, this new image can now be used by Beebop.
+
+### Deployment
+
+A pull request can be created so GHA pushes the images to the docker hub. Add `--with-dev` to the build & push commands `pipeline.yaml`.
+**Ensure to remove the `--with-dev` flag before merging the PR.**
+Then on the `beebop-deploy` the api image can be updated with the new dev image.

beebop/app.py

@@ -303,7 +303,7 @@ def run_poppunk_internal(sketches: dict,
                             depends_on=job_assign, **queue_kwargs)
     redis.hset("beebop:hash:job:network", p_hash, job_network.id)
     # microreact
-    # delete all previous microreact cluster job results
+    # delete all previous microreact cluster job results for this project
     redis.delete(f"beebop:hash:job:microreact:{p_hash}")
     job_microreact = q.enqueue(
         visualise.microreact,

docker/Dockerfile.dev

@@ -0,0 +1,61 @@
+FROM continuumio/miniconda3
+
+# ensure conda does not install packages for the wrong architecture
+ENV CONDA_OVERRIDE_ARCHSPEC=skylake
+
+ARG POPPUNK_VERSION="v2.6.7"
+
+# Make RUN commands use the new environment:
+SHELL ["conda", "run", "-n", "base", "/bin/bash", "-c"]
+
+# Set up conda environment
+RUN conda install python=3.10
+RUN conda config --append channels conda-forge && \
+    conda config --append channels bioconda
+
+# Install mamba: which is a faster package manager than conda
+RUN conda install -c conda-forge mamba
+RUN conda config --set channel_priority flexible
+
+#Install PopPUNK conda dependencies
+RUN mamba install -y -c conda-forge -y graph-tool mandrake 
+RUN mamba install -y \
+    # Core data packages
+    pandas \
+    requests \
+    networkx \
+    scikit-learn \
+    # Bioinformatics tools
+    pp-sketchlib \
+    biopython \
+    treeswift \
+    rapidnj \
+    # Analysis tools
+    hdbscan \
+    # Progress tracking
+    tqdm \
+    && mamba clean -afy
+
+# System dependencies 
+RUN apt-get update && \
+    apt-get install -y \
+    build-essential \
+    cmake \
+    libeigen3-dev \
+    libhdf5-dev \
+    libopenblas-dev
+
+
+RUN pip install git+https://github.com/bacpop/PopPUNK@${POPPUNK_VERSION}#egg=PopPUNK 
+
+# Poetry setup
+RUN pip install poetry
+COPY *.toml *.lock /
+RUN poetry config virtualenvs.create false && \
+    poetry install
+
+COPY . /beebop
+WORKDIR /beebop
+EXPOSE 5000
+
+CMD ["conda", "run" ,"--no-capture-output", "-n", "base", "poetry", "run", "waitress-serve", "--port=5000", "beebop.app:app"]

docker/build

@@ -4,11 +4,21 @@ set -ex
 HERE=$(dirname $0)
 . $HERE/common
 
+# Build and push the production image
 docker build --pull \
        --tag $TAG_SHA \
-       -f docker/Dockerfile \
+       -f docker/Dockerfile.prod \
        $PACKAGE_ROOT
-
 # We always push the SHA tagged versions, for debugging if the tests
 # after this step fail
 docker push $TAG_SHA
+
+# Build and push dev image only if --with-dev flag is set
+if [ "$1" == "--with-dev" ]; then
+    docker build --pull \
+           --build-arg POPPUNK_VERSION=$POPPUNK_VERSION \
+           --tag $TAG_DEV_SHA \
+           -f docker/Dockerfile.dev \
+           $PACKAGE_ROOT
+    docker push $TAG_DEV_SHA
+fi

docker/common

@@ -1,25 +1,24 @@
 #!/usr/bin/env bash
 set -e
-
+REGISTRY=ghcr.io
 PACKAGE_ROOT=$(realpath $HERE/..)
 PACKAGE_NAME=beebop-py
-PACKAGE_ORG=mrcide
+PACKAGE_ORG=bacpop
+PACKAGE_DEV=dev
 
-# Buildkite doesn't check out a full history from the remote (just the
-# single commit) so you end up with a detached head and git rev-parse
-# doesn't work
-if [ "$BUILDKITE" = "true" ]; then
-    GIT_SHA=${BUILDKITE_COMMIT:0:7}
+GIT_SHA=$(git -C "$PACKAGE_ROOT" rev-parse --short=7 HEAD)
+if [[ -v "BRANCH_NAME" ]]; then
+    GIT_BRANCH=${BRANCH_NAME}
 else
-    GIT_SHA=$(git -C "$PACKAGE_ROOT" rev-parse --short=7 HEAD)
+    GIT_BRANCH=$(git symbolic-ref --short HEAD)
 fi
 
-if [ "$BUILDKITE" = "true" ]; then
-    GIT_BRANCH=$BUILDKITE_BRANCH
-else
-    GIT_BRANCH=$(git -C "$PACKAGE_ROOT" symbolic-ref --short HEAD)
-fi
+# production image
+TAG_SHA="${REGISTRY}/${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_SHA}"
+TAG_BRANCH="${REGISTRY}/${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}"
+TAG_LATEST="${REGISTRY}/${PACKAGE_ORG}/${PACKAGE_NAME}:latest"
 
-TAG_SHA="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_SHA}"
-TAG_BRANCH="${PACKAGE_ORG}/${PACKAGE_NAME}:${GIT_BRANCH}"
-TAG_LATEST="${PACKAGE_ORG}/${PACKAGE_NAME}:latest"
+# development image
+TAG_DEV_SHA="${REGISTRY}/${TAG_SHA}-${PACKAGE_DEV}"
+TAG_DEV_BRANCH="${REGISTRY}/${TAG_BRANCH}-${PACKAGE_DEV}"
+POPPUNK_VERSION=v2.6.7 # can be version, branch or commit

docker/push

@@ -5,11 +5,16 @@ HERE=$(dirname $0)
 . $HERE/common
 
 # In case we switch agents between steps
-[ ! -z $(docker images -q $TAG_SHA) ] || docker pull $TAG_SHA
 
 docker tag $TAG_SHA $TAG_BRANCH
 docker push $TAG_BRANCH
 
+if [ "$1" == "--with-dev" ]; then
+    [ ! -z $(docker images -q $TAG_DEV_SHA) ] || docker pull $TAG_DEV_SHA
+    docker tag $TAG_DEV_SHA $TAG_DEV_BRANCH
+    docker push $TAG_DEV_BRANCH
+fi
+
 if [ $GIT_BRANCH == "main" ]; then
    docker tag $TAG_SHA $TAG_LATEST
    docker push $TAG_LATEST

scripts/run_app

@@ -10,4 +10,4 @@ function cleanup() {
 trap cleanup INT
 trap cleanup ERR
 
-STORAGE_LOCATION=./storage DBS_LOCATION=./storage/dbs FLASK_APP=beebop/app.py poetry run flask run
+rq worker & STORAGE_LOCATION=./storage DBS_LOCATION=./storage/dbs FLASK_APP=beebop/app.py poetry run flask run