From 669eb24fd82a9d3cb18ad0e73673ecb26827f683 Mon Sep 17 00:00:00 2001 From: Edmund Miller <20095261+edmundmiller@users.noreply.github.com> Date: Thu, 7 Nov 2024 09:59:45 -0600 Subject: [PATCH] Add CI to build mulled containers with Wave (#4080) * build: Add wave * build: Set strategy to dockerfile, conda then container * refactor: Remove container * build: Add a repo to push to * ci(wave): Add wave build https://github.com/nodejs/docker-node/blob/3c4fa6daf06a4786d202f2f610351837806a0380/.github/workflows/build-test.yml#L29 * ci(wave): Switch to all_changed_files * ci(wave): Only look for envronment.ymls * dummy: Change env * ci(wave): Remove raw format * ci(wave): Try a bunch of different things at once * ci(wave): Remove redundant fromJson and wrap in an array * ci(wave): I have no idea what I'm doing * ci(wave): Wrap it * ci(wave): Found an example https://github.com/tj-actions/changed-files/blob/main/.github/workflows/matrix-test.yml * ci(wave): Maybe quotes? * ci(wave): That'll do it * ci(wave): Fix wave install * ci(wave): Hard code an image * ci(wave): Add secrets * feat: Try a different files structure * ci(wave): First stab at building singularity images * fixup! feat: Try a different files structure * ci(wave): Add profile to matrix * ci(wave): Give up on fancy substitution * ci(wave): Add await Co-authored-by: ewels * ci(wave): Switch to quay * test(wave): Add freeze and update build repo * refactor(wave): What happens if I add a container? * refactor(wave): Have both bowtie modules use the same env For the sake of demonstration * test: Cut out using wave on tests * refactor: What happens if we use the singularity one? * refactor: Keep container directives for offline download https://github.com/seqeralabs/wave/issues/323 * feat: Try new singularity OCI setting https://github.com/nextflow-io/nextflow/commit/f5362a7b067173a29d684663df22bb48fbbf5659 * build: Update container name Guess #4327 broke that * chore: Bump wave-cli version * ci: Install runc * ci: Switch to singularityhub action https://github.com/nextflow-io/nextflow/issues/4543 * ci: Install new singularity manually Why that action trys to build from source, idk. * ci: Install dependancies for singularity * ci: runc => crun * ci: Fix cgroup error https://blog.misharov.pro/2021-05-16/systemd-github-actions * ci: That'll do it * ci: Remove Dockerfile We'll have a seperate action for this I think * ci: Update name * ci: Push to the correct repos * ci: Remove OCI stuff * ci: Need a full URL * ci: Fix // in container name * ci: Remove push Build once, renovate should bump the images automagically * build: Add containers back * ci: Add cache repos Idk what this does exactly * ci: Change registry name to use _ Because "build" is a api end point on quay.io. So `bowtie/build` doesn't work. Other plus is this matches the conda env name. * build: / => _ in container name * Try ociAutoPull * chore: Add renovate comments to samtools Just to trigger wave build * test: Add ociAutoPull to nf-test * ci: Bump wave version * chore: Bump containers with new wave version Not sure why that's happening... * build: Update to use commity.wave.seqera.io * ci: Bump wave-cli to 1.4.1 * ci: Try apptainer * ci: Remove build-repo to see what happens * build: Bump Nextflow version requirement * fix: Get rid of the environment name? Maybe this will get the auto generated tag? * ci: Bump action versions * ci: Try name-strategy tagPrefix https://github.com/seqeralabs/wave-cli/commit/269df0e52e18c9dbd919a5a5b8934a37b5300ad1 * ci: Remove singularity build for now * ci: Try imageSuffix * ci: Try none * ci: What is the bowtie container name * ci: Remove --name-strategy * style: Add back in container elvis operator * ci: Remove cache repo * Revert "build: Bump Nextflow version requirement" This reverts commit 69e1ea59a079752946682a04c2a3c066b21bc283. * Revert "test: Add ociAutoPull to nf-test" This reverts commit 5a3d54609b67b1814c545b7e3dd8b49a82cb00e6. * test(#6505): Snapshot the versions contents, not the hash * ci(#6505): Update version snapshot after building containers * test(#6505): Attempt a topic channel with tests https://github.com/askimed/nf-test/issues/258 * chore: Bump to 1.5.0 * fix: Remove shard and filter on test bumping * build: Bump images to match environment * ci: Fix nf-test setup * ci: Remove snapshot bumping * build: Fix containers in bowtie --------- Co-authored-by: ewels --- .github/workflows/wave.yml | 138 ++++++++++++++++++ modules/nf-core/bowtie/align/environment.yml | 6 +- modules/nf-core/bowtie/align/main.nf | 4 +- modules/nf-core/bowtie/build/environment.yml | 5 +- modules/nf-core/bowtie/build/main.nf | 4 +- .../nf-core/bowtie/build/tests/main.nf.test | 4 +- modules/nf-core/samtools/view/environment.yml | 2 + modules/nf-core/samtools/view/main.nf | 4 +- 8 files changed, 157 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/wave.yml diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml new file mode 100644 index 00000000000..d450ef24d34 --- /dev/null +++ b/.github/workflows/wave.yml @@ -0,0 +1,138 @@ +name: Wave +# When environment.yml is changed +on: + pull_request: + paths: + - "**/environment.yml" + +# TODO On complete call testing CI +# TODO Skip testing CI if any changes to environment.yml + +env: + WAVE_VER: "1.5.0" + NFTEST_VER: "0.9.1" + +jobs: + gen-matrix: + name: generate-matrix + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + + - name: Calculate file differences + id: diff + uses: tj-actions/changed-files@v44 + with: + json: true + quotepath: false + # TODO Add Dockerfiles + files: | + modules/**/environment.yml + - name: Debug + run: echo ${{ steps.diff.outputs.all_changed_files }} + - id: set-matrix + run: echo "matrix={\"profile\":[\"docker\", \"singularity\"],\"files\":${{ steps.diff.outputs.all_changed_files }} }" >> "$GITHUB_OUTPUT" + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + + build: + if: "${{ fromJson(needs.gen-matrix.outputs.matrix) }}" + needs: gen-matrix + name: build + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: "${{ fromJson(needs.gen-matrix.outputs.matrix) }}" + steps: + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + + - name: Install wave-cli + run: | + wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64 + sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave + chmod +x /usr/local/bin/wave + + - name: Create a registry name + uses: actions/github-script@v7 + id: registry-name + with: + result-encoding: string + script: | + return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/environment.yml', '').replace('/', '_'); + + - name: Build container + if: matrix.profile == 'docker' + run: | + wave --conda-file "${{ matrix.files }}" \ + --freeze \ + --await \ + --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ + --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} + + - name: Build Singularity + if: matrix.profile == 'singularity' + run: | + wave --conda-file "${{ matrix.files }}" \ + --freeze \ + --await \ + --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ + --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ + --singularity + + # TODO Build from Dockerfiles + + # bump-versions: + # needs: gen-matrix + # name: bump-versions + # runs-on: ubuntu-latest + # steps: + # - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 + # with: + # distribution: "temurin" + # java-version: "17" + + # - uses: nf-core/setup-nextflow@v2 + + # - uses: nf-core/setup-nf-test@v1 + # with: + # version: ${{ env.NFTEST_VER }} + + # - name: Bump Snapshot Versions + # env: + # # NFT_DIFF: "pdiff" + # # NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" + # SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} + # SENTIEON_AUTH_MECH: "GitHub Actions - token" + # run: | + # # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker + # if [ "${{ matrix.profile }}" == "docker" ]; then + # PROFILE="docker_self_hosted" + # else + # PROFILE=${{ matrix.profile }} + # fi + + # NFT_WORKDIR=~ \ + # nf-test test \ + # --profile=${{ matrix.profile }} \ + # --tap=test.tap \ + # # --ci \ + # --verbose \ + # --only-changed \ + # # --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \ + # # --filter ${{ matrix.filter }} \ + # --follow-dependencies \ + # --tag version \ + # --update-snapshot + + # - name: Commit & push version bumps + # run: | + # git config user.email "core@nf-co.re" + # git config user.name "nf-core-bot" + # git config push.default upstream + # git add . + # git status + # git commit -m "[automated] Bump versions snapshot" + # git push diff --git a/modules/nf-core/bowtie/align/environment.yml b/modules/nf-core/bowtie/align/environment.yml index 4434c7e7143..61bd69c2c7a 100644 --- a/modules/nf-core/bowtie/align/environment.yml +++ b/modules/nf-core/bowtie/align/environment.yml @@ -2,5 +2,7 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::bowtie=1.3.0 - - bioconda::samtools=1.16.1 + # renovate: datasource=conda depName=bioconda/bowtie + - bioconda::bowtie=1.3.1 + # renovate: datasource=conda depName=bioconda/samtools + - bioconda::samtools=1.20 diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index 5e72b02a678..3dee85ffb4d 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -4,8 +4,8 @@ process BOWTIE_ALIGN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-ffbf83a6b0ab6ec567a336cf349b80637135bca3:c84c7c55c45af231883d9ff4fe706ac44c479c36-0' : - 'biocontainers/mulled-v2-ffbf83a6b0ab6ec567a336cf349b80637135bca3:c84c7c55c45af231883d9ff4fe706ac44c479c36-0' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c8/c8c0819a9b1f520c49c933e667ae50de2a0730ece4c8b9efe79ac5e403963a9f/data' : + 'community​.wave​.seqera​.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" input: tuple val(meta), path(reads) diff --git a/modules/nf-core/bowtie/build/environment.yml b/modules/nf-core/bowtie/build/environment.yml index ab5a842215c..61bd69c2c7a 100644 --- a/modules/nf-core/bowtie/build/environment.yml +++ b/modules/nf-core/bowtie/build/environment.yml @@ -2,4 +2,7 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::bowtie=1.3.0 + # renovate: datasource=conda depName=bioconda/bowtie + - bioconda::bowtie=1.3.1 + # renovate: datasource=conda depName=bioconda/samtools + - bioconda::samtools=1.20 diff --git a/modules/nf-core/bowtie/build/main.nf b/modules/nf-core/bowtie/build/main.nf index d5b4c690404..0f6b9d4d1bf 100644 --- a/modules/nf-core/bowtie/build/main.nf +++ b/modules/nf-core/bowtie/build/main.nf @@ -4,8 +4,8 @@ process BOWTIE_BUILD { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/bowtie:1.3.0--py38hed8969a_1' : - 'biocontainers/bowtie:1.3.0--py38hed8969a_1' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c8/c8c0819a9b1f520c49c933e667ae50de2a0730ece4c8b9efe79ac5e403963a9f/data' : + 'community​.wave​.seqera​.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" input: tuple val(meta), path(fasta) diff --git a/modules/nf-core/bowtie/build/tests/main.nf.test b/modules/nf-core/bowtie/build/tests/main.nf.test index 25fb3dad83b..81209a7b1fc 100644 --- a/modules/nf-core/bowtie/build/tests/main.nf.test +++ b/modules/nf-core/bowtie/build/tests/main.nf.test @@ -34,6 +34,7 @@ nextflow_process { test("sarscov2 - fasta - stub") { options "-stub" + tag "stub" when { process { @@ -48,7 +49,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot(process.out).match() }, + { assert snapshot(path(process.out.versions.get(0)).yaml).match("versions") }, ) } diff --git a/modules/nf-core/samtools/view/environment.yml b/modules/nf-core/samtools/view/environment.yml index 62054fc97ac..02cda6e6a3b 100644 --- a/modules/nf-core/samtools/view/environment.yml +++ b/modules/nf-core/samtools/view/environment.yml @@ -4,5 +4,7 @@ channels: - conda-forge - bioconda dependencies: + # renovate: datasource=conda depName=bioconda/htslib - bioconda::htslib=1.21 + # renovate: datasource=conda depName=bioconda/samtools - bioconda::samtools=1.21 diff --git a/modules/nf-core/samtools/view/main.nf b/modules/nf-core/samtools/view/main.nf index 37e05cec887..41fa3d6a0b9 100644 --- a/modules/nf-core/samtools/view/main.nf +++ b/modules/nf-core/samtools/view/main.nf @@ -4,8 +4,8 @@ process SAMTOOLS_VIEW { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.21--h50ea8bc_0' : - 'biocontainers/samtools:1.21--h50ea8bc_0' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/9e/9edc2564215d5cd137a8b25ca8a311600987186d406b092022444adf3c4447f7/data' : + 'community​.wave​.seqera​.io/library/htslib_samtools:1​.21--6cb89bfd40cbaabf' }" input: tuple val(meta), path(input), path(index)