diff --git a/.circleci/config.yml b/.circleci/config.yml index c4a3117dd6022..725c78f5e7e8a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,10 +2,10 @@ version: 2.1 executors: - # osx-arm64: - # macos: - # xcode: 14.2.0 # indicate your selected version of Xcode - # resource_class: macos.m1.large.gen1 + osx-arm64: + macos: + xcode: 15.4.0 + resource_class: macos.m1.medium.gen1 linux-aarch64: machine: image: ubuntu-2204:current @@ -22,7 +22,13 @@ jobs: # a basic unit of work in a run - run: name: Check for Additional Platforms - command: ./.circleci/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" + command: | + result=$(./scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${CIRCLE_JOB}") + if [[ ${result} != "build" ]] + then + echo "No recipes using this platform, skipping rest of job." + circleci-agent step halt + fi - run: name: Fetch bioconda install script @@ -47,14 +53,29 @@ jobs: # a basic unit of work in a run - run: name: Build and test command: | - . common.sh - source /opt/mambaforge/etc/profile.d/conda.sh - source /opt/mambaforge/etc/profile.d/mamba.sh - mamba activate bioconda - bioconda-utils build recipes config.yml \ - --lint --docker --mulled-test \ - --docker-base-image "quay.io/bioconda/bioconda-utils-build-env-cos7-$(arch):${BIOCONDA_UTILS_TAG#v}" \ - --git-range origin/master HEAD + set -xe + eval "$(conda shell.bash hook)" + conda activate bioconda + source common.sh + + # Sets up OSX SDK + if [[ $(uname) = "Darwin" ]]; then + if [[ "${OSX_SDK_DIR:-}" == "" ]]; then + OSX_SDK_DIR="$(xcode-select -p)/Platforms/MacOSX.platform/Developer/SDKs" + fi + sudo chmod o+rwx ${OSX_SDK_DIR} + run_conda_forge_build_setup + fi + + if command -V docker >/dev/null; then + bioconda-utils build recipes config.yml \ + --lint --docker --mulled-test \ + --docker-base-image "quay.io/bioconda/bioconda-utils-build-env-cos7-$(arch):${BIOCONDA_UTILS_TAG#v}" \ + --git-range origin/master HEAD + else + bioconda-utils build recipes config.yml \ + --lint --git-range origin/master HEAD + fi - run: name: Prepare artifacts @@ -63,7 +84,7 @@ jobs: # a basic unit of work in a run mkdir -p /tmp/artifacts/packages cd /opt/mambaforge/envs/bioconda/conda-bld || exit 0 find -name .cache | xargs rm -rf || true - for n in index.html channeldata.json linux-aarch64 linux-64 osx-64 noarch; do + for n in index.html channeldata.json linux-aarch64 linux-64 osx-64 osx-arm64 noarch; do cp -rv $n /tmp/artifacts/packages || true done if command -V docker >/dev/null; then @@ -93,7 +114,13 @@ jobs: # a basic unit of work in a run - run: name: Check for Additional Platforms - command: ./.circleci/check-for-additional-platforms.sh "${CIRCLE_SHA1}~1 ${CIRCLE_SHA1}" "build_and_upload" + command: | + result=$(./scripts/check-for-additional-platforms.sh "${CIRCLE_SHA1}~1 ${CIRCLE_SHA1}" "build_and_upload" "${CIRCLE_JOB}") + if [[ ${result} != "build" ]] + then + echo "No recipes using this platform, skipping rest of job." + circleci-agent step halt + fi - run: name: Fetch bioconda install script @@ -118,9 +145,20 @@ jobs: # a basic unit of work in a run - run: name: Build and push command: | - source /opt/mambaforge/etc/profile.d/conda.sh - source /opt/mambaforge/etc/profile.d/mamba.sh - mamba activate bioconda + set -xe + eval "$(conda shell.bash hook)" + conda activate bioconda + source common.sh + + # Sets up OSX SDK + if [[ $(uname) = "Darwin" ]]; then + if [[ "${OSX_SDK_DIR:-}" == "" ]]; then + OSX_SDK_DIR="$(xcode-select -p)/Platforms/MacOSX.platform/Developer/SDKs" + fi + sudo chmod o+rwx ${OSX_SDK_DIR} + run_conda_forge_build_setup + fi + bioconda-utils handle-merged-pr recipes config.yml \ --repo bioconda/bioconda-recipes \ --git-range ${CIRCLE_SHA1}~1 ${CIRCLE_SHA1} \ @@ -184,10 +222,10 @@ jobs: # a basic unit of work in a run name: Build and upload command: | set -e - . common.sh - source /opt/mambaforge/etc/profile.d/conda.sh - source /opt/mambaforge/etc/profile.d/mamba.sh - mamba activate bioconda + eval "$(conda shell.bash hook)" + conda activate bioconda + source common.sh + echo '============' conda info --all conda config --show-sources @@ -214,7 +252,7 @@ workflows: matrix: parameters: os: - #- osx-arm64 + - osx-arm64 - linux-aarch64 build and upload (ARM): @@ -226,7 +264,7 @@ workflows: matrix: parameters: os: - #- osx-arm64 + - osx-arm64 - linux-aarch64 @@ -239,6 +277,6 @@ workflows: matrix: parameters: os: - #- osx-arm64 + # - osx-arm64 Bulk is on GitHub Actions - linux-aarch64 runner: [0, 1, 2, 3, 4, 5] diff --git a/.github/workflows/Bulk.yml b/.github/workflows/Bulk.yml index ec689ce742449..38e3f8248cf8b 100644 --- a/.github/workflows/Bulk.yml +++ b/.github/workflows/Bulk.yml @@ -62,7 +62,7 @@ jobs: conda clean -y --all build-osx: - name: Bulk OSX Builds + name: Bulk OSX-64 Builds if: "contains(github.event.head_commit.message, '[ci run]')" runs-on: macos-13 strategy: @@ -121,3 +121,65 @@ jobs: --lint --anaconda-upload --record-build-failures --skiplist-leafs \ --exclude bioconda-repodata-patches conda clean -y --all + + build-osx-arm: + name: Bulk OSX-ARM64 Builds + if: "contains(github.event.head_commit.message, '[ci run]')" + runs-on: macOS-14 # M1 + strategy: + fail-fast: false + max-parallel: 4 + matrix: + runner: [0, 1, 2, 3] + steps: + - uses: actions/checkout@v4 + with: + # checkout as BiocondaBot in order to have the permission to push fail logs + token: ${{secrets.BIOCONDA_BOT_REPO_TOKEN}} + + - name: set git user + run: | + git config user.name BiocondaBot + git config user.email BiocondaBot@users.noreply.github.com + + - name: set path + run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH + + - name: Fetch conda install script + run: | + wget https://raw.githubusercontent.com/bioconda/bioconda-common/bulk/{common,install-and-set-up-conda,configure-conda}.sh + + - name: Set up bioconda-utils + run: bash install-and-set-up-conda.sh + + - name: Configure conda + run: bash configure-conda.sh + + - name: Build and upload + env: + ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + INVOLUCRO_AUTH: ${{ secrets.INVOLUCRO_AUTH }} + QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_OAUTH_TOKEN }} + # Mimic circleci + OSTYPE: "darwin" + CI: "true" + run: | + set -xe + eval "$(conda shell.bash hook)" + conda activate bioconda + + source common.sh + # Sets up OSX SDK + run_conda_forge_build_setup + + echo '============' + conda info --all + conda config --show-sources + conda activate bioconda + python -c 'import bioconda_utils.utils as u ; import pathlib as p ; print(*(f"{f}:\n{p.Path(f).read_text()}" for f in u.load_conda_build_config().exclusive_config_files), sep="\n")' + echo '============' + bioconda-utils build recipes config.yml \ + --worker-offset ${{ matrix.runner }} --n-workers 4 \ + --lint --anaconda-upload --record-build-failures --skiplist-leafs \ + --exclude bioconda-repodata-patches + conda clean -y --all diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index 9418eba60943d..82f17b7c4a736 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -13,7 +13,7 @@ jobs: fail-fast: true max-parallel: 13 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -24,15 +24,7 @@ jobs: run: | wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - name: Restore cache - id: cache - uses: actions/cache@v3 - with: - path: /opt/mambaforge - key: ${{ runner.os }}--bulk--${{ hashFiles('**/install-and-set-up-conda.sh') }} - - name: Set up bioconda-utils - if: steps.cache.outputs.cache-hit != 'true' run: bash install-and-set-up-conda.sh - name: Configure conda @@ -67,7 +59,7 @@ jobs: max-parallel: 13 needs: lint steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -78,23 +70,10 @@ jobs: run: | wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - name: Restore cache - id: cache - uses: actions/cache@v3 - with: - path: /opt/mambaforge - key: ${{ runner.os }}--bulk--${{ hashFiles('**/install-and-set-up-conda.sh') }} - - name: Set up bioconda-utils - if: steps.cache.outputs.cache-hit != 'true' run: bash install-and-set-up-conda.sh # This script can be used to reconfigure conda to use the right channel setup. - # This has to be done after the cache is restored, because - # the channel setup is not cached as it resides in the home directory. - # We could use a system-wide (and therefore cached) channel setup, - # but mamba does not support that at the time of implementation - # (it ignores settings made with --system). - name: Configure conda run: bash configure-conda.sh @@ -151,28 +130,28 @@ jobs: ) || true - name: Archive packages - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: linux-packages path: | /tmp/artifacts/packages - name: Archive images - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: docker-images path: | /tmp/artifacts/images - build-osx: - name: OSX Tests + build-osx-64: + name: OSX-64 Tests runs-on: macos-13 strategy: fail-fast: true max-parallel: 4 needs: build-linux steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -183,23 +162,10 @@ jobs: run: | wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - name: Restore cache - id: cache - uses: actions/cache@v3 - with: - path: /opt/mambaforge - key: ${{ runner.os }}--bulk--${{ hashFiles('**/install-and-set-up-conda.sh') }} - - name: Set up bioconda-utils - if: steps.cache.outputs.cache-hit != 'true' run: bash install-and-set-up-conda.sh # This script can be used to reconfigure conda to use the right channel setup. - # This has to be done after the cache is restored, because - # the channel setup is not cached as it resides in the home directory. - # We could use a system-wide (and therefore cached) channel setup, - # but mamba does not support that at the time of implementation - # (it ignores settings made with --system). - name: Configure conda run: bash configure-conda.sh @@ -241,8 +207,90 @@ jobs: ) || true - name: Archive packages - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: osx-packages path: | /tmp/artifacts/packages + + build_and_test-osx-arm64: + name: OSX-ARM64 Tests + runs-on: macOS-14 # M1 + strategy: + fail-fast: true + max-parallel: 4 + needs: build-linux + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # bail if there's no osx-arm64 recipes + - name: Check for Additional Platforms + id: additional_platforms + run: | + result=$(./scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${GITHUB_JOB}") + if [[ ${result} != "build" ]] + then + echo "No recipes using this platform, skipping rest of job." + echo "skip_build=true" >> $GITHUB_OUTPUT + fi + + - name: set path + run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH + + - name: Fetch conda install script + if: steps.additional_platforms.outputs.skip_build != 'true' + run: | + wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh + + - name: Set up bioconda-utils + if: steps.additional_platforms.outputs.skip_build != 'true' + run: bash install-and-set-up-conda.sh + + - name: Configure conda + if: steps.additional_platforms.outputs.skip_build != 'true' + run: bash configure-conda.sh + + - name: Build and Test + if: steps.additional_platforms.outputs.skip_build != 'true' + env: + # Mimic circleci + OSTYPE: "darwin" + CI: "true" + run: | + set -xe + eval "$(conda shell.bash hook)" + conda activate bioconda + source common.sh + # Sets up OSX SDK + run_conda_forge_build_setup + + if [ -z "$GITHUB_BASE_REF" ] ; then + export GITHUB_BASE_REF="master" + fi + git fetch origin "$GITHUB_BASE_REF" + + bioconda-utils build recipes config.yml \ + --lint --git-range origin/"$GITHUB_BASE_REF" HEAD + + - name: Prepare artifacts + if: steps.additional_platforms.outputs.skip_build != 'true' + run: | + ( + rm -rf /tmp/artifacts + mkdir -p /tmp/artifacts/packages + cd /opt/mambaforge/envs/bioconda/conda-bld || exit 0 + find -name .cache | xargs rm -rf || true + for n in index.html channeldata.json osx-arm64 noarch; do + cp -rv $n /tmp/artifacts/packages || true + done + ) || true + + - name: Archive packages + if: steps.additional_platforms.outputs.skip_build != 'true' + uses: actions/upload-artifact@v4 + with: + name: osx-arm64-packages + path: | + /tmp/artifacts diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index e698443590e50..7e237b314da22 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false max-parallel: 13 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -23,23 +23,10 @@ jobs: run: | wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - name: Restore cache - id: cache - uses: actions/cache@v3 - with: - path: /opt/mambaforge - key: ${{ runner.os }}--bulk--${{ hashFiles('**/install-and-set-up-conda.sh') }} - - name: Set up bioconda-utils - if: steps.cache.outputs.cache-hit != 'true' run: bash install-and-set-up-conda.sh # This script can be used to reconfigure conda to use the right channel setup. - # This has to be done after the cache is restored, because - # the channel setup is not cached as it resides in the home directory. - # We could use a system-wide (and therefore cached) channel setup, - # but mamba does not support that at the time of implementation - # (it ignores settings made with --system). - name: Configure conda run: bash configure-conda.sh @@ -68,15 +55,15 @@ jobs: --docker --mulled-test --anaconda-upload --mulled-upload-target biocontainers docker rmi quay.io/dpryan79/mulled_container:latest - build-osx: - name: OSX Upload + build-osx-64: + name: OSX-64 Upload if: github.repository == 'bioconda/bioconda-recipes' runs-on: macos-13 strategy: fail-fast: false max-parallel: 4 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -87,23 +74,10 @@ jobs: run: | wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - name: Restore cache - id: cache - uses: actions/cache@v3 - with: - path: /opt/mambaforge - key: ${{ runner.os }}--bulk--${{ hashFiles('**/install-and-set-up-conda.sh') }} - - name: Set up bioconda-utils - if: steps.cache.outputs.cache-hit != 'true' run: bash install-and-set-up-conda.sh # This script can be used to reconfigure conda to use the right channel setup. - # This has to be done after the cache is restored, because - # the channel setup is not cached as it resides in the home directory. - # We could use a system-wide (and therefore cached) channel setup, - # but mamba does not support that at the time of implementation - # (it ignores settings made with --system). - name: Configure conda run: bash configure-conda.sh @@ -122,7 +96,7 @@ jobs: eval "$(conda shell.bash hook)" conda activate bioconda - # The SDK isn't actually cached, so reinstall it + # Sets up OSX SDK run_conda_forge_build_setup # bioconda-utils handle-merged-pr recipes config.yml \ @@ -131,3 +105,68 @@ jobs: # --fallback build bioconda-utils build recipes config.yml \ --anaconda-upload --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} + + build_and_upload-osx-arm64: + name: OSX-ARM64 Upload + if: github.repository == 'bioconda/bioconda-recipes' + runs-on: macOS-14 # M1 + strategy: + fail-fast: false + max-parallel: 4 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # bail if there's no osx-arm64 recipes + - name: Check for Additional Platforms + id: additional_platforms + run: | + result=$(./scripts/check-for-additional-platforms.sh "${GITHUB_SHA}~1 ${GITHUB_SHA}" "build_and_upload" "${GITHUB_JOB}") + if [[ ${result} != "build" ]] + then + echo "No recipes using this platform, skipping rest of job." + echo "skip_build=true" >> $GITHUB_OUTPUT + fi + + - name: set path + run: | + echo "/opt/mambaforge/bin" >> $GITHUB_PATH + + - name: Fetch conda install script + if: steps.additional_platforms.outputs.skip_build != 'true' + run: | + wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh + + - name: Set up bioconda-utils + if: steps.additional_platforms.outputs.skip_build != 'true' + run: bash install-and-set-up-conda.sh + + - name: Configure conda + if: steps.additional_platforms.outputs.skip_build != 'true' + run: bash configure-conda.sh + + - name: Build and Upload + if: steps.additional_platforms.outputs.skip_build != 'true' + env: + QUAY_LOGIN: ${{ secrets.QUAY_LOGIN }} + QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_OAUTH_TOKEN }} + ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + INVOLUCRO_AUTH: ${{ secrets.INVOLUCRO_AUTH }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Mimic circleci + OSTYPE: "darwin" + CI: "true" + run: | + set -xe + eval "$(conda shell.bash hook)" + conda activate bioconda + source common.sh + # Sets up OSX SDK + run_conda_forge_build_setup + + bioconda-utils handle-merged-pr recipes config.yml \ + --repo bioconda/bioconda-recipes \ + --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} \ + --fallback build \ + --artifact-source github-actions diff --git a/recipes/bamtools/meta.yaml b/recipes/bamtools/meta.yaml index 803497b6c9a4b..0f5b309f2a25e 100644 --- a/recipes/bamtools/meta.yaml +++ b/recipes/bamtools/meta.yaml @@ -12,7 +12,7 @@ source: # - 0001-Const-qualify-all-functors-operator-member-functions.patch build: - number: 3 + number: 5 run_exports: - {{ pin_subpackage(name, max_pin='x.x') }} @@ -37,5 +37,6 @@ about: extra: additional-platforms: - linux-aarch64 + - osx-arm64 identifiers: - biotools:bamtools diff --git a/recipes/bioawk/meta.yaml b/recipes/bioawk/meta.yaml index c5dd0a636caf2..70ec45f58e648 100644 --- a/recipes/bioawk/meta.yaml +++ b/recipes/bioawk/meta.yaml @@ -7,7 +7,7 @@ source: sha256: 5cbef3f39b085daba45510ff450afcf943cfdfdd483a546c8a509d3075ff51b5 build: - number: 10 + number: 12 run_exports: - {{ pin_subpackage("bioawk", max_pin="x.x.x") }} @@ -34,3 +34,4 @@ about: extra: additional-platforms: - linux-aarch64 + - osx-arm64 diff --git a/.circleci/check-for-additional-platforms.sh b/scripts/check-for-additional-platforms.sh similarity index 87% rename from .circleci/check-for-additional-platforms.sh rename to scripts/check-for-additional-platforms.sh index 50641d2b63052..d088794f14422 100755 --- a/.circleci/check-for-additional-platforms.sh +++ b/scripts/check-for-additional-platforms.sh @@ -7,6 +7,7 @@ # arguments git_range=$1 job_name=$2 +current_job=$3 # Download ARM version of yq yq_platform=$(uname) @@ -21,7 +22,6 @@ files=`git diff --name-only --diff-filter AMR ${git_range} | grep -E 'meta.yaml$ build=0 for file in $files; do - echo $file # To create a properly-formatted yaml that yq can parse, comment out jinja2 # variable setting with {% %} and remove variable use with {{ }}. additional_platforms=$(cat $file \ @@ -37,7 +37,7 @@ for file in $files; do # Check if any additional platforms match this job for additional_platform in $additional_platforms; do - if [ "${CIRCLE_JOB}" = "${job_name}-${additional_platform}" ] + if [ "${current_job}" = "${job_name}-${additional_platform}" ] then build=1 break @@ -46,8 +46,7 @@ for file in $files; do done # If no changed recipes apply to this platform, skip remaining steps -if [[ build -lt 1 ]] +if [[ build -gt 0 ]] then - echo "No recipes using this platform, skipping rest of job." - circleci-agent step halt + echo "build" fi