concrete-python release-cpu #26
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: concrete-python release-cpu | |
| on: | |
| workflow_dispatch: | |
| push: | |
| tags: | |
| - 'v[0-9]+.[0-9]+.[0-9]+*' | |
| schedule: | |
| # Nightly Release @ 3AM after each work day | |
| - cron: "0 3 * * 2-6" | |
| env: | |
| DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler | |
| ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} | |
| SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} | |
| RELEASE_TYPE: ${{ (github.event_name == 'push' && contains(github.ref, 'refs/tags/')) && 'public' || 'nightly' }} | |
| concurrency: | |
| group: concrete_python_release_cpu_${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| jobs: | |
| setup-instance: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| runner-name: ${{ steps.start-instance.outputs.label }} | |
| steps: | |
| - name: Start instance | |
| id: start-instance | |
| uses: zama-ai/slab-github-runner@98f0788261a7323d5d695a883e20df36591a92b7 # v1.3.0 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| backend: aws | |
| profile: release | |
| release-checks: | |
| needs: setup-instance | |
| runs-on: ${{ needs.setup-instance.outputs.runner-name }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Check python api doc is up to date | |
| run: ci/scripts/make_apidocs.sh | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "release-checks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| build-linux-x86: | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| needs: setup-instance | |
| runs-on: ${{ needs.setup-instance.outputs.runner-name }} | |
| steps: | |
| - name: Set up GitHub environment | |
| run: | | |
| echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" | |
| - name: Checkout | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Set release version (nightly) | |
| if: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| run: | | |
| NIGHTLY_VERSION=$(date +"%Y.%m.%d") | |
| NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d") | |
| LATEST_RELEASE_VERSION=$(git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o) | |
| echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt | |
| git tag "nightly-${NIGHTLY_VERSION}" || true | |
| git push origin "nightly-${NIGHTLY_VERSION}" || true | |
| - name: Set release version (public) | |
| if: ${{ env.RELEASE_TYPE == 'public' }} | |
| run: echo "__version__ = \"$(git describe --tags --abbrev=0 | grep -e '[0-9].*' -o)\"" >| frontends/concrete-python/version.txt | |
| - name: Expose release version from Python | |
| run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py | |
| - name: Build wheel | |
| uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 | |
| id: build-compiler-bindings | |
| with: | |
| registry: ghcr.io | |
| image: ${{ env.DOCKER_IMAGE_TEST }} | |
| username: ${{ secrets.GHCR_LOGIN }} | |
| password: ${{ secrets.GHCR_PASSWORD }} | |
| options: >- | |
| -v ${{ github.workspace }}:/concrete | |
| -v ${{ github.workspace }}/build:/build | |
| shell: bash | |
| run: | | |
| set -e | |
| rm -rf /build/* | |
| export PYTHON=${{ format('python{0}', matrix.python-version) }} | |
| echo "Using $PYTHON" | |
| dnf -y install graphviz graphviz-devel | |
| cd /concrete/frontends/concrete-python | |
| make PYTHON=$PYTHON venv | |
| source .venv/bin/activate | |
| cd /concrete/compilers/concrete-compiler/compiler | |
| make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python) python-bindings | |
| echo "Debug: ccache statistics (after the build):" | |
| ccache -s | |
| cd /concrete/frontends/concrete-python | |
| export COMPILER_BUILD_DIRECTORY="/build" | |
| make whl | |
| deactivate | |
| - name: Upload wheel | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| with: | |
| name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} | |
| path: frontends/concrete-python/dist/*manylinux*.whl | |
| retention-days: 3 | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "build-linux-x86 finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| build-macos: | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| runs-on: ["aws-mac1-metal", "aws-mac2-metal"] | |
| runs-on: ${{ matrix.runs-on }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Install OS Dependencies | |
| run: | | |
| brew install ninja ccache | |
| - name: Setup rust toolchain for concrete-cpu | |
| uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu | |
| - name: Set release version (nightly) | |
| if: ${{ env.RELEASE_TYPE == 'nightly' }} | |
| run: | | |
| NIGHTLY_VERSION=$(date +"%Y%m%d") | |
| LATEST_RELEASE_VERSION=$(git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o) | |
| echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION}\"" >| frontends/concrete-python/version.txt | |
| - name: Set release version (public) | |
| if: ${{ env.RELEASE_TYPE == 'public' }} | |
| run: echo "__version__ = \"$(git describe --tags --abbrev=0 | grep -e '[0-9].*' -o)\"" >| frontends/concrete-python/version.txt | |
| - name: Expose release version from Python | |
| run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py | |
| - name: Build wheel | |
| run: | | |
| CONCRETE_PYTHON=$(pwd)/frontends/concrete-python | |
| CONCRETE_COMPILER=$(pwd)/compilers/concrete-compiler/compiler | |
| export COMPILER_BUILD_DIRECTORY=$CONCRETE_COMPILER/build | |
| export PYTHON=${{ format('python{0}', matrix.python-version) }} | |
| echo "Using $PYTHON" | |
| # Setup pkg-config to find OpenBLAS (scipy need it) | |
| export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" | |
| # Setup vitual environment | |
| rm -rf .venv | |
| $PYTHON -m venv .venv && . .venv/bin/activate | |
| # Install requirements | |
| pip install -r "${CONCRETE_PYTHON}"/requirements.txt | |
| pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt | |
| # Build python bindings of concrete compiler | |
| cd "${CONCRETE_COMPILER}" || exit | |
| echo "Debug: ccache statistics (prior to the build):" && ccache -s | |
| make Python3_EXECUTABLE="$(which python)" python-bindings | |
| echo "Debug: ccache statistics (after the build):" && ccache -s | |
| # Build wheel | |
| cd "${CONCRETE_PYTHON}" || exit | |
| rm -rf dist | |
| mkdir -p dist | |
| pip wheel -v --no-deps -w dist . | |
| delocate-wheel -v dist/*macos*.whl | |
| deactivate | |
| - name: Upload wheel | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| with: | |
| name: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} | |
| path: frontends/concrete-python/dist/*macos*.whl | |
| retention-days: 3 | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "build-macos finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| hash: | |
| # Generate hashes for the wheels, used later for provenance. | |
| needs: [build-linux-x86, build-macos] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| hash: ${{ steps.hash.outputs.hash }} | |
| steps: | |
| - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| path: frontends/concrete-python/dist | |
| pattern: wheel-* | |
| merge-multiple: true | |
| - name: generate hash | |
| id: hash | |
| run: cd frontends/concrete-python/dist && echo "hash=$(sha256sum ./*.whl | base64 -w0)" >> "${GITHUB_OUTPUT}" | |
| provenance: | |
| needs: [hash] | |
| permissions: | |
| actions: read | |
| id-token: write | |
| contents: write | |
| # Can't pin with hash due to how this workflow works. | |
| uses: slsa-framework/slsa-github-generator/.github/workflows/[email protected] | |
| with: | |
| base64-subjects: ${{ needs.hash.outputs.hash }} | |
| push: | |
| needs: [build-linux-x86, build-macos, provenance] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| path: wheels | |
| pattern: 'wheel-*' | |
| merge-multiple: true | |
| - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| pattern: '*.intoto.jsonl' | |
| # When building a new public tag, create a new draft release. | |
| - name: create draft release | |
| if: ${{ env.RELEASE_TYPE == 'public'}} | |
| run: | | |
| TAG=$(git describe --tags --abbrev=0) | |
| export TAG | |
| echo "${TAG}" | |
| gh release create --draft --repo ${{ github.repository }} \ | |
| --verify-tag "${TAG}" \ | |
| --title "${TAG}" \ | |
| wheels/* ./*.intoto.jsonl/* | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Upload wheels to S3 | |
| if: ${{ env.RELEASE_TYPE == 'public' || env.RELEASE_TYPE == 'nightly' }} | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_IAM_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_IAM_KEY }} | |
| AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} | |
| S3_BUCKET_NAME: ${{ secrets.AWS_S3_PYPI_BUCKET_NAME }} | |
| CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.AWS_CLOUDFRONT_PYPI_DISTRIBUTION_ID }} | |
| run: | | |
| pip install boto3 bigtree | |
| # upload wheels | |
| aws s3 sync ./wheels/ "s3://${S3_BUCKET_NAME}/cpu/concrete-python" | |
| # update indexes and invalidate cloudfront cache | |
| python .github/workflows/scripts/s3_update_html_indexes.py | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "push finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| test-linux-x86: | |
| needs: [setup-instance, build-linux-x86] | |
| continue-on-error: true | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| runs-on: ${{ needs.setup-instance.outputs.runner-name }} | |
| steps: | |
| # HOME is needed by actions-rs/toolchain | |
| - run: | | |
| echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" | |
| - name: Install rust | |
| uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7 | |
| with: | |
| toolchain: nightly | |
| default: true | |
| - name: Setup Python | |
| uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Download wheels | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} | |
| path: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} | |
| - name: Checkout the repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| submodules: recursive | |
| path: repo | |
| - name: Test wheel | |
| run: | | |
| WHEEL_DIR=$(pwd)/${{ format('wheel-{0}-linux-x86', matrix.python-version) }} | |
| CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python | |
| # Initialize an empty test environment | |
| cd "$(mktemp -d)" | |
| python -m venv .testenv && source .testenv/bin/activate | |
| # Install the concrete-python wheel | |
| pip install "${WHEEL_DIR}"/*.whl | |
| # Install extra requirements for tests | |
| sudo apt update -y | |
| sudo apt install -y graphviz libgraphviz-dev | |
| pip install -r "${CONCRETE_PYTHON}"/requirements.extra-full.txt | |
| pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt | |
| # TODO - check for version | |
| # Copy test files | |
| cp -R "${CONCRETE_PYTHON}"/tests . | |
| cp -R "${CONCRETE_PYTHON}"/examples . | |
| cp -R "${CONCRETE_PYTHON}"/pytest.ini . | |
| cp "${CONCRETE_PYTHON}"/Makefile . | |
| # Running tests | |
| make tfhers-utils | |
| pytest tests -svv -n auto | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "test-linux-x86 (${{ matrix.python-version }}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| teardown-instance: | |
| needs: [ setup-instance, test-linux-x86 ] | |
| if: ${{ always() && needs.setup-instance.result != 'skipped' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Stop instance | |
| id: stop-instance | |
| uses: zama-ai/slab-github-runner@98f0788261a7323d5d695a883e20df36591a92b7 # v1.3.0 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| label: ${{ needs.setup-instance.outputs.runner-name }} | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" | |
| test-macos: | |
| needs: [build-macos] | |
| continue-on-error: true | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| runs-on: ["aws-mac1-metal", "aws-mac2-metal"] | |
| runs-on: ${{ matrix.runs-on }} | |
| outputs: | |
| slack_message: ${{ steps.prepare_slack_notif.outputs.slack_message }} | |
| slack_color: ${{ steps.prepare_slack_notif.outputs.slack_color }} | |
| steps: | |
| - name: Download wheels | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} | |
| path: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} | |
| - name: Checkout the repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| submodules: recursive | |
| path: repo | |
| - name: Test wheel | |
| run: | | |
| WHEEL_DIR=$(pwd)/${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} | |
| CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python | |
| PYTHON=${{ format('python{0}', matrix.python-version) }} | |
| # Initialize an empty test environment | |
| TEST_TMP_DIR=$(mktemp -d) | |
| echo "TEST_TMP_DIR=${TEST_TMP_DIR}" >> "${GITHUB_ENV}" | |
| cd "${TEST_TMP_DIR}" || exit | |
| # Activate virtual environment | |
| $PYTHON -m venv .testenv && source .testenv/bin/activate | |
| # Install extra requirements for tests | |
| pip install "${WHEEL_DIR}"/*macos*.whl | |
| pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt | |
| "${CONCRETE_PYTHON}"/../../.github/workflows/scripts/fix_multi_omp_bug_macos.sh | |
| # Copy test files | |
| cp -R "${CONCRETE_PYTHON}"/tests . | |
| cp -R "${CONCRETE_PYTHON}"/examples . | |
| cp -R "${CONCRETE_PYTHON}"/pytest.ini . | |
| cp "${CONCRETE_PYTHON}"/Makefile . | |
| # Fix: this is because cargo cannot be found | |
| . "$HOME/.cargo/env" | |
| # Running tests | |
| make tfhers-utils | |
| mkdir ./KeySetCache | |
| pytest tests -svv -n auto --key-cache "./KeySetCache" -m "not dataflow and not graphviz" | |
| - name: Cleanup host | |
| if: success() || failure() | |
| run: | | |
| rm -rf "${TEST_TMP_DIR}" | |
| - name: Prepare Slack Notification | |
| id: prepare_slack_notif | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| run: | | |
| echo "slack_message=test-macos (${{matrix.runs-on}}/${{ matrix.python-version }}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" >> "$GITHUB_OUTPUT" | |
| echo "slack_color=${{ job.status }}" >> "$GITHUB_OUTPUT" | |
| slack-notif-macos: | |
| needs: ["test-macos"] | |
| runs-on: "ubuntu-latest" | |
| if: always() | |
| steps: | |
| - name: Slack Notification | |
| # we want to check that prepare_slack_notif was run | |
| if: ${{ needs.test-macos.outputs.slack_color != '' }} | |
| uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 | |
| env: | |
| SLACK_COLOR: ${{ needs.test-macos.outputs.slack_color }} | |
| SLACK_MESSAGE: ${{ needs.test-macos.outputs.slack_message }} |