f apply review

zama-ai · Nov 7, 2024 · e36f4eb · e36f4eb
1 parent 72f63ac
commit e36f4eb
Show file tree

Hide file tree

Showing 9 changed files with 89 additions and 117 deletions.
diff --git a/.github/workflows/commit_compliance.yml b/.github/workflows/commit_compliance.yml
@@ -1,10 +1,6 @@
 name: check commit compliance
 on:
   pull_request:
-  push:
-    branches:
-      - 'main'
-      - 'release/*'
 
 jobs:
   format:
@@ -20,16 +16,6 @@ jobs:
           excludeTitle: 'true' # optional: this excludes the title of a pull request
           checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request
           accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true
-
-  signatures:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check commit signatures
-        uses: 1Password/check-signed-commits-action@ed2885f3ed2577a4f5d3c3fe895432a557d23d52
-
-  files-lint:
-    runs-on: ubuntu-latest
-    steps:
       - name: checkout
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: linelint
@@ -40,10 +26,3 @@ jobs:
         with:
           use-quiet-mode: 'yes'
           use-verbose-mode: 'yes'
-      - name: Slack Notification
-        if: ${{ failure() && github.ref == 'refs/heads/main' }}
-        continue-on-error: true
-        uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
-        env:
-          SLACK_COLOR: ${{ job.status }}
-          SLACK_MESSAGE: "files-lint finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
diff --git a/.github/workflows/concrete_compiler_benchmark.yml b/.github/workflows/concrete_compiler_benchmark.yml
@@ -62,30 +62,23 @@ jobs:
           git config --global --add safe.directory '*'
           cd compilers/concrete-compiler/compiler
           make BINDINGS_PYTHON_ENABLED=OFF build-benchmarks
-      - name: Slack Notification
-        if: ${{ failure() && github.ref == 'refs/heads/main' }}
-        continue-on-error: true
-        uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
-        env:
-          SLACK_COLOR: ${{ job.status }}
-          SLACK_MESSAGE: "Build compiler benchmark finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
       - name: Run compiler benchmarks
         run: |
           set -e
           cd compilers/concrete-compiler/compiler
           make run-cpu-benchmarks
+      - name: Upload raw results artifact
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+        with:
+          name: compiler-benchmarks-result
+          path: compilers/concrete-compiler/compiler/benchmarks_results.json
       - name: Slack Notification
         if: ${{ failure() && github.ref == 'refs/heads/main' }}
         continue-on-error: true
         uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
         env:
           SLACK_COLOR: ${{ job.status }}
-          SLACK_MESSAGE: "Run compiler benchmark finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
-      - name: Upload raw results artifact
-        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
-        with:
-          name: compiler-benchmarks-result
-          path: compilers/concrete-compiler/compiler/benchmarks_results.json
+          SLACK_MESSAGE: "build-and-run-benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
 
   parse-and-send-results:
     name: Parse and send results
@@ -99,16 +92,19 @@ jobs:
       - name: Parse results
         shell: bash
         run: |
-          COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})"
-          COMMIT_HASH="$(git describe --tags --dirty)"
+          {
+            echo "BENCH_DATE=$(date --iso-8601=seconds)";
+            echo "COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})";
+            echo "COMMIT_HASH=$(git describe --tags --dirty)";
+          } >> "${GITHUB_ENV}"
           # TODO output setup-instance (https://github.com/zama-ai/slab-github-runner/issues/38)
           python3 ./ci/benchmark_parser.py benchmarks_results.json parsed_benchmark_results.json \
           --database compiler_benchmarks \
           --hardware "hpc7a.96xlarge" \
           --project-version ${COMMIT_HASH} \
           --branch ${{ github.ref_name }} \
           --commit-date ${COMMIT_DATE} \
-          --bench-date "$(date --iso-8601=seconds)" \
+          --bench-date "${BENCH_DATE}" \
           --throughput
       - name: Upload parsed results artifact
         uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
@@ -124,16 +120,7 @@ jobs:
       - name: Send data to Slab
         shell: bash
         run: |
-          echo "Computing HMac on downloaded artifact"
-          SIGNATURE="$(slab/scripts/hmac_calculator.sh parsed_benchmark_results.json '${{ secrets.JOB_SECRET }}')"
-          echo "Sending results to Slab..."
-          curl -v -k \
-          -H "Content-Type: application/json" \
-          -H "X-Slab-Repository: ${{ github.repository }}" \
-          -H "X-Slab-Command: store_data" \
-          -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \
-          -d @parsed_benchmark_results.json \
-          ${{ secrets.SLAB_URL }}
+          python3 slab/scripts/data_sender.py parsed_benchmark_results.json "${{ secrets.JOB_SECRET }}" --slab-url "${{ secrets.SLAB_URL }}"
       - name: Slack Notification
         if: ${{ failure() && github.ref == 'refs/heads/main' }}
         continue-on-error: true

diff --git a/.github/workflows/concrete_compiler_publish_docker_images.yml b/.github/workflows/concrete_compiler_publish_docker_images.yml
@@ -2,11 +2,7 @@ name: concrete-compiler publish docker images
 
 on:
   workflow_dispatch:
-  pull_request:
-
   push:
-    paths:
-      - .github/workflows/concrete_compiler_publish_docker_images.yml
     branches:
       - 'main'
       - 'force-docker-images'
@@ -19,7 +15,7 @@ env:
   THIS_FILE: .github/workflows/concrete_compiler_publish_docker_images.yml
 
 concurrency:
-  group: concrete_compiler_publich_docker_images
+  group: concrete_compiler_publish_docker_images
   cancel-in-progress: true
 
 jobs:
@@ -39,52 +35,6 @@ jobs:
           backend: aws
           profile: cpu-test
 
-  compiler-image:
-    needs: [setup-instance, hpx-image, cuda-image]
-    runs-on: ${{ needs.setup-instance.outputs.runner-name }}
-    env:
-      image: ghcr.io/zama-ai/concrete-compiler
-      dockerfile: docker/Dockerfile.concrete-compiler-env
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 0
-          submodules: recursive
-      - name: Login to Registry
-        run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
-      # label was initially a need from the frontend CI
-      - name: Build Image
-        run: |
-          DOCKER_BUILDKIT=1 docker build --no-cache \
-            --label "commit-sha=${{ github.sha }}" -t ${{ env.image }} -f ${{ env.dockerfile }} .
-      # disabled because of https://github.com/aquasecurity/trivy/discussions/7668
-      # - name: Run Trivy vulnerability scanner
-      #   uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0
-      #   with:
-      #     image-ref: '${{ matrix.image }}'
-      #     format: 'table'
-      #     exit-code: '1'
-      #     ignore-unfixed: true
-      #     vuln-type: 'os,library'
-      #     severity: 'CRITICAL,HIGH'
-      - name: Tag and Publish Image
-        run: |
-          docker image tag ${{ env.image }} ${{ env.image }}:${{ github.sha }}
-          docker image push ${{ env.image }}:latest
-          docker image push ${{ env.image }}:${{ github.sha }}
-      - name: Tag and Publish Release Image
-        if: startsWith(github.ref, 'refs/tags/v')
-        run: |
-          docker image tag ${{ env.image }} ${{ env.image }}:${{ github.ref_name }}
-          docker image push ${{ env.image }}:${{ github.ref_name }}
-      - name: Slack Notification
-        if: ${{ failure() && github.ref == 'refs/heads/main' }}
-        continue-on-error: true
-        uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
-        env:
-          SLACK_COLOR: ${{ job.status }}
-          SLACK_MESSAGE: "compiler-image finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
-
   hpx-image:
     needs: [setup-instance]
     runs-on: ${{ needs.setup-instance.outputs.runner-name }}
@@ -100,7 +50,7 @@ jobs:
         uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 # v44.5.24
       - name: Login
         id: login
-        if: contains(steps.changed-files.outputs.modified_files, '${{ env.dockerfile }}') || contains(steps.changed-files.outputs.modified_files, env.THIS_FILE)
+        if: contains(steps.changed-files.outputs.modified_files, env.dockerfile) || contains(steps.changed-files.outputs.modified_files, env.THIS_FILE)
         run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
       - name: Build
         if: ${{ steps.login.conclusion != 'skipped' }}
@@ -186,6 +136,64 @@ jobs:
           SLACK_COLOR: ${{ job.status }}
           SLACK_MESSAGE: "cuda-image finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
 
+  compiler-image:
+    needs: [setup-instance, hpx-image, cuda-image]
+    runs-on: ${{ needs.setup-instance.outputs.runner-name }}
+    env:
+      image: ghcr.io/zama-ai/concrete-compiler
+      dockerfile: docker/Dockerfile.concrete-compiler-env
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+          submodules: recursive
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 # v44.5.24
+        with:
+          files: |
+            backends/**
+            compilers/**
+            third_party/**
+            tools/**
+      - name: Login to Registry
+        id: login
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
+      - name: Build Image
+        if: steps.login.conclusion != 'skipped'
+        run: |
+          DOCKER_BUILDKIT=1 docker build --no-cache \
+            --label "commit-sha=${{ github.sha }}" -t ${{ env.image }} -f ${{ env.dockerfile }} .
+      # disabled because of https://github.com/aquasecurity/trivy/discussions/7668
+      # - name: Run Trivy vulnerability scanner
+      #   uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0
+      #   with:
+      #     image-ref: '${{ matrix.image }}'
+      #     format: 'table'
+      #     exit-code: '1'
+      #     ignore-unfixed: true
+      #     vuln-type: 'os,library'
+      #     severity: 'CRITICAL,HIGH'
+      - name: Tag and Publish Image
+        if: steps.login.conclusion != 'skipped'
+        run: |
+          docker image tag ${{ env.image }} ${{ env.image }}:${{ github.sha }}
+          docker image push ${{ env.image }}:latest
+          docker image push ${{ env.image }}:${{ github.sha }}
+      - name: Tag and Publish Release Image
+        if: steps.login.conclusion != 'skipped' && startsWith(github.ref, 'refs/tags/v')
+        run: |
+          docker image tag ${{ env.image }} ${{ env.image }}:${{ github.ref_name }}
+          docker image push ${{ env.image }}:${{ github.ref_name }}
+      - name: Slack Notification
+        if: ${{ failure() && github.ref == 'refs/heads/main' }}
+        continue-on-error: true
+        uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
+        env:
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_MESSAGE: "compiler-image finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
+
   teardown-instane:
     name: Teardown instance
     needs: [ setup-instance, compiler-image ]
@@ -207,4 +215,4 @@ jobs:
         uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
         env:
           SLACK_COLOR: ${{ job.status }}
-          SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
+          SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
diff --git a/.github/workflows/concrete_compiler_test_cpu.yml b/.github/workflows/concrete_compiler_test_cpu.yml
@@ -42,7 +42,7 @@ jobs:
           profile: cpu-test
 
   format-and-lint:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Format with clang-format (Cpp)
@@ -127,10 +127,10 @@ jobs:
             for i in `ls *Dialect.md`; do diff $i /concrete/docs/explanations/$i; done;
       - name: Enable complete tests on push to main
         if: github.ref == 'refs/heads/main'
-        run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV
+        run: echo "MINIMAL_TESTS=OFF" >> ${GITHUB_ENV}
       - name: Enable minimal tests otherwise
         if: github.ref != 'refs/heads/main'
-        run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV
+        run: echo "MINIMAL_TESTS=ON" >> ${GITHUB_ENV}
       - name: Run compiler tests
         uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
         with:

diff --git a/.github/workflows/concrete_optimizer.yml b/.github/workflows/concrete_optimizer.yml
@@ -28,7 +28,7 @@ jobs:
   tests:
     strategy:
       matrix:
-        runson: ["ubuntu-22.04", "aws-mac1-metal", "aws-mac2-metal"]
+        runson: ["ubuntu-latest", "aws-mac1-metal", "aws-mac2-metal"]
     runs-on: ${{ matrix.runson }}
     env:
       RUSTFLAGS: -D warnings
@@ -57,7 +57,7 @@ jobs:
           make -C concrete-optimizer-cpp test-ci
 
   benchmarks:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

diff --git a/.github/workflows/concrete_python_release_gpu.yml b/.github/workflows/concrete_python_release_gpu.yml
@@ -214,7 +214,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install CP
+      - name: Install concrete-python
         run: pip install --pre --extra-index-url https://pypi.zama.ai/gpu/ "concrete-python==${{ env.CP_VERSION }}"
 
       - name: Checkout the repository

diff --git a/.github/workflows/concrete_python_tests_linux.yml b/.github/workflows/concrete_python_tests_linux.yml
@@ -82,8 +82,6 @@ jobs:
             rustup toolchain install nightly-2024-09-30
             dnf -y install graphviz graphviz-devel
 
-            #rm -rf /build/*
-
             cd /concrete/frontends/concrete-python
             make venv
             source .venv/bin/activate
@@ -93,13 +91,6 @@ jobs:
 
             echo "Debug: ccache statistics (after the build):"
             ccache -s
-      - name: Slack Notification
-        if: ${{ failure() && github.ref == 'refs/heads/main' }}
-        continue-on-error: true
-        uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
-        env:
-          SLACK_COLOR: ${{ job.status }}
-          SLACK_MESSAGE: "build-python-bindings finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
       - name: Create artifact archive
         run: |
             cd build
@@ -111,6 +102,13 @@ jobs:
           include-hidden-files: true
           retention-days: 3
           path: build/artifacts.tgz
+      - name: Slack Notification
+        if: ${{ failure() && github.ref == 'refs/heads/main' }}
+        continue-on-error: true
+        uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
+        env:
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_MESSAGE: "build-python-bindings finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
 
   test-pytest:
     needs: [setup-instance, build-python-bindings]

diff --git a/.github/workflows/scripts/teardown-check.sh b/.github/workflows/scripts/teardown-check.sh
@@ -7,4 +7,4 @@ if [ -s missing-teardown.txt ]; then
     echo
     cat missing-teardown.txt
     exit 1
-fi
+fi
diff --git a/ci/slab.toml b/ci/slab.toml
@@ -26,4 +26,4 @@ security_group= ["sg-02dd8470fa845f31b", ]
 [backend.aws.release]
 region = "eu-west-1"
 image_id = "ami-002bdcd64b8472cf9"
-instance_type = "hpc7a.96xlarge"
+instance_type = "hpc7a.96xlarge"