diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 6b9c96d0426f..53a3d136f87d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -203,13 +203,13 @@ jobs: run: echo "${{ github.event.pull_request.number || 0 }}" > pr_number.txt - name: "Upload PR number" - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: path: "pr_number.txt" name: "pr_number" - name: "Upload result artifact" - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: path: "benchmark-results" name: "benchmark-results" diff --git a/.github/workflows/build_pyvelox.yml b/.github/workflows/build_pyvelox.yml index b1882528ae1c..f57ed568531e 100644 --- a/.github/workflows/build_pyvelox.yml +++ b/.github/workflows/build_pyvelox.yml @@ -157,7 +157,7 @@ jobs: cd wheelhouse rename 's/11_0/10_15/g' *.whl - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: wheels path: | @@ -170,7 +170,7 @@ jobs: needs: build_wheels runs-on: ubuntu-22.04 steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: wheels path: ./wheelhouse diff --git a/.github/workflows/conbench_upload.yml b/.github/workflows/conbench_upload.yml index 7dd54dfb3520..518aacd25826 100644 --- a/.github/workflows/conbench_upload.yml +++ b/.github/workflows/conbench_upload.yml @@ -32,54 +32,29 @@ permissions: jobs: upload: runs-on: ubuntu-latest - # Disabling workflow due to potential security sev - if: false steps: - - name: 'Download artifacts' + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + run-id: ${{ github.event.workflow_run.id || inputs.run_id }} + merge-multiple: true + path: /tmp/artifacts/ + github-token: ${{ github.token }} + + - name: Set meta data outputs id: 'download' - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: + # Have to get the run data explicitly instead of using the payload + # for workflow_dispatch to work. script: | - const run_id = "${{ github.event.workflow_run.id || inputs.run_id }}"; let benchmark_run = await github.rest.actions.getWorkflowRun({ owner: context.repo.owner, repo: context.repo.repo, - run_id: run_id, - }); - - let artifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: run_id, + run_id: "${{ github.event.workflow_run.id || inputs.run_id }}", }); - let result_artifact = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == "benchmark-results" - })[0]; - - let pr_artifact = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == "pr_number" - })[0]; - - let result_download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: result_artifact.id, - archive_format: 'zip', - }); - - let pr_download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: pr_artifact.id, - archive_format: 'zip', - }); - - var fs = require('fs'); - fs.writeFileSync('${{github.workspace}}/benchmark-results.zip', Buffer.from(result_download.data)); - fs.writeFileSync('${{github.workspace}}/pr_number.zip', Buffer.from(pr_download.data)); - core.setOutput('contender_sha', benchmark_run.data.head_sha); if (benchmark_run.data.event == 'push') { @@ -88,13 +63,10 @@ jobs: core.setOutput('merge_commit_message', ''); } - - name: Extract artifact + - name: Extract PR Number id: extract run: | - unzip benchmark-results.zip -d benchmark-results - unzip pr_number.zip - - pr_number=$(grep -ox '[[:digit:]]*' pr_number.txt | head -1) + pr_number=$(grep -ox '[[:digit:]]*' /tmp/artifacts/pr_number.txt | head -1) if [ "$pr_number" -ge 0 ]; then echo "Found PR number: $pr_number" @@ -105,9 +77,11 @@ jobs: echo "pr_number=$pr_number" >> $GITHUB_OUTPUT - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: path: velox + persist-credentials: false + - uses: actions/setup-python@v4 with: python-version: '3.8' @@ -115,7 +89,7 @@ jobs: cache-dependency-path: "velox/scripts/*" - name: "Install dependencies" - run: python -m pip install -r velox/scripts/benchmark-requirements.txt + run: pip install -r velox/scripts/benchmark-requirements.txt - name: "Upload results" env: @@ -126,7 +100,7 @@ jobs: CONBENCH_PROJECT_REPOSITORY: "${{ github.repository }}" CONBENCH_PROJECT_COMMIT: "${{ steps.download.outputs.contender_sha }}" run: | - if [ "${{ steps.extract.outputs.pr_number }}" -gt 0]; then + if [ "${{ steps.extract.outputs.pr_number }}" -gt 0 ]; then export CONBENCH_PROJECT_PR_NUMBER="${{ steps.extract.outputs.pr_number }}" fi @@ -134,7 +108,7 @@ jobs: --run_id "GHA-${{ github.run_id }}-${{ github.run_attempt }}" \ --pr_number "${{ steps.extract.outputs.pr_number }}" \ --sha "${{ steps.download.outputs.contender_sha }}" \ - --output_dir "${{ github.workspace }}/benchmark-results/contender/" + --output_dir "/tmp/artifacts/benchmark-results/contender/" - name: "Check the status of the upload" # Status functions like failure() only work in `if:` @@ -143,7 +117,7 @@ jobs: run: echo "failed=true" >> $GITHUB_OUTPUT - name: "Create a GitHub Status on the contender commit (whether the upload was successful)" - uses: actions/github-script@v6 + uses: actions/github-script@v7 if: ${{ !cancelled() && steps.extract.conclusion != 'failure' }} with: script: | diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index fbca400247ec..b02bdaed4e60 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -80,37 +80,31 @@ jobs: ccache -s - name: Upload aggregation fuzzer - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: aggregation path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_aggregation_fuzzer_test - name: Upload spark aggregation fuzzer - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: spark_aggregation_fuzzer path: velox/_build/debug/velox/functions/sparksql/fuzzer/spark_aggregation_fuzzer_test - - name: Upload aggregation fuzzer - uses: actions/upload-artifact@v3 - with: - name: aggregation - path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_aggregation_fuzzer_test - - name: Upload join fuzzer - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: join path: velox/_build/debug/velox/exec/tests/velox_join_fuzzer_test - name: Upload Presto expression fuzzer - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: presto_expression_fuzzer path: velox/_build/debug/velox/expression/fuzzer/velox_expression_fuzzer_test - name: Upload Spark expression fuzzer - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: spark_expression_fuzzer path: velox/_build/debug/velox/expression/fuzzer/spark_expression_fuzzer_test @@ -173,7 +167,7 @@ jobs: - name: Archive aggregate production artifacts if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: aggregate-fuzzer-failure-artifacts path: | @@ -195,7 +189,7 @@ jobs: run: source ./scripts/setup-ubuntu.sh && install_apt_deps - name: Download spark aggregation fuzzer - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: spark_aggregation_fuzzer @@ -215,7 +209,7 @@ jobs: - name: Archive Spark aggregate production artifacts if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: spark-agg-fuzzer-failure-artifacts path: | @@ -236,7 +230,7 @@ jobs: run: source ./scripts/setup-ubuntu.sh && install_apt_deps - name: Download join fuzzer - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: join @@ -256,7 +250,7 @@ jobs: - name: Archive aggregate production artifacts if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: join-fuzzer-failure-artifacts path: | @@ -277,7 +271,7 @@ jobs: run: source ./scripts/setup-ubuntu.sh && install_apt_deps - name: Download presto fuzzer - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: presto_expression_fuzzer @@ -306,7 +300,7 @@ jobs: - name: Archive Presto expression production artifacts if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: presto-fuzzer-failure-artifacts path: | @@ -327,7 +321,7 @@ jobs: run: source ./scripts/setup-ubuntu.sh && install_apt_deps - name: Download spark fuzzer - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: spark_expression_fuzzer @@ -349,7 +343,7 @@ jobs: - name: Archive Spark expression production artifacts if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: spark-fuzzer-failure-artifacts path: |