Scanner versioned vulnerabilities update #523
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scanner versioned vulnerabilities update | |
| on: | |
| schedule: | |
| - cron: "30 */4 * * *" | |
| pull_request: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| workflow_dispatch: | |
| inputs: | |
| job: | |
| type: choice | |
| description: "Choose the NVD source" | |
| options: | |
| - nvd-api | |
| - nvd-feeds | |
| required: true | |
| default: nvd-feeds | |
| jobs: | |
| parse-versions: | |
| if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-vulns') | |
| runs-on: ubuntu-latest | |
| outputs: | |
| versions: ${{ steps.set-versions.outputs.versions }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Parse VULNERABILITY_BUNDLE_VERSION | |
| id: set-versions | |
| run: | | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) | |
| echo "versions<<$EOF" >> "$GITHUB_OUTPUT" | |
| ./.github/workflows/scripts/scanner-get-released-tags.sh | tee -a "$GITHUB_OUTPUT" | |
| echo "$EOF" >> "$GITHUB_OUTPUT" | |
| prepare-environment: | |
| if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-vulns') | |
| runs-on: ubuntu-latest | |
| outputs: | |
| manual_url: ${{ steps.set-manual-url.outputs.manual_url }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Download NVD | |
| run: | | |
| set -eu | |
| since_time=$(date -u -d '24 hours ago' '+%a, %d %b %Y %H:%M:%S GMT') | |
| NVD_BUNDLE_TYPE=${{ github.event.inputs.job || 'nvd-feeds' }} | |
| case "$NVD_BUNDLE_TYPE" in | |
| nvd-api) | |
| nvd_file=nvd-api.zip | |
| ;; | |
| nvd-feeds) | |
| nvd_file=nvd-feeds.zip | |
| ;; | |
| *) | |
| echo >&2 "Warning: invalid NVD bundle type '$NVD_BUNDLE_TYPE'" | |
| exit 1 | |
| esac | |
| url="https://definitions.stackrox.io/v4/nvd/$nvd_file" | |
| code=$(curl \ | |
| -o nvd.zip \ | |
| -w "%{http_code}" \ | |
| -H "If-Modified-Since: $since_time" \ | |
| "$url") | |
| echo "code: $code" | |
| echo "$code" | grep -q 200 | |
| - name: Set MANUAL_URL | |
| id: set-manual-url | |
| run: | | |
| if [ "${{ github.event_name }}" == "schedule" ]; then | |
| # Scheduled workflows should use the master branch | |
| echo "manual_url=https://raw.githubusercontent.com/stackrox/stackrox/master/scanner/updater/manual/vulns.yaml" >> "$GITHUB_ENV" | |
| else | |
| # Determine the SHA to use | |
| if [ -z "${{ github.event.pull_request.head.sha }}" ]; then | |
| # Fallback to the current ref or SHA if it's not a pull request event | |
| sha=${{ github.sha }} | |
| else | |
| sha=${{ github.event.pull_request.head.sha }} | |
| fi | |
| echo "manual_url=https://raw.githubusercontent.com/stackrox/stackrox/$sha/scanner/updater/manual/vulns.yaml" >> "$GITHUB_ENV" | |
| fi | |
| - uses: ./.github/actions/upload-artifact-with-retry | |
| with: | |
| name: nvd | |
| path: nvd.zip | |
| if-no-files-found: error | |
| build-and-run: | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| needs: | |
| - parse-versions | |
| - prepare-environment | |
| runs-on: ubuntu-latest | |
| container: | |
| image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.4 | |
| volumes: | |
| # The updater makes heavy use of /tmp files. | |
| - /tmp:/tmp | |
| - /usr:/mnt/usr | |
| - /opt:/mnt/opt | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| include: ${{ fromJson(needs.parse-versions.outputs.versions) }} | |
| env: | |
| SCANNER_BUNDLE_VERSION: ${{ matrix.version }} | |
| ROX_GIT_REF: ${{ matrix.ref }} | |
| steps: | |
| - name: Free up disk space | |
| shell: bash | |
| run: | | |
| set +e | |
| set -x | |
| df -h | |
| for delete in /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL; do | |
| rm -rf "/mnt${delete:?}" | |
| done | |
| df -h | |
| - name: Checkout specific reference | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ env.ROX_GIT_REF }} | |
| - uses: ./.github/actions/job-preamble | |
| with: | |
| gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} | |
| - name: Download NVD | |
| uses: ./.github/actions/download-artifact-with-retry | |
| with: | |
| name: nvd | |
| path: . | |
| - name: Build updater | |
| run: | | |
| echo "Building updater for version ${{ env.SCANNER_BUNDLE_VERSION }} based on git ref ${{ env.ROX_GIT_REF }}..." | |
| make tag | |
| make -C scanner bin/updater | |
| - name: Create bundle output directory | |
| run: mkdir -p definitions/${{ env.SCANNER_BUNDLE_VERSION }} | |
| - name: Sanity check NVD zip | |
| run: | | |
| path="$PWD/nvd.zip" | |
| echo "checking contents of $path" | |
| unzip -l "$path" | |
| echo "STACKROX_NVD_ZIP_PATH=$path" >> "$GITHUB_ENV" | |
| - name: Run Updater (single bundle) | |
| if: ${{ env.SCANNER_BUNDLE_VERSION == 'v1' }} | |
| run: | | |
| scanner/bin/updater export --manual-url "${{ needs.prepare-environment.outputs.manual_url }}" "definitions/${SCANNER_BUNDLE_VERSION}" | |
| - name: Run updater (multi bundle) | |
| run: | | |
| scanner/bin/updater export --manual-url "${{ needs.prepare-environment.outputs.manual_url }}" --split bundles | |
| zip definitions/${{ env.SCANNER_BUNDLE_VERSION }}/vulnerabilities.zip bundles/*.json.zst | |
| - name: Upload definitions artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: artifact_${{ env.SCANNER_BUNDLE_VERSION }} | |
| path: definitions | |
| build-upload-pr-vulnerabilities: | |
| if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'pr-update-scanner-vulns') | |
| needs: | |
| - prepare-environment | |
| runs-on: ubuntu-latest | |
| container: | |
| image: quay.io/stackrox-io/apollo-ci:scanner-test-0.4.4 | |
| volumes: | |
| # The updater makes heavy use of /tmp files. | |
| - /tmp:/tmp | |
| - /usr:/mnt/usr | |
| - /opt:/mnt/opt | |
| steps: | |
| - name: Free up disk space | |
| shell: bash | |
| run: | | |
| set +e | |
| set -x | |
| df -h | |
| for delete in /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL; do | |
| rm -rf "/mnt${delete:?}" | |
| done | |
| df -h | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - uses: ./.github/actions/job-preamble | |
| with: | |
| gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} | |
| - name: Authenticate with test GCS bucket | |
| if: github.event_name == 'pull_request' | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GOOGLE_SA_CIRCLECI_SCANNER }} | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Download NVD | |
| uses: ./.github/actions/download-artifact-with-retry | |
| with: | |
| name: nvd | |
| path: . | |
| - name: Build updater | |
| run: | | |
| echo "Building updater for pull request ${{ env.PR_NAME }}..." | |
| make tag | |
| make -C scanner bin/updater | |
| - name: Sanity check NVD zip | |
| run: | | |
| path="$PWD/nvd.zip" | |
| echo "checking contents of $path" | |
| unzip -l "$path" | |
| echo "STACKROX_NVD_ZIP_PATH=$path" >> "$GITHUB_ENV" | |
| - name: Create bundle output directory | |
| run: mkdir -p definitions/${{ github.event.pull_request.number }} | |
| - name: Run updater (multi bundle) | |
| run: | | |
| scanner/bin/updater export --manual-url "${{ needs.prepare-environment.outputs.manual_url }}" --split bundles | |
| zip definitions/${{ github.event.pull_request.number }}/vulnerabilities.zip bundles/*.json.zst | |
| # PR owner is responsible for verifying vulnerability bundles are generated | |
| - name: Upload PR vulnerabilities | |
| run: | | |
| branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} | |
| # Replace / with -, so the branch name isn't truncated when pushed to GCS. | |
| dir=${branch////-} | |
| case "$dir" in | |
| dev|1.0.0) | |
| echo "Error: branch $dir is protected. Choose a different branch name." | |
| exit 1 | |
| esac | |
| mkdir -p "$dir" | |
| cp -r "definitions/${{ github.event.pull_request.number }}/." "$dir/" | |
| gsutil -m cp -r "$dir" "gs://scanner-v4-test/vulnerability-bundles/" | |
| upload-definitions: | |
| needs: | |
| - build-and-run | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Checkout to run ./.github/actions/download-artifact-with-retry | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/download-artifact-with-retry | |
| with: | |
| # Only download vulnerability bundles. | |
| # This avoids accidentally downloading nvd.zip. | |
| pattern: "artifact_*" | |
| path: downloaded_artifacts | |
| - name: Move files artifacts | |
| run: rsync -av downloaded_artifacts/*/ definitions_files/ | |
| - name: Authenticate with Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GOOGLE_SA_STACKROX_HUB_VULN_DUMP_UPLOADER }} | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Upload to GCS | |
| run: | | |
| ls -lr definitions_files | |
| for dir in definitions_files/*; do | |
| if [ -d "$dir" ]; then | |
| echo "Copy $dir" | |
| gsutil -m cp -r "$dir" gs://definitions.stackrox.io/v4/vulnerability-bundles/ | |
| fi | |
| done | |
| echo "Copy upstream dev (dev) to downstream dev (1.0.0)" | |
| gsutil cp -r gs://definitions.stackrox.io/v4/vulnerability-bundles/dev/* gs://definitions.stackrox.io/v4/vulnerability-bundles/1.0.0/ | |
| - name: Copy v1 to pre-versioned bundles | |
| run: | | |
| # Using v1 bundle for released versions listed in RELEASE_VERSION. | |
| single=gs://definitions.stackrox.io/v4/vulnerability-bundles/v1/vulns.json.zst | |
| multi=gs://definitions.stackrox.io/v4/vulnerability-bundles/v1/vulnerabilities.zip | |
| # Parse all supported pre-4.6 releases and copy the versioned bundle v1 to ensure | |
| # these releases get updates. | |
| grep -E "^4\.(4|5)\.[0-9]+$" scanner/updater/version/RELEASE_VERSION | while read -r release; do | |
| case "$release" in | |
| 4.4.*) | |
| gsutil cp "$single" "gs://definitions.stackrox.io/v4/vulnerability-bundles/$release/" | |
| ;; | |
| 4.5.*) | |
| gsutil cp "$single" "gs://definitions.stackrox.io/v4/vulnerability-bundles/$release/" | |
| gsutil cp "$multi" "gs://definitions.stackrox.io/v4/vulnerability-bundles/$release/" | |
| ;; | |
| *) | |
| echo "Should not happen!" | |
| echo "Error: unexpected release version: $release" | |
| echo "Ignoring..." | |
| ;; | |
| esac | |
| echo "Copied v1 into $release" | |
| done | |
| send-notification: | |
| needs: | |
| - build-and-run | |
| - upload-definitions | |
| runs-on: ubuntu-latest | |
| if: ${{ failure() && github.ref_name == 'master' }} | |
| steps: | |
| - name: Send Slack notification on workflow failure | |
| run: | | |
| curl -X POST -H 'Content-type: application/json' --data '{"text":"<${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|Workflow ${{ github.workflow }}> failed in repository ${{ github.repository }}: Failed to update vulnerabilities"}' ${{ secrets.SLACK_ONCALL_SCANNER_WEBHOOK }} |