From d79a1ff109bcba773bf533d17b0b20312e90e7cb Mon Sep 17 00:00:00 2001 From: ikolomi Date: Sun, 10 Nov 2024 20:58:16 +0200 Subject: [PATCH] ORT workflows rework: 1. Added ort-sweeper workflow that triggers periodic (once a day) ORT runs for branches main and release-* 2. Fixed ORT invocation for glide-core/redis-rs by creating a virtual workspace 3. Updated license-generating script to prevent the creation of unapproved_package_list.txt file for unapproved/unknown licenses 4. Configured ORT checker workflow to run on each pull request without file filters, producing warnings if licenses need to be updated 5. TODO: Modify ORT checker to fail pull requests introducing unapproved/unknown licenses, deferred until unsupported licenses are removed to avoid blocking PRs 6. Added support to trigger the ORT checker manually with a specified target branch Signed-off-by: ikolomi --- .github/workflows/ort-sweeper.yml | 57 ++++ .github/workflows/ort.yml | 432 ++++++++++++++++-------------- glide-core/redis-rs/Cargo.toml | 8 +- glide-core/redis-rs/src/main.rs | 3 + utils/get_licenses_from_ort.py | 4 + 5 files changed, 299 insertions(+), 205 deletions(-) create mode 100644 .github/workflows/ort-sweeper.yml create mode 100644 glide-core/redis-rs/src/main.rs diff --git a/.github/workflows/ort-sweeper.yml b/.github/workflows/ort-sweeper.yml new file mode 100644 index 0000000000..91889c2239 --- /dev/null +++ b/.github/workflows/ort-sweeper.yml @@ -0,0 +1,57 @@ +name: ORT - Trigger periodic checks for relevant branches + +on: + schedule: + - cron: '0 0 * * *' # Runs daily at 00:00 UTC + +jobs: + trigger-ort-check: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Fetch relevant branches + id: get-branches + run: | + # Get all branches matching 'release-*' and include 'main' + branches=$(git ls-remote --heads origin | awk -F'/' '/refs\/heads\/release-/ {print $NF}') + branches="main $branches" + echo "::set-output name=branches::$branches" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Trigger ORT Check workflows + uses: actions/github-script@v6 + with: + script: | + const branches = "${{ steps.get-branches.outputs.branches }}".split(" "); + const workflowFile = "ort.yml"; + + const triggerWorkflow = async (branch) => { + try { + console.log(`Triggering workflow for branch: ${branch}`); + await github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: workflowFile, + ref: branch, // The branch where workflow_dispatch is triggered + inputs: { + branch_name: branch + } + }); + console.log(`Successfully triggered workflow for branch: ${branch}`); + } catch (error) { + core.setFailed(error.message); + } + }; + + // Fire all workflow dispatch requests concurrently + const promises = branches + .filter(branch => branch) // Skip empty branches + .map(branch => triggerWorkflow(branch)); + + await Promise.allSettled(promises); + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ort.yml b/.github/workflows/ort.yml index 2eff2a3f1a..486229e733 100644 --- a/.github/workflows/ort.yml +++ b/.github/workflows/ort.yml @@ -1,236 +1,260 @@ name: The OSS Review Toolkit (ORT) on: - schedule: - - cron: "0 0 * * *" pull_request: - paths: - - .github/workflows/ort.yml - - .github/workflows/run-ort-tools/action.yml - - utils/get_licenses_from_ort.py + workflow_dispatch: inputs: - branch: - description: "The branch to run against the ORT tool" - required: true - version: - description: "The release version of GLIDE" + branch_name: + description: 'The branch to run against the ORT tool' required: true + default: 'main' + jobs: run-ort: - if: github.repository_owner == 'valkey-io' name: Create attribution files runs-on: ubuntu-latest + + # 1. For workflow_dispatch, always allow + # 2. For pull_request, run if branch is not autogenerated ort-diff-for- branches + if: > + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && !startsWith(github.head_ref, 'ort-diff-for-')) strategy: fail-fast: false - env: + env: PYTHON_ATTRIBUTIONS: "python/THIRD_PARTY_LICENSES_PYTHON" NODE_ATTRIBUTIONS: "node/THIRD_PARTY_LICENSES_NODE" RUST_ATTRIBUTIONS: "glide-core/THIRD_PARTY_LICENSES_RUST" JAVA_ATTRIBUTIONS: "java/THIRD_PARTY_LICENSES_JAVA" + INPUT_TARGET_BRANCH: ${{ github.event.inputs.branch_name }} + EVENT_NAME: ${{ github.event_name }} + HEAD_REF: ${{ github.head_ref }} + steps: - - name: Set the release version - shell: bash - run: | - export version=`if [ "$EVENT_NAME" == 'schedule' ] || [ "$EVENT_NAME" == 'pull_request' ]; then echo '255.255.255'; else echo "$INPUT_VERSION"; fi` - echo "RELEASE_VERSION=${version}" >> $GITHUB_ENV - env: - EVENT_NAME: ${{ github.event_name }} - INPUT_VERSION: ${{ github.event.inputs.version }} - - - name: Set the base branch - run: | - export BASE_BRANCH=`if [ "$EVENT_NAME" == 'schedule' ]; then echo 'main'; elif [ "$EVENT_NAME" == 'workflow_dispatch' ]; then echo "$INPUT_BRANCH"; else echo ""; fi` - echo "Base branch is: ${BASE_BRANCH}" - echo "BASE_BRANCH=${BASE_BRANCH}" >> $GITHUB_ENV - env: - EVENT_NAME: ${{ github.event_name }} - INPUT_BRANCH: ${{ github.event.inputs.branch }} - - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: "true" - ref: ${{ env.BASE_BRANCH }} - - - name: Set up JDK 11 for the ORT package - uses: actions/setup-java@v4 - with: - distribution: "temurin" - java-version: 11 - - - name: Cache ORT and Gradle packages - uses: actions/cache@v4 - id: cache-ort - with: - path: | - ./ort - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-ort - - - name: Checkout ORT Repository - if: steps.cache-ort.outputs.cache-hit != 'true' - uses: actions/checkout@v4 - with: - repository: "oss-review-toolkit/ort" - path: "./ort" - ref: "26.0.0" - submodules: recursive - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@1.78 - - - name: Install ORT - if: steps.cache-ort.outputs.cache-hit != 'true' - working-directory: ./ort/ - run: | - export JAVA_OPTS="$JAVA_OPTS -Xmx8g" - ./gradlew installDist - - - name: Create ORT config file - run: | - mkdir -p ~/.ort/config - cat << EOF > ~/.ort/config/config.yml - ort: - analyzer: - allowDynamicVersions: true - enabledPackageManagers: [Cargo, NPM, PIP, GradleInspector] - EOF - cat ~/.ort/config/config.yml + - name: Setup target branch and commit + run: | + if [ "$EVENT_NAME" == 'workflow_dispatch' ]; then + echo "TARGET_BRANCH=$INPUT_TARGET_BRANCH" >> $GITHUB_ENV + elif [ "$EVENT_NAME" == 'pull_request' ]; then + echo "TARGET_BRANCH=$HEAD_REF" >> $GITHUB_ENV + fi + + - name: Checkout target branch + uses: actions/checkout@v4 + with: + ref: ${{ env.TARGET_BRANCH }} + + - name: Setup target commit + run: | + echo "TARGET_COMMIT=`git rev-parse HEAD`" >> $GITHUB_ENV + + - name: Set up JDK 11 for the ORT package + uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: 11 + + - name: Cache ORT and Gradle packages + uses: actions/cache@v4 + id: cache-ort + with: + path: | + ./ort + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-ort + + - name: Checkout ORT Repository + if: steps.cache-ort.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + repository: "oss-review-toolkit/ort" + path: "./ort" + ref: "26.0.0" + submodules: recursive + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@1.78 + + - name: Build and install ORT + if: steps.cache-ort.outputs.cache-hit != 'true' + working-directory: ./ort/ + run: | + export JAVA_OPTS="$JAVA_OPTS -Xmx8g" + ./gradlew installDist + + - name: Create ORT config file + run: | + mkdir -p ~/.ort/config + cat << EOF > ~/.ort/config/config.yml + ort: + analyzer: + allowDynamicVersions: true + enabledPackageManagers: [Cargo, NPM, PIP, GradleInspector] + EOF + cat ~/.ort/config/config.yml ### NodeJS ### + - name: Set up Node.js 16.x + uses: actions/setup-node@v4 + with: + node-version: 16.x - - name: Set up Node.js 16.x - uses: actions/setup-node@v4 - with: - node-version: 16.x - - - name: Create package.json file for the Node wrapper - uses: ./.github/workflows/node-create-package-file - with: - release_version: ${{ env.RELEASE_VERSION }} - os: "ubuntu-latest" - - - name: Fix Node base NPM package.json file for ORT - working-directory: ./node/npm/glide - run: | - # Remove the glide-rs dependency to avoid duplication - sed -i '/ "glide-rs":/d' ../../package.json - export pkg_name=valkey-glide-base - export package_version="${{ env.RELEASE_VERSION }}" - export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` - mv package.json package.json.tmpl - envsubst < package.json.tmpl > "package.json" - cat package.json - - - name: Run ORT tools for Node - uses: ./.github/workflows/run-ort-tools - with: - folder_path: "${{ github.workspace }}/node" + - name: Create package.json file for the Node wrapper + uses: ./.github/workflows/node-create-package-file + with: + release_version: ${{ env.TARGET_COMMIT }} + os: "ubuntu-latest" - ### Python ### + - name: Fix Node base NPM package.json file for ORT + working-directory: ./node/npm/glide + run: | + # Remove the glide-rs dependency to avoid duplication + sed -i '/ "glide-rs":/d' ../../package.json + export pkg_name=valkey-glide-base + export package_version="${{ env.TARGET_COMMIT }}" + export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` + mv package.json package.json.tmpl + envsubst < package.json.tmpl > "package.json" + cat package.json - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" + - name: Run ORT tools for Node + uses: ./.github/workflows/run-ort-tools + with: + folder_path: "${{ github.workspace }}/node" - - name: Install python-inspector - working-directory: ./python - run: | - python -m pip install --upgrade pip - pip install git+https://github.com/nexB/python-inspector + ### Python ### + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" - - name: Run ORT tools for Python - uses: ./.github/workflows/run-ort-tools - with: - folder_path: "${{ github.workspace }}/python" + - name: Install python-inspector + working-directory: ./python + run: | + python -m pip install --upgrade pip + pip install git+https://github.com/nexB/python-inspector - ### Rust ### + - name: Run ORT tools for Python + uses: ./.github/workflows/run-ort-tools + with: + folder_path: "${{ github.workspace }}/python" - - name: Run ORT tools for Rust - uses: ./.github/workflows/run-ort-tools - with: - folder_path: "${{ github.workspace }}/glide-core" + ### Rust glide-core ### + - name: Run ORT tools for glide-core + uses: ./.github/workflows/run-ort-tools + with: + folder_path: "${{ github.workspace }}/glide-core" ### Java ### + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: 11 + + - name: Run ORT tools for Java + uses: ./.github/workflows/run-ort-tools + with: + folder_path: "${{ github.workspace }}/java" + + + ### Get licenses ### + - name: Retrieve licenses list + working-directory: ./utils + run: | + { + echo 'LICENSES_LIST<> "$GITHUB_ENV" + + ### Upload licenses ### + - name: Get current date + id: date + run: | + CURR_DATE=$(date +'%Y-%m-%d-%H') + echo "date=${CURR_DATE}" >> $GITHUB_OUTPUT + + - name: Upload the final package list + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: final-package-list-${{ steps.date.outputs.date }} + path: | + utils/final_package_list.txt + retention-days: 30 + + - name: Upload the skipped package list + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: skipped-package-list-${{ steps.date.outputs.date }} + path: | + utils/skipped_package_list.txt + retention-days: 30 - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - distribution: "temurin" - java-version: 11 - - - name: Run ORT tools for Java - uses: ./.github/workflows/run-ort-tools - with: - folder_path: "${{ github.workspace }}/java" - - ### Process results ### - - - name: Check for diff - run: | - cp python/ort_results/NOTICE_DEFAULT $PYTHON_ATTRIBUTIONS - cp node/ort_results/NOTICE_DEFAULT $NODE_ATTRIBUTIONS - cp glide-core/ort_results/NOTICE_DEFAULT $RUST_ATTRIBUTIONS - cp java/ort_results/NOTICE_DEFAULT $JAVA_ATTRIBUTIONS - GIT_DIFF=`git diff $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS` - if [ -n "$GIT_DIFF" ]; then + - name: Upload the unknown/unapproved package list + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: unapproved-package-list-${{ steps.date.outputs.date }} + path: | + utils/unapproved_package_list.txt + retention-days: 30 + + ### TODO: Fail if there are unapproved packages ### + + ### Check for attributions diff ### + - name: Check for diff + run: | + cp python/ort_results/NOTICE_DEFAULT $PYTHON_ATTRIBUTIONS + cp node/ort_results/NOTICE_DEFAULT $NODE_ATTRIBUTIONS + cp glide-core/ort_results/NOTICE_DEFAULT $RUST_ATTRIBUTIONS + cp java/ort_results/NOTICE_DEFAULT $JAVA_ATTRIBUTIONS + GIT_DIFF=`git diff $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS` + if [ -n "$GIT_DIFF" ]; then echo "FOUND_DIFF=true" >> $GITHUB_ENV - else + else echo "FOUND_DIFF=false" >> $GITHUB_ENV - fi - - - name: Retrieve licenses list - working-directory: ./utils - run: | - { - echo 'LICENSES_LIST<> "$GITHUB_ENV" - - ### Create PR ### - - - name: Create pull request - if: ${{ env.FOUND_DIFF == 'true' && github.event_name != 'pull_request' }} - run: | - export BRANCH_NAME=`if [ "$EVENT_NAME" == 'schedule' ] || [ "$EVENT_NAME" == 'pull_request' ]; then echo 'scheduled-ort'; else echo "ort-v$INPUT_VERSION"; fi` - echo "Creating pull request from branch ${BRANCH_NAME} to branch ${{ env.BASE_BRANCH }}" - git config --global user.email "valkey-glide@lists.valkey.io" - git config --global user.name "ort-bot" - git checkout -b ${BRANCH_NAME} - git add $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS - git commit -m "Updated attribution files" -s - git push --set-upstream origin ${BRANCH_NAME} -f - title="Updated attribution files for ${BRANCH_NAME}" - gh pr create -B ${{ env.BASE_BRANCH }} -H ${BRANCH_NAME} --title "${title}" --body 'Created by Github action.\n${{ env.LICENSES_LIST }}' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - EVENT_NAME: ${{ github.event_name }} - INPUT_VERSION: ${{ github.event.inputs.version }} - - - name: Get current date - id: date - run: | - CURR_DATE=$(date +'%Y-%m-%d-%H') - echo "date=${CURR_DATE}" >> $GITHUB_OUTPUT - - - name: Upload the final package list - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: final-package-list-${{ steps.date.outputs.date }} - path: | - utils/final_package_list.txt - retention-days: 30 - - - name: Upload the skipped package list - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: skipped-package-list-${{ steps.date.outputs.date }} - path: | - utils/skipped_package_list.txt - retention-days: 30 + fi + + ### Create PR, Note a potential race on the source branch ### + - name: Create pull request + if: ${{ env.FOUND_DIFF == 'true' && github.event_name != 'pull_request' }} + run: | + export ORT_DIFF_BRANCH_NAME="ort-diff-for-$TARGET_BRANCH" + echo "Creating pull request from branch $ORT_DIFF_BRANCH_NAME to branch $TARGET_BRANCH" + git config --global user.email "valkey-glide@lists.valkey.io" + git config --global user.name "ort-bot" + git checkout -b ${ORT_DIFF_BRANCH_NAME} + git add $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS + git commit -m "Updated attribution files" -s + git push --set-upstream origin ${ORT_DIFF_BRANCH_NAME} -f + + # Check if PR already exists + existing_pr=$(gh pr list --base ${TARGET_BRANCH} --head ${ORT_DIFF_BRANCH_NAME} --json number --jq '.[0].number') + + if [ -z "$existing_pr" ]; then + # Create a new PR if none exists + title="Updated attribution files for commit ${TARGET_COMMIT}" + gh pr create -B ${TARGET_BRANCH} -H ${ORT_DIFF_BRANCH_NAME} --title "${title}" --body "Created by Github action. ${{ env.LICENSES_LIST }}" + echo "Pull request created successfully." + else + # Update the existing PR + echo "Pull request #$existing_pr already exists. Updating branch." + gh pr edit $existing_pr --title "Updated attribution files for commit ${TARGET_COMMIT}" --body "Created by Github action. ${{ env.LICENSES_LIST }}" + echo "Pull request updated successfully." + fi + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + INPUT_VERSION: ${{ github.event.inputs.version }} + + ### Warn of outdated attributions for PR ### + - name: Warn of outdated attributions due to the PR + if: ${{ env.FOUND_DIFF == 'true' && github.event_name == 'pull_request' }} + uses: actions/github-script@v6 + with: + script: | + core.warning('WARNING! Note the attribution files differ with this PR, make sure an updating PR is issued using scheduled or manual run of this workflow!'); diff --git a/glide-core/redis-rs/Cargo.toml b/glide-core/redis-rs/Cargo.toml index 2f4ebbcbbe..fa989b93cd 100644 --- a/glide-core/redis-rs/Cargo.toml +++ b/glide-core/redis-rs/Cargo.toml @@ -1,3 +1,9 @@ +# Dummy package so ORT tool will not fail on virtual workspace +[package] +name = "dummy-for-ort" +version = "0.1.0" +edition = "2021" + [workspace] -members = ["redis", "redis-test"] +members = ["redis", "redis-test", "afl/parser"] resolver = "2" diff --git a/glide-core/redis-rs/src/main.rs b/glide-core/redis-rs/src/main.rs new file mode 100644 index 0000000000..a5610f8be9 --- /dev/null +++ b/glide-core/redis-rs/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Dummy source to bypass ORT OSS Tool virtual workspace restrictions."); +} diff --git a/utils/get_licenses_from_ort.py b/utils/get_licenses_from_ort.py index 3d9853ea53..9c7d7b62ba 100644 --- a/utils/get_licenses_from_ort.py +++ b/utils/get_licenses_from_ort.py @@ -124,6 +124,10 @@ def __str__(self): with open(skipped_list_file_path, mode="wt", encoding="utf-8") as f: f.writelines(f"{package}\n" for package in skipped_packages) +unapproved_list_file_path = f"{SCRIPT_PATH}/unapproved_package_list.txt" +with open(unapproved_list_file_path, mode="wt", encoding="utf-8") as f: + f.writelines(f"{package}\n" for package in unknown_licenses) + print("\n\n#### Found Licenses #####\n") all_licenses_set = set(sorted(all_licenses_set)) for license in all_licenses_set: