From 41c39e6561c7f4ddafcb391bdfbc0dee34c00d80 Mon Sep 17 00:00:00 2001 From: Samir Rashid Date: Thu, 26 Sep 2024 20:07:26 +0000 Subject: [PATCH] Add ci --- .github/workflows/benchmarks.yml | 39 ++++++++ .github/workflows/ci-nightly.yml | 88 ++++++++++++++++ .github/workflows/ci.yml | 126 +++++++++++++++++++++++ .github/workflows/labeler.yml | 21 ++++ .github/workflows/litex_sim.yml | 138 ++++++++++++++++++++++++++ .github/workflows/mergequeue_docs.yml | 34 +++++++ .github/workflows/tockbot-nightly.yml | 134 +++++++++++++++++++++++++ 7 files changed, 580 insertions(+) create mode 100644 .github/workflows/benchmarks.yml create mode 100644 .github/workflows/ci-nightly.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/labeler.yml create mode 100644 .github/workflows/litex_sim.yml create mode 100644 .github/workflows/mergequeue_docs.yml create mode 100644 .github/workflows/tockbot-nightly.yml diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000000..2e179aa664 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 or the MIT License. +# SPDX-License-Identifier: Apache-2.0 OR MIT +# Copyright Tock Contributors 2023. + +# This workflow calculates size diffs for the compiled binary of each supported tock board + +name: Benchmarks + +# Controls when the action will run. Triggers the workflow on pull request +# events but only for the master branch +on: + pull_request: + branches: master + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +# If you add additional jobs, remember to add them to bors.toml +permissions: + contents: read + +jobs: + benchmarks: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + pip install --user cxxfilt + sudo apt install llvm + - name: size report + run: | + ./tools/github_actions_size_changes.sh diff --git a/.github/workflows/ci-nightly.yml b/.github/workflows/ci-nightly.yml new file mode 100644 index 0000000000..46d25f08bd --- /dev/null +++ b/.github/workflows/ci-nightly.yml @@ -0,0 +1,88 @@ +# Licensed under the Apache License, Version 2.0 or the MIT License. +# SPDX-License-Identifier: Apache-2.0 OR MIT +# Copyright Tock Contributors 2024. + +name: tock-nightly-ci + +on: + schedule: + - cron: "0 0 * * *" + +env: + TERM: xterm # Makes tput work in actions output + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +# If you add additional jobs, remember to add them to bors.toml +permissions: + contents: read + issues: write + +jobs: + ci-build: + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v3 + + - name: ci-job-syntax + run: make ci-job-syntax + - name: ci-job-compilation + run: make ci-job-compilation + - name: ci-job-debug-support-targets + run: make ci-job-debug-support-targets + + - name: ci-job-collect-artifacts + run: make ci-job-collect-artifacts + - name: upload-build-artifacts + uses: actions/upload-artifact@v3 + with: + name: build-artifacts + path: tools/ci-artifacts + + ci-tests: + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + + steps: + - name: Update package repositories + run: | + sudo apt update + if: matrix.os == 'ubuntu-latest' + - name: Install dependencies for ubuntu-latest + run: | + sudo apt install libudev-dev libzmq3-dev + if: matrix.os == 'ubuntu-latest' + - name: Install dependencies for macos-latest + run: | + brew install zeromq + if: matrix.os == 'macos-latest' + - uses: actions/checkout@v3 + - name: ci-job-libraries + run: make ci-job-libraries + - name: ci-job-archs + run: make ci-job-archs + - name: ci-job-kernel + run: make ci-job-kernel + - name: ci-job-chips + run: make ci-job-chips + - name: ci-job-tools + run: make ci-job-tools + - name: Create Issue on Failed workflow + if: failure() + uses: dacbd/create-issue-action@main + with: + token: ${{ github.token }} + title: Nightly CI failed + body: | + ### Context + [Failed Run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) + [Codebase](https://github.com/${{ github.repository }}/tree/${{ github.sha }}) + Workflow name - `${{ github.workflow }}` + Job - `${{ github.job }}` + status - `${{ job.status }}` + assignees: tock/core-wg diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..6732130e97 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,126 @@ +# Licensed under the Apache License, Version 2.0 or the MIT License. +# SPDX-License-Identifier: Apache-2.0 OR MIT +# Copyright Tock Contributors 2023. + +# This workflow contains all tock-ci, separated into jobs + +name: tock-ci +env: + TERM: xterm # Makes tput work in actions output + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: # Run CI for all branches except GitHub merge queue tmp branches + branches-ignore: + - "gh-readonly-queue/**" + pull_request: # Run CI for PRs on any branch + merge_group: # Run CI for the GitHub merge queue + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +# If you add additional jobs, remember to add them to bors.toml +permissions: + contents: read + +jobs: + ci-format: + strategy: + matrix: + os: [ubuntu-latest] + # The type of runner that the job will run on + runs-on: ${{ matrix.os }} + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + - name: ci-job-format + run: make ci-job-format + - name: ci-job-markdown-toc + run: make ci-job-markdown-toc + - name: ci-job-readme-check + run: make ci-job-readme-check + + ci-clippy: + strategy: + matrix: + os: [ubuntu-latest] + # The type of runner that the job will run on + runs-on: ${{ matrix.os }} + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - uses: actions/checkout@v3 + - name: ci-job-clippy + run: make ci-job-clippy + + ci-build: + strategy: + matrix: + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v3 + - name: ci-job-syntax + run: make ci-job-syntax + - name: ci-job-compilation + run: make ci-job-compilation + - name: ci-job-debug-support-targets + run: make ci-job-debug-support-targets + - name: ci-job-collect-artifacts + run: make ci-job-collect-artifacts + - name: upload-build-artifacts + uses: actions/upload-artifact@v3 + with: + name: build-artifacts + path: tools/ci-artifacts + + ci-tests: + strategy: + matrix: + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + + steps: + - name: Update package repositories + run: | + sudo apt update + if: matrix.os == 'ubuntu-latest' + - name: Install dependencies for ubuntu-latest + run: | + sudo apt install libudev-dev libzmq3-dev + if: matrix.os == 'ubuntu-latest' + - uses: actions/checkout@v3 + - name: ci-job-libraries + run: make ci-job-libraries + - name: ci-job-archs + run: make ci-job-archs + - name: ci-job-kernel + run: make ci-job-kernel + - name: ci-job-capsules + run: make ci-job-capsules + - name: ci-job-chips + run: make ci-job-chips + - name: ci-job-tools + run: make ci-job-tools + - name: ci-job-cargo-test-build + run: make ci-job-cargo-test-build + + ci-qemu: + strategy: + matrix: + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + + steps: + - name: Update package repositories + run: | + sudo apt update + - name: Install dependencies + continue-on-error: true + run: | + sudo apt install meson + - uses: actions/checkout@v3 + - name: ci-job-qemu + run: make ci-job-qemu diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 0000000000..d7d87957db --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,21 @@ +# Licensed under the Apache License, Version 2.0 or the MIT License. +# SPDX-License-Identifier: Apache-2.0 OR MIT +# Copyright Tock Contributors 2023. + +name: "Pull Request Labeler" +on: +- pull_request_target + +permissions: + contents: read + +jobs: + triage: + permissions: + contents: read # for actions/labeler to determine modified files + pull-requests: write # for actions/labeler to add labels to PRs + runs-on: ubuntu-latest + steps: + - uses: actions/labeler@v4.3.0 + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/litex_sim.yml b/.github/workflows/litex_sim.yml new file mode 100644 index 0000000000..f2877aea3f --- /dev/null +++ b/.github/workflows/litex_sim.yml @@ -0,0 +1,138 @@ +# Licensed under the Apache License, Version 2.0 or the MIT License. +# SPDX-License-Identifier: Apache-2.0 OR MIT +# Copyright Tock Contributors 2023. + +# This workflow contains the litex-ci-runner job, which uses the LiteX Verilated +# simulation to run a Tock kernel and perform various tests using libtock-c +# example applications. + +name: litex-sim-ci +env: + TERM: xterm # Makes tput work in actions output + +# Controls when the action will run. Triggers the workflow on push or pull +# request events but only for the master branch +on: + push: # Run CI for all branches except GitHub merge queue tmp branches + branches-ignore: + - "gh-readonly-queue/**" + pull_request: # Run CI for PRs on any branch + merge_group: # Run CI for the GitHub merge queue + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +# If you add additional jobs, remember to add them to bors.toml +permissions: + contents: read + +jobs: + litex-sim-ci: + strategy: + matrix: + os: [ubuntu-latest] + + # The type of runner that the job will run on + runs-on: ${{ matrix.os }} + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checkout the Tock repo, needs to happen at the beginning given + # that other steps (such as the Rust toolchain) depend on files + # in this repo. + - name: Checkout the current repository + uses: actions/checkout@v3 + + # Install basic packages required for the GitHub actions workflow + - name: Update packages and install dependencies + run: | + sudo apt update + sudo apt install python3-pip python3-venv gcc-riscv64-unknown-elf \ + verilator libevent-dev libjson-c-dev libz-dev libzmq3-dev + + # Install elf2tab to be able to build userspace apps + - name: Install elf2tab + run: | + cargo install elf2tab@0.12.0 + + # Install tockloader, which is used to prepare binaries with userspace + # applications. + - name: Install tockloader + run: | + pip3 install tockloader==1.12.0 + + # Clone tock-litex support repository under ./tock-litex, check out the + # targeted release. + - name: Checkout the tock-litex repository + uses: actions/checkout@v3 + with: + repository: lschuermann/tock-litex + # The pinned revision is different from the targeted release as + # documented in the LiteX boards, as the CI requires special patches + # to LiteX for interacting with the simulation: + ref: 2024011101-tock-ci-1 + path: tock-litex + + # Install all of the required Python packages from the tock-litex' + # requirements.txt file + - name: Install Python packages pinned by the tock-litex revision + run: | + pushd tock-litex + # Migen is the DSL which the LiteX ecosystem uses as its + # hardware-description language. It effectively provides a set of + # Python classes and constructs which can be translated into Verilog. + # It is not a package of the LiteX ecosystem, and thus not in the + # requirements.txt, but it is required to be present on the system. + # It should not require any specific or patched version. + pip3 install migen==0.9.2 + pip3 install -r requirements.txt + popd + + # Build the LiteX simulator Tock kernel. This kernel is never touched, the + # litex-ci-runner will use its own temporary flash files. + - name: Build the LiteX simulator Tock kernel + run: | + pushd boards/litex/sim + make + popd + + # Revision to checkout defined in the main tock repository in + # .libtock_c_ci_rev + - name: Checkout libtock-c CI revision + uses: actions/checkout@v3 + with: + repository: tock/libtock-c + # Pins a libtock-c revision for LiteX CI tests. In case of + # bugs fixed in libtock-c, backwards-incompatible changes in + # Tock or new tests this might need to be updated. + # + # libtock-c of Feb 9, 2024, 3:08 PM EST + ref: 0e72c922e13cd83a1aaae7a5f587099d45815e6a + path: libtock-c + + - name: Build libtock-c apps + run: | + # We only need to build for a single target, but at multiple flash and + # memory addresses such that tockloader can place the non-PIC apps + # into the kernel binary properly. + export TOCK_TARGETS="\ + rv32imc|rv32imc.0x00080080.0x40008000|0x00080080|0x40008000 + rv32imc|rv32imc.0x00088080.0x40010000|0x00088080|0x40010000" + export LIBTOCK_C_APPS="\ + c_hello \ + tests/console_timeout \ + tests/mpu_walk_region \ + tests/printf_long \ + rot13_service \ + rot13_client \ + tests/console_recv_short \ + tests/console_recv_long" + pushd libtock-c/examples + for APP in $LIBTOCK_C_APPS; do + make -C "$APP" + done + popd + + # Run the LiteX simulation with required options for Tock + - name: Run various tests in the LiteX simulation using the litex-ci-runner + run: | + pushd tools/litex-ci-runner + cargo run diff --git a/.github/workflows/mergequeue_docs.yml b/.github/workflows/mergequeue_docs.yml new file mode 100644 index 0000000000..6cf8794b62 --- /dev/null +++ b/.github/workflows/mergequeue_docs.yml @@ -0,0 +1,34 @@ +# Licensed under the Apache License, Version 2.0 or the MIT License. +# SPDX-License-Identifier: Apache-2.0 OR MIT +# Copyright Tock Contributors 2023. + +# Netlify's own CI builds both deploy previews for PRs, as well as the +# production deploy for the master branch. We use this workflow purely as we +# can't have Netlify build the wildcard gh-readonly-queue/* branches. This +# workflow thus ensures that docs build successfully (albeit not in the exact +# same environment as Netlify's). +# +# See issue #3428 for more information. + +name: docs-ci +env: + TERM: dumb # Identical to Netlify build environment + +on: + merge_group: + +permissions: + contents: read + +jobs: + ci-docs: + strategy: + matrix: + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v3 + - # This also sets up the rustup environment + name: ci-netlify-build + run: tools/netlify-build.sh diff --git a/.github/workflows/tockbot-nightly.yml b/.github/workflows/tockbot-nightly.yml new file mode 100644 index 0000000000..158373505d --- /dev/null +++ b/.github/workflows/tockbot-nightly.yml @@ -0,0 +1,134 @@ +# Licensed under the Apache License, Version 2.0 or the MIT License. +# SPDX-License-Identifier: Apache-2.0 OR MIT +# Copyright Tock Contributors 2024. + +name: "Tockbot" + +on: + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + inputs: + dispatch-job: + description: 'Which job to execute (choose between "all", "maint-nightly")' + required: true + default: 'all' + dry-run: + description: 'Whether to execute the jobs as dry-run' + required: true + default: true + +jobs: + dispatcher: + runs-on: ubuntu-latest + + # This job determines which other jobs should be run: + outputs: + run-maint-nightly: ${{ steps.dispatch-logic.outputs.run-maint-nightly }} + dry-run: ${{ steps.dispatch-logic.outputs.dry-run }} + + steps: + # On pushes we want to check whether any changes have been made + # to the Tockbot code base. Disabled for now: + - uses: actions/checkout@v4 + + # Dispatcher business logic: + - name: Dispatch Tockbot Jobs + id: dispatch-logic + env: + DISPATCH_JOB: ${{ github.event.inputs.dispatch-job }} + DISPATCH_DRY_RUN: ${{ github.event.inputs.dry-run }} + run: | + if [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]; then + if [ "$DISPATCH_DRY_RUN" == "true" ]; then + echo "dry-run=true" >> $GITHUB_OUTPUT + elif [ "$DISPATCH_DRY_RUN" == "false" ]; then + echo "dry-run=false" >> $GITHUB_OUTPUT + else + echo "Error: dry-run not a boolean: \"$DISPATCH_DRY_RUN\"" >&2 + exit 1 + fi + + if [ "$DISPATCH_JOB" == "all" ]; then + echo "run-maint-nightly=true" >> $GITHUB_OUTPUT + elif [ "$DISPATCH_JOB" == "maint-nightly" ]; then + echo "run-maint-nightly=true" >> $GITHUB_OUTPUT + else + echo "Error: unknown job \"$DISPATCH_JOB\"" >&2 + exit 1 + fi + elif [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then + echo "dry-run=true" >> $GITHUB_OUTPUT + echo "run-maint-nightly=true" >> $GITHUB_OUTPUT + elif [ "$GITHUB_EVENT_NAME" == "schedule" ]; then + echo "dry-run=false" >> $GITHUB_OUTPUT + echo "run-maint-nightly=true" >> $GITHUB_OUTPUT + else + echo "Error: unknown event name \"$GITHUB_EVENT_NAME\"" >&2 + exit 1 + fi + + maint-nightly: + runs-on: ubuntu-latest + + # Only run this job if the dispatcher determined to schedule the + # "maint-nightly" or "dry-run" jobs: + needs: dispatcher + if: ${{ needs.dispatcher.outputs.run-maint-nightly == 'true' && needs.dispatcher.outputs.dry-run != 'true' }} + + permissions: + # Give GITHUB_TOKEN write permissions to modify PRs and issues: + pull-requests: write + issues: write + + steps: + # Requires a tock checkout to run from: + - uses: actions/checkout@v4 + + # Setup Python and install dependencies: + - uses: actions/setup-python@v5 + - name: Install Python Dependencies + run: pip install -r tools/tockbot/requirements.txt + + # Run nightly tockbot maintenance: + - name: Nightly Tockbot Maintenance + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DRY_RUN: ${{ needs.dispatcher.outputs.dry-run == 'true' && '-n' || '' }} + run: | + cd tools/tockbot/ + ./tockbot.py -v $DRY_RUN maint-nightly -c ./maint_nightly.yaml + + # We'd like to avoid duplicating this, either by using conditionals in the + # permissions key, or by using YAML anchors, neither of which are supported by + # GH Actions... + maint-nightly-dry-run: + runs-on: ubuntu-latest + + # Only run this job if the dispatcher determined to schedule the + # "maint-nightly" or "dry-run" jobs: + needs: dispatcher + if: ${{ needs.dispatcher.outputs.run-maint-nightly == 'true' && needs.dispatcher.outputs.dry-run == 'true' }} + + permissions: + # Dry-run, read-only access: + pull-requests: read + issues: read + + steps: + # Requires a tock checkout to run from: + - uses: actions/checkout@v4 + + # Setup Python and install dependencies: + - uses: actions/setup-python@v5 + - name: Install Python Dependencies + run: pip install -r tools/tockbot/requirements.txt + + # Run nightly tockbot maintenance: + - name: Nightly Tockbot Maintenance + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DRY_RUN: ${{ needs.dispatcher.outputs.dry-run == 'true' && '-n' || '' }} + run: | + cd tools/tockbot/ + ./tockbot.py -v $DRY_RUN maint-nightly -c ./maint_nightly.yaml