From 8c36a250e4128d3d9fc9c1c04af3f2cae4ba991f Mon Sep 17 00:00:00 2001 From: "Marlon (Esolitos) Saglia" Date: Thu, 8 Aug 2024 15:10:43 +0200 Subject: [PATCH] feat: Add workflow to verify guides This change adds a new GitHub Actions workflow to verify the guides in the repository. The workflow checks out the code, installs the required dependencies (including the Vespa CLI), builds the Jekyll site, and runs a set of tests to ensure the guides are working as expected. The workflow is triggered on pull requests and pushes to a temporary branch, to allow for testing the configuration before merging to the main branch. Disable container and service configuration for CI job remove verbose flag setup java feat: Add free-disk-space action to verify-guides workflow feat: create separate workflows for small and large guide verification feat: Simplify workflows to only watch "master" branch disable jobs on Screwdriver temp run large too Update .github/workflows/verify-guides-large.yml Co-authored-by: Eirik Nygaard Update .github/workflows/verify-guides-large.yml Co-authored-by: Eirik Nygaard --- .github/actions/install-deps/action.yml | 56 ++++++++++++++++ .github/workflows/verify-guides-large.yml | 18 +++++ .github/workflows/verify-guides-small.yml | 16 +++++ .github/workflows/verify-guides.yml | 44 +++++++++++++ screwdriver.yaml | 80 +++++++++++------------ 5 files changed, 174 insertions(+), 40 deletions(-) create mode 100644 .github/actions/install-deps/action.yml create mode 100644 .github/workflows/verify-guides-large.yml create mode 100644 .github/workflows/verify-guides-small.yml create mode 100644 .github/workflows/verify-guides.yml diff --git a/.github/actions/install-deps/action.yml b/.github/actions/install-deps/action.yml new file mode 100644 index 0000000000..327c3e4017 --- /dev/null +++ b/.github/actions/install-deps/action.yml @@ -0,0 +1,56 @@ +name: "Setup Vespa CLI and dependencies" +description: "Install Vespa CLI and dependencies" + +inputs: + # The version of the Vespa CLI to install. + vespa-cli-version: + description: "The version of the Vespa CLI to install." + required: true + default: "latest" + +runs: + using: "composite" + steps: + - name: Select CLI version + shell: bash + id: vespa-cli-version + run: | + if [ "${{ inputs.vespa-cli-version }}" == "latest" ]; then + VESPA_CLI_VERSION=$(curl -fsSL https://api.github.com/repos/vespa-engine/vespa/releases/latest | grep -Po '"tag_name": "v\K.*?(?=")') + else + VESPA_CLI_VERSION="${{ inputs.vespa-cli-version }}" + fi + + echo "version=${VESPA_CLI_VERSION}" >> "$GITHUB_OUTPUT" + + # - name: Install dnf dependencies + # shell: bash + # run: | + # dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + # dnf -y install docker-ce protobuf-compiler protobuf-devel python3 python3-pip --nobest + + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: 3.1 + bundler-cache: true + + - uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: "17" + + - name: Install python dependencies + shell: bash + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -qqq -r test/requirements.txt --user + python3 -m pip install -qqq pytest nbmake --user + + - name: Install Vespa CLI + shell: bash + env: + VESPA_CLI_VERSION: ${{ steps.vespa-cli-version.outputs.version }} + run: | + curl -fsSL https://github.com/vespa-engine/vespa/releases/download/v${VESPA_CLI_VERSION}/vespa-cli_${VESPA_CLI_VERSION}_linux_amd64.tar.gz | tar -zxf - -C /opt && \ + ln -sf /opt/vespa-cli_${VESPA_CLI_VERSION}_linux_amd64/bin/vespa /usr/local/bin/ diff --git a/.github/workflows/verify-guides-large.yml b/.github/workflows/verify-guides-large.yml new file mode 100644 index 0000000000..a450ad0fa1 --- /dev/null +++ b/.github/workflows/verify-guides-large.yml @@ -0,0 +1,18 @@ +name: Verify Giudes (Large) + +on: + workflow_dispatch: # Allow manual triggering of this workflow. + + push: + paths: ["en/tutorials/"] + branches: ["master"] + + pull_request: + paths: ["en/tutorials/"] # Enable before merge! + branches: ["master"] + +jobs: + large: + uses: ./.github/workflows/verify-guides.yml + with: + test-config-path: "test/_test_config-large.yml" diff --git a/.github/workflows/verify-guides-small.yml b/.github/workflows/verify-guides-small.yml new file mode 100644 index 0000000000..d781eab7e7 --- /dev/null +++ b/.github/workflows/verify-guides-small.yml @@ -0,0 +1,16 @@ +name: Verify Giudes (Small) + +on: + workflow_dispatch: # Allow manual triggering of this workflow. + + push: + branches: ["master"] + + pull_request: + branches: ["master"] + +jobs: + small: + uses: ./.github/workflows/verify-guides.yml + with: + test-config-path: "test/_test_config.yml" diff --git a/.github/workflows/verify-guides.yml b/.github/workflows/verify-guides.yml new file mode 100644 index 0000000000..65aa3e4302 --- /dev/null +++ b/.github/workflows/verify-guides.yml @@ -0,0 +1,44 @@ +name: Verify Giudes + +on: + workflow_call: + inputs: + test-config-path: + description: | + The path to the test configuration file, relative to the repository root. + + Example: `test/_test_config.yml` + type: string + required: true + +defaults: + run: + # Specify to ensure "pipefail and errexit" are set. + # Ref: https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#defaultsrunshell + shell: bash + +jobs: + test: + runs-on: ubuntu-latest + env: + LANG: "C.UTF-8" + steps: + - uses: jlumbroso/free-disk-space@main + + - uses: actions/checkout@v4 + + - uses: ./.github/actions/install-deps + with: + vespa-cli-version: latest + + - name: build-site + run: | + bundle exec jekyll build + + - name: check-queries + run: | + ./test/test_queries.py _site + + - name: run-tests + run: | + ./test/test.py -c ./test/_test_config.yml -w $GITHUB_WORKSPACE diff --git a/screwdriver.yaml b/screwdriver.yaml index a5d0bbd9a3..885244e741 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -19,47 +19,47 @@ shared: ln -sf /opt/vespa-cli_${VESPA_CLI_VERSION}_linux_amd64/bin/vespa /usr/local/bin/ jobs: - verify-guides: - requires: [~pr, ~commit] - image: vespaengine/vespa-build-almalinux-8:latest - annotations: - screwdriver.cd/cpu: HIGH - screwdriver.cd/ram: HIGH - screwdriver.cd/dockerEnabled: true - screwdriver.cd/dockerCpu: HIGH - screwdriver.cd/dockerRam: HIGH - screwdriver.cd/buildPeriodically: H H(0-5) * * 1-5 # some time between 12:00 AM UTC (midnight) to 5:59 AM UTC Mon-Fri - steps: - - *install-deps-new - - install-bundler: | - gem install bundler - export LANG=C.UTF-8 - bundle install - - build-site: | - bundle exec jekyll build - - check-queries: | - ./test/test_queries.py _site - - run-tests: | - cd $SD_DIND_SHARE_PATH - $SD_SOURCE_DIR/test/test.py -c $SD_SOURCE_DIR/test/_test_config.yml -w $SD_SOURCE_DIR + # verify-guides: + # requires: [~pr, ~commit] + # image: vespaengine/vespa-build-almalinux-8:latest + # annotations: + # screwdriver.cd/cpu: HIGH + # screwdriver.cd/ram: HIGH + # screwdriver.cd/dockerEnabled: true + # screwdriver.cd/dockerCpu: HIGH + # screwdriver.cd/dockerRam: HIGH + # screwdriver.cd/buildPeriodically: H H(0-5) * * 1-5 # some time between 12:00 AM UTC (midnight) to 5:59 AM UTC Mon-Fri + # steps: + # - *install-deps-new + # - install-bundler: | + # gem install bundler + # export LANG=C.UTF-8 + # bundle install + # - build-site: | + # bundle exec jekyll build + # - check-queries: | + # ./test/test_queries.py _site + # - run-tests: | + # cd $SD_DIND_SHARE_PATH + # $SD_SOURCE_DIR/test/test.py -c $SD_SOURCE_DIR/test/_test_config.yml -w $SD_SOURCE_DIR - verify-guides-large: - requires: [~pr, ~commit] - sourcePaths: ["en/tutorials/"] - image: vespaengine/vespa-build-almalinux-8:latest - annotations: - screwdriver.cd/cpu: TURBO - screwdriver.cd/ram: TURBO - screwdriver.cd/dockerEnabled: true - screwdriver.cd/dockerCpu: TURBO - screwdriver.cd/dockerRam: TURBO - screwdriver.cd/timeout: 180 - screwdriver.cd/buildPeriodically: H H(0-5) * * 1-5 # some time between 12:00 AM UTC (midnight) to 5:59 AM UTC Mon-Fri - steps: - - *install-deps-new - - run-tests: | - cd $SD_DIND_SHARE_PATH - $SD_SOURCE_DIR/test/test.py -c $SD_SOURCE_DIR/test/_test_config-large.yml -w $SD_SOURCE_DIR + # verify-guides-large: + # requires: [~pr, ~commit] + # sourcePaths: ["en/tutorials/"] + # image: vespaengine/vespa-build-almalinux-8:latest + # annotations: + # screwdriver.cd/cpu: TURBO + # screwdriver.cd/ram: TURBO + # screwdriver.cd/dockerEnabled: true + # screwdriver.cd/dockerCpu: TURBO + # screwdriver.cd/dockerRam: TURBO + # screwdriver.cd/timeout: 180 + # screwdriver.cd/buildPeriodically: H H(0-5) * * 1-5 # some time between 12:00 AM UTC (midnight) to 5:59 AM UTC Mon-Fri + # steps: + # - *install-deps-new + # - run-tests: | + # cd $SD_DIND_SHARE_PATH + # $SD_SOURCE_DIR/test/test.py -c $SD_SOURCE_DIR/test/_test_config-large.yml -w $SD_SOURCE_DIR update-vespa-version: image: buildpack-deps