One off hosted runners (#9016)

* On-demand hosted action runner, and use runtime logs to improve test speed
* Add support for uploading capybara screenshots to S3

.github/workflows/test-core.yml

@@ -1,25 +1,39 @@
 name: Core/Test
+# https://securitylab.github.com/research/github-actions-preventing-pwn-requests
 on:
   push:
+    branches:
+      - dev
+      - release/*
+    paths-ignore:
+      - 'docs/**'
+      - 'help/**'
+  pull_request:
+    types: [opened, reopened, synchronize]
+    paths-ignore:
+      - 'docs/**'
+      - 'help/**'
 
 jobs:
   units:
     name: Units
     if: github.repository == 'opf/openproject'
-    runs-on: [self-hosted,public]
+    runs-on: [self-hosted-autoscale]
     timeout-minutes: 30
     env:
-      CI_CACHE_PATH: "/tmp/${{ github.sha }}/"
-      CI_JOBS: 16
+      CI_CACHE_PATH: "/tmp/cache"
       CI_RETRY_COUNT: 3
       LOCAL_DEV_CHECK: 1
     steps:
     - uses: actions/checkout@v2
     - name: cache
       uses: actions/cache@v2
       with:
-        path: /tmp/${{ github.sha }}
-        key: ${{ runner.os }}-core-tests
+        path: /tmp/cache
+        key: ${{ runner.os }}-core-tests-units-${{ hashFiles('**/Gemfile.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-core-tests-units-
+          ${{ runner.os }}-core-tests-
     - name: test
       run: |
         cp .env.example .env
@@ -31,23 +45,29 @@ jobs:
         docker-compose -f docker-compose.ci.yml down --remove-orphans -t 10
         sudo chown -R $(whoami):$(id -ng) $CI_CACHE_PATH
   features:
-    needs: units
     name: Features
     if: github.repository == 'opf/openproject'
-    runs-on: [self-hosted,public]
-    timeout-minutes: 200
+    runs-on: [self-hosted-autoscale]
+    timeout-minutes: 60
     env:
-      CI_JOBS: 16
-      CI_CACHE_PATH: "/tmp/${{ github.sha }}/"
+      CI_CACHE_PATH: "/tmp/cache"
       LOCAL_DEV_CHECK: 1
       CI_RETRY_COUNT: 3
+      OPENPROJECT_ENABLE_CAPYBARA_SCREENSHOT_S3_UPLOADS: true
+      AWS_ACCESS_KEY_ID: "${{ secrets.CAPYBARA_AWS_ACCESS_KEY_ID }}"
+      AWS_SECRET_ACCESS_KEY: "${{ secrets.CAPYBARA_AWS_SECRET_ACCESS_KEY }}"
+      AWS_REGION: "eu-west-1"
+      S3_BUCKET_NAME: "openproject-ci-public-logs"
     steps:
     - uses: actions/checkout@v2
     - name: cache
       uses: actions/cache@v2
       with:
-        path: /tmp/${{ github.sha }}
-        key: ${{ runner.os }}-core-tests
+        path: /tmp/cache
+        key: ${{ runner.os }}-core-tests-features-${{ hashFiles('**/Gemfile.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-core-tests-features-
+          ${{ runner.os }}-core-tests-
     - name: test
       run: |
         cp .env.example .env

.rspec_parallel

@@ -1,2 +1,3 @@
 --format progress
---format ParallelTests::RSpec::SummaryLogger --out tmp/parallel_runtime_rspec.log
+--format ParallelTests::RSpec::SummaryLogger --out tmp/parallel_summary.log
+--format ParallelTests::RSpec::RuntimeLogger --out tmp/parallel_runtime.log

docker-compose.ci.yml

@@ -15,7 +15,7 @@ services:
       - "/tmp"
     volumes:
       - ".:/home/dev/openproject"
-      - "${CI_CACHE_PATH:-/tmp/}op-bundle:/usr/local/bundle"
-      - "${CI_CACHE_PATH:-/tmp/}op-node:/home/dev/openproject/frontend/node_modules"
+      - "${CI_CACHE_PATH:-/tmp}/op-bundle:/usr/local/bundle"
+      - "${CI_CACHE_PATH:-/tmp}/op-node:/home/dev/openproject/frontend/node_modules"
     networks:
       - testing

docker/ci/Dockerfile

@@ -1,20 +1,27 @@
 FROM ruby:2.7.2-buster
 MAINTAINER [email protected]
 
+ENV NODE_VERSION="12.18.3"
+ENV CHROME_SOURCE_URL=https://dl.google.com/dl/linux/direct/google-chrome-stable_current_amd64.deb
 ENV USER=dev
+
 RUN useradd -d /home/$USER -m $USER -s /bin/bash
 WORKDIR /home/$USER
 
+RUN curl -s https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.gz | tar xzf - -C /usr/local --strip-components=1
+
 RUN wget --quiet -O- https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
 RUN echo "deb http://apt.postgresql.org/pub/repos/apt buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list
 
 RUN apt-get update -qq && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y \
     postgresql-9.6 postgresql-client-9.6 time pandoc imagemagick libpq-dev default-jre-headless firefox-esr
 
+RUN wget --no-verbose -O /tmp/$(basename $CHROME_SOURCE_URL) $CHROME_SOURCE_URL && \
+  apt install -y /tmp/$(basename $CHROME_SOURCE_URL) && rm -f /tmp/$(basename $CHROME_SOURCE_URL)
+
 ENV CI=true
 ENV RAILS_ENV=test
-ENV NODE_VERSION="12.18.3"
 ENV BUNDLER_VERSION="2.1.4"
 ENV BUNDLE_WITHOUT="development:production:docker"
 ENV OPENPROJECT_DISABLE_DEV_ASSET_PROXY=1
@@ -23,13 +30,7 @@ ENV CAPYBARA_DOWNLOADED_FILE_DIR=/tmp
 # disable deprecations and other warnings in output
 ENV RUBYOPT="-W0"
 ENV DATABASE_URL=postgres://app:[email protected]/app
-ENV CHROME_SOURCE_URL=https://dl.google.com/dl/linux/direct/google-chrome-stable_current_amd64.deb
-ENV JOBS=4
 
-RUN wget --no-verbose -O /tmp/$(basename $CHROME_SOURCE_URL) $CHROME_SOURCE_URL && \
-  apt install -y /tmp/$(basename $CHROME_SOURCE_URL) && rm -f /tmp/$(basename $CHROME_SOURCE_URL)
-
-RUN curl -s https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.gz | tar xzf - -C /usr/local --strip-components=1
 RUN gem install bundler --version "$BUNDLER_VERSION" --no-document
 
 COPY ./entrypoint.sh /usr/sbin/entrypoint.sh

docker/ci/entrypoint.sh

@@ -2,6 +2,7 @@
 set -e
 
 export PGBIN="$(pg_config --bindir)"
+export JOBS="${CI_JOBS:=$(nproc)}"
 # for parallel rspec
 export PARALLEL_TEST_PROCESSORS=$JOBS
 
@@ -30,10 +31,15 @@ execute() {
 	fi
 }
 
+cleanup() {
+        rm -rf tmp/cache/parallel*
+}
+
 if [ "$1" == "setup-tests" ]; then
 	echo "Preparing environment for running tests..."
 	shift
 
+	execute "mkdir -p tmp"
 	execute "cp docker/ci/database.yml config/"
 
 	for i in $(seq 0 $JOBS); do
@@ -53,10 +59,13 @@ if [ "$1" == "run-units" ]; then
 	shift
 	execute "cd frontend && npm install && npm run test"
 	execute "time bundle exec rspec -I spec_legacy spec_legacy"
-	execute "time bundle exec rake parallel:units"
-	if [ ! $? -eq 0 ]; then
-		execute "cat tmp/parallel_runtime_rspec.log | grep -Ev 'passed|unknown|pending'"
+	if ! execute "time bundle exec rake parallel:units" ; then
+		execute "cat tmp/parallel_summary.log"
+		cleanup
 		exit 1
+	else
+		cleanup
+		exit 0
 	fi
 fi
 
@@ -65,10 +74,13 @@ if [ "$1" == "run-features" ]; then
 	execute "cd frontend; npm install ; cd -"
 	execute "bundle exec rake assets:precompile assets:clean"
 	execute "cp -rp config/frontend_assets.manifest.json public/assets/frontend_assets.manifest.json"
-	execute "time bundle exec rake parallel:features"
-	if [ ! $? -eq 0 ]; then
-		execute "cat tmp/parallel_runtime_rspec.log | grep -Ev 'passed|unknown|pending'"
+	if ! execute "time bundle exec rake parallel:features" ; then
+		execute "cat tmp/parallel_summary.log"
+		cleanup
 		exit 1
+	else
+		cleanup
+		exit 0
 	fi
 fi