Skip to content
CI

Merge pull request #445 from ral-facilities/dependabot/github_actions… #931

Sign in to view logs

Merge pull request #445 from ral-facilities/dependabot/github_actions…

Merge pull request #445 from ral-facilities/dependabot/github_actions… #931

Workflow file for this run

.github/workflows/ci-build.yml at 60c539c
name: CI
on:
workflow_dispatch:
pull_request:
push:
branches:
- main
jobs:
tests:
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
name: Python ${{ matrix.python-version }} Build & Tests
steps:
- name: Add apt repo
run: sudo add-apt-repository universe
# Setup Java & Python
- name: Setup Java
uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 # v3.12.0
with:
distribution: 'temurin'
java-version: 8
- name: Setup Python
uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: ${{ matrix.python-version }}
architecture: x64
- name: Checkout DataGateway API
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
# Install Nox, Poetry and API's dependencies
- name: Install Nox
run: pip install nox==2020.8.22
- name: Install Poetry
run: pip install poetry==1.1.9
# This fixes the issue with cachecontrol (https://github.com/psf/cachecontrol/issues/292).
# We will not be facing this issue when we upgrade to a newer Poetry version.
- name: Install Requests
run: pip install 'requests<2.30'
# This command is a workaround for getting Poetry working with Python 3.10. An
# fix is made in Poetry 1.2.0a2 but there is currently no official release for
# Poetry 1.2 and am apprehensive to moving to a pre-release. Disabling the
# experimental installer is a workaround for Poetry 1.1.x
# See https://github.com/python-poetry/poetry/issues/4210 for more details
- name: Disable Poetry's experimental new installer
if: ${{ matrix.python-version == '3.10' }}
run: poetry config experimental.new-installer false
# Prep for using the API for tests
- name: Create log file
run: touch logs.log
- name: Configure log file location
run: echo "`yq \
'.log_location="${GITHUB_WORKSPACE}/logs.log"' datagateway_api/config.yaml.example | envsubst`" > datagateway_api/config.yaml.example
- name: Configure datagateway extension
run: echo "`yq \
'.datagateway_api.extension="/datagateway_api"' datagateway_api/config.yaml.example`" > datagateway_api/config.yaml.example
- name: Create config.yaml
run: cp datagateway_api/config.yaml.example datagateway_api/config.yaml
- name: Create search_api_mapping.json
run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json
# Run Unit tests
- name: Run Nox unit tests session
run: nox -p ${{ matrix.python-version }} -s unit_tests -- --cov=datagateway_api --cov-report=xml
- name: Upload unit test code coverage report
if: matrix.python-version == '3.6'
uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # v3.1.4
# ICAT Ansible clone and install dependencies
- name: Checkout icat-ansible
if: success()
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
with:
repository: icatproject-contrib/icat-ansible
ref: master
path: icat-ansible
- name: Install Ansible
run: pip install -r icat-ansible/requirements.txt
# Prep for running the playbook
- name: Create hosts file
run: echo -e "[icatdb_minimal_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
- name: Prepare vault pass
run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
- name: Move vault to directory it'll get detected by Ansible
run: mv icat-ansible/vault.yml icat-ansible/group_vars/all
- name: Replace default payara user with Actions user
run: |
sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml
# Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions
- name: Change hostname to localhost
run: sudo hostname -b localhost
# Remove existing MySQL installation so it doesn't interfere with GitHub Actions
- name: Remove existing mysql
run: |
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
sudo apt-get remove --purge "mysql*"
sudo rm -rf /var/lib/mysql* /etc/mysql
# Create local instance of ICAT
- name: Run ICAT Ansible Playbook
run: |
ansible-playbook icat-ansible/icatdb_minimal_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv
# rootUserNames needs editing as anon/anon is used in search API and required to pass endpoint tests
- name: Add anon user to rootUserNames
run: |
awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp
- name: Apply rootUserNames change
run: |
mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties
- name: Reinstall ICAT Server
run: |
cd /home/runner/install/icat.server/ && ./setup -vv install
- name: Checkout DataGateway API
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
# Prep for using the API for tests
- name: Create log file
run: touch logs.log
- name: Configure log file location
run: echo "`yq \
'.log_location="${GITHUB_WORKSPACE}/logs.log"' datagateway_api/config.yaml.example | envsubst`" > datagateway_api/config.yaml.example
- name: Configure datagateway extension
run: echo "`yq \
'.datagateway_api.extension="/datagateway_api"' datagateway_api/config.yaml.example`" > datagateway_api/config.yaml.example
- name: Create config.yaml
run: cp datagateway_api/config.yaml.example datagateway_api/config.yaml
- name: Create search_api_mapping.json
run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json
- name: Install dependencies
run: poetry install
- name: Add dummy data to icatdb
run: |
poetry run python -m util.icat_db_generator
# Run Nox integration tests session, saves and uploads a coverage report to codecov
- name: Run Nox Integration tests session
if: success()
run: nox -p ${{ matrix.python-version }} -s integration_tests -- --cov=datagateway_api --cov-report=xml
- name: Upload integration test code coverage report
if: matrix.python-version == '3.6'
uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # v3.1.4
linting:
runs-on: ubuntu-20.04
name: Linting
steps:
- name: Setup Python
uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: "3.9.7"
architecture: x64
- name: Checkout DataGateway API
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
- name: Install Nox
run: pip install nox==2020.8.22
- name: Install Poetry
run: pip install poetry==1.1.9
- name: Run Nox lint session
run: nox -s lint
formatting:
runs-on: ubuntu-20.04
name: Code Formatting
steps:
- name: Setup Python
uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: "3.9.7"
architecture: x64
- name: Checkout DataGateway API
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
- name: Install Nox
run: pip install nox==2020.8.22
- name: Install Poetry
run: pip install poetry==1.1.9
- name: Run Nox black session
run: nox -s black
safety:
runs-on: ubuntu-20.04
name: Dependency Safety
steps:
- name: Setup Python
uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: "3.9.7"
architecture: x64
- name: Checkout DataGateway API
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
- name: Install Nox
run: pip install nox==2020.8.22
- name: Install Poetry
run: pip install poetry==1.1.9
- name: Run Nox safety session
run: nox -s safety
generator-script-testing:
runs-on: ubuntu-20.04
continue-on-error: true
name: icatdb Generator Script Consistency Test
steps:
- name: Add apt repo
run: sudo add-apt-repository universe
# Setup Java & Python
- name: Setup Java
uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 # v3.12.0
with:
distribution: 'temurin'
java-version: 8
- name: Setup Python
uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: "3.9.7"
architecture: x64
# ICAT Ansible clone and install dependencies
- name: Checkout icat-ansible
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
with:
repository: icatproject-contrib/icat-ansible
ref: master
path: icat-ansible
- name: Install Ansible
run: pip install -r icat-ansible/requirements.txt
# Prep for running the playbook
- name: Create hosts file
run: echo -e "[icatdb_minimal_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
- name: Prepare vault pass
run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
- name: Move vault to directory it'll get detected by Ansible
run: mv icat-ansible/vault.yml icat-ansible/group_vars/all
- name: Replace default payara user with Actions user
run: |
sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml
# Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions
- name: Change hostname to localhost
run: sudo hostname -b localhost
# Remove existing MySQL installation so it doesn't interfere with GitHub Actions
- name: Remove existing mysql
run: |
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
sudo apt-get remove --purge "mysql*"
sudo rm -rf /var/lib/mysql* /etc/mysql
# Create local instance of ICAT
- name: Run ICAT Ansible Playbook
run: |
ansible-playbook icat-ansible/icatdb_minimal_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv
- name: Checkout DataGateway API
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
- name: Create config.yaml
run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/config.yaml.example datagateway_api/config.yaml
- name: Create search_api_mapping.json
run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json
- name: Install Poetry
run: pip install poetry==1.1.9
# This fixes the issue with cachecontrol (https://github.com/psf/cachecontrol/issues/292).
# We will not be facing this issue when we upgrade to a newer Poetry version.
- name: Install Requests
run: pip install 'requests<2.30'
- name: Install dependencies
run: poetry install
- name: Add dummy data to icatdb
run: poetry run python -m util.icat_db_generator
- name: Drop modTime and createTime
run: mysql -picatdbuserpw -uicatdbuser icatdb < /home/runner/work/datagateway-api/datagateway-api/util/columns_to_drop.sql
- name: Get SQL dump of dummy data
run: mysqldump -picatdbuserpw -uicatdbuser --skip-comments icatdb > ~/generator_script_dump_1.sql
# Drop and re-create icatdb to remove generated data
- name: Drop icatdb
run: mysql -picatdbuserpw -uicatdbuser -e 'DROP DATABASE icatdb;'
- name: Create icatdb
run: mysql -picatdbuserpw -uicatdbuser -e 'CREATE DATABASE icatdb;'
# Regenerate table structure of icatdb
- name: Reinstall ICAT Server
run: cd /home/runner/install/icat.server; ./setup -vv install
- name: Add ICAT 5 triggers
run: cd /home/runner/install/icat.server; sudo mysql -uroot -D icatdb < create_triggers_mysql_5_0.sql
- name: Add (new) dummy data to icatdb
run: |
cd /home/runner/work/datagateway-api/datagateway-api; poetry run python -m util.icat_db_generator
- name: Drop modTime and createTime
run: mysql -picatdbuserpw -uicatdbuser icatdb < /home/runner/work/datagateway-api/datagateway-api/util/columns_to_drop.sql
- name: Get SQL dump of new dummy data
run: mysqldump -picatdbuserpw -uicatdbuser --skip-comments icatdb > ~/generator_script_dump_2.sql
# Tests that the generator script produces consistent data over two separate runs
- name: Diff SQL dumps
run: diff -s ~/generator_script_dump_1.sql ~/generator_script_dump_2.sql
# Drop and re-create icatdb to remove generated data
- name: Drop icatdb
run: mysql -picatdbuserpw -uicatdbuser -e 'DROP DATABASE icatdb;'
- name: Create icatdb
run: mysql -picatdbuserpw -uicatdbuser -e 'CREATE DATABASE icatdb;'
# Regenerate table structure of icatdb
- name: Reinstall ICAT Server
run: cd /home/runner/install/icat.server; ./setup -vv install
- name: Add ICAT 5 triggers
run: cd /home/runner/install/icat.server; sudo mysql -uroot -D icatdb < create_triggers_mysql_5_0.sql
- name: Checkout DataGateway API (default branch)
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
with:
ref: ${{ github.event.repository.default_branch }}
- name: Create config.yaml
run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/config.yaml.example datagateway_api/config.yaml
- name: Create search_api_mapping.json
run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json
- name: Install dependencies
run: poetry install
- name: Add dummy data to icatdb
run: poetry run python -m util.icat_db_generator
- name: Drop modTime and createTime
run: mysql -picatdbuserpw -uicatdbuser icatdb < /home/runner/work/datagateway-api/datagateway-api/util/columns_to_drop.sql
- name: Get SQL dump of dummy data from main's generator script
run: mysqldump -picatdbuserpw -uicatdbuser --skip-comments icatdb > ~/generator_script_dump_main.sql
# Tests that the generator script produces the same data as is produced with main's version
# NOTE: If a delibrate change is made to the script that will change the data generated,
# the diff (and therefore this job) will fail. If this is the case, don't be alarmed.
# The `continue-on-error` keyword has been added to this job so the workflow should
# pass when the PR is merged in, even if this job fails.
# But, if you didn't mean to change the output of the script, there is likely a
# problem with the changes made that will make the generator script's data
# different to SciGateway preprod
- name: Diff SQL dumps
run: diff -s ~/generator_script_dump_main.sql ~/generator_script_dump_1.sql
docker:
# This job triggers only if all the other jobs succeed and does different things depending on the context.
# The job builds the Docker image in all cases and also pushes the image to Harbor only if something is
# pushed to the main branch.
needs: [tests, linting, formatting, safety, generator-script-testing]
name: Docker
runs-on: ubuntu-20.04
steps:
- name: Check out repo
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.5.3
- name: Login to Harbor
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0
with:
registry: harbor.stfc.ac.uk/datagateway
username: ${{ secrets.HARBOR_USERNAME }}
password: ${{ secrets.HARBOR_TOKEN }}
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@c4ee3adeed93b1fa6a762f209fb01608c1a22f1e # v4.4.0
with:
images: harbor.stfc.ac.uk/datagateway/datagateway-api
- name: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'Build and push Docker image to Harbor' || 'Build Docker image' }}
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v4.0.0
with:
context: .
push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}