From 78bcf83796b61b399c4f20c3b7904cb6932a409a Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Sat, 7 Dec 2024 17:43:13 +0100 Subject: [PATCH 1/8] Webstandard job has been moved to GHA The GitLab folder will be removed so this cleanup was needed. Also fixed displaying the sections for the GHA job output. --- gitlab/jsontogitlab.py => .github/jobs/jsontogha.py | 4 ++-- .github/jobs/webstandard.sh | 2 +- .github/workflows/database-upgrade.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename gitlab/jsontogitlab.py => .github/jobs/jsontogha.py (93%) diff --git a/gitlab/jsontogitlab.py b/.github/jobs/jsontogha.py similarity index 93% rename from gitlab/jsontogitlab.py rename to .github/jobs/jsontogha.py index a1e8e69327..9e259e74c1 100644 --- a/gitlab/jsontogitlab.py +++ b/.github/jobs/jsontogha.py @@ -10,10 +10,10 @@ def cleanHash(toHash): return hashlib.sha224(toHash).hexdigest() def sec_start(job,header): - print('section_start:{}:{}{}{}'.format(int(time.time()),cleanHash(job),'\r\033[0K',header)) + print('section_start\r\033[0K'+header) def sec_end(job): - print('section_end:{}:{}'.format(int(time.time()),cleanHash(job)+'\r\033[0K')) + print('section_end\r\033[0K') with open(sys.argv[1],'r') as f: data = json.load(f) diff --git a/.github/jobs/webstandard.sh b/.github/jobs/webstandard.sh index 66d5641aa6..c1fd988eca 100755 --- a/.github/jobs/webstandard.sh +++ b/.github/jobs/webstandard.sh @@ -123,7 +123,7 @@ if [ "$TEST" = "w3cval" ]; then NEWFOUNDERRORS=$("$DIR"/vnu-runtime-image/bin/vnu --errors-only --exit-zero-always --skip-non-$typ --format gnu $FLTR "$URL" 2>&1 | wc -l) FOUNDERR=$((NEWFOUNDERRORS+FOUNDERR)) python3 -m "json.tool" < result.json > "$ARTIFACTS/w3c$typ$URL.json" - trace_off; python3 gitlab/jsontogitlab.py "$ARTIFACTS/w3c$typ$URL.json"; trace_on + trace_off; python3 .github/jobs/jsontogha.py "$ARTIFACTS/w3c$typ$URL.json"; trace_on section_end done else diff --git a/.github/workflows/database-upgrade.yml b/.github/workflows/database-upgrade.yml index dd3fe7a351..47b6a14ed1 100644 --- a/.github/workflows/database-upgrade.yml +++ b/.github/workflows/database-upgrade.yml @@ -32,5 +32,5 @@ jobs: run: echo "pass" > /opt/domjudge/domserver/etc/initial_admin_password.secret - name: Check for Errors in the Upgrade run: mysql -hsqlserver -uroot -proot -e "SHOW TABLES FROM domjudge;" - - name: Check for Errors in Domjudge Web + - name: Check for Errors in DOMjudge Webinterface run: .github/jobs/webstandard.sh none admin From 8976cee6b5eae91b72783aff217601d6f90b7748 Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Mon, 9 Dec 2024 19:35:08 +0100 Subject: [PATCH 2/8] Allow passing the root password --- .github/jobs/baseinstall.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/jobs/baseinstall.sh b/.github/jobs/baseinstall.sh index d292fc639c..e1fb60d9f5 100755 --- a/.github/jobs/baseinstall.sh +++ b/.github/jobs/baseinstall.sh @@ -5,6 +5,8 @@ export version="$1" db=${2:-install} +MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-root} + set -eux PHPVERSION=$(php -r 'echo PHP_MAJOR_VERSION.".".PHP_MINOR_VERSION."\n";') @@ -72,11 +74,11 @@ section_end if [ "${db}" = "install" ]; then section_start "Install DOMjudge database" - /opt/domjudge/domserver/bin/dj_setup_database -uroot -proot bare-install + /opt/domjudge/domserver/bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} bare-install section_end elif [ "${db}" = "upgrade" ]; then section_start "Upgrade DOMjudge database" - /opt/domjudge/domserver/bin/dj_setup_database -uroot -proot upgrade + /opt/domjudge/domserver/bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} upgrade section_end fi @@ -114,7 +116,7 @@ section_end if [ "${db}" = "install" ]; then section_start "Install the example data" - /opt/domjudge/domserver/bin/dj_setup_database -uroot -proot install-examples | tee -a "$ARTIFACTS/mysql.txt" + /opt/domjudge/domserver/bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} install-examples | tee -a "$ARTIFACTS/mysql.txt" section_end fi From 6992e1bbab209e32c0aa3e61e26c77ddb661478f Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Sun, 8 Dec 2024 13:59:46 +0100 Subject: [PATCH 3/8] Fix python linting issues --- .github/jobs/jsontogha.py | 40 +++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/.github/jobs/jsontogha.py b/.github/jobs/jsontogha.py index 9e259e74c1..b2916904b2 100644 --- a/.github/jobs/jsontogha.py +++ b/.github/jobs/jsontogha.py @@ -3,31 +3,35 @@ import time import hashlib -storage1 ={} +storage1 = {} storage2 = {} + def cleanHash(toHash): return hashlib.sha224(toHash).hexdigest() -def sec_start(job,header): + +def sec_start(job, header): print('section_start\r\033[0K'+header) + def sec_end(job): print('section_end\r\033[0K') -with open(sys.argv[1],'r') as f: + +with open(sys.argv[1], 'r') as f: data = json.load(f) for message in data['messages']: mtyp = message['type'].encode('utf-8', 'ignore') murl = message['url'].encode('utf-8', 'ignore') mmes = message['message'].encode('utf-8', 'ignore') if mtyp not in storage1.keys(): - storage1[mtyp] = {"messages":{}, "cnt":0} - storage2[mtyp] = {"urls":{}, "cnt":0} + storage1[mtyp] = {"messages": {}, "cnt": 0} + storage2[mtyp] = {"urls": {}, "cnt": 0} if mmes not in storage1[mtyp]["messages"].keys(): - storage1[mtyp]["messages"][mmes] = {"urls":{}, "cnt":0} + storage1[mtyp]["messages"][mmes] = {"urls": {}, "cnt": 0} if murl not in storage2[mtyp]["urls"].keys(): - storage2[mtyp]["urls"][murl] = {"messages":{}, "cnt":0} + storage2[mtyp]["urls"][murl] = {"messages": {}, "cnt": 0} if murl not in storage1[mtyp]["messages"][mmes]["urls"].keys(): storage1[mtyp]["messages"][mmes]["urls"][murl] = 0 if mmes not in storage2[mtyp]["urls"][murl]["messages"].keys(): @@ -38,18 +42,18 @@ def sec_end(job): storage2[mtyp]["urls"][murl]["messages"][mmes] += 1 storage2[mtyp]["urls"][murl]["cnt"] += 1 storage2[mtyp]["cnt"] += 1 - # Stats -for key,value in sorted(storage1.items(), key=lambda x:x[1]['cnt']): - print("Type: {}, Totalfound: {}".format(key, value["cnt"])) - for key2,value2 in sorted(storage1[key]["messages"].items(), key=lambda x:x[1]['cnt'], reverse=True): + +for key, value in sorted(storage1.items(), key=lambda x: x[1]['cnt']): + print("Type: {}, Totalfound: {}".format(key, value["cnt"])) + for key2, value2 in sorted(storage1[key]["messages"].items(), key=lambda x: x[1]['cnt'], reverse=True): sec_start(key+key2, key2) - print("[{}] [{}%] Message: {}".format(key, round(100*value2["cnt"]/value["cnt"],2), key2)) - for key3,value3 in sorted(storage1[key]["messages"][key2]["urls"].items(), key=lambda x:x[1], reverse=True): - print("[{}%] URL: {}".format(round(100*value3/value2["cnt"],2), key3)) + print("[{}] [{}%] Message: {}".format(key, round(100*value2["cnt"]/value["cnt"], 2), key2)) + for key3, value3 in sorted(storage1[key]["messages"][key2]["urls"].items(), key=lambda x: x[1], reverse=True): + print("[{}%] URL: {}".format(round(100*value3/value2["cnt"], 2), key3)) sec_end(key+key2) - for key2,value2 in sorted(storage2[key]["urls"].items(), key=lambda x:x[1]['cnt'], reverse=True): + for key2, value2 in sorted(storage2[key]["urls"].items(), key=lambda x: x[1]['cnt'], reverse=True): sec_start(key+key2, key2) - print("[{}] [{}%] URL: {}".format(key, round(100*value2["cnt"]/value["cnt"],2), key2)) - for key3,value3 in sorted(storage2[key]["urls"][key2]["messages"].items(), key=lambda x:x[1], reverse=True): - print("[{}%] Message: {}".format(round(100*value3/value2["cnt"],2), key3)) + print("[{}] [{}%] URL: {}".format(key, round(100*value2["cnt"]/value["cnt"], 2), key2)) + for key3, value3 in sorted(storage2[key]["urls"][key2]["messages"].items(), key=lambda x: x[1], reverse=True): + print("[{}%] Message: {}".format(round(100*value3/value2["cnt"], 2), key3)) sec_end(key+key2) From 01cbbba4c64ee7cfee97c06362c2f40ea058dd80 Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Wed, 4 Dec 2024 20:00:29 +0100 Subject: [PATCH 4/8] Add extra debugging about MySQL This is useful for when the tests suddenly break because the upstream images have changed. --- .github/jobs/baseinstall.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/jobs/baseinstall.sh b/.github/jobs/baseinstall.sh index e1fb60d9f5..9cf5d1fe63 100755 --- a/.github/jobs/baseinstall.sh +++ b/.github/jobs/baseinstall.sh @@ -67,6 +67,8 @@ mysql_root "SELECT CURRENT_USER();" mysql_root "SELECT USER();" mysql_root "SELECT user,host FROM mysql.user" mysql_root "SET GLOBAL max_allowed_packet=1073741824" +mysql_root "SHOW GLOBAL STATUS LIKE 'Connection_errors_%'" +mysql_root "SHOW VARIABLES LIKE '%_timeout'" echo "unused:sqlserver:domjudge:domjudge:domjudge:3306" > /opt/domjudge/domserver/etc/dbpasswords.secret mysql_user "SELECT CURRENT_USER();" mysql_user "SELECT USER();" From b0ebb1c6a321ff89cc9bad23c6bdda11df0302ae Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Sun, 8 Dec 2024 13:13:17 +0100 Subject: [PATCH 5/8] Explicit use the new latest image We keep getting a warning about this so it's better to explicit test against that version. --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index b26abc8118..f351558f6a 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -8,7 +8,7 @@ on: jobs: check-static-codecov: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Download latest codecov upload script From e3e07f8ecf809e533183433eafcb77510199f8a4 Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Wed, 4 Dec 2024 19:20:05 +0100 Subject: [PATCH 6/8] Allow changing the PHP version for CI tests Both useful for the integration & unit tests. --- .github/jobs/baseinstall.sh | 6 +++++- .github/jobs/ci_settings.sh | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/jobs/baseinstall.sh b/.github/jobs/baseinstall.sh index 9cf5d1fe63..ddde844cbe 100755 --- a/.github/jobs/baseinstall.sh +++ b/.github/jobs/baseinstall.sh @@ -4,13 +4,17 @@ export version="$1" db=${2:-install} +phpversion="${3}" MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-root} set -eux +if [ -z "$phpversion" ]; then PHPVERSION=$(php -r 'echo PHP_MAJOR_VERSION.".".PHP_MINOR_VERSION."\n";') -export PHPVERSION +fi + +show_phpinfo "$PHPVERSION" section_start "Run composer" export APP_ENV="dev" diff --git a/.github/jobs/ci_settings.sh b/.github/jobs/ci_settings.sh index 350b07cf09..391996752b 100644 --- a/.github/jobs/ci_settings.sh +++ b/.github/jobs/ci_settings.sh @@ -34,6 +34,15 @@ mysql_user () { echo "$1" | mysql -udomjudge -pdomjudge ${2:-} | tee -a "$ARTIFACTS"/mysql.txt } +show_phpinfo() { + phpversion=$1 + section_start "Show the new PHP info" + update-alternatives --set php /usr/bin/php"${phpversion}" + php -v + php -m + section_end +} + section_start () { if [ "$#" -ne 1 ]; then echo "Only 1 argument is needed for GHA, 2 was needed for GitLab." From 82391376eda773948610f8a944fd610adedfac2d Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Sat, 7 Dec 2024 12:15:15 +0100 Subject: [PATCH 7/8] Run the unit tests We'll need to play around with the unit test upload features (https://github.com/EnricoMi/publish-unit-test-result-action#configuration) what works best. This is based on what we do for the integration test. Most shared code is handled by baseinstall.sh. We can now also install a database with a different name as the unit tests use that locally. As side-effect we directly test this in CI. The CI ran on GitLab before and needed extra steps to push the result to GitHub. As everything is in GitHub Actions now we can remove those extra steps. --- .github/jobs/baseinstall.sh | 52 +++++++++++--------- .github/jobs/unit-tests.sh | 63 +++++++++++++++++++++++++ .github/workflows/unit-tests.yml | 63 +++++++++++++++++++++++++ gitlab/base.sh | 8 ---- gitlab/unit-tests.sh | 81 -------------------------------- 5 files changed, 157 insertions(+), 110 deletions(-) create mode 100755 .github/jobs/unit-tests.sh delete mode 100755 gitlab/unit-tests.sh diff --git a/.github/jobs/baseinstall.sh b/.github/jobs/baseinstall.sh index ddde844cbe..2064339d94 100755 --- a/.github/jobs/baseinstall.sh +++ b/.github/jobs/baseinstall.sh @@ -4,17 +4,22 @@ export version="$1" db=${2:-install} -phpversion="${3}" +phpversion="${3:-8.1}" +# If this script is called from unit-tests.sh, we use the test environment +export APP_ENV="${4:-prod}" + +# In the test environment, we need to use a different database +[ "$APP_ENV" = "prod" ] && DATABASE_NAME=domjudge || DATABASE_NAME=domjudge_test MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-root} set -eux if [ -z "$phpversion" ]; then -PHPVERSION=$(php -r 'echo PHP_MAJOR_VERSION.".".PHP_MINOR_VERSION."\n";') +phpversion=$(php -r 'echo PHP_MAJOR_VERSION.".".PHP_MINOR_VERSION."\n";') fi -show_phpinfo "$PHPVERSION" +show_phpinfo "$phpversion" section_start "Run composer" export APP_ENV="dev" @@ -47,8 +52,10 @@ else --enable-judgehost-build=no | tee "$ARTIFACTS"/configure.txt make domserver make install-domserver + rm -rf /opt/domjudge/domserver/webapp/public/doc + cp -r doc /opt/domjudge/domserver/webapp/public/ + find /opt/domjudge/domserver -name DOMjudgelogo.pdf fi - section_end section_start "SQL settings" @@ -56,13 +63,13 @@ cat > ~/.my.cnf < /opt/domjudge/domserver/etc/dbpasswords.secret +echo "unused:sqlserver:$DATABASE_NAME:domjudge:domjudge:3306" > /opt/domjudge/domserver/etc/dbpasswords.secret mysql_user "SELECT CURRENT_USER();" mysql_user "SELECT USER();" section_end @@ -100,7 +107,7 @@ cp /proc/cmdline "$ARTIFACTS"/cmdline.txt section_end section_start "Setup webserver" -cp /opt/domjudge/domserver/etc/domjudge-fpm.conf /etc/php/"$PHPVERSION"/fpm/pool.d/domjudge.conf +cp /opt/domjudge/domserver/etc/domjudge-fpm.conf /etc/php/"$phpversion"/fpm/pool.d/domjudge.conf rm -f /etc/nginx/sites-enabled/* cp /opt/domjudge/domserver/etc/nginx-conf /etc/nginx/sites-enabled/domjudge @@ -114,7 +121,7 @@ nginx -t section_end section_start "Show webserver is up" -for service in nginx php${PHPVERSION}-fpm; do +for service in nginx php${phpversion}-fpm; do service "$service" restart service "$service" status done @@ -122,30 +129,33 @@ section_end if [ "${db}" = "install" ]; then section_start "Install the example data" + if [ "$version" = "unit" ]; then + # Make sure admin has no team associated so we will not insert submissions during unit tests. + mysql_root "UPDATE user SET teamid=null WHERE userid=1;" $DATABASE_NAME + fi /opt/domjudge/domserver/bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} install-examples | tee -a "$ARTIFACTS/mysql.txt" section_end fi section_start "Setup user" # We're using the admin user in all possible roles -mysql_root "DELETE FROM userrole WHERE userid=1;" domjudge +mysql_root "DELETE FROM userrole WHERE userid=1;" $DATABASE_NAME if [ "$version" = "team" ]; then # Add team to admin user - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" domjudge - mysql_root "UPDATE user SET teamid = 1 WHERE userid = 1;" domjudge + mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" $DATABASE_NAME + mysql_root "UPDATE user SET teamid = 1 WHERE userid = 1;" $DATABASE_NAME elif [ "$version" = "jury" ]; then # Add jury to admin user - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 2);" domjudge + mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 2);" $DATABASE_NAME elif [ "$version" = "balloon" ]; then # Add balloon to admin user - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 4);" domjudge + mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 4);" $DATABASE_NAME elif [ "$version" = "admin" ]; then # Add admin to admin user - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 1);" domjudge -elif [ "$version" = "all" ]; then - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 1);" domjudge - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" domjudge - mysql_root "UPDATE user SET teamid = 1 WHERE userid = 1;" domjudge + mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 1);" $DATABASE_NAME +elif [ "$version" = "all" ] || [ "$version" = "unit" ]; then + mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 1);" $DATABASE_NAME + mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" $DATABASE_NAME + mysql_root "UPDATE user SET teamid = 1 WHERE userid = 1;" $DATABASE_NAME fi section_end - diff --git a/.github/jobs/unit-tests.sh b/.github/jobs/unit-tests.sh new file mode 100755 index 0000000000..b55fb86ce3 --- /dev/null +++ b/.github/jobs/unit-tests.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +. .github/jobs/ci_settings.sh + +DIR="$PWD" + +export version=$1 +unittest=$2 +[ "$version" = "8.1" ] && CODECOVERAGE=1 || CODECOVERAGE=0 + +# Set up +export unit=1 + +# Add team to admin user +echo "UPDATE user SET teamid = 1 WHERE userid = 1;" | mysql domjudge_test + +# Copy the .env.test file, as this is normally not done during +# installation and we need it. +cp webapp/.env.test /opt/domjudge/domserver/webapp/ + +# We also need the composer.json for PHPunit to detect the correct directory. +cp webapp/composer.json /opt/domjudge/domserver/webapp/ + +cd /opt/domjudge/domserver + +# Run phpunit tests. +pcov="" +phpcov="" +if [ "$CODECOVERAGE" -eq 1 ]; then + phpcov="-dpcov.enabled=1 -dpcov.directory=webapp/src" + pcov="--coverage-html=${DIR}/coverage-html --coverage-clover coverage.xml" +fi +set +e +echo "unused:sqlserver:domjudge:domjudge:domjudge:3306" > /opt/domjudge/domserver/etc/dbpasswords.secret +php $phpcov webapp/bin/phpunit -c webapp/phpunit.xml.dist webapp/tests/$unittest --log-junit ${ARTIFACTS}/unit-tests.xml --colors=never $pcov > "$ARTIFACTS"/phpunit.out +UNITSUCCESS=$? + +# Store the unit tests also in the root for the GHA +cp $ARTIFACTS/unit-tests.xml $DIR/ + +# Make sure the log exists before copy +touch ${DIR}/webapp/var/log/test.log +cp ${DIR}/webapp/var/log/*.log "$ARTIFACTS"/ + +set -e +CNT=0 +THRESHOLD=32 +if [ $CODECOVERAGE -eq 1 ]; then + CNT=$(sed -n '/Generating code coverage report/,$p' "$ARTIFACTS"/phpunit.out | grep -v DoctrineTestBundle | grep -cv ^$) +fi + +if [ $UNITSUCCESS -ne 0 ] || [ $CNT -gt $THRESHOLD ]; then + exit 1 +fi + +if [ $CODECOVERAGE -eq 1 ]; then + section_start "Upload code coverage" + # Only upload when we got working unit-tests. + set +u # Uses some variables which are not set + # shellcheck disable=SC1090 + . $DIR/.github/jobs/uploadcodecov.sh &>> "$ARTIFACTS"/codecov.log + section_end +fi diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index f351558f6a..7455125a7f 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,4 +1,5 @@ name: Unit tests +# We can speedup with: https://github.com/actions/cache on: merge_group: pull_request: @@ -16,3 +17,65 @@ jobs: - name: Detect changes to manually verify run: diff newcodecov .github/jobs/uploadcodecov.sh + unit-tests: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + container: + image: domjudge/gitlabci:24.04 + services: + sqlserver: + image: mariadb + ports: + - 3306:3306 + env: + MYSQL_ROOT_PASSWORD: root + MYSQL_USER: domjudge + MYSQL_PASSWORD: domjudge + options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3 + strategy: + matrix: + PHPVERSION: [8.1, 8.4] + TEST: [Unit, E2E] + steps: + - uses: actions/checkout@v4 + - name: info + run: | + cat /proc/cmdline && echo && + cat /proc/mounts && echo && + ls -al /sys/fs/cgroup && echo && + uname -a && echo && + stat -fc %T /sys/fs/cgroup && echo && + cat /proc/self/cgroup && echo && + cat /proc/cpuinfo + - name: pstree + run: pstree -p + - name: Install DOMjudge + run: .github/jobs/baseinstall.sh unit install ${{ matrix.PHPVERSION }} test + - name: Check nginx + run: curl -v https://localhost/domjudge/ + - name: Run the unit-tests + run: .github/jobs/unit-tests.sh ${{ matrix.PHPVERSION }} ${{ matrix.TEST }} + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: ${{ !cancelled() }} + with: + comment_mode: changes in failures + check_name: unit-tests-${{ matrix.PHPVERSION }}-${{ matrix.TEST }}.xml + files: unit-tests-${{ matrix.PHPVERSION }}-${{ matrix.TEST }}.xml + - name: Get SQL logs + run: docker logs "${{ job.services.sqlserver.id }}" + - name: Collect docker logs on failure + if: ${{ !cancelled() }} + uses: jwalton/gh-docker-logs@v1 + with: + dest: '/tmp/docker-logs' + - name: Upload all logs/artifacts + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: Logs-${{ matrix.PHPVERSION }}-${{ matrix.TEST }} + path: | + /var/log/nginx + /opt/domjudge/domserver/webapp/var/log/*.log + /tmp/docker-logs + /tmp/artifacts diff --git a/gitlab/base.sh b/gitlab/base.sh index a07f026d5b..7300b8e865 100755 --- a/gitlab/base.sh +++ b/gitlab/base.sh @@ -10,14 +10,6 @@ export APP_ENV="${1:-prod}" lsb_release -a -cat > ~/.my.cnf < "$GITLABARTIFACTS"/phpunit.out -UNITSUCCESS=$? -set -e -CNT=0 -THRESHOLD=32 -if [ $CODECOVERAGE -eq 1 ]; then - CNT=$(sed -n '/Generating code coverage report/,$p' "$GITLABARTIFACTS"/phpunit.out | grep -v DoctrineTestBundle | grep -cv ^$) - FILE=deprecation.txt - sed -n '/Generating code coverage report/,$p' "$GITLABARTIFACTS"/phpunit.out > ${CI_PROJECT_DIR}/$FILE - if [ $CNT -le $THRESHOLD ]; then - STATE=success - else - STATE=failure - fi - ORIGINAL="gitlab.com/DOMjudge" - REPLACETO="domjudge.gitlab.io/-" - # Copied from CCS - curl https://api.github.com/repos/domjudge/domjudge/statuses/$CI_COMMIT_SHA \ - -X POST \ - -H "Authorization: token $GH_BOT_TOKEN_OBSCURED" \ - -H "Accept: application/vnd.github.v3+json" \ - -d "{\"state\": \"$STATE\", \"target_url\": \"${CI_JOB_URL/$ORIGINAL/$REPLACETO}/artifacts/$FILE\", \"description\":\"Symfony deprecations ($version)\", \"context\": \"Symfony deprecation ($version)\"}" -fi -if [ $UNITSUCCESS -eq 0 ]; then - STATE=success -else - STATE=failure -fi -cp webapp/var/log/test.log "$GITLABARTIFACTS"/test.log - -curl https://api.github.com/repos/domjudge/domjudge/statuses/$CI_COMMIT_SHA \ - -X POST \ - -H "Authorization: token $GH_BOT_TOKEN_OBSCURED" \ - -H "Accept: application/vnd.github.v3+json" \ - -d "{\"state\": \"$STATE\", \"target_url\": \"${CI_PIPELINE_URL}/test_report\", \"description\":\"Unit tests\", \"context\": \"unit_tests ($version)\"}" -if [ $UNITSUCCESS -ne 0 ] || [ $CNT -gt $THRESHOLD ]; then - exit 1 -fi - -if [ $CODECOVERAGE -eq 1 ]; then - section_start_collap uploadcoverage "Upload code coverage" - # Only upload when we got working unit-tests. - set +u # Uses some variables which are not set - # shellcheck disable=SC1090 - . $DIR/.github/jobs/uploadcodecov.sh 1>/dev/zero 2>/dev/zero - section_end uploadcoverage -fi From 0e8cb7b67b3905b4727ef5a5255b0a4b2dce28e4 Mon Sep 17 00:00:00 2001 From: Michael Vasseur <14887731+vmcj@users.noreply.github.com> Date: Sun, 1 Dec 2024 00:07:46 +0100 Subject: [PATCH 8/8] Ignore old GitLab jobs --- .gitlab-ci.yml | 11 ----- gitlab/base.sh | 100 ----------------------------------------- gitlab/ci/template.yml | 64 -------------------------- gitlab/ci/unit.yml | 42 ----------------- gitlab/ci_settings.sh | 55 ----------------------- gitlab/default-nginx | 12 ----- 6 files changed, 284 deletions(-) delete mode 100644 .gitlab-ci.yml delete mode 100755 gitlab/base.sh delete mode 100644 gitlab/ci/template.yml delete mode 100644 gitlab/ci/unit.yml delete mode 100755 gitlab/ci_settings.sh delete mode 100644 gitlab/default-nginx diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 9f73622022..0000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,11 +0,0 @@ -include: - - '/gitlab/ci/unit.yml' - - '/gitlab/ci/template.yml' - -stages: - - test - - unit - - style - - ci_checks - -image: domjudge/gitlabci:24.04 diff --git a/gitlab/base.sh b/gitlab/base.sh deleted file mode 100755 index 7300b8e865..0000000000 --- a/gitlab/base.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash - -. gitlab/ci_settings.sh - -# If this script is called from unit.sh, we use the test environment -export APP_ENV="${1:-prod}" - -# In the test environment, we need to use a different database -[ "$APP_ENV" = "prod" ] && DATABASE_NAME=domjudge || DATABASE_NAME=domjudge_test - -lsb_release -a - -# FIXME: This chicken-egg problem is annoying but let us bootstrap for now. -echo "CREATE DATABASE IF NOT EXISTS \`${DATABASE_NAME}\` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" | mysql -echo "CREATE USER 'domjudge'@'%' IDENTIFIED BY 'domjudge';" | mysql -echo "GRANT SELECT, INSERT, UPDATE, DELETE ON \`${DATABASE_NAME}\`.* TO 'domjudge'@'%';" | mysql - -# Increase max_allowed_packet for following connections. -echo "SET GLOBAL max_allowed_packet = 100*1024*1024;" | mysql - -# Test that SQL upgrade scripts also work with this setting -if [ -n "${MYSQL_REQUIRE_PRIMARY_KEY:-}" ]; then - echo 'SET GLOBAL sql_require_primary_key = 1;' | mysql -fi - -# Generate a dbpasswords file -# Note that this does not use ${DATABASE_NAME} since Symfony adds the _test postfix itself -echo "unused:sqlserver:domjudge:domjudge:domjudge:3306" > etc/dbpasswords.secret - -# Generate APP_SECRET for symfony -# shellcheck disable=SC2164 -( cd etc ; ./gensymfonysecret > symfony_app.secret ) - -cat > webapp/config/static.yaml <> ~www-data/.netrc -sudo -Eu www-data bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} bare-install - -# shellcheck disable=SC2154 -if [ -n "${integration:-}" ]; then - # Make sure admin has a team associated to insert submissions as well. - echo "UPDATE user SET teamid=1 WHERE userid=1;" | mysql domjudge -elif [ -n "${unit:-}" ]; then - # Make sure admin has no team associated so we will not insert submissions during unit tests. - echo "UPDATE user SET teamid=null WHERE userid=1;" | mysql domjudge_test -fi - -sudo -Eu www-data bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} install-examples diff --git a/gitlab/ci/template.yml b/gitlab/ci/template.yml deleted file mode 100644 index e810df854c..0000000000 --- a/gitlab/ci/template.yml +++ /dev/null @@ -1,64 +0,0 @@ -# This placeholder job tries to start as soon as possible -.clean_ordering: - needs: [] - retry: - max: 2 #Max is 2, set when gitlab is flacky - when: - - always - script: - - /bin/true - -.tiny_job: - extends: [.clean_ordering] - timeout: 4m - -.short_job: - extends: [.clean_ordering] - timeout: 7m - -.normal_job: - extends: [.clean_ordering] - timeout: 20m - -.long_job: - extends: [.clean_ordering] - timeout: 30m - -.cached_vendor: - extends: [.clean_ordering] - cache: - key: webappvendor-20240623 - paths: - - webapp/vendor/ - -.mysql_job: - script: - - /bin/true - services: - - name: mysql - alias: sqlserver - -.mariadb_job: - script: - - /bin/true - services: - - name: mariadb - alias: sqlserver - -.phpsupported_job: - script: - - /bin/true - parallel: - matrix: - - PHPVERSION: ["8.1","8.2","8.3", "8.4"] - TEST: ["E2E","Unit"] - CRAWL_SHADOW_MODE: ["0","1"] - -.phpsupported_job_pr: - script: - - /bin/true - parallel: - matrix: - - PHPVERSION: ["8.3"] - TEST: ["E2E","Unit"] - CRAWL_SHADOW_MODE: ["0"] diff --git a/gitlab/ci/unit.yml b/gitlab/ci/unit.yml deleted file mode 100644 index 5bf7aab8c7..0000000000 --- a/gitlab/ci/unit.yml +++ /dev/null @@ -1,42 +0,0 @@ -.unit_job: - extends: [.normal_job,.cached_vendor] - stage: unit - # Disabled for now as it drastically speeds up running unit tests and we don't use it yet - # before_script: - # - apt-get update -yqq - # - apt-get install php-xdebug -yqq - variables: - MYSQL_ROOT_PASSWORD: password - MARIADB_PORT_3306_TCP_ADDR: sqlserver - script: - - set -eux - - if [ -z ${PHPVERSION+x} ]; then export PHPVERSION=8.1; fi - - if [ -z ${TEST+x} ]; then export TEST="UNIT"; fi - - if [ "$TEST" = "UNIT" ] && [ "$CRAWL_SHADOW_MODE" != "0" ]; then exit 0; fi - - if [ "$TEST" = "E2E" ] && [ "$CRAWL_SHADOW_MODE" != "0" ] && [ "$CI_COMMIT_BRANCH" != "main" ]; then exit 0; fi - - export CRAWL_SHADOW_MODE - - ./gitlab/unit-tests.sh $PHPVERSION $TEST - artifacts: - when: always - paths: - - unit-tests.xml - - coverage-html - - deprecation.txt - - duration - - gitlabartifacts - reports: - junit: - - unit-tests.xml - -run unit tests: - extends: [.mariadb_job,.phpsupported_job,.unit_job] - -run unit tests (PR): - extends: [.mariadb_job,.phpsupported_job_pr,.unit_job] - -run unit tests (MySQL): - extends: [.mysql_job,.unit_job] - parallel: - matrix: - - TEST: ["E2E","Unit"] - CRAWL_SHADOW_MODE: ["0"] diff --git a/gitlab/ci_settings.sh b/gitlab/ci_settings.sh deleted file mode 100755 index dfea8a1925..0000000000 --- a/gitlab/ci_settings.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -# Fail pipeline when variable is not set or individual command has an non-zero exitcode. -set -euo pipefail - -shopt -s expand_aliases - -# Shared constants between jobs -DIR=$(pwd) -GITSHA=$(git rev-parse HEAD || true) -export DIR -export GITSHA -export PS4='(${BASH_SOURCE}:${LINENO}): - [$?] $ ' -export LOGFILE="/opt/domjudge/domserver/webapp/var/log/prod.log" - -CCS_SPECS_PINNED_SHA1='a68aff54c4e60fc2bff2fc5c36c119bffa4d30f1' - -# Shared storage for all artifacts -export GITLABARTIFACTS="$DIR/gitlabartifacts" -mkdir -p "$GITLABARTIFACTS" - -# Functions to annotate the GitLab logs -alias trace_on='set -x' -alias trace_off='{ set +x; } 2>/dev/null' -function section_start_internal() { - echo -e "section_start:$(date +%s):$2[collapsed=$1]\r\e[0K$3" - trace_on -} -function section_end_internal() { - echo -e "section_end:$(date +%s):$1\r\e[0K" - trace_on -} -alias section_start_collap='trace_off ; section_start_internal true' -alias section_start='trace_off ; section_start_internal false' -alias section_end='trace_off ; section_end_internal ' - -function log_on_err() { - echo -e "\\n\\n=======================================================\\n" - echo "Symfony log:" - if sudo test -f "$LOGFILE" ; then - sudo cat "$LOGFILE" - fi -} - -function show_phpinfo() { - phpversion=$1 - section_start_collap phpinfo "Show the new PHP info" - update-alternatives --set php /usr/bin/php"${phpversion}" - php -v - php -m - section_end phpinfo -} - -# Show running command -set -x diff --git a/gitlab/default-nginx b/gitlab/default-nginx deleted file mode 100644 index f27f32a0ec..0000000000 --- a/gitlab/default-nginx +++ /dev/null @@ -1,12 +0,0 @@ -server { - listen 80 default_server; - listen [::]:80 default_server; - root /var/www/html; - index index.html index.htm index.nginx-debian.html; - server_name _; - location / { - # First attempt to serve request as file, then - # as directory, then fall back to displaying a 404. - try_files $uri $uri/ =404; - } -}