From 872174e28d92556a2384525e674f0f809d2b2cc9 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 9 Sep 2024 09:58:19 -0400 Subject: [PATCH 1/4] build: Switch to ubuntu-latest for builds This code does not have any dependencies that are specific to any specific version of ubuntu. So instead of testing on a specific version and then needing to do work to keep the versions up-to-date, we switch to the ubuntu-latest target which should be sufficient for testing purposes. This work is being done as a part of https://github.com/openedx/platform-roadmap/issues/377 closes https://github.com/openedx/edx-platform/issues/35314 --- .github/workflows/ci-static-analysis.yml | 2 +- .github/workflows/compile-python-requirements.yml | 2 +- .github/workflows/js-tests.yml | 2 +- .github/workflows/lint-imports.yml | 2 +- .github/workflows/migrations-check.yml | 2 +- .github/workflows/publish-ci-docker-image.yml | 2 +- .github/workflows/pylint-checks.yml | 2 +- .github/workflows/quality-checks.yml | 2 +- .github/workflows/semgrep.yml | 2 +- .github/workflows/static-assets-check.yml | 2 +- .github/workflows/unit-tests.yml | 12 ++++++------ .github/workflows/upgrade-one-python-dependency.yml | 2 +- .github/workflows/verify-dunder-init.yml | 2 +- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml index 7e768a456463..a3b0527aad72 100644 --- a/.github/workflows/ci-static-analysis.yml +++ b/.github/workflows/ci-static-analysis.yml @@ -10,7 +10,7 @@ jobs: matrix: python-version: - "3.11" - os: ["ubuntu-20.04"] + os: ["ubuntu-latest"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/compile-python-requirements.yml b/.github/workflows/compile-python-requirements.yml index 0ff99b9c685a..21cb80083f1d 100644 --- a/.github/workflows/compile-python-requirements.yml +++ b/.github/workflows/compile-python-requirements.yml @@ -15,7 +15,7 @@ defaults: jobs: recompile-python-dependencies: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Check out target branch diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 4d025e540163..c9d2d7ab1191 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -12,7 +12,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04] + os: [ubuntu-latest] node-version: [18, 20] python-version: - "3.11" diff --git a/.github/workflows/lint-imports.yml b/.github/workflows/lint-imports.yml index 8ead8396bf39..e3c59ec09304 100644 --- a/.github/workflows/lint-imports.yml +++ b/.github/workflows/lint-imports.yml @@ -9,7 +9,7 @@ on: jobs: lint-imports: name: Lint Python Imports - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Check out branch diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index 183b90effa29..ec3ff21e60bc 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04] + os: [ubuntu-latest] python-version: - "3.11" # 'pinned' is used to install the latest patch version of Django diff --git a/.github/workflows/publish-ci-docker-image.yml b/.github/workflows/publish-ci-docker-image.yml index 0a9f50f6daf9..6a0f3768b7e6 100644 --- a/.github/workflows/publish-ci-docker-image.yml +++ b/.github/workflows/publish-ci-docker-image.yml @@ -7,7 +7,7 @@ on: jobs: push: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml index eeb53c24ed98..58560bf3073f 100644 --- a/.github/workflows/pylint-checks.yml +++ b/.github/workflows/pylint-checks.yml @@ -8,7 +8,7 @@ on: jobs: run-pylint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index cf8ffd5d2910..5445d70e3b4b 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04] + os: [ubuntu-latest] python-version: - "3.11" node-version: [20] diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 7f2b4925af8e..d880d7351766 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -17,7 +17,7 @@ jobs: runs-on: "${{ matrix.os }}" strategy: matrix: - os: ["ubuntu-20.04"] + os: ["ubuntu-latest"] python-version: - "3.11" diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 7bbfd3369b6b..177416770400 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -12,7 +12,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04] + os: [ubuntu-latest] python-version: - "3.11" node-version: [18, 20] diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 3e442b75d4e7..a697700898de 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -15,7 +15,7 @@ concurrency: jobs: run-tests: name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }}) - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: python-version: @@ -142,7 +142,7 @@ jobs: overwrite: true collect-and-verify: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Setup Python @@ -207,7 +207,7 @@ jobs: # https://github.com/orgs/community/discussions/33579 success: name: Unit tests successful - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: always() needs: [run-tests] steps: @@ -218,7 +218,7 @@ jobs: jobs: ${{ toJSON(needs) }} compile-warnings-report: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest needs: [run-tests] steps: - uses: actions/checkout@v4 @@ -246,7 +246,7 @@ jobs: overwrite: true merge-artifacts: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest needs: [compile-warnings-report] steps: - name: Merge Pytest Warnings JSON Artifacts @@ -266,7 +266,7 @@ jobs: # Combine and upload coverage reports. coverage: if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest needs: [run-tests] strategy: matrix: diff --git a/.github/workflows/upgrade-one-python-dependency.yml b/.github/workflows/upgrade-one-python-dependency.yml index 6ca5dfcb355e..84a00266e99f 100644 --- a/.github/workflows/upgrade-one-python-dependency.yml +++ b/.github/workflows/upgrade-one-python-dependency.yml @@ -28,7 +28,7 @@ defaults: jobs: upgrade-one-python-dependency: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Check out target branch diff --git a/.github/workflows/verify-dunder-init.yml b/.github/workflows/verify-dunder-init.yml index 611fc0afc6e3..9d920238ebd4 100644 --- a/.github/workflows/verify-dunder-init.yml +++ b/.github/workflows/verify-dunder-init.yml @@ -8,7 +8,7 @@ on: jobs: verify_dunder_init: name: Verify __init__.py Files - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Check out branch From d8eef6e347b36c61221c455fdf1931290939e298 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 11 Sep 2024 09:26:57 -0400 Subject: [PATCH 2/4] build: Run mongosh commands within the container. This is no longer installed by default on ubuntu and so we have to either manually install it or just run the relevant commands in the container here it's already available. This lets us do some of the test setup in a more robust way. --- .github/workflows/migrations-check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index ec3ff21e60bc..f253d48e4f41 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -52,7 +52,7 @@ jobs: steps: - name: Setup mongodb user run: | - mongosh edxapp --eval ' + docker exec ${{ job.services.mongo.id }} mongosh edxapp --eval ' db.createUser( { user: "edxapp", @@ -67,7 +67,7 @@ jobs: - name: Verify mongo and mysql db credentials run: | mysql -h 127.0.0.1 -uedxapp001 -ppassword -e "select 1;" edxapp - mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp + docker exec ${{ job.services.mongo.id }} mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp - name: Checkout repo uses: actions/checkout@v4 From e6e5bedf63fc393b95fc74501298fba72099d5bb Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 11 Sep 2024 10:05:23 -0400 Subject: [PATCH 3/4] fix: Don't start the mongo service. We stopped using mongo on the runner directly a while ago so this is just an errant start that should have been removed. --- .github/workflows/static-assets-check.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 177416770400..0a417f9b1c79 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -72,9 +72,6 @@ jobs: run: | pip install -r requirements/edx/assets.txt - - name: Initiate Mongo DB Service - run: sudo systemctl start mongod - - name: Add node_modules bin to $Path run: echo $GITHUB_WORKSPACE/node_modules/.bin >> $GITHUB_PATH From 1804fbb13107bd9681c5c51c5c20e90edf9e09d2 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Tue, 17 Sep 2024 15:10:12 -0400 Subject: [PATCH 4/4] build!: enable md4 for testing. Operators Note: In newer versions of ubuntu the MD4 hashing algorithm is disabled by default. To enable it the openssl config needs to be updated in a manner similar to what's being done here. Alternatively, you can set the `FEATURES['ENABLE_BLAKE2B_HASHING']` setting to `True` which will switch to a newer hashing algorithm where MD4 was previously used. Because this hashing is being used as a part of the edx-platform caching mechanism, this will effectively clear the cache for the items that use this hash. The will impact any items where the cache key might have been too big to store in memcache so it's hard to predict exactly which items will be impacted. BREAKING CHANGE: See the operator note above for more details as this may break for users transitioning from Ubuntu 20.04 to newer versions. --- .github/workflows/unit-tests.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index a697700898de..5fef1c8352ce 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -66,7 +66,29 @@ jobs: - name: install system requirements run: | - sudo apt-get update && sudo apt-get install libmysqlclient-dev libxmlsec1-dev lynx + sudo apt-get update && sudo apt-get install libmysqlclient-dev libxmlsec1-dev lynx openssl + + # This is needed until the ENABLE_BLAKE2B_HASHING can be removed and we + # can stop using MD4 by default. + - name: enable md4 hashing in libssl + run: | + cat <