From 8eacdc5e6b5f1a4fb30339be960eaf451dfcb034 Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Wed, 16 Oct 2024 10:49:03 -0700 Subject: [PATCH 1/2] update_python_test_versions fixes update_python_test_versions.py needed update to handle python 3.12, and to match recent yaml format changes. --- .github/workflows/build.yml | 36 +++++++++------ .github/workflows/update_python_test.yml | 2 +- ci/update_python_test_versions.py | 58 +++++++++++++++--------- 3 files changed, 59 insertions(+), 37 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ffa9aa1a..9dd86160 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -188,6 +188,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: fail-fast: false + # automatically generated by ci/update_python_test_versions.py matrix: python-version: [ @@ -200,20 +201,6 @@ jobs: 3.9.0, 3.9.20, 3.10.0, - 3.10.1, - 3.10.2, - 3.10.3, - 3.10.4, - 3.10.5, - 3.10.6, - 3.10.7, - 3.10.8, - 3.10.9, - 3.10.10, - 3.10.11, - 3.10.12, - 3.10.13, - 3.10.14, 3.10.15, 3.11.0, 3.11.1, @@ -227,6 +214,13 @@ jobs: 3.11.9, 3.11.10, 3.12.0, + 3.12.1, + 3.12.2, + 3.12.3, + 3.12.4, + 3.12.5, + 3.12.6, + 3.12.7, ] # TODO: also test windows os: [ubuntu-20.04, macos-13] @@ -256,6 +250,20 @@ jobs: python-version: 3.11.10 - os: macos-13 python-version: 3.12.0 + - os: macos-13 + python-version: 3.12.1 + - os: macos-13 + python-version: 3.12.2 + - os: macos-13 + python-version: 3.12.3 + - os: macos-13 + python-version: 3.12.4 + - os: macos-13 + python-version: 3.12.5 + - os: macos-13 + python-version: 3.12.6 + - os: macos-13 + python-version: 3.12.7 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/update_python_test.yml b/.github/workflows/update_python_test.yml index 1c904ccf..1a9b3d3c 100644 --- a/.github/workflows/update_python_test.yml +++ b/.github/workflows/update_python_test.yml @@ -14,7 +14,7 @@ jobs: with: python-version: 3.9 - name: Install - run: pip install --upgrade requests + run: pip install --upgrade requests yaml - name: Scan for new python versions run: python ci/update_python_test_versions.py - name: Format results diff --git a/ci/update_python_test_versions.py b/ci/update_python_test_versions.py index 6a53286d..d9a8df69 100644 --- a/ci/update_python_test_versions.py +++ b/ci/update_python_test_versions.py @@ -1,6 +1,7 @@ from collections import defaultdict import requests import pathlib +import yaml import re @@ -37,10 +38,10 @@ def get_github_python_versions(): # for older versions of python, don't test all patches # (just test first and last) to keep the test matrix down - if (major == 2 or minor < 10): + if (major == 2 or minor <= 10): patches = [patches[0], patches[-1]] - if (major == 3 and minor >= 12): + if (major == 3 and minor >= 13): continue versions.extend(f"{major}.{minor}.{patch}" for patch in patches) @@ -48,36 +49,49 @@ def get_github_python_versions(): return versions -if __name__ == "__main__": +def update_python_test_versions(): versions = sorted( get_github_python_versions(), key=parse_version) - build_yml = ( + build_yml_path = ( pathlib.Path(__file__).parent.parent / ".github" / "workflows" / "build.yml" ) + build_yml = yaml.safe_load(open(".github/workflows/build.yml")) + test_matrix = build_yml["jobs"]["test-wheels"]["strategy"]["matrix"] + existing_python_versions = test_matrix["python-version"] + if versions == existing_python_versions: + return + + print("Adding new versions") + print("Old:", existing_python_versions) + print("New:", versions) - transformed = [] - for line in open(build_yml): - if line.startswith(" python-version: ["): - newversions = f" python-version: [{', '.join(v for v in versions)}]\n" - if newversions != line: - print("Adding new versions") - print("Old:", line) - print("New:", newversions) - line = newversions - transformed.append(line) + # we can't use the yaml package to update the GHA script, since + # the data in build_yml is treated as an unordered dictionary. + # instead modify the file in place + lines = list(open(build_yml_path)) + first_line = lines.index(" # automatically generated by ci/update_python_test_versions.py\n") - # also automatically exclude v3.11.* from running on OSX, + first_version_line = lines.index(" [\n", first_line) + last_version_line = lines.index(" ]\n", first_version_line) + new_versions = [f" {v},\n" for v in versions] + lines = lines[:first_version_line+1] + new_versions + lines[last_version_line:] + + # also automatically exclude >= v3.11.* from running on OSX, # since it currently fails in GHA on SIP errors exclusions = [] for v in versions: - if v.startswith("3.11"): + if v.startswith("3.11") or v.startswith("3.12"): exclusions.append(" - os: macos-13\n") exclusions.append(f" python-version: {v}\n") - test_wheels = transformed.index(" test-wheels:\n") - first_line = transformed.index(" exclude:\n", test_wheels) - last_line = transformed.index("\n", first_line) - transformed = transformed[:first_line+1] + exclusions + transformed[last_line:] + first_exclude_line = lines.index(" exclude:\n", first_line) + last_exclude_line = lines.index("\n", first_exclude_line) + lines = lines[:first_exclude_line + 1] + exclusions + lines[last_exclude_line:] + + with open(build_yml_path, "w") as o: + o.write("".join(lines)) + + +if __name__ == "__main__": + update_python_test_versions() - with open(build_yml, "w") as o: - o.write("".join(transformed)) From c8580cae9c83c4c8bfeca7e09ed538e73fdac192 Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Wed, 16 Oct 2024 10:54:41 -0700 Subject: [PATCH 2/2] style --- ci/update_python_test_versions.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ci/update_python_test_versions.py b/ci/update_python_test_versions.py index d9a8df69..adba5fc9 100644 --- a/ci/update_python_test_versions.py +++ b/ci/update_python_test_versions.py @@ -38,10 +38,10 @@ def get_github_python_versions(): # for older versions of python, don't test all patches # (just test first and last) to keep the test matrix down - if (major == 2 or minor <= 10): + if major == 2 or minor <= 10: patches = [patches[0], patches[-1]] - if (major == 3 and minor >= 13): + if major == 3 and minor >= 13: continue versions.extend(f"{major}.{minor}.{patch}" for patch in patches) @@ -50,8 +50,7 @@ def get_github_python_versions(): def update_python_test_versions(): - versions = sorted( - get_github_python_versions(), key=parse_version) + versions = sorted(get_github_python_versions(), key=parse_version) build_yml_path = ( pathlib.Path(__file__).parent.parent / ".github" / "workflows" / "build.yml" ) @@ -70,12 +69,14 @@ def update_python_test_versions(): # the data in build_yml is treated as an unordered dictionary. # instead modify the file in place lines = list(open(build_yml_path)) - first_line = lines.index(" # automatically generated by ci/update_python_test_versions.py\n") + first_line = lines.index( + " # automatically generated by ci/update_python_test_versions.py\n" + ) first_version_line = lines.index(" [\n", first_line) - last_version_line = lines.index(" ]\n", first_version_line) + last_version_line = lines.index(" ]\n", first_version_line) new_versions = [f" {v},\n" for v in versions] - lines = lines[:first_version_line+1] + new_versions + lines[last_version_line:] + lines = lines[: first_version_line + 1] + new_versions + lines[last_version_line:] # also automatically exclude >= v3.11.* from running on OSX, # since it currently fails in GHA on SIP errors @@ -86,7 +87,7 @@ def update_python_test_versions(): exclusions.append(f" python-version: {v}\n") first_exclude_line = lines.index(" exclude:\n", first_line) last_exclude_line = lines.index("\n", first_exclude_line) - lines = lines[:first_exclude_line + 1] + exclusions + lines[last_exclude_line:] + lines = lines[: first_exclude_line + 1] + exclusions + lines[last_exclude_line:] with open(build_yml_path, "w") as o: o.write("".join(lines)) @@ -94,4 +95,3 @@ def update_python_test_versions(): if __name__ == "__main__": update_python_test_versions() -