Merge pull request #250 from iiasa/enh/py3.13

Confirm support for Python 3.13 and drop 3.8
iiasa · Nov 21, 2024 · 8c4f995 · 8c4f995
2 parents 012b501 + 0c40618
commit 8c4f995
Show file tree

Hide file tree

Showing 58 changed files with 280 additions and 270 deletions.
diff --git a/.github/workflows/pytest-snapshots.yaml b/.github/workflows/pytest-snapshots.yaml
@@ -13,14 +13,16 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+env:
+  version:
+    gams: "43.4.1"
+    python: "3.12"
+    upstream: main
+
 jobs:
   snapshots:
     runs-on: ubuntu-latest
     name: Test snapshots ubuntu-latest
-    env:
-      upstream-version: main
-      python-version: "3.12"
-      extra-deps: 'dask[dataframe]'
 
     steps:
     - name: Cache test data
@@ -33,17 +35,16 @@ jobs:
       uses: actions/checkout@v4
       with:
         lfs: true
-        fetch-depth: ${{ env.depth }}
 
     - uses: actions/setup-python@v5
       with:
-        python-version: ${{ env.python-version }}
+        python-version: ${{ env.version.python }}
         cache: pip
         cache-dependency-path: "**/pyproject.toml"
 
     - uses: iiasa/actions/setup-gams@main
       with:
-        version: 29.1.0
+        version: ${{ env.version.gams }}
         license: ${{ secrets.GAMS_LICENSE }}
 
     - uses: ts-graphviz/setup-graphviz@v2
@@ -52,19 +53,19 @@ jobs:
 
     - name: Install packages and dependencies
       # By default, install:
-      # - ixmp, message_ix: from GitHub branches/tags per env.upstream-version (above)
+      # - ixmp, message_ix: from GitHub branches/tags per env.version.upstream (above)
       # - other dependencies including genno: from PyPI.
       #
       # To test against unreleased code (on `main`, or other branches
       # for open PRs), temporarily uncomment, add, or edit lines below
       # as needed. DO NOT merge such changes to `main`.
       run: |
         # pip install --upgrade "genno @ git+https://github.com/khaeru/genno.git@main"
-        pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ env.upstream-version }}"
-        # pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ env.upstream-version }}"
+        pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ env.version.upstream }}"
+        # pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ env.version.upstream }}"
         pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@issue/723"
 
-        pip install .[docs,tests] ${{ env.extra-deps }}
+        pip install .[docs,tests] dask[dataframe]
 
     - name: Configure local data path
       run: |
@@ -86,4 +87,4 @@ jobs:
     - name: Upload test coverage to Codecov.io
       uses: codecov/codecov-action@v5
       with:
-        token: ${{ secrets.CODECOV_TOKEN }} # required
+        token: ${{ secrets.CODECOV_TOKEN }} # required
diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml
@@ -16,10 +16,10 @@ jobs:
   warm-lfs-cache:
     strategy:
       matrix:
-        os: [ macos-13, ubuntu-latest, windows-latest ]
+        os: [ macos-13, macos-latest, ubuntu-latest, windows-latest ]
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: nschloe/action-cached-lfs-checkout@v1
+    - uses: francisbilham11/action-cached-lfs-checkout@v3
 
   pytest:
     needs: warm-lfs-cache
@@ -28,52 +28,40 @@ jobs:
       matrix:
         os:
         - macos-13
+        - macos-latest
         - ubuntu-latest
         - windows-latest
-        upstream:
+        version:
         # In each group:
         # - Versions of ixmp and message_ix to test.
         # - Latest supported Python version for those or other dependencies.
-        # - Extra dependencies, in particular fixed/maximum versions to resolve conficts.
-        #   - dask[dataframe] >= 2024.3.0 requires dask-expr and in turn pandas >= 2.0.
-        #     https://github.com/iiasa/message-ix-models/pull/156#issuecomment-2020152360
-        #   - genno: upstream versions < 3.8.0 import genno.computations, removed in 1.25.0.
-        #     https://github.com/iiasa/message-ix-models/pull/156
-        #   - pytest: upstream versions < 3.9.0 use a hook argument removed in pytest 8.1.0.
-        #     https://github.com/iiasa/message-ix-models/pull/155
-        #
-        # Minimum version given in pyproject.toml
-        - version: v3.4.0
-          python-version: "3.11"
-          extra-deps: '"dask < 2024.3.0" "genno < 1.25" "pandas < 2.0" "pytest == 8.0.0"' #
-          dask-dataframe: false
-        - version: v3.5.0
-          python-version: "3.11"
-          extra-deps: '"dask < 2024.3.0" "genno < 1.25" "pandas < 2.0" "pytest == 8.0.0"' #
-          dask-dataframe: false
-        - version: v3.6.0
-          python-version: "3.11"
-          extra-deps: '"dask < 2024.3.0" "genno < 1.25" "pandas < 2.0" "pytest == 8.0.0"' #
-          dask-dataframe: false
-        - version: v3.7.0
-          python-version: "3.11"
-          extra-deps: '                  "genno < 1.25"                "pytest == 8.0.0"' #
-          dask-dataframe: true
-        # Latest released version
-        - version: v3.8.0
-          python-version: "3.12"
-          extra-deps: '                                                "pytest == 8.0.0"' #
-          dask-dataframe: true
-        # Development version
-        - version: main
-          python-version: "3.12"
-          extra-deps: '' #
-          dask-dataframe: true
+        # Minimum version given in pyproject.toml + earlier version of Python
+        - { upstream: v3.4.0, python: "3.11" }  # 2022-01-27
+        - { upstream: v3.5.0, python: "3.11" }  # 2022-05-06
+        - { upstream: v3.6.0, python: "3.11" }  # 2022-08-18
+        - { upstream: v3.7.0, python: "3.11" }  # 2023-05-17
+        - { upstream: v3.8.0, python: "3.12" }  # 2024-01-12
+        # Latest released version + latest released Python
+        - { upstream: v3.9.0, python: "3.13" }  # 2024-06-04
+        # Development version + latest released Python
+        - { upstream: main,   python: "3.13" }
+
+        exclude:
+        # Specific version combinations that are invalid / not to be used
+        # These versions of ixmp are not able locate the arm64 GAMS API binaries
+        - { os: macos-latest, version: {upstream: v3.4.0 }}
+        - { os: macos-latest, version: {upstream: v3.5.0 }}
+        - { os: macos-latest, version: {upstream: v3.6.0 }}
+        - { os: macos-latest, version: {upstream: v3.7.0 }}
+        - { os: macos-latest, version: {upstream: v3.8.0 }}
+        - { os: macos-latest, version: {upstream: v3.9.0 }}
+        # Redundant with macos-latest
+        - { os: macos-13, version: {upstream: main }}
 
       fail-fast: false
 
     runs-on: ${{ matrix.os }}
-    name: ${{ matrix.os }}-py${{ matrix.upstream.python-version }}-upstream-${{ matrix.upstream.version }}
+    name: ${{ matrix.os }}-py${{ matrix.version.python }}-upstream-${{ matrix.version.upstream }}
 
     steps:
     - name: Cache test data
@@ -83,45 +71,59 @@ jobs:
         key: ${{ matrix.os }}
 
     - name: Check out message-ix-models
-      uses: nschloe/action-cached-lfs-checkout@v1
+      uses: francisbilham11/action-cached-lfs-checkout@v3
 
     - uses: actions/setup-python@v5
       with:
-        python-version: ${{ matrix.upstream.python-version }}
+        python-version: ${{ matrix.version.python }}
         cache: pip
         cache-dependency-path: "**/pyproject.toml"
 
     - uses: iiasa/actions/setup-gams@main
       with:
-        version: 29.1.0
+        version: 43.4.1
         license: ${{ secrets.GAMS_LICENSE }}
 
     - uses: ts-graphviz/setup-graphviz@v2
-      # TEMPORARY Work around ts-graphviz/setup-graphviz#630
-      if: ${{ ! startswith(matrix.os, 'macos-') }}
+      # Work around ts-graphviz/setup-graphviz#630
+      if: ${{ matrix.os != 'macos-13' }}
+
+    - name: Determine extra dependencies
+      id: extra-deps
+      run : |
+        from os import environ
+        from pathlib import Path
+
+        v, result = "${{ matrix.version.upstream }}".replace("main", "vmain"), []
+        for condition, dependency in (
+            (v <= "v3.6.0", "dask < 2024.3.0"),  # dask[dataframe] >= 2024.3.0 requires dask-expr and in turn pandas >= 2.0 (#156)
+            (v <= "v3.6.0", "pandas < 2.0"),
+            (v >= "v3.7.0", "dask[dataframe] < 2024.11.0"),  # dask >= 2024.11.0 changes handling of dict (will be addressed in #225)
+            (v <= "v3.7.0", "genno < 1.25"),  # Upstream versions < 3.8.0 import genno.computations, removed in 1.25.0 (#156)
+            (v < "v3.9.0", "pytest == 8.0.0"),  # Upstream versions < 3.9.0 use a hook argument removed in pytest 8.1.0 (#155)
+        ):
+            result.extend([f'"{dependency}"'] if condition else [])
+
+        Path(environ["GITHUB_OUTPUT"]).write_text(f"value={' '.join(result)}\n")
+      shell: python
 
     - name: Install packages and dependencies
       # By default, install:
-      # - ixmp, message_ix: from GitHub branches/tags per matrix.upstream-version (above)
+      # - ixmp, message_ix: from GitHub branches/tags per matrix.version.upstream (above)
       # - other dependencies including genno: from PyPI.
       #
       # To test against unreleased code (on `main`, or other branches
       # for open PRs), temporarily uncomment, add, or edit lines below
       # as needed. DO NOT merge such changes to `main`.
       run: |
-        # pip install --upgrade "genno @ git+https://github.com/khaeru/genno.git@main"
-        pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ matrix.upstream.version }}"
-        pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ matrix.upstream.version }}"
+        pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ matrix.version.upstream }}"
+        pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ matrix.version.upstream }}"
 
-        pip install .[docs,tests] ${{ matrix.upstream.extra-deps }}
+        pip install .[docs,tests] ${{ steps.extra-deps.outputs.value }}
 
-
-    - name: Install specific dask versions as workaround
-      if: ${{ matrix.upstream.dask-dataframe }}
-      run: |
-        # TEMPORARY Work around dask v2024.11.0;
-        # see https://github.com/khaeru/genno/issues/149
-        pip install "dask[dataframe] < 2024.11.0"
+        # TEMPORARY With Python 3.13 pyam-iamc resolves to 1.3.1, which in turn
+        # limits pint < 0.17. Override.
+        pip install --upgrade pint
 
     - name: Configure local data path
       run: |
@@ -153,7 +155,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
-      with: { python-version: "3.12" }
+      with: { python-version: "3.13" }
 
     - name: Force recreation of pre-commit virtual environment for mypy
       if: github.event_name == 'schedule'  # Comment this line to run on a PR

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -10,7 +10,7 @@ repos:
     language: python
     entry: bash -c ". ${PRE_COMMIT_MYPY_VENV:-/dev/null}/bin/activate 2>/dev/null; mypy $0 $@; python -m pip list"
     additional_dependencies:
-    - mypy >= 1.11.0
+    - mypy >= 1.13.0
     - plotnine
     - pytest
     - sdmx1
@@ -20,7 +20,7 @@ repos:
     - "message-ix @ git+https://github.com/iiasa/message_ix.git@main"
     args: ["."]
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.5
+  rev: v0.7.4
   hooks:
   - id: ruff
   - id: ruff-format

diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst
@@ -4,8 +4,10 @@ What's new
 Next release
 ============
 
+- :mod:`message_ix_models` is tested and compatible with `Python 3.13 <https://www.python.org/downloads/release/python-3130/>`__ (:pull:`250`).
+- Support for Python 3.8 is dropped (:pull:`250`), as it has reached end-of-life.
 - Connect the water module to the cost module for cooling technologies (:pull:`245`).
-- Make setup of constraints for cooling technologies flexible and update solar csp tech. name (:pull:`242`). 
+- Make setup of constraints for cooling technologies flexible and update solar csp tech. name (:pull:`242`).
 - Fix the nexus/cooling function and add test for checking some input data (:pull:`236`).
 - Add :doc:`/project/circeular` project code and documentation (:pull:`232`).
 - Update water availability data and major code editing to allow a new test suite for the water module (:pull:`106`).

diff --git a/message_ix_models/model/build.py b/message_ix_models/model/build.py
@@ -1,5 +1,6 @@
 import logging
-from typing import Callable, Dict, List, Mapping, Optional, Union
+from collections.abc import Callable, Mapping
+from typing import Optional, Union
 
 import ixmp
 import pandas as pd
@@ -77,7 +78,7 @@ def apply_spec(  # noqa: C901
             pass
         maybe_check_out(scenario)
 
-    dump: Dict[str, pd.DataFrame] = {}  # Removed data
+    dump: dict[str, pd.DataFrame] = {}  # Removed data
 
     # Sort the list of sets by the number of dimensions; this places basic (non-indexed)
     # sets first. Elements for these sets must be added before elements for indexed
@@ -166,7 +167,7 @@ def apply_spec(  # noqa: C901
     )
 
 
-def ellipsize(elements: List) -> str:
+def ellipsize(elements: list) -> str:
     """Generate a short string representation of `elements`.
 
     If the list has more than 5 elements, only the first two and last two are shown,

diff --git a/message_ix_models/model/disutility.py b/message_ix_models/model/disutility.py
@@ -1,9 +1,9 @@
 import logging
 from collections import defaultdict
+from collections.abc import Mapping, MutableMapping, Sequence
 from copy import copy
 from functools import partial
 from itertools import product
-from typing import List, Mapping, MutableMapping, Sequence
 
 import message_ix
 import pandas as pd
@@ -166,10 +166,10 @@ def data_conversion(info, spec: Spec) -> MutableMapping[str, pd.DataFrame]:
     )
 
     # Use the spec to retrieve information
-    technology: List[Code] = spec.add.set["technology"]
+    technology: list[Code] = spec.add.set["technology"]
 
     # Data to return
-    data0: Mapping[str, List[pd.DataFrame]] = defaultdict(list)
+    data0: Mapping[str, list[pd.DataFrame]] = defaultdict(list)
 
     # Loop over conversion technologies
     for t in technology:

diff --git a/message_ix_models/model/emissions.py b/message_ix_models/model/emissions.py
@@ -1,6 +1,6 @@
 import logging
 import re
-from typing import Optional, Tuple
+from typing import Optional
 
 import pandas as pd
 from genno import Quantity
@@ -153,7 +153,7 @@ def add_tax_emission(
         scen.add_par(name, data)
 
 
-def split_species(unit_expr: str) -> Tuple[str, Optional[str]]:
+def split_species(unit_expr: str) -> tuple[str, Optional[str]]:
     """Split `unit_expr` to an expression without a unit mention, and maybe species."""
     if match := re.fullmatch("(.*)(CO2|C)(.*)", unit_expr):
         return f"{match.group(1)}{match.group(3)}", match.group(2)

diff --git a/message_ix_models/model/macro.py b/message_ix_models/model/macro.py
@@ -5,10 +5,11 @@
 """
 
 import logging
+from collections.abc import Mapping
 from functools import lru_cache
 from itertools import product
 from pathlib import Path
-from typing import TYPE_CHECKING, List, Literal, Mapping, Optional, Union
+from typing import TYPE_CHECKING, Literal, Optional, Union
 
 import pandas as pd
 
@@ -29,7 +30,7 @@
 def generate(
     parameter: Literal["aeei", "config", "depr", "drate", "lotol"],
     context: "Context",
-    commodities: Union[List[str], List["Code"]] = COMMODITY,
+    commodities: Union[list[str], list["Code"]] = COMMODITY,
     value: Optional[float] = None,
 ) -> pd.DataFrame:
     """Generate uniform data for one :mod:`message_ix.macro` `parameter`.

diff --git a/message_ix_models/model/material/build.py b/message_ix_models/model/material/build.py
@@ -1,5 +1,6 @@
 import logging
-from typing import Any, Dict, Mapping
+from collections.abc import Mapping
+from typing import Any
 
 import message_ix
 import pandas as pd
@@ -206,7 +207,7 @@ def get_spec() -> Mapping[str, ScenarioInfo]:
 
 
 def make_spec(regions: str, materials: str or None = SPEC_LIST) -> Spec:
-    sets: Dict[str, Any] = dict()
+    sets: dict[str, Any] = dict()
     materials = ["common"] if not materials else materials
     # Overrides specific to regional versions
     tmp = dict()