Merge branch 'main' into c19-backend-config

GridTools · Feb 1, 2024 · cfbef72 · cfbef72
2 parents 7c9e306 + d6dfd6f
commit cfbef72
Show file tree

Hide file tree

Showing 87 changed files with 1,420 additions and 1,003 deletions.
diff --git a/.github/workflows/test-notebooks.yml b/.github/workflows/test-notebooks.yml
@@ -0,0 +1,42 @@
+name: "Test Jupyter Notebooks"
+
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+    branches:
+    - main
+
+jobs:
+  test-notebooks:
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11"]
+        os: ["ubuntu-latest"]
+      fail-fast: false
+
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          **/pyproject.toml
+          **/constraints.txt
+          **/requirements-dev.txt
+    - name: Install python dependencies
+      run: |
+        python -m pip install -c ./constraints.txt pip setuptools wheel
+        python -m pip install -r ./requirements-dev.txt
+    - name: Run tox tests
+      env:
+        NUM_PROCESSES: auto
+      shell: bash
+      run: |
+        pyversion=${{ matrix.python-version }}
+        pyversion_no_dot=${pyversion//./}
+        tox run -e notebooks-py${pyversion_no_dot}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -148,7 +148,7 @@ repos:
   ##]]]
   #========= FROM constraints.txt: v1.8.0 =========
   ##[[[end]]]
-  rev: v1.7.1  # MUST match version ^^^^ in constraints.txt (if the mirror is up-to-date)
+  rev: v1.8.0  # MUST match version ^^^^ in constraints.txt (if the mirror is up-to-date)
   hooks:
   - id: mypy
     additional_dependencies:  # versions from constraints.txt

diff --git a/constraints.txt b/constraints.txt
@@ -6,10 +6,12 @@
 #
 aenum==3.1.15             # via dace
 alabaster==0.7.13         # via sphinx
-asttokens==2.4.1          # via devtools
+appnope==0.1.3            # via ipykernel, ipython
+asttokens==2.4.1          # via devtools, stack-data
 astunparse==1.6.3 ; python_version < "3.9"  # via dace, gt4py (pyproject.toml)
 attrs==23.2.0             # via flake8-bugbear, flake8-eradicate, gt4py (pyproject.toml), hypothesis, jsonschema, referencing
 babel==2.14.0             # via sphinx
+backcall==0.2.0           # via ipython
 black==24.1.1             # via gt4py (pyproject.toml)
 blinker==1.7.0            # via flask
 boltons==23.1.1           # via gt4py (pyproject.toml)
@@ -26,11 +28,16 @@ click==8.1.7              # via black, flask, gt4py (pyproject.toml), pip-tools
 cmake==3.28.1             # via gt4py (pyproject.toml)
 cogapp==3.3.0             # via -r requirements-dev.in
 colorama==0.4.6           # via tox
+comm==0.2.1               # via ipykernel
+contourpy==1.1.1          # via matplotlib
 coverage==7.4.1           # via -r requirements-dev.in, pytest-cov
 cryptography==42.0.2      # via types-paramiko, types-pyopenssl, types-redis
+cycler==0.12.1            # via matplotlib
 cytoolz==0.12.3           # via gt4py (pyproject.toml)
 dace==0.15.1              # via gt4py (pyproject.toml)
 darglint==1.8.1           # via -r requirements-dev.in
+debugpy==1.8.0            # via ipykernel
+decorator==5.1.1          # via ipython
 deepdiff==6.7.1           # via gt4py (pyproject.toml)
 devtools==0.12.2          # via gt4py (pyproject.toml)
 dill==0.3.8               # via dace
@@ -39,7 +46,7 @@ docutils==0.20.1          # via restructuredtext-lint, sphinx, sphinx-rtd-theme
 eradicate==2.3.0          # via flake8-eradicate
 exceptiongroup==1.2.0     # via hypothesis, pytest
 execnet==2.0.2            # via pytest-cache, pytest-xdist
-executing==2.0.1          # via devtools
+executing==2.0.1          # via devtools, stack-data
 factory-boy==3.3.0        # via -r requirements-dev.in, gt4py (pyproject.toml), pytest-factoryboy
 faker==22.6.0             # via factory-boy
 fastjsonschema==2.19.1    # via nbformat
@@ -54,67 +61,87 @@ flake8-mutable==1.2.0     # via -r requirements-dev.in
 flake8-pyproject==1.2.3   # via -r requirements-dev.in
 flake8-rst-docstrings==0.3.0  # via -r requirements-dev.in
 flask==3.0.1              # via dace
+fonttools==4.47.2         # via matplotlib
 fparser==0.1.4            # via dace
 frozendict==2.4.0         # via gt4py (pyproject.toml)
 gridtools-cpp==2.3.2      # via gt4py (pyproject.toml)
 hypothesis==6.97.4        # via -r requirements-dev.in, gt4py (pyproject.toml)
 identify==2.5.33          # via pre-commit
 idna==3.6                 # via requests
 imagesize==1.4.1          # via sphinx
-importlib-metadata==7.0.1  # via build, flask, sphinx
-importlib-resources==6.1.1 ; python_version < "3.9"  # via gt4py (pyproject.toml), jsonschema, jsonschema-specifications
+importlib-metadata==7.0.1  # via build, flask, jupyter-client, sphinx
+importlib-resources==6.1.1 ; python_version < "3.9"  # via gt4py (pyproject.toml), jsonschema, jsonschema-specifications, matplotlib
 inflection==0.5.1         # via pytest-factoryboy
 iniconfig==2.0.0          # via pytest
+ipykernel==6.29.0         # via nbmake
+ipython==8.12.3           # via ipykernel
 isort==5.13.2             # via -r requirements-dev.in
 itsdangerous==2.1.2       # via flask
+jedi==0.19.1              # via ipython
 jinja2==3.1.3             # via flask, gt4py (pyproject.toml), sphinx
 jsonschema==4.21.1        # via nbformat
 jsonschema-specifications==2023.12.1  # via jsonschema
-jupyter-core==5.7.1       # via nbformat
+jupyter-client==8.6.0     # via ipykernel, nbclient
+jupyter-core==5.7.1       # via ipykernel, jupyter-client, nbformat
 jupytext==1.16.1          # via -r requirements-dev.in
+kiwisolver==1.4.5         # via matplotlib
 lark==1.1.9               # via gt4py (pyproject.toml)
 mako==1.3.2               # via gt4py (pyproject.toml)
 markdown-it-py==3.0.0     # via jupytext, mdit-py-plugins
 markupsafe==2.1.4         # via jinja2, mako, werkzeug
+matplotlib==3.7.4         # via -r requirements-dev.in
+matplotlib-inline==0.1.6  # via ipykernel, ipython
 mccabe==0.7.0             # via flake8
 mdit-py-plugins==0.4.0    # via jupytext
 mdurl==0.1.2              # via markdown-it-py
 mpmath==1.3.0             # via sympy
 mypy==1.8.0               # via -r requirements-dev.in
 mypy-extensions==1.0.0    # via black, mypy
 nanobind==1.8.0           # via gt4py (pyproject.toml)
-nbformat==5.9.2           # via jupytext
+nbclient==0.6.8           # via nbmake
+nbformat==5.9.2           # via jupytext, nbclient, nbmake
+nbmake==1.5.0             # via -r requirements-dev.in
+nest-asyncio==1.6.0       # via ipykernel, nbclient
 networkx==3.1             # via dace
 ninja==1.11.1.1           # via gt4py (pyproject.toml)
 nodeenv==1.8.0            # via pre-commit
-numpy==1.24.4             # via dace, gt4py (pyproject.toml), types-jack-client
+numpy==1.24.4             # via contourpy, dace, gt4py (pyproject.toml), matplotlib, types-jack-client
 ordered-set==4.1.0        # via deepdiff
-packaging==23.2           # via black, build, gt4py (pyproject.toml), jupytext, pyproject-api, pytest, setuptools-scm, sphinx, tox
+packaging==23.2           # via black, build, gt4py (pyproject.toml), ipykernel, jupytext, matplotlib, pyproject-api, pytest, setuptools-scm, sphinx, tox
+parso==0.8.3              # via jedi
 pathspec==0.12.1          # via black
+pexpect==4.9.0            # via ipython
+pickleshare==0.7.5        # via ipython
+pillow==10.2.0            # via matplotlib
 pip-tools==7.3.0          # via -r requirements-dev.in
 pipdeptree==2.13.2        # via -r requirements-dev.in
 pkgutil-resolve-name==1.3.10  # via jsonschema
 platformdirs==4.2.0       # via black, jupyter-core, tox, virtualenv
 pluggy==1.4.0             # via pytest, tox
 ply==3.11                 # via dace
 pre-commit==3.5.0         # via -r requirements-dev.in
-psutil==5.9.8             # via -r requirements-dev.in, pytest-xdist
+prompt-toolkit==3.0.43    # via ipython
+psutil==5.9.8             # via -r requirements-dev.in, ipykernel, pytest-xdist
+ptyprocess==0.7.0         # via pexpect
+pure-eval==0.2.2          # via stack-data
 pybind11==2.11.1          # via gt4py (pyproject.toml)
 pycodestyle==2.11.1       # via flake8, flake8-debugger
 pycparser==2.21           # via cffi
 pydocstyle==6.3.0         # via flake8-docstrings
 pyflakes==3.2.0           # via flake8
-pygments==2.17.2          # via -r requirements-dev.in, devtools, flake8-rst-docstrings, sphinx
+pygments==2.17.2          # via -r requirements-dev.in, devtools, flake8-rst-docstrings, ipython, nbmake, sphinx
+pyparsing==3.1.1          # via matplotlib
 pyproject-api==1.6.1      # via tox
 pyproject-hooks==1.0.0    # via build
-pytest==8.0.0             # via -r requirements-dev.in, gt4py (pyproject.toml), pytest-cache, pytest-cov, pytest-factoryboy, pytest-xdist
+pytest==8.0.0             # via -r requirements-dev.in, gt4py (pyproject.toml), nbmake, pytest-cache, pytest-cov, pytest-factoryboy, pytest-xdist
 pytest-cache==1.0         # via -r requirements-dev.in
 pytest-cov==4.1.0         # via -r requirements-dev.in
 pytest-factoryboy==2.6.0  # via -r requirements-dev.in
 pytest-xdist==3.5.0       # via -r requirements-dev.in
-python-dateutil==2.8.2    # via faker
+python-dateutil==2.8.2    # via faker, jupyter-client, matplotlib
 pytz==2023.4              # via babel
 pyyaml==6.0.1             # via dace, jupytext, pre-commit
+pyzmq==25.1.2             # via ipykernel, jupyter-client
 referencing==0.33.0       # via jsonschema, jsonschema-specifications
 requests==2.31.0          # via dace, sphinx
 restructuredtext-lint==1.4.0  # via flake8-rst-docstrings
@@ -133,13 +160,15 @@ sphinxcontrib-jquery==4.1  # via sphinx-rtd-theme
 sphinxcontrib-jsmath==1.0.1  # via sphinx
 sphinxcontrib-qthelp==1.0.3  # via sphinx
 sphinxcontrib-serializinghtml==1.1.5  # via sphinx
+stack-data==0.6.3         # via ipython
 sympy==1.9                # via dace, gt4py (pyproject.toml)
 tabulate==0.9.0           # via gt4py (pyproject.toml)
 toml==0.10.2              # via jupytext
 tomli==2.0.1              # via -r requirements-dev.in, black, build, coverage, flake8-pyproject, mypy, pip-tools, pyproject-api, pyproject-hooks, pytest, setuptools-scm, tox
 toolz==0.12.1             # via cytoolz
+tornado==6.4              # via ipykernel, jupyter-client
 tox==4.12.1               # via -r requirements-dev.in
-traitlets==5.14.1         # via jupyter-core, nbformat
+traitlets==5.14.1         # via comm, ipykernel, ipython, jupyter-client, jupyter-core, matplotlib-inline, nbclient, nbformat
 types-aiofiles==23.2.0.20240106  # via types-all
 types-all==1.0.0          # via -r requirements-dev.in
 types-annoy==1.17.8.4     # via types-all
@@ -229,9 +258,10 @@ types-ujson==5.9.0.0      # via types-all
 types-waitress==2.1.4.20240106  # via types-all
 types-werkzeug==1.0.9     # via types-all, types-flask
 types-xxhash==3.0.5.2     # via types-all
-typing-extensions==4.5.0  # via black, faker, gt4py (pyproject.toml), mypy, pytest-factoryboy, setuptools-scm
+typing-extensions==4.5.0  # via black, faker, gt4py (pyproject.toml), ipython, mypy, pytest-factoryboy, setuptools-scm
 urllib3==2.2.0            # via requests, types-requests
 virtualenv==20.25.0       # via pre-commit, tox
+wcwidth==0.2.13           # via prompt-toolkit
 websockets==12.0          # via dace
 werkzeug==3.0.1           # via flask
 wheel==0.42.0             # via astunparse, pip-tools

diff --git a/docs/development/ADRs/0015-Test_Exclusion_Matrices.md b/docs/development/ADRs/0015-Test_Exclusion_Matrices.md
@@ -1,13 +1,13 @@
 ---
-tags: []
+tags: [testing]
 ---
 
 # Test-Exclusion Matrices
 
 - **Status**: valid
 - **Authors**: Edoardo Paone (@edopao), Enrique G. Paredes (@egparedes)
 - **Created**: 2023-09-21
-- **Updated**: 2023-09-21
+- **Updated**: 2024-01-25
 
 In the context of Field View testing, lacking support for specific ITIR features while a certain backend
 is being developed, we decided to use `pytest` fixtures to exclude unsupported tests.
@@ -22,7 +22,7 @@ the supported backends, while keeping the test code clean.
 ## Decision
 
 It was decided to apply fixtures and markers from `pytest` module. The fixture is the same used to execute the test
-on different backends (`fieldview_backend` and `program_processor`), but it is extended with a check on the available feature markers.
+on different backends (`exec_alloc_descriptor` and `program_processor`), but it is extended with a check on the available feature markers.
 If a test is annotated with a feature marker, the fixture will check if this feature is supported on the selected backend.
 If no marker is specified, the test is supposed to run on all backends.
 
@@ -33,7 +33,7 @@ In the example below, `test_offset_field` requires the backend to support dynami
 def test_offset_field(cartesian_case):
 ```
 
-In order to selectively enable the backends, the dictionary `next_tests.exclusion_matrices.BACKEND_SKIP_TEST_MATRIX`
+In order to selectively enable the backends, the dictionary `next_tests.definitions.BACKEND_SKIP_TEST_MATRIX`
 lists for each backend the features that are not supported.
 The fixture will check if the annotated feature is present in the exclusion-matrix for the selected backend.
 If so, the exclusion matrix will also specify the action `pytest` should take (e.g. `SKIP` or `XFAIL`).

diff --git a/examples/lap_cartesian_vs_next.ipynb b/examples/lap_cartesian_vs_next.ipynb
@@ -81,7 +81,7 @@
    "source": [
     "import gt4py.next as gtx\n",
     "\n",
-    "allocator = gtx.itir_embedded # should match the executor\n",
+    "allocator = gtx.itir_python # should match the executor\n",
     "# allocator = gtx.gtfn_cpu\n",
     "# allocator = gtx.gtfn_gpu\n",
     "\n",
@@ -137,7 +137,7 @@
    "source": [
     "from gt4py.next import Field\n",
     "\n",
-    "next_backend = gtx.itir_embedded\n",
+    "next_backend = gtx.itir_python\n",
     "# next_backend = gtx.gtfn_cpu\n",
     "# next_backend = gtx.gtfn_gpu\n",
     "\n",

diff --git a/min-extra-requirements-test.txt b/min-extra-requirements-test.txt
@@ -41,7 +41,7 @@ flake8-pyproject==1.2.2
 flake8-rst-docstrings==0.0.14
 flake8==5.0.4
 frozendict==2.3
-gridtools-cpp==2.3.1
+gridtools-cpp==2.3.2
 hypothesis==6.0.0
 importlib-resources==5.0;python_version<'3.9'
 isort==5.10
@@ -50,8 +50,10 @@ jinja2==3.0.0
 jupytext==1.14
 lark==1.1.2
 mako==1.1
+matplotlib==3.3
 mypy==1.0
 nanobind==1.4.0
+nbmake==1.4.6
 ninja==1.10
 numpy==1.21.2
 packaging==20.0
@@ -60,7 +62,7 @@ pipdeptree==2.3
 pre-commit==2.17
 psutil==5.0
 pybind11==2.5
-pygments==2.7
+pygments==2.7.3
 pytest-cache==1.0
 pytest-cov==2.8
 pytest-factoryboy==2.0.3

diff --git a/min-requirements-test.txt b/min-requirements-test.txt
@@ -39,16 +39,18 @@ flake8-pyproject==1.2.2
 flake8-rst-docstrings==0.0.14
 flake8==5.0.4
 frozendict==2.3
-gridtools-cpp==2.3.1
+gridtools-cpp==2.3.2
 hypothesis==6.0.0
 importlib-resources==5.0;python_version<'3.9'
 isort==5.10
 jinja2==3.0.0
 jupytext==1.14
 lark==1.1.2
 mako==1.1
+matplotlib==3.3
 mypy==1.0
 nanobind==1.4.0
+nbmake==1.4.6
 ninja==1.10
 numpy==1.21.2
 packaging==20.0
@@ -57,7 +59,7 @@ pipdeptree==2.3
 pre-commit==2.17
 psutil==5.0
 pybind11==2.5
-pygments==2.7
+pygments==2.7.3
 pytest-cache==1.0
 pytest-cov==2.8
 pytest-factoryboy==2.0.3

diff --git a/pyproject.toml b/pyproject.toml
@@ -35,7 +35,7 @@ dependencies = [
   'devtools>=0.6',
   'factory-boy>=3.3.0',
   'frozendict>=2.3',
-  'gridtools-cpp>=2.3.1,==2.*',
+  'gridtools-cpp>=2.3.2,==2.*',
   "importlib-resources>=5.0;python_version<'3.9'",
   'jinja2>=3.0.0',
   'lark>=1.1.2',
@@ -163,7 +163,9 @@ ignore = [
   'DAR',  # Disable dargling errors by default
   'E203',  # Whitespace before ':' (black formatter breaks this sometimes)
   'E501',  # Line too long (using Bugbear's B950 warning)
-  'W503'  # Line break occurred before a binary operator
+  'W503',  # Line break occurred before a binary operator
+  'E701',  # Multiple statements on one line, see https://github.com/psf/black/issues/3887
+  'E704'  # Multiple statements on one line, see https://github.com/psf/black/issues/3887
 ]
 max-complexity = 15
 max-line-length = 100  # It should be the same as in `tool.black.line-length` above
@@ -341,7 +343,11 @@ markers = [
   'uses_origin: tests that require backend support for domain origin',
   'uses_reduction_over_lift_expressions: tests that require backend support for reduction over lift expressions',
   'uses_reduction_with_only_sparse_fields: tests that require backend support for with sparse fields',
+  'uses_scan: tests that uses scan',
   'uses_scan_in_field_operator: tests that require backend support for scan in field operator',
+  'uses_scan_without_field_args: tests that require calls to scan that do not have any fields as arguments',
+  'uses_scan_nested: tests that use nested scans',
+  'uses_scan_requiring_projector: tests need a projector implementation in gtfn',
   'uses_sparse_fields: tests that require backend support for sparse fields',
   'uses_sparse_fields_as_output: tests that require backend support for writing sparse fields',
   'uses_strided_neighbor_offset: tests that require backend support for strided neighbor offset',
@@ -350,7 +356,8 @@ markers = [
   'uses_zero_dimensional_fields: tests that require backend support for zero-dimensional fields',
   'uses_cartesian_shift: tests that use a Cartesian connectivity',
   'uses_unstructured_shift: tests that use a unstructured connectivity',
-  'uses_scan: tests that uses scan',
+  'uses_max_over: tests that use the max_over builtin',
+  'uses_mesh_with_skip_values: tests that use a mesh with skip values',
   'checks_specific_error: tests that rely on the backend to produce a specific error message'
 ]
 norecursedirs = ['dist', 'build', 'cpp_backend_tests/build*', '_local/*', '.*']

diff --git a/requirements-dev.in b/requirements-dev.in
@@ -26,11 +26,13 @@ flake8-rst-docstrings>=0.0.14
 isort>=5.10
 jupytext>=1.14
 mypy>=1.0
+matplotlib>=3.3
+nbmake>=1.4.6
 pipdeptree>=2.3
 pip-tools>=6.10
 pre-commit>=2.17
 psutil>=5.0
-pygments>=2.7
+pygments>=2.7.3
 pytest-cache>=1.0
 pytest-cov>=2.8
 pytest-factoryboy>=2.0.3