Introduce skipexitcode mode with tox so that we can run the PySide te…

…sts without failing due to a non-zero pytest exit code even when all tests pass

.github/workflows/check_failures.py

@@ -0,0 +1,20 @@
+import os
+import sys
+from xml.etree import ElementTree
+
+if len(sys.argv) != 2:
+    print("Usage: python check_failures.py filename.xml")
+    sys.exit(1)
+
+filename = sys.argv[1]
+
+if not os.path.exists(filename):
+    print("Output file does not exist, pytest command did not complete")
+    sys.exit(1)
+
+et = ElementTree.parse(filename)
+results = et.find("testsuite").attrib
+
+if int(results["errors"]) > 0 or int(results["failures"]) > 0:
+    print("There were failures/errors as part of the test suite")
+    sys.exit(1)

.github/workflows/ci_workflows.yml

@@ -86,19 +86,19 @@ jobs:
       envs: |
 
         # Non-deterministic QThread exceptions
-        - linux: py38-test-pyside514
-        - linux: py39-test-pyside515
-        - linux: py310-test-pyside63
+        - linux: py38-test-pyside514-skipexitcode
+        - linux: py39-test-pyside515-skipexitcode
+        - linux: py310-test-pyside63-skipexitcode
 
         # PySide6 6.4 failures due to https://github.com/spyder-ide/qtpy/issues/373
         # and https://github.com/matplotlib/matplotlib/issues/24155
         # Python 3.11.0 failing on Windows in test_image.py on
         # > assert df.find_factory(fname) is df.img_data
-        - linux: py310-test-pyside64
-        - linux: py311-test-pyside64
-        - macos: py310-test-pyside63
-        - macos: py311-test-pyside64
-        - windows: py310-test-pyside64
+        - linux: py310-test-pyside64-skipexitcode
+        - linux: py311-test-pyside64-skipexitcode
+        - macos: py310-test-pyside63-skipexitcode
+        - macos: py311-test-pyside64-skipexitcode
+        - windows: py310-test-pyside64-skipexitcode
         - windows: py311-test-pyqt515
 
         # Windows docs build

tox.ini

@@ -56,9 +56,15 @@ extras =
     test
     all: all
     docs: docs
+allowlist_externals =
+    sh
 commands =
     test: pip freeze
-    test: pytest --pyargs glue_qt --cov glue_qt --cov-config={toxinidir}/setup.cfg {posargs}
+    test-!skipexitcode: pytest --pyargs glue_qt --cov glue_qt --cov-config={toxinidir}/setup.cfg {posargs}
+    # The skipexitcode mode consists of ignoring the exit code of the pytest
+    # command and just checking whether any errors or failures were present.
+    test-skipexitcode: sh -c 'pytest --pyargs glue_qt --cov glue_qt --cov-config={toxinidir}/setup.cfg --junit-xml={toxinidir}/{envname}.xml {posargs} || true'
+    test-skipexitcode: python {toxinidir}/.github/workflows/check_failures.py {toxinidir}/{envname}.xml
     docs: sphinx-build -W -n -b html -d _build/doctrees   . _build/html
 
 [testenv:codestyle]