refactor: testing suite (random order) (#3519)

* ci: using random order when doing the tests

* ci: using random order when doing the tests

* chore: adding changelog file 3519.dependencies.md [dependabot-skip]

* test: to get more info

* fix: making sure we are plotting something

* feat: avoid mixing class tests

* feat: removing scoping to module only.

* feat: testing resuming between class methods

* test: fixing some tests

* fix: more tests

* fix: tests

* fix: tests

* fix: logging test

* fix: pymapdl::tests::test_mapdl.py::test_equal_in_comments_and_title

* test: uncommenting test

* feat: improving cleared fixture to clean format too. Using the fixture over clear() calls.

* fix: double fixture

* fix: fixtures

* feat: making sure the MAPDL database is clean up on class Test creation.
refactor: to use fixtures instead of functions when possible.

* fix: get_port test

* fix: using lower scope fixture on high scope fixture in database module

* chore: adding test dependency in minimal

* fix: missing staticmethods

* fix: tests

* fix: tests

* fix: logging test

* test: check we are not changing state

* chore: update the image cache

* chore: empty commit to trigger CICD

* fix: codecov issues

* fix: test

* fix: typos

* fix: update golden data

* fix: not calling a method and reducing tolerance for matching.

* fix: setting graphics, full as default for testing

* refactor: test organization

* fix: test

* chore: update the image cache

* chore: empty comment to trigger CICD

* chore: update the image cache

* chore: empty comment to trigger CICD

* chore: update the image cache

* fix: test_igesin_whitespace

* chore: update the image cache

* feat: clearing all tests before running.

* chore: adding changelog file 3519.documentation.md [dependabot-skip]

* chore: update the image cache

* fix: test

* test: rebuilding the plotting config every time

* chore: update the image cache

* fix: codecov issues

* ci: reducing the amount of version tested to only last 2.

* fix: adding missing import

* feat: assert no changes on type of plotting

* fix: method name

* fix: function replacement

* feat: restoring default in test_command

* fix: test_freq

* fix: removing duplicated function

* fix: show issue

* test: remove mute

* chore: apply suggestions from code review

* ci: auto fixes from pre-commit.com hooks.

for more information, see https://pre-commit.ci

* chore: adding changelog file 3519.added.md [dependabot-skip]

---------

Co-authored-by: pyansys-ci-bot <[email protected]>
Co-authored-by: germa89 <[email protected]>
Co-authored-by: pyansys-ci-bot <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/ci.yml

@@ -33,7 +33,8 @@ env:
   DPF_PORT: 21004
   MAPDL_PACKAGE: ghcr.io/ansys/mapdl
   ON_CI: True
-  PYTEST_ARGUMENTS: '-vvv --color=yes -ra --durations=10 --maxfail=3 --reruns 3 --reruns-delay 4 --cov=ansys.mapdl.core --cov-report=html'
+  PYTEST_ARGUMENTS: '-vvv -rxXsa --color=yes --durations=10 --random-order --random-order-bucket=class --maxfail=10 --reruns 3 --reruns-delay 4 --cov=ansys.mapdl.core --cov-report=html'
+
   BUILD_CHEATSHEET: True
 
   # Following env vars when changed will "reset" the mentioned cache,
@@ -820,7 +821,7 @@ jobs:
 
       - name: "Unit testing requirements installation"
         run: |
-          python -m pip install pytest pytest-rerunfailures pytest-cov
+          python -m pip install pytest pytest-rerunfailures pytest-cov pytest-random-order
 
       - name: "Unit testing"
         env:

doc/changelog.d/3519.added.md

@@ -0,0 +1 @@
+refactor: testing suite (random order)

doc/source/getting_started/develop_pymapdl.rst

@@ -230,12 +230,14 @@ It is executed upstream of each test and not within all tests.
 
 .. code:: python
 
-   def test_my_new_feature(mapdl):  # pass the 'mapdl' fixture as an argument.
+   def test_my_new_feature(mapdl, cleared):  # pass the 'mapdl' fixture as an argument.
        mapdl.prep7()
        # .... more code
 
        return True  # if everything goes ok until here
 
+Passing the ``cleared`` fixture is also useful since it clears up the MAPDL database
+and configuration before performing the test.
 If you do not have MAPDL installed locally but still want to run the
 unit testing, you must set up the following environment variables.
 

pyproject.toml

@@ -17,8 +17,8 @@ dependencies = [
     "ansys-mapdl-reader>=0.51.7",
     "ansys-math-core>=0.1.2",
     "ansys-platform-instancemanagement~=1.0",
-    "ansys-tools-visualization-interface>=0.2.6",
     "ansys-tools-path>=0.3.1",
+    "ansys-tools-visualization-interface>=0.2.6",
     "click>=8.1.3", # for CLI interface
     "grpcio>=1.30.0",  # tested up to grpcio==1.35
     "matplotlib>=3.0.0",  # for colormaps for pyvista
@@ -67,6 +67,7 @@ tests = [
     "pytest-cov==6.0.0",
     "pytest-memprof<0.3.0",
     "pytest-pyvista==0.1.9",
+    "pytest-random-order==1.1.1",
     "pytest-rerunfailures==15.0",
     "pytest==8.3.3",
     "scipy==1.14.1",
@@ -75,13 +76,13 @@ tests = [
 doc = [
     "ansys-dpf-core==0.10.1",
     "ansys-mapdl-reader==0.54.2",
-    "ansys-tools-visualization-interface==0.5.0",
     "ansys-sphinx-theme==1.2.2",
+    "ansys-tools-visualization-interface==0.5.0",
     "grpcio==1.68.0",
     "imageio-ffmpeg==0.5.1",
     "imageio==2.36.0",
-    "jupyter==1.1.1",
     "jupyter_sphinx==0.5.3",
+    "jupyter==1.1.1",
     "jupyterlab>=3.2.8",
     "matplotlib==3.9.2",
     "nbformat==5.10.4",
@@ -119,7 +120,7 @@ pymapdl_convert_script = "ansys.mapdl.core.cli:old_pymapdl_convert_script_entry_
 pymapdl = "ansys.mapdl.core.cli:main"
 
 [tool.pytest.ini_options]
-addopts = "-rxXsa -vvv --maxfail=10"
+addopts = "-rxXsa -vvv --maxfail=10 --random-order-bucket=class --random-order --durations=10"
 junit_family = "legacy"
 filterwarnings = [
     "ignore::FutureWarning",

src/ansys/mapdl/core/mapdl_core.py

@@ -1560,6 +1560,7 @@ def open_apdl_log(
         """
         if self._apdl_log is not None:
             raise MapdlRuntimeError("APDL command logging already enabled")
+
         self._log.debug("Opening ANSYS log file at %s", filename)
 
         if mode not in ["w", "a", "x"]:

src/ansys/mapdl/core/plotting/plotting_defaults.py

@@ -72,7 +72,7 @@ def __init__(self):
         self._configured = False
 
     def __call__(self, name):
-        if not self._configured:
+        if True:  # not self._configured: # Temporal patch pending on #3568
             self._set_configuration()
             self._configured = True
 

src/ansys/mapdl/core/plotting/visualizer.py

@@ -819,6 +819,7 @@ def show(
         if savefig:
             self._off_screen = True
             self._notebook = False
+
         # permit user to save the figure as a screenshot
         if self._savefig or savefig:
             self._backend.show(
@@ -833,7 +834,7 @@ def show(
             if return_plotter:
                 return self
 
-            # ifplotter.scene.set_background("paraview") not returning plotter, close right away
+            # if plotter.scene.set_background("paraview") not returning plotter, close right away
             self.scene.close()
 
         else:

tests/common.py

@@ -23,9 +23,28 @@
 """Shared testing module"""
 from collections import namedtuple
 import os
-from typing import Dict
+import subprocess
+from typing import Dict, List
+
+import psutil
+
+from ansys.mapdl.core import LOG, Mapdl
+from ansys.mapdl.core.errors import MapdlConnectionError, MapdlExitedError
+from ansys.mapdl.core.launcher import (
+    _is_ubuntu,
+    get_start_instance,
+    is_ansys_process,
+    launch_mapdl,
+)
 
-from ansys.mapdl.core.launcher import _is_ubuntu
+PROCESS_OK_STATUS = [
+    psutil.STATUS_RUNNING,  #
+    psutil.STATUS_SLEEPING,  #
+    psutil.STATUS_DISK_SLEEP,
+    psutil.STATUS_DEAD,
+    psutil.STATUS_PARKED,  # (Linux)
+    psutil.STATUS_IDLE,  # (Linux, macOS, FreeBSD)
+]
 
 Node = namedtuple("Node", ["number", "x", "y", "z", "thx", "thy", "thz"])
 Element = namedtuple(
@@ -199,3 +218,94 @@ def get_details_of_elements(mapdl_) -> Dict[int, Node]:
             if len(args) == 6:
                 elements[args[0]] = Element(*args, node_numbers=None)
     return elements
+
+
+def log_test_start(mapdl: Mapdl) -> None:
+    """Print the current test to the MAPDL log file and console output."""
+    test_name = os.environ.get(
+        "PYTEST_CURRENT_TEST", "**test id could not get retrieved.**"
+    )
+
+    mapdl.run("!")
+    mapdl.run(f"! PyMAPDL running test: {test_name}")
+    mapdl.run("!")
+
+    # To see it also in MAPDL terminal output
+    if len(test_name) > 75:
+        # terminal output is limited to 75 characters
+        test_name = test_name.split("::")
+        if len(test_name) > 2:
+            types_ = ["File path", "Test class", "Method"]
+        else:
+            types_ = ["File path", "Test function"]
+
+        mapdl._run("/com,Running test in:", mute=True)
+        for type_, name_ in zip(types_, test_name):
+            mapdl._run(f"/com,    {type_}: {name_}", mute=True)
+
+    else:
+        mapdl._run(f"/com,Running test: {test_name}", mute=True)
+
+
+def restart_mapdl(mapdl: Mapdl) -> Mapdl:
+    """Restart MAPDL after a failed test"""
+
+    def is_exited(mapdl: Mapdl):
+        try:
+            _ = mapdl._ctrl("VERSION")
+            return False
+        except MapdlExitedError:
+            return True
+
+    if get_start_instance() and (is_exited(mapdl) or mapdl._exited):
+        # Backing up the current local configuration
+        local_ = mapdl._local
+        ip = mapdl.ip
+        port = mapdl.port
+        try:
+            # to connect
+            mapdl = Mapdl(port=port, ip=ip)
+
+        except MapdlConnectionError as err:
+            # Registering error.
+            LOG.info(str(err))
+
+            # we cannot connect.
+            # Kill the instance
+            mapdl.exit()
+
+            # Relaunching MAPDL
+            mapdl = launch_mapdl(
+                port=mapdl._port,
+                override=True,
+                run_location=mapdl._path,
+                cleanup_on_exit=mapdl._cleanup,
+                log_apdl=log_apdl(),
+            )
+
+        # Restoring the local configuration
+        mapdl._local = local_
+        mapdl._exited = False
+
+    return mapdl
+
+
+def make_sure_not_instances_are_left_open(valid_ports: List) -> None:
+    """Make sure we leave no MAPDL running behind"""
+
+    if is_on_local():
+        for proc in psutil.process_iter():
+            try:
+                if (
+                    psutil.pid_exists(proc.pid)
+                    and proc.status() in PROCESS_OK_STATUS
+                    and is_ansys_process(proc)
+                ):
+                    cmdline = proc.cmdline()
+                    port = int(cmdline[cmdline.index("-port") + 1])
+
+                    if port not in valid_ports:
+                        subprocess.run(["pymapdl", "stop", "--port", f"{port}"])
+                        time.sleep(1)
+            except psutil.NoSuchProcess:
+                continue