diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 4d7a51ccb..e2d9e164e 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -131,7 +131,7 @@ jobs: run: | python3 -m pytest tests/unit/ - - name: Run integration test suite + - name: Run integration test suite for local tests run: | python3 tests/integration/run_tests.py --verbose --local @@ -179,7 +179,7 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt - - name: Install merlin to run unit tests + - name: Install merlin and setup redis as the broker run: | pip3 install -e . merlin config --broker redis @@ -189,12 +189,12 @@ jobs: merlin example feature_demo pip3 install -r feature_demo/requirements.txt - - name: Run integration test suite for Redis + - name: Run integration test suite for distributed tests env: REDIS_HOST: redis REDIS_PORT: 6379 run: | - python3 tests/integration/run_tests.py --verbose --ids 31 32 + python3 tests/integration/run_tests.py --verbose --distributed # - name: Setup rabbitmq config # run: | diff --git a/.gitignore b/.gitignore index e427f8ad4..8b0fb8ad2 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,9 @@ ARCHIVE_DIR *_OUTPUT/ *_OUTPUT_D/ *_ensemble_*/ +studies/ +appendonlydir/ +cli_test_studies/ # Scheduler logs flux.out diff --git a/CHANGELOG.md b/CHANGELOG.md index 502d48379..3aa643c8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,15 +8,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Pip wheel wasn't including .sh files for merlin examples - The learn.py script in the openfoam_wf* examples will now create the missing Energy v Lidspeed plot + ### Added - Now loads np.arrays of dtype='object', allowing mix-type sample npy - Added a singularity container openfoam_wf example - Added flux native worker launch support - Added PBS flux launch support - Added check_for_flux, check_for_slurm, check_for_lsf, and check_for_pbs utility functions +- Tests for the `stop-workers` command +- A function in `run_tests.py` to check that an integration test definition is formatted correctly +- A new dev_workflow example `multiple_workers.yaml` that's used for testing the `stop-workers` command +- Ability to start 2 subprocesses for a single test +- Added the --distributed and --display-table flags to run_tests.py + - --distributed: only run distributed tests + - --display-tests: displays a table of all existing tests and the id associated with each test ### Changed - Changed celery_regex to celery_slurm_regex in test_definitions.py +- Reformatted how integration tests are defined and part of how they run + - Test values are now dictionaries rather than tuples + - Stopped using `subprocess.Popen()` and `subprocess.communicate()` to run tests and now instead use `subprocess.run()` for simplicity and to keep things up-to-date with the latest subprocess release (`run()` will call `Popen()` and `communicate()` under the hood so we don't have to handle that anymore) +- Rewrote the README in the integration tests folder to explain the new integration test format ## [1.9.1] ### Fixed diff --git a/LICENSE b/LICENSE index 746aac58b..3adc85cb9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 Lawrence Livermore National Laboratory +Copyright (c) 2023 Lawrence Livermore National Laboratory Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index fd6949bcf..897b44a31 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # @@ -38,6 +38,8 @@ include config.mk .PHONY : e2e-tests-diagnostic .PHONY : e2e-tests-local .PHONY : e2e-tests-local-diagnostic +.PHONY : e2e-tests-distributed +.PHONY : e2e-tests-distributed-diagnostic .PHONY : tests .PHONY : check-flake8 .PHONY : check-black @@ -109,6 +111,16 @@ e2e-tests-local-diagnostic: $(PYTHON) $(TEST)/integration/run_tests.py --local --verbose +e2e-tests-distributed: + . $(VENV)/bin/activate; \ + $(PYTHON) $(TEST)/integration/run_tests.py --distributed; \ + + +e2e-tests-distributed-diagnostic: + . $(VENV)/bin/activate; \ + $(PYTHON) $(TEST)/integration/run_tests.py --distributed --verbose + + # run unit and CLI tests tests: unit-tests e2e-tests @@ -185,6 +197,17 @@ version: find tests/ -type f -print0 | xargs -0 sed -i 's/Version: $(VSTRING)/Version: $(VER)/g' find Makefile -type f -print0 | xargs -0 sed -i 's/Version: $(VSTRING)/Version: $(VER)/g' +# Increment copyright year +year: +# do LICENSE (no comma after year) + sed -i 's/$(YEAR) Lawrence Livermore/$(NEW_YEAR) Lawrence Livermore/g' LICENSE + +# do all file headers (works on linux) + find merlin/ -type f -print0 | xargs -0 sed -i 's/$(YEAR), Lawrence Livermore/$(NEW_YEAR), Lawrence Livermore/g' + find *.py -type f -print0 | xargs -0 sed -i 's/$(YEAR), Lawrence Livermore/$(NEW_YEAR), Lawrence Livermore/g' + find tests/ -type f -print0 | xargs -0 sed -i 's/$(YEAR), Lawrence Livermore/$(NEW_YEAR), Lawrence Livermore/g' + find Makefile -type f -print0 | xargs -0 sed -i 's/$(YEAR), Lawrence Livermore/$(NEW_YEAR), Lawrence Livermore/g' + # Make a list of all dependencies/requirements reqlist: johnnydep merlin --output-format pinned diff --git a/config.mk b/config.mk index 38a87a9bf..f1cfbcea3 100644 --- a/config.mk +++ b/config.mk @@ -19,6 +19,8 @@ endif VER?=1.0.0 VSTRING=[0-9]\+\.[0-9]\+\.[0-9]\+ +YEAR=20[0-9][0-9] +NEW_YEAR?=2023 CHANGELOG_VSTRING="## \[$(VSTRING)\]" INIT_VSTRING="__version__ = \"$(VSTRING)\"" diff --git a/merlin/__init__.py b/merlin/__init__.py index eaf3ff668..1222a38f6 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index b961afe50..a521d2cc5 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/celery.py b/merlin/celery.py index 0041523b2..52d4e4589 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index 30fea7f47..f242b15fb 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index da214275b..5e82abbc2 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -1,7 +1,7 @@ #!/usr/bin/env python ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 274ddb062..c2391ca6d 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -1,7 +1,7 @@ #!/usr/bin/env python ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 591f9a49f..dd93cf5a7 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 4861a757b..45ddb2ed1 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 229f00851..6a0766427 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 1ba552626..930c67b5f 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 4ec7c5dbe..b0eca1b6c 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 352da5550..bd19ac539 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index b460be069..08a0362ae 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index e30a18ca0..ea26edc8f 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index 7822e7086..38fba1ddc 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -3,7 +3,7 @@ """ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index bd22ddaa3..bb7f79875 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index e435e1ba0..40ea19af7 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 626b776d5..3cce32883 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/display.py b/merlin/display.py index 30140981e..0e0e11e66 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # @@ -45,7 +45,27 @@ from merlin.config.configfile import default_config_info +# TODO: make these color blind compliant +# (see https://mikemol.github.io/technique/colorblind/2018/02/11/color-safe-palette.html) +ANSI_COLORS = { + "RESET": "\033[0m", + "GREY": "\033[90m", + "RED": "\033[91m", + "GREEN": "\033[92m", + "YELLOW": "\033[93m", + "BLUE": "\033[94m", + "MAGENTA": "\033[95m", + "CYAN": "\033[96m", + "WHITE": "\033[97m", +} + + class ConnProcess(Process): + """ + An extension of Multiprocessing's Process class in order + to overwrite the run and exception defintions. + """ + def __init__(self, *args, **kwargs): Process.__init__(self, *args, **kwargs) self._pconn, self._cconn = Pipe() @@ -55,19 +75,24 @@ def run(self): try: Process.run(self) self._cconn.send(None) - except Exception as e: - tb = traceback.format_exc() - self._cconn.send((e, tb)) + except Exception as e: # pylint: disable=W0718,C0103 + trace_back = traceback.format_exc() + self._cconn.send((e, trace_back)) # raise e # You can still rise this exception if you need to @property def exception(self): + """Create custom exception""" if self._pconn.poll(): self._exception = self._pconn.recv() return self._exception def check_server_access(sconf): + """ + Check if there are any issues connecting to the servers. + If there are, output the errors. + """ servers = ["broker server", "results server"] if sconf.keys(): @@ -75,25 +100,25 @@ def check_server_access(sconf): print("-" * 28) excpts = {} - for s in servers: - if s in sconf: - _examine_connection(s, sconf, excpts) + for server in servers: + if server in sconf: + _examine_connection(server, sconf, excpts) if excpts: print("\nExceptions:") - for k, v in excpts.items(): - print(f"{k}: {v}") + for key, val in excpts.items(): + print(f"{key}: {val}") -def _examine_connection(s, sconf, excpts): +def _examine_connection(server, sconf, excpts): connect_timeout = 60 try: ssl_conf = None - if "broker" in s: + if "broker" in server: ssl_conf = broker.get_ssl_config() - if "results" in s: + if "results" in server: ssl_conf = results_backend.get_ssl_config() - conn = Connection(sconf[s], ssl=ssl_conf) + conn = Connection(sconf[server], ssl=ssl_conf) conn_check = ConnProcess(target=conn.connect) conn_check.start() counter = 0 @@ -102,16 +127,16 @@ def _examine_connection(s, sconf, excpts): counter += 1 if counter > connect_timeout: conn_check.kill() - raise Exception(f"Connection was killed due to timeout ({connect_timeout}s)") + raise TimeoutError(f"Connection was killed due to timeout ({connect_timeout}server)") conn.release() if conn_check.exception: - error, traceback = conn_check.exception + error, _ = conn_check.exception raise error - except Exception as e: - print(f"{s} connection: Error") - excpts[s] = e + except Exception as e: # pylint: disable=W0718,C0103 + print(f"{server} connection: Error") + excpts[server] = e else: - print(f"{s} connection: OK") + print(f"{server} connection: OK") def display_config_info(): @@ -129,7 +154,7 @@ def display_config_info(): conf["broker server"] = broker.get_connection_string(include_password=False) sconf["broker server"] = broker.get_connection_string() conf["broker ssl"] = broker.get_ssl_config() - except Exception as e: + except Exception as e: # pylint: disable=W0718,C0103 conf["broker server"] = "Broker server error." excpts["broker server"] = e @@ -137,7 +162,7 @@ def display_config_info(): conf["results server"] = results_backend.get_connection_string(include_password=False) sconf["results server"] = results_backend.get_connection_string() conf["results ssl"] = results_backend.get_ssl_config() - except Exception as e: + except Exception as e: # pylint: disable=W0718,C0103 conf["results server"] = "No results server configured or error." excpts["results server"] = e @@ -145,8 +170,8 @@ def display_config_info(): if excpts: print("\nExceptions:") - for k, v in excpts.items(): - print(f"{k}: {v}") + for key, val in excpts.items(): + print(f"{key}: {val}") check_server_access(sconf) @@ -170,7 +195,7 @@ def display_multiple_configs(files, configs): pprint.pprint(config) -def print_info(args): +def print_info(args): # pylint: disable=W0613 """ Provide version and location information about python and pip to facilitate user troubleshooting. 'merlin info' is a CLI tool only for @@ -187,9 +212,30 @@ def print_info(args): print("") info_calls = ["which python3", "python3 --version", "which pip3", "pip3 --version"] info_str = "" - for x in info_calls: - info_str += 'echo " $ ' + x + '" && ' + x + "\n" + for cmd in info_calls: + info_str += 'echo " $ ' + cmd + '" && ' + cmd + "\n" info_str += "echo \n" info_str += r"echo \"echo \$PYTHONPATH\" && echo $PYTHONPATH" _ = subprocess.run(info_str, shell=True) print("") + + +def tabulate_info(info, headers=None, color=None): + """ + Display info in a table. Colorize the table if you'd like. + Intended for use for functions outside of this file so they don't + need to import tabulate. + :param `info`: The info you want to tabulate. + :param `headers`: A string or list stating what you'd like the headers to be. + Options: "firstrow", "keys", or List[str] + :param `color`: An ANSI color. + """ + # Adds the color at the start of the print + if color: + print(color, end="") + + # \033[0m resets color to white + if headers: + print(tabulate(info, headers=headers), ANSI_COLORS["RESET"]) + else: + print(tabulate(info), ANSI_COLORS["RESET"]) diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/examples/dev_workflows/multiple_workers.yaml b/merlin/examples/dev_workflows/multiple_workers.yaml new file mode 100644 index 000000000..f393f87d3 --- /dev/null +++ b/merlin/examples/dev_workflows/multiple_workers.yaml @@ -0,0 +1,56 @@ +description: + name: multiple_workers + description: a very simple merlin workflow with multiple workers + +global.parameters: + GREET: + values : ["hello","hola"] + label : GREET.%% + WORLD: + values : ["world","mundo"] + label : WORLD.%% + +study: + - name: step_1 + description: say hello + run: + cmd: | + echo "$(GREET), $(WORLD)!" + task_queue: hello_queue + + - name: step_2 + description: step 2 + run: + cmd: | + echo "step_2" + depends: [step_1_*] + task_queue: echo_queue + + - name: step_3 + description: stop workers + run: + cmd: | + echo "stop workers" + depends: [step_2] + task_queue: other_queue + + - name: step_4 + description: another step + run: + cmd: | + echo "another step" + depends: [step_3] + task_queue: other_queue + +merlin: + resources: + workers: + step_1_merlin_test_worker: + args: -l INFO + steps: [step_1] + step_2_merlin_test_worker: + args: -l INFO + steps: [step_2] + other_merlin_test_worker: + args: -l INFO + steps: [step_3, step_4] \ No newline at end of file diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 43573b416..7fbd502b1 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 2796465ab..c97cc2757 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 816546f67..225e69d28 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index f540e6f1f..475982d8c 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -1,7 +1,7 @@ """This module handles setting up the extensive logging system in Merlin.""" ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/main.py b/merlin/main.py index 5de8423b0..47f471dd8 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -1,7 +1,7 @@ """The top level main function for invoking Merlin.""" ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index a1e4de372..a1af0298b 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/router.py b/merlin/router.py index 90711756c..3ec9122ff 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 31115004f..5653cba21 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 38359938c..045ef22e3 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -1,7 +1,7 @@ """Main functions for instantiating and running Merlin server containers.""" ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 298a360b7..e62e12e4f 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 16d2c3686..65f0b2abb 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 46db0fd4f..ac0031823 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 46f71882f..7b54e5200 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 710bc2500..f4ea42e24 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 5ca2eea6b..4c9291693 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 9d93197bf..287fab1f6 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index af6ed22b5..9a42873f1 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 97fc40289..f6d344a25 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 37a429055..f7ba3532f 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index ab998140a..027ebaff9 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 3a55df606..031302480 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/study.py b/merlin/study/study.py index dd108005a..6bf07653f 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/utils.py b/merlin/utils.py index 5217e1d7a..48def3a10 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/setup.py b/setup.py index 06706ca57..1a1ad94ba 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/tests/integration/README.md b/tests/integration/README.md index ef402c9e5..1d575e022 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,21 +1,160 @@ -# Integration test script: run_tests.py +# Integration Tests -To run command line-level tests of Merlin, follow these steps: +This directory contains 3 key files for testing: +1. `run_tests.py` - script to launch tests +2. `test_definitions.py` - test definitions +3. `conditions.py` - test conditions + +## How to Run + +To run command-line-level tests of Merlin, follow these steps: 1. activate the Merlin virtual environment 2. navigate to the top-level `merlin` directory 3. run `python tests/integration/run_tests.py` -This will run all tests found in the `define_tests` function. -A test is a python dict where the key is the test name, and the -value is a tuple holding the test shell command, and a regexp string -to search for, if any. Without a regexp string, the script will -output 'FAIL' on non-zero return codes. With a regexp string, the -script outputs 'FAIL' if the string cannot be found in the -test's stdout. +This will run all tests found in the `define_tests` function located within `test_definitions.py`. + +## How Tests are Defined + +A test is a python dict where the key is the test name and the +value is another dict. The value dict can currently have 5 keys: + +Required: + +1. `cmds` + - Type: Str or List[Str] + - Functionality: Defines the CLI commands to run for a test + - Limitations: The number of strings here should be equal to `num procs` (see `5.num procs` below) +2. `conditions` + - Type: Condition or List[Condition] + - Functionality: Defines the conditions to check against for this test + - Condition classes can be found in `conditions.py` + +Optional: + +3. `run type` + - Type: Str + - Functionality: Defines the type of run (either `local` or `distributed`) + - Default: None +4. `cleanup` + - Type: Str + - Functionality: Defines a CLI command to run that will clean the output of your test + - Default: None +5. `num procs` + - Type: int + - Functionality: Defines the number of subprocesses required for a test + - Default: 1 + - Limitations: + - Currently the value here can only be 1 or 2 + - The number of `cmds` must be equal to `num procs` (i.e. one command will be run per subprocess launched) + +## Examples + +This section will show both valid and invalid test definitions. + +### Valid Test Definitions + +The most basic test you can provide can be written 4 ways since `cmds` and `conditions` can both be 2 different types: + + "cmds as string, conditions as Condition": { + "cmds": "echo hello", + "conditions": HasReturnCode(), + } + + "cmds as list, conditions as Condition": { + "cmds": ["echo hello"], + "conditions": HasReturnCode(), + } + + "cmds as string, conditions as list": { + "cmds": "echo hello", + "conditions": [HasReturnCode()], + } + + "cmds as list, conditions as list": { + "cmds": ["echo hello"], + "conditions": [HasReturnCode()], + } + +Adding slightly more complexity, we provide a run type: + + "basic test with run type": { + "cmds": "echo hello", + "conditions": HasReturnCode(), + "run type": "local" # This could also be "distributed" + } + +Now we'll add a cleanup command: + + "basic test with cleanup": { + "cmds": "mkdir output_dir/", + "conditions": HasReturnCode(), + "run type": "local", + "cleanup": "rm -rf output_dir/" + } + +Finally we'll play with the number of processes to start: + + "test with 1 process": { + "cmds": "mkdir output_dir/", + "conditions": HasReturnCode(), + "run type": "local", + "cleanup": "rm -rf output_dir/", + "num procs": 1 + } + + "test with 2 processes": { + "cmds": ["mkdir output_dir/", "touch output_dir/new_file.txt"], + "conditions": HasReturnCode(), + "run type": "local", + "cleanup": "rm -rf output_dir/", + "num procs": 2 + } + +Similarly, the test with 2 processes can be condensed to a test with 1 process +by placing them in the same string and separating them with a semi-colon: + + "condensing test with 2 processes into 1 process": { + "cmds": ["mkdir output_dir/ ; touch output_dir/new_file.txt"], + "conditions": HasReturnCode(), + "run type": "local", + "cleanup": "rm -rf output_dir/", + "num procs": 1 + } + + +### Invalid Test Definitions + +No `cmds` provided: + + "no cmd": { + "conditions": HasReturnCode(), + } + +No `conditions` provided: + + "no conditions": { + "cmds": "echo hello", + } + +Number of `cmds` does not match `num procs`: + + "num cmds != num procs": { + "cmds": ["echo hello; echo goodbye"], + "conditions": HasReturnCode(), + "num procs": 2 + } + +Note: Technically 2 commands were provided here ("echo hello" and "echo goodbye") +but since they were placed in one string it will be viewed as one command. +Changing the `cmds` section here to be: + + "cmds": ["echo hello", "echo goodbye"] + +would fix this issue and create a valid test definition. -### Continuous integration -Currently run from [Bamboo](https://lc.llnl.gov/bamboo/chain/admin/config/defaultStages.action?buildKey=MLSI-TES). +## Continuous Integration -Our Bamboo agents make the virtual environment, staying at the `merlin/` location, then run: `python tests/integration/run_tests.py`. +Merlin's CI is currently done through [GitHub Actions](https://github.com/features/actions). If you're needing to modify this CI, you'll need to update `/.github/workflows/push-pr_workflow.yml`. diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 3448ec61a..9c44e1f5f 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -1,3 +1,33 @@ +############################################################################### +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.1. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### +"""This module defines the different conditions to test against.""" import os from abc import ABC, abstractmethod from glob import glob @@ -6,6 +36,8 @@ # TODO when moving command line tests to pytest, change Condition boolean returns to assertions class Condition(ABC): + """Abstract Condition class that other conditions will inherit from""" + def ingest_info(self, info): """ This function allows child classes of Condition @@ -14,11 +46,14 @@ def ingest_info(self, info): for key, val in info.items(): setattr(self, key, val) + @property @abstractmethod def passes(self): + """The method that will check if the test passes or not""" pass +# pylint: disable=no-member class HasReturnCode(Condition): """ A condition that some process must return 0 @@ -80,7 +115,7 @@ def is_within(self, text): @property def passes(self): if self.negate: - return not self.is_within(self.stdout) + return not self.is_within(self.stdout) and not self.is_within(self.stderr) return self.is_within(self.stdout) or self.is_within(self.stderr) @@ -99,6 +134,9 @@ def __init__(self, study_name, output_path): self.dirpath_glob = f"{self.output_path}/{self.study_name}" f"_[0-9]*-[0-9]*" def glob(self, glob_string): + """ + Returns a regex string for the glob library to recursively find files with. + """ candidates = glob(glob_string) if isinstance(candidates, list): return sorted(candidates)[-1] @@ -110,7 +148,7 @@ class StepFileExists(StudyOutputAware): A StudyOutputAware that checks for a particular file's existence. """ - def __init__(self, step, filename, study_name, output_path, params=False): + def __init__(self, step, filename, study_name, output_path, params=False): # pylint: disable=R0913 """ :param `step`: the name of a step :param `filename`: name of file to search for in step's workspace directory @@ -127,12 +165,16 @@ def __str__(self): @property def glob_string(self): + """ + Returns a regex string for the glob library to recursively find files with. + """ param_glob = "" if self.params: param_glob = "*/" return f"{self.dirpath_glob}/{self.step}/{param_glob}{self.filename}" def file_exists(self): + """Check if the file path created by glob_string exists""" glob_string = self.glob_string try: filename = self.glob(glob_string) @@ -150,7 +192,7 @@ class StepFileHasRegex(StudyOutputAware): A StudyOutputAware that checks that a particular file contains a regex. """ - def __init__(self, step, filename, study_name, output_path, regex): + def __init__(self, step, filename, study_name, output_path, regex): # pylint: disable=R0913 """ :param `step`: the name of a step :param `filename`: name of file to search for in step's workspace directory @@ -163,20 +205,25 @@ def __init__(self, step, filename, study_name, output_path, regex): self.regex = regex def __str__(self): - return f"{__class__.__name__} expected to find '{self.regex}' regex match in file '{self.glob_string}', but match was not found" + return f"""{__class__.__name__} expected to find '{self.regex}' + regex match in file '{self.glob_string}', but match was not found""" @property def glob_string(self): + """ + Returns a regex string for the glob library to recursively find files with. + """ return f"{self.dirpath_glob}/{self.step}/{self.filename}" def contains(self): + """See if the regex is within the filetext""" glob_string = self.glob_string try: filename = self.glob(glob_string) with open(filename, "r") as textfile: filetext = textfile.read() return self.is_within(filetext) - except Exception: + except Exception: # pylint: disable=W0718 return False def is_within(self, text): @@ -196,7 +243,7 @@ class ProvenanceYAMLFileHasRegex(HasRegex): MUST contain a given regular expression. """ - def __init__(self, regex, name, output_path, provenance_type, negate=False): + def __init__(self, regex, name, output_path, provenance_type, negate=False): # pylint: disable=R0913 """ :param `regex`: a string regex pattern :param `name`: the name of a study @@ -214,14 +261,19 @@ def __init__(self, regex, name, output_path, provenance_type, negate=False): def __str__(self): if self.negate: - return f"{__class__.__name__} expected to find no '{self.regex}' regex match in provenance spec '{self.glob_string}', but match was found" - return f"{__class__.__name__} expected to find '{self.regex}' regex match in provenance spec '{self.glob_string}', but match was not found" + return f"""{__class__.__name__} expected to find no '{self.regex}' + regex match in provenance spec '{self.glob_string}', but match was found""" + return f"""{__class__.__name__} expected to find '{self.regex}' + regex match in provenance spec '{self.glob_string}', but match was not found""" @property def glob_string(self): + """ + Returns a regex string for the glob library to recursively find files with. + """ return f"{self.output_path}/{self.name}" f"_[0-9]*-[0-9]*/merlin_info/{self.name}.{self.prov_type}.yaml" - def is_within(self): + def is_within(self): # pylint: disable=W0221 """ Uses glob to find the correct provenance yaml spec. Returns True if that file contains a match to this @@ -249,6 +301,7 @@ def __init__(self, pathname) -> None: self.pathname = pathname def path_exists(self) -> bool: + """Check if a path exists""" return os.path.exists(self.pathname) def __str__(self) -> str: @@ -270,18 +323,21 @@ def __init__(self, filename, regex) -> None: self.regex = regex def contains(self) -> bool: + """Checks if the regex matches anywhere in the filetext""" try: - with open(self.filename, "r") as f: + with open(self.filename, "r") as f: # pylint: disable=C0103 filetext = f.read() return self.is_within(filetext) - except Exception: + except Exception: # pylint: disable=W0718 return False def is_within(self, text): + """Check if there's a match for the regex in text""" return search(self.regex, text) is not None def __str__(self) -> str: - return f"{__class__.__name__} expected to find {self.regex} regex match within {self.filename} file but no match was found" + return f"""{__class__.__name__} expected to find {self.regex} + regex match within {self.filename} file but no match was found""" @property def passes(self): @@ -295,7 +351,8 @@ class FileHasNoRegex(FileHasRegex): """ def __str__(self) -> str: - return f"{__class__.__name__} expected to find {self.regex} regex to not match within {self.filename} file but a match was found" + return f"""{__class__.__name__} expected to find {self.regex} + regex to not match within {self.filename} file but a match was found""" @property def passes(self): diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 913a2ab08..0282f888b 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # @@ -37,36 +37,89 @@ import sys import time from contextlib import suppress -from subprocess import PIPE, Popen +from subprocess import TimeoutExpired, run -from test_definitions import OUTPUT_DIR, define_tests +from test_definitions import OUTPUT_DIR, define_tests # pylint: disable=E0401 -def run_single_test(name, test, test_label="", buffer_length=50): - dot_length = buffer_length - len(name) - len(str(test_label)) - print(f"TEST {test_label}: {name}{'.'*dot_length}", end="") - command = test[0] - conditions = test[1] +def get_definition_issues(test): + """ + Function to make sure the test definition was written properly. + :param `test`: The test definition we're checking + :returns: A list of errors found with the test definition + """ + errors = [] + # Check that commands were provided + try: + commands = test["cmds"] + if not isinstance(commands, list): + commands = [commands] + except KeyError: + errors.append("'cmds' flag not defined") + commands = None + + # Check that conditions were provided + if "conditions" not in test: + errors.append("'conditions' flag not defined") + + # Check that correct number of cmds were given depending on + # the number of processes we'll need to start + if commands: + if "num procs" not in test: + num_procs = 1 + else: + num_procs = test["num procs"] + + if num_procs == 1 and len(commands) != 1: + errors.append(f"Need 1 'cmds' since 'num procs' is 1 but {len(commands)} 'cmds' were given") + elif num_procs == 2 and len(commands) != 2: + errors.append(f"Need 2 'cmds' since 'num procs' is 2 but {len(commands)} 'cmds' were given") + + return errors + + +def run_single_test(test): + """ + Runs a single test and returns whether it passed or not + and information about the test for logging purposes. + :param `test`: A dictionary that defines the test + :returns: A tuple of type (bool, dict) where the bool + represents if the test passed and the dict + contains info about the test. + """ + # Parse the test definition + commands = test.pop("cmds", None) + if not isinstance(commands, list): + commands = [commands] + conditions = test.pop("conditions", None) if not isinstance(conditions, list): conditions = [conditions] + cleanup = test.pop("cleanup", None) + num_procs = test.pop("num procs", 1) start_time = time.time() - process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) - stdout, stderr = process.communicate() + # As of now the only time we need 2 processes is to test stop-workers + # Therefore we only care about the result of the second process + if num_procs == 2: + # First command should start the workers + try: + run(commands[0], timeout=8, capture_output=True, shell=True) + except TimeoutExpired: + pass + # Second command should stop the workers + result = run(commands[1], capture_output=True, text=True, shell=True) + else: + # Run the commands + result = run(commands[0], capture_output=True, text=True, shell=True) end_time = time.time() total_time = end_time - start_time - if stdout is not None: - stdout = stdout.decode("utf-8") - if stderr is not None: - stderr = stderr.decode("utf-8") - return_code = process.returncode info = { "total_time": total_time, - "command": command, - "stdout": stdout, - "stderr": stderr, - "return_code": return_code, + "command": commands, + "stdout": result.stdout, + "stderr": result.stderr, + "return_code": result.returncode, "violated_condition": None, } @@ -78,11 +131,10 @@ def run_single_test(name, test, test_label="", buffer_length=50): info["violated_condition"] = (condition, i, len(conditions)) break - if len(test) == 4: - end_process = Popen(test[3], stdout=PIPE, stderr=PIPE, shell=True) - end_stdout, end_stderr = end_process.communicate() - info["end_stdout"] = end_stdout - info["end_stderr"] = end_stderr + if cleanup: + end_process = run(cleanup, capture_output=True, text=True, shell=True) + info["end_stdout"] = end_process.stdout + info["end_stderr"] = end_process.stderr return passed, info @@ -96,7 +148,7 @@ def clear_test_studies_dir(): shutil.rmtree(f"./{OUTPUT_DIR}") -def process_test_result(passed, info, is_verbose, exit): +def process_test_result(passed, info, is_verbose, exit_on_failure): """ Process and print test results to the console. """ @@ -104,9 +156,9 @@ def process_test_result(passed, info, is_verbose, exit): if passed is False and "merlin: command not found" in info["stderr"]: print(f"\nMissing from environment:\n\t{info['stderr']}") return None - elif passed is False: + if passed is False: print("FAIL") - if exit is True: + if exit_on_failure is True: return None else: print("pass") @@ -127,11 +179,18 @@ def process_test_result(passed, info, is_verbose, exit): return passed -def run_tests(args, tests): +def filter_tests_to_run(args, tests): """ - Run all inputted tests. - :param `tests`: a dictionary of - {"test_name" : ("test_command", [conditions])} + Filter which tests to run based on args. The tests to + run will be what makes up the args.ids list. This function + will return whether we're being selective with what tests + we run and also the number of tests that match the filter. + :param `args`: CLI args given by user + :param `tests`: a dict of all the tests that exist + :returns: a tuple where the first entry is a bool on whether + we filtered the tests at all and the second entry + is an int representing the number of tests we're + going to run. """ selective = False n_to_run = len(tests) @@ -140,19 +199,27 @@ def run_tests(args, tests): raise ValueError(f"Test ids must be between 1 and {len(tests)}, inclusive.") selective = True n_to_run = len(args.ids) - elif args.local is not None: + elif args.local is not None or args.distributed is not None: args.ids = [] n_to_run = 0 selective = True for test_id, test in enumerate(tests.values()): - # Ensures that test definitions are atleast size 3. - # 'local' variable is stored in 3rd element of the test definitions, - # but an optional 4th element can be provided for an ending command - # to be ran after all checks have been made. - if len(test) >= 3 and test[2] == "local": + run_type = test.pop("run type", None) + if (args.local and run_type == "local") or (args.distributed and run_type == "distributed"): args.ids.append(test_id + 1) n_to_run += 1 + return selective, n_to_run + + +def run_tests(args, tests): # pylint: disable=R0914 + """ + Run all inputted tests. + :param `tests`: a dictionary of + {"test_name" : ("test_command", [conditions])} + """ + selective, n_to_run = filter_tests_to_run(args, tests) + print(f"Running {n_to_run} integration tests...") start_time = time.time() @@ -163,14 +230,32 @@ def run_tests(args, tests): if selective and test_label not in args.ids: total += 1 continue - try: - passed, info = run_single_test(test_name, test, test_label) - except Exception as e: - print(e) + dot_length = 50 - len(test_name) - len(str(test_label)) + print(f"TEST {test_label}: {test_name}{'.'*dot_length}", end="") + # Check the format of the test definition + definition_issues = get_definition_issues(test) + if definition_issues: + print("FAIL") + print(f"\tTest with name '{test_name}' has problems with its' test definition. Skipping...") + if args.verbose: + print(f"\tFound {len(definition_issues)} problems with the definition of '{test_name}':") + for error in definition_issues: + print(f"\t- {error}") + total += 1 passed = False - info = None + if args.exit: + result = None + else: + result = False + else: + try: + passed, info = run_single_test(test) + except Exception as e: # pylint: disable=C0103,W0718 + print(e) + passed = False + info = None + result = process_test_result(passed, info, args.verbose, args.exit) - result = process_test_result(passed, info, args.verbose, args.exit) clear_test_studies_dir() if result is None: print("Exiting early") @@ -190,6 +275,10 @@ def run_tests(args, tests): def setup_argparse(): + """ + Using ArgumentParser, define the arguments allowed for this script. + :returns: An ArgumentParser object + """ parser = argparse.ArgumentParser(description="run_tests cli parser") parser.add_argument( "--exit", @@ -198,6 +287,7 @@ def setup_argparse(): ) parser.add_argument("--verbose", action="store_true", help="Flag for more detailed output messages") parser.add_argument("--local", action="store_true", default=None, help="Run only local tests") + parser.add_argument("--distributed", action="store_true", default=None, help="Run only distributed tests") parser.add_argument( "--ids", action="store", @@ -207,9 +297,29 @@ def setup_argparse(): default=None, help="Provide space-delimited ids of tests you want to run. Example: '--ids 1 5 8 13'", ) + parser.add_argument( + "--display-tests", action="store_true", default=False, help="Display a table format of test names and ids" + ) return parser +def display_tests(tests): + """ + Helper function to display a table of tests and associated ids. + Helps choose which test to run if you're trying to debug and use + the --id flag. + :param `tests`: A dict of tests (Dict) + """ + from merlin.display import tabulate_info # pylint: disable=C0415 + + test_names = list(tests.keys()) + test_table = [(i + 1, test_names[i]) for i in range(len(test_names))] + test_table.insert(0, ("ID", "Test Name")) + print() + tabulate_info(test_table, headers="firstrow") + print() + + def main(): """ High-level CLI test operations. @@ -219,6 +329,10 @@ def main(): tests = define_tests() + if args.display_tests: + display_tests(tests) + return + clear_test_studies_dir() result = run_tests(args, tests) sys.exit(result) diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index feb5168ba..752b3650d 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -1,4 +1,46 @@ -from conditions import ( +############################################################################### +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.1. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### +""" +This module defines all the integration tests to be ran through run_tests.py. + +Each test looks like: +"test name": { + "cmds": the commands to run, + "conditions": the conditions to check for, + "run type": the type of test (local or distributed), + "cleanup": the command to run after your test in order to clean output, + "num procs": the number of processes you need to start for a test (1 or 2) +} +""" + +from conditions import ( # pylint: disable=E0401 FileHasNoRegex, FileHasRegex, HasRegex, @@ -14,9 +56,11 @@ OUTPUT_DIR = "cli_test_studies" CLEAN_MERLIN_SERVER = "rm -rf appendonly.aof dump.rdb merlin_server/" +# KILL_WORKERS = "pkill -9 -f '.*merlin_test_worker'" +KILL_WORKERS = "pkill -9 -f 'celery'" -def define_tests(): +def define_tests(): # pylint: disable=R0914 """ Returns a dictionary of tests, where the key is the test's name, and the value is a tuple @@ -48,62 +92,64 @@ def define_tests(): pbs_path = f"{examples}/flux/scripts/pbs_test" workers_pbs = f"""PATH="{pbs_path}:$PATH";merlin {err_lvl} run-workers""" lsf = f"{examples}/lsf/lsf_par.yaml" + mul_workers_demo = f"{dev_examples}/multiple_workers.yaml" black = "black --check --target-version py36" config_dir = "./CLI_TEST_MERLIN_CONFIG" release_dependencies = "./requirements/release.txt" basic_checks = { - "merlin": ("merlin", HasReturnCode(1), "local"), - "merlin help": ("merlin --help", HasReturnCode(), "local"), - "merlin version": ("merlin --version", HasReturnCode(), "local"), - "merlin config": ( - f"merlin config -o {config_dir}; rm -rf {config_dir}", - HasReturnCode(), - "local", - ), + "merlin": {"cmds": "merlin", "conditions": HasReturnCode(1), "run type": "local"}, + "merlin help": {"cmds": "merlin --help", "conditions": HasReturnCode(), "run type": "local"}, + "merlin version": {"cmds": "merlin --version", "conditions": HasReturnCode(), "run type": "local"}, + "merlin config": { + "cmds": f"merlin config -o {config_dir}", + "conditions": HasReturnCode(), + "run type": "local", + "cleanup": f"rm -rf {config_dir}", + }, } server_basic_tests = { - "merlin server init": ( - "merlin server init", - HasRegex(".*successful"), - "local", - CLEAN_MERLIN_SERVER, - ), - "merlin server start/stop": ( - """merlin server init; - merlin server start; - merlin server status; - merlin server stop;""", - [ + "merlin server init": { + "cmds": "merlin server init", + "conditions": HasRegex(".*successful"), + "run type": "local", + "cleanup": CLEAN_MERLIN_SERVER, + }, + "merlin server start/stop": { + "cmds": """merlin server init ; + merlin server start ; + merlin server status ; + merlin server stop""", + "conditions": [ HasRegex("Server started with PID [0-9]*"), HasRegex("Merlin server is running"), HasRegex("Merlin server terminated"), ], - "local", - CLEAN_MERLIN_SERVER, - ), - "merlin server restart": ( - """merlin server init; + "run type": "local", + "cleanup": CLEAN_MERLIN_SERVER, + }, + "merlin server restart": { + "cmds": """merlin server init; merlin server start; merlin server restart; merlin server status; merlin server stop;""", - [ + "conditions": [ HasRegex("Server started with PID [0-9]*"), HasRegex("Merlin server is running"), HasRegex("Merlin server terminated"), ], - "local", - CLEAN_MERLIN_SERVER, - ), + "run type": "local", + "cleanup": CLEAN_MERLIN_SERVER, + }, } server_config_tests = { - "merlin server change config": ( - """merlin server init; + "merlin server change config": { + "cmds": """merlin server init; merlin server config -p 8888 -pwd new_password -d ./config_dir -ss 80 -sc 8 -sf new_sf -am always -af new_af.aof; merlin server start; merlin server stop;""", - [ + "conditions": [ FileHasRegex("merlin_server/redis.conf", "port 8888"), FileHasRegex("merlin_server/redis.conf", "requirepass new_password"), FileHasRegex("merlin_server/redis.conf", "dir ./config_dir"), @@ -116,11 +162,11 @@ def define_tests(): HasRegex("Server started with PID [0-9]*"), HasRegex("Merlin server terminated"), ], - "local", - "rm -rf appendonly.aof dump.rdb merlin_server/ config_dir/", - ), - "merlin server config add/remove user": ( - """merlin server init; + "run type": "local", + "cleanup": "rm -rf appendonly.aof dump.rdb merlin_server/ config_dir/", + }, + "merlin server config add/remove user": { + "cmds": """merlin server init; merlin server start; merlin server config --add-user new_user new_password; merlin server stop; @@ -129,129 +175,132 @@ def define_tests(): merlin server config --remove-user new_user; merlin server stop; """, - [ + "conditions": [ FileHasRegex("./merlin_server/redis.users_new", "new_user"), FileHasNoRegex("./merlin_server/redis.users", "new_user"), ], - "local", - CLEAN_MERLIN_SERVER, - ), + "run type": "local", + "cleanup": CLEAN_MERLIN_SERVER, + }, } examples_check = { - "example list": ( - "merlin example list", - HasReturnCode(), - "local", - ), + "example list": { + "cmds": "merlin example list", + "conditions": HasReturnCode(), + "run type": "local", + }, } run_workers_echo_tests = { - "run-workers echo simple_chain": ( - f"{workers} {simple} --echo", - [HasReturnCode(), HasRegex(celery_slurm_regex)], - "local", - ), - "run-workers echo feature_demo": ( - f"{workers} {demo} --echo", - [HasReturnCode(), HasRegex(celery_slurm_regex)], - "local", - ), - "run-workers echo slurm_test": ( - f"{workers} {slurm} --echo", - [HasReturnCode(), HasRegex(celery_slurm_regex)], - "local", - ), - "run-workers echo flux_test": ( - f"{workers} {flux} --echo", - [HasReturnCode(), HasRegex(celery_slurm_regex)], - "local", - ), - "run-workers echo flux_native_test": ( - f"{workers_flux} {flux_native} --echo", - [HasReturnCode(), HasRegex(celery_flux_regex)], - "local", - ), - "run-workers echo pbs_test": ( - f"{workers_pbs} {flux_native} --echo", - [HasReturnCode(), HasRegex(celery_pbs_regex)], - "local", - ), - "run-workers echo override feature_demo": ( - f"{workers} {demo} --echo --vars VERIFY_QUEUE=custom_verify_queue", - [HasReturnCode(), HasRegex("custom_verify_queue")], - "local", - ), + "run-workers echo simple_chain": { + "cmds": f"{workers} {simple} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_slurm_regex)], + "run type": "local", + }, + "run-workers echo feature_demo": { + "cmds": f"{workers} {demo} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_slurm_regex)], + "run type": "local", + }, + "run-workers echo slurm_test": { + "cmds": f"{workers} {slurm} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_slurm_regex)], + "run type": "local", + }, + "run-workers echo flux_test": { + "cmds": f"{workers} {flux} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_slurm_regex)], + "run type": "local", + }, + "run-workers echo flux_native_test": { + "cmds": f"{workers_flux} {flux_native} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_flux_regex)], + "run type": "local", + }, + "run-workers echo pbs_test": { + "cmds": f"{workers_pbs} {flux_native} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_pbs_regex)], + "run type": "local", + }, + "run-workers echo override feature_demo": { + "cmds": f"{workers} {demo} --echo --vars VERIFY_QUEUE=custom_verify_queue", + "conditions": [HasReturnCode(), HasRegex("custom_verify_queue")], + "run type": "local", + }, } wf_format_tests = { - "local minimum_format": ( - f"mkdir {OUTPUT_DIR} ; cd {OUTPUT_DIR} ; merlin run ../{dev_examples}/minimum_format.yaml --local", - StepFileExists( + "local minimum_format": { + "cmds": f"mkdir {OUTPUT_DIR}; cd {OUTPUT_DIR}; merlin run ../{dev_examples}/minimum_format.yaml --local", + "conditions": StepFileExists( "step1", "MERLIN_FINISHED", "minimum_format", OUTPUT_DIR, params=False, ), - "local", - ), - "local no_description": ( - f"mkdir {OUTPUT_DIR} ; cd {OUTPUT_DIR} ; merlin run ../merlin/examples/dev_workflows/no_description.yaml --local", - HasReturnCode(1), - "local", - ), - "local no_steps": ( - f"mkdir {OUTPUT_DIR} ; cd {OUTPUT_DIR} ; merlin run ../merlin/examples/dev_workflows/no_steps.yaml --local", - HasReturnCode(1), - "local", - ), - "local no_study": ( - f"mkdir {OUTPUT_DIR} ; cd {OUTPUT_DIR} ; merlin run ../merlin/examples/dev_workflows/no_study.yaml --local", - HasReturnCode(1), - "local", - ), + "run type": "local", + }, + "local no_description": { + "cmds": f"""mkdir {OUTPUT_DIR}; cd {OUTPUT_DIR}; + merlin run ../merlin/examples/dev_workflows/no_description.yaml --local""", + "conditions": HasReturnCode(1), + "run type": "local", + }, + "local no_steps": { + "cmds": f"mkdir {OUTPUT_DIR}; cd {OUTPUT_DIR}; merlin run ../merlin/examples/dev_workflows/no_steps.yaml --local", + "conditions": HasReturnCode(1), + "run type": "local", + }, + "local no_study": { + "cmds": f"mkdir {OUTPUT_DIR}; cd {OUTPUT_DIR}; merlin run ../merlin/examples/dev_workflows/no_study.yaml --local", + "conditions": HasReturnCode(1), + "run type": "local", + }, } example_tests = { - "example failure": ("merlin example failure", HasRegex("not found"), "local"), - "example simple_chain": ( - f"merlin example simple_chain ; {run} simple_chain.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR} ; rm simple_chain.yaml", - HasReturnCode(), - "local", - ), + "example failure": {"cmds": "merlin example failure", "conditions": HasRegex("not found"), "run type": "local"}, + "example simple_chain": { + "cmds": f"""merlin example simple_chain; + {run} simple_chain.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}; rm simple_chain.yaml""", + "conditions": HasReturnCode(), + "run type": "local", + }, } restart_step_tests = { - "local restart_shell": ( - f"{run} {dev_examples}/restart_shell.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileExists( + "local restart_shell": { + "cmds": f"{run} {dev_examples}/restart_shell.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileExists( "step2", "MERLIN_FINISHED", "restart_shell", OUTPUT_DIR, params=False, ), - "local", - ), - "local restart": ( - f"{run} {dev_examples}/restart.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileExists( + "run type": "local", + }, + "local restart": { + "cmds": f"{run} {dev_examples}/restart.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileExists( "final_check_for_no_hard_fails", "MERLIN_FINISHED", "restart", OUTPUT_DIR, params=False, ), - "local", - ), + "run type": "local", + }, } restart_wf_tests = { - "restart local simple_chain": ( - f"{run} {simple} --local --vars OUTPUT_PATH=./{OUTPUT_DIR} ; {restart} $(find ./{OUTPUT_DIR} -type d -name 'simple_chain_*') --local", - HasReturnCode(), - "local", - ), + "restart local simple_chain": { + "cmds": f"""{run} {simple} --local --vars OUTPUT_PATH=./{OUTPUT_DIR}; + {restart} $(find ./{OUTPUT_DIR} -type d -name 'simple_chain_*') --local""", + "conditions": HasReturnCode(), + "run type": "local", + }, } dry_run_tests = { - "dry feature_demo": ( - f"{run} {demo} --local --dry --vars OUTPUT_PATH=./{OUTPUT_DIR}", - [ + "dry feature_demo": { + "cmds": f"{run} {demo} --local --dry --vars OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": [ StepFileExists( "verify", "verify_*.sh", @@ -261,61 +310,61 @@ def define_tests(): ), HasReturnCode(), ], - "local", - ), - "dry launch slurm": ( - f"{run} {slurm} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex("runs", "*/runs.slurm.sh", "slurm_test", OUTPUT_DIR, "srun "), - "local", - ), - "dry launch flux": ( - f"{run} {flux} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex( + "run type": "local", + }, + "dry launch slurm": { + "cmds": f"{run} {slurm} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileHasRegex("runs", "*/runs.slurm.sh", "slurm_test", OUTPUT_DIR, "srun "), + "run type": "local", + }, + "dry launch flux": { + "cmds": f"{run} {flux} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileHasRegex( "runs", "*/runs.slurm.sh", "flux_test", OUTPUT_DIR, get_flux_cmd("flux", no_errors=True), ), - "local", - ), - "dry launch lsf": ( - f"{run} {lsf} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex("runs", "*/runs.slurm.sh", "lsf_par", OUTPUT_DIR, "jsrun "), - "local", - ), - "dry launch slurm restart": ( - f"{run} {slurm_restart} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex( + "run type": "local", + }, + "dry launch lsf": { + "cmds": f"{run} {lsf} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileHasRegex("runs", "*/runs.slurm.sh", "lsf_par", OUTPUT_DIR, "jsrun "), + "run type": "local", + }, + "dry launch slurm restart": { + "cmds": f"{run} {slurm_restart} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileHasRegex( "runs", "*/runs.restart.slurm.sh", "slurm_par_restart", OUTPUT_DIR, "srun ", ), - "local", - ), - "dry launch flux restart": ( - f"{run} {flux_restart} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex( + "run type": "local", + }, + "dry launch flux restart": { + "cmds": f"{run} {flux_restart} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileHasRegex( "runs_rs", "*/runs_rs.restart.slurm.sh", "flux_par_restart", OUTPUT_DIR, get_flux_cmd("flux", no_errors=True), ), - "local", - ), + "run type": "local", + }, } other_local_tests = { - "local simple_chain": ( - f"{run} {simple} --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", - HasReturnCode(), - "local", - ), - "local override feature_demo": ( - f"{run} {demo} --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR} --local", - [ + "local simple_chain": { + "cmds": f"{run} {simple} --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": HasReturnCode(), + "run type": "local", + }, + "local override feature_demo": { + "cmds": f"{run} {demo} --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR} --local", + "conditions": [ HasReturnCode(), ProvenanceYAMLFileHasRegex( regex=r"HELLO: \$\(SCRIPTS\)/hello_world.py", @@ -356,10 +405,11 @@ def define_tests(): params=True, ), ], - "local", - ), + "run type": "local", + }, # "local restart expand name": ( - # f"{run} {demo} --local --vars OUTPUT_PATH=./{OUTPUT_DIR} NAME=test_demo ; {restart} $(find ./{OUTPUT_DIR} -type d -name 'test_demo_*') --local", + # f"""{run} {demo} --local --vars OUTPUT_PATH=./{OUTPUT_DIR} NAME=test_demo; + # {restart} $(find ./{OUTPUT_DIR} -type d -name 'test_demo_*') --local""", # [ # HasReturnCode(), # ProvenanceYAMLFileHasRegex( @@ -374,19 +424,23 @@ def define_tests(): # ], # "local", # ), - "local csv feature_demo": ( - f"echo 42.0,47.0 > foo_testing_temp.csv; merlin run {demo} --samplesfile foo_testing_temp.csv --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; rm -f foo_testing_temp.csv", - [HasRegex("1 sample loaded."), HasReturnCode()], - "local", - ), - "local tab feature_demo": ( - f"echo '42.0\t47.0\n7.0 5.3' > foo_testing_temp.tab; merlin run {demo} --samplesfile foo_testing_temp.tab --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; rm -f foo_testing_temp.tab", - [HasRegex("2 samples loaded."), HasReturnCode()], - "local", - ), - "local pgen feature_demo": ( - f"{run} {demo} --pgen {demo_pgen} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local", - [ + "local csv feature_demo": { + "cmds": f"""echo 42.0,47.0 > foo_testing_temp.csv; + merlin run {demo} --samplesfile foo_testing_temp.csv --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; + rm -f foo_testing_temp.csv""", + "conditions": [HasRegex("1 sample loaded."), HasReturnCode()], + "run type": "local", + }, + "local tab feature_demo": { + "cmds": f"""echo '42.0\t47.0\n7.0 5.3' > foo_testing_temp.tab; + merlin run {demo} --samplesfile foo_testing_temp.tab --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; + rm -f foo_testing_temp.tab""", + "conditions": [HasRegex("2 samples loaded."), HasReturnCode()], + "run type": "local", + }, + "local pgen feature_demo": { + "cmds": f"{run} {demo} --pgen {demo_pgen} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local", + "conditions": [ ProvenanceYAMLFileHasRegex( regex=r"\[0.3333333", name="feature_demo", @@ -402,35 +456,118 @@ def define_tests(): ), HasReturnCode(), ], - "local", - ), + "run type": "local", + }, } - provenence_equality_checks = { # noqa: F841 - "local provenance spec equality": ( - f"{run} {simple} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local ; cp $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml') ./{OUTPUT_DIR}/FILE1 ; rm -rf ./{OUTPUT_DIR}/simple_chain_* ; {run} ./{OUTPUT_DIR}/FILE1 --vars OUTPUT_PATH=./{OUTPUT_DIR} --local ; cmp ./{OUTPUT_DIR}/FILE1 $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml')", - HasReturnCode(), - "local", - ), + provenence_equality_checks = { # noqa: F841 pylint: disable=W0612 + "local provenance spec equality": { + "cmds": f"""{run} {simple} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; + cp $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml') ./{OUTPUT_DIR}/FILE1; + rm -rf ./{OUTPUT_DIR}/simple_chain_*; + {run} ./{OUTPUT_DIR}/FILE1 --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; + cmp ./{OUTPUT_DIR}/FILE1 $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml')""", # pylint: disable=C0301 + "conditions": HasReturnCode(), + "run type": "local", + }, } - style_checks = { # noqa: F841 - "black check merlin": (f"{black} merlin/", HasReturnCode(), "local"), - "black check tests": (f"{black} tests/", HasReturnCode(), "local"), + style_checks = { # noqa: F841 pylint: disable=W0612 + "black check merlin": {"cmds": f"{black} merlin/", "conditions": HasReturnCode(), "run type": "local"}, + "black check tests": {"cmds": f"{black} tests/", "conditions": HasReturnCode(), "run type": "local"}, } dependency_checks = { - "deplic no GNU": ( - f"deplic {release_dependencies}", - [HasRegex("GNU", negate=True), HasRegex("GPL", negate=True)], - "local", - ), + "deplic no GNU": { + "cmds": f"deplic {release_dependencies}", + "conditions": [HasRegex("GNU", negate=True), HasRegex("GPL", negate=True)], + "run type": "local", + }, + } + stop_workers_tests = { + "stop workers no workers": { + "cmds": "merlin stop-workers", + "conditions": [ + HasReturnCode(), + HasRegex("No workers found to stop"), + HasRegex("step_1_merlin_test_worker", negate=True), + HasRegex("step_2_merlin_test_worker", negate=True), + HasRegex("other_merlin_test_worker", negate=True), + ], + "run type": "distributed", + }, + "stop workers no flags": { + "cmds": [ + f"{workers} {mul_workers_demo}", + "merlin stop-workers", + ], + "conditions": [ + HasReturnCode(), + HasRegex("No workers found to stop", negate=True), + HasRegex("step_1_merlin_test_worker"), + HasRegex("step_2_merlin_test_worker"), + HasRegex("other_merlin_test_worker"), + ], + "run type": "distributed", + "cleanup": KILL_WORKERS, + "num procs": 2, + }, + "stop workers with spec flag": { + "cmds": [ + f"{workers} {mul_workers_demo}", + f"merlin stop-workers --spec {mul_workers_demo}", + ], + "conditions": [ + HasReturnCode(), + HasRegex("No workers found to stop", negate=True), + HasRegex("step_1_merlin_test_worker"), + HasRegex("step_2_merlin_test_worker"), + HasRegex("other_merlin_test_worker"), + ], + "run type": "distributed", + "cleanup": KILL_WORKERS, + "num procs": 2, + }, + "stop workers with workers flag": { + "cmds": [ + f"{workers} {mul_workers_demo}", + "merlin stop-workers --workers step_1_merlin_test_worker step_2_merlin_test_worker", + ], + "conditions": [ + HasReturnCode(), + HasRegex("No workers found to stop", negate=True), + HasRegex("step_1_merlin_test_worker"), + HasRegex("step_2_merlin_test_worker"), + HasRegex("other_merlin_test_worker", negate=True), + ], + "run type": "distributed", + "cleanup": KILL_WORKERS, + "num procs": 2, + }, + "stop workers with queues flag": { + "cmds": [ + f"{workers} {mul_workers_demo}", + "merlin stop-workers --queues hello_queue", + ], + "conditions": [ + HasReturnCode(), + HasRegex("No workers found to stop", negate=True), + HasRegex("step_1_merlin_test_worker"), + HasRegex("step_2_merlin_test_worker", negate=True), + HasRegex("other_merlin_test_worker", negate=True), + ], + "run type": "distributed", + "cleanup": KILL_WORKERS, + "num procs": 2, + }, } distributed_tests = { # noqa: F841 - "run and purge feature_demo": ( - f"{run} {demo} ; {purge} {demo} -f", - HasReturnCode(), - ), - "remote feature_demo": ( - f"{run} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; {workers} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers", - [ + "run and purge feature_demo": { + "cmds": f"{run} {demo}; {purge} {demo} -f", + "conditions": HasReturnCode(), + "run type": "distributed", + }, + "remote feature_demo": { + "cmds": f"""{run} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers; + {workers} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers""", + "conditions": [ HasReturnCode(), ProvenanceYAMLFileHasRegex( regex="cli_test_demo_workers:", @@ -446,27 +583,8 @@ def define_tests(): params=True, ), ], - ), - # this test is deactivated until the --spec option for stop-workers is active again - # "stop workers for distributed feature_demo": ( - # f"{run} {demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; {workers} {demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; sleep 20 ; merlin stop-workers --spec {demo}", - # [ - # HasReturnCode(), - # ProvenanceYAMLFileHasRegex( - # regex="cli_test_demo_workers:", - # name="feature_demo", - # output_path=OUTPUT_DIR, - # provenance_type="expanded", - # ), - # StepFileExists( - # "verify", - # "MERLIN_FINISHED", - # "feature_demo", - # OUTPUT_DIR, - # params=True, - # ), - # ], - # ), + "run type": "distributed", + }, } # combine and return test dictionaries @@ -486,6 +604,7 @@ def define_tests(): # provenence_equality_checks, # omitting provenance equality check because it is broken # style_checks, # omitting style checks due to different results on different machines dependency_checks, + stop_workers_tests, distributed_tests, ]: all_tests.update(test_dict)