From a40ed4625d7eb3ad296e5194c3ebd36992f9ad76 Mon Sep 17 00:00:00 2001 From: Ryan Lee <44886374+ryannova@users.noreply.github.com> Date: Wed, 21 Dec 2022 15:01:41 -0800 Subject: [PATCH 1/2] Version 1.9.1 (#395) * Create python-publish.yml (#353) * Adding GitHub action for publishing to pypi * Workflows Community Initiative Metadata (#355) * Added Workflows Community Initiative metadata info; fixed some old links * Run black * Add updates for lgtm CI security site (#357) * Update code to remove LGTM Errors and Warnings and implement Recommendations. * Change BaseException to Exception. * Add lgtm config file. * Changes for flake8. * Add TypeError yo yam read. * Add TypeError to yaml read. * Just return when successful on the yaml read. * Fix typo. * Add merlin/examples to lgtm exclude list as well. * Add ssl comment. * Fix typo. * Update version to 1.8.5. * Update conf.py for new sphinx versions. * Added Server Command Feature to Merlin (#360) * Added merlin server capability to initalize, monitor and stop redis containers * Added configuration for singularity, docker, and podman in merlin/server/ * Created documentation for "merlin server" command * Added tests to initialize, start and stop a singularity+redis server to the local test suite. (Future: add to the "distributed tests" connecting to that server and running merlin) Co-authored-by: Ryan Lee & Joe Koning * Fix lgtm returns (#363) * Changed script sys.exit commands to use try/catch, per lgtm recommendation * Allow for flux exec arguments to limit the number of celery workers. (#366) * Added the flux_exec batch argument to allow for flux exec arguments, e.g. flux_exec: flux exec -r "0-1" to run celery workers only on ranks 0 and 1 of a multi-rank allocation. * Remove period. * Merlin server configuration through commands (#365) * Reorganized functions within server_setup and server_config * Rename server_setup file to server_commands * Added password generation for redis container in merlin server * Changed redis configuration to require password authentication * Added merlin config flags ipaddress, port, password, directory, snapshot_seconds, snapshot_changes, snapshot_file, append_mode, append_file * Added server_util.py * Added merlin user file into merlin server config * Added RedisConfig class to interact and change config values within the redis config file * Added merlin server restart * Updated info messages * Added function to add/remove users and store info to user file * Update running container with new users and removed users * Added ServerConfig, ProcessConfig, ContainerConfig, and ContainerFormatConfig classes to interact with configuration files * Adjusted adding user and password to use values in config files * Updated host in redis.conf * Updated pull_server_image step * Moved creation of local merlin config to create_server_config() * Added placeholder for documentation of restart and config commands for merlin server * Bugfix/changelog ci (#370) * remove deprecated gitlab ci file * Change CHANGELOG test to work for PRs other than to main * App.yaml for merlin server (#369) * Added AppYaml class to pull app.yaml and make changes required for merlin server configuration * Applied AppYaml class and added log message to inform users to use new app.yaml to use merlin server * Update LOG messages to inform users regarding local runs and instruct users of how to use app.yaml for local configuration * Changed type to image type in ContainerConfig * Shorten CHANGELOG.md for merlin server changes * Updated read in AppYaml to utilize merlin.util.load_yaml * Updated merlin server unit testing (#372) * Added additional tests for merlin server in test definitions * Fixed directory change to create a new directory if one doesn't exist * Updated redis version to provide acl user channel support * Addition of new shortcuts in specification file (#375) * Added five shortcuts to the specification definition MERLIN_SAMPLE_VECTOR, MERLIN_SAMPLE_NAMES, MERLIN_SPEC_ORIGINAL_TEMPLATE, MERLIN_SPEC_EXECUTED_RUN, MERLIN_SPEC_ARCHIVED_COPY * Added documentation for the above shortcuts. Co-authored-by: Jim Gaffney * Remove emoji from issue templates (#377) * Update bug_report.md remove "buggy" emoji * Update feature_request.md * Update question.md * Update CHANGELOG.md * Update CHANGELOG.md typo fix Co-authored-by: Ryan Lee * Update contribute.rst Remove more emoji from docs that are breaking pdf builds * Update cert_req to cert_regs in the docs. (#379) Co-authored-by: Ryan Lee <44886374+ryannova@users.noreply.github.com> * Ssl server check fixes (#380) * Add ssl to the Connection object for checking broker and results server acess. * Update CHANGELOG * Update documentation in tutorial and merlin server (#378) * Updated installation in instroduction and removed redis requirements * Removed pip headers and added commands for merlin server into installation * Removed additional references to old redis way and update description of merlin server * Remove more emoji from docs that are breaking pdf builds * Updated CHANGELOG to reflect changes to documentation Co-authored-by: Luc Peterson * Update MANIFEST.in (#381) * Update MANIFEST.in Add .temp to examples in MANIFEST, so that they get bundled with pypi releases * Update CHANGELOG.md * Add support for non-merlin blocks in specification file (#376) * Adding support for "user" block in _dict_to_string method * Updated CHANGELOG * Updated Merlin Spec docs * Added user block in feature_demo.yaml example Co-authored-by: Jim Gaffney * Update Merlin Server (#385) * Added condition for fatal error from redis server * Update default value for config_dir * Updated fix-style target in Makefile to be consistent with other style related targets * Update default password to use generated password * Updated run user to be default rather than created user * Updated singularity command to specify configuration directory as home directory to solve unaccessible directory issue * Update merlin to use app.yaml configuration rather than its own configuration file * Docs/install changes (#383) Many modifications to documentation, including installation instructions and formatting fixes. * Maestro v 1.1.9dev1 Compatibility (#388) Maestro up to date compatibility Also unpacked maestro DAG to just use what we need, which should help reduce task message size and perhaps allow us to use other serializers in the future. * Bump certifi from 2022.9.24 to 2022.12.7 in /docs (#387) Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.9.24 to 2022.12.7. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2022.09.24...2022.12.07) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Release 1.9.0 (#390) * Make CHANGELOG more concise * Updated Merlin version and added License to files missing it * Incremented python version for workflow test * fix merlinspec not being installed with pip and python 3.7 issues with celery * update changelog and version to 1.9.1 * fix a numpy issue on new numpy version release * modify changelog to show numpy fix * add version change to all files * re-add fix for numpy since it got removed in the last commit by accident * revert utils.py back to previous implementation * change dtype to python str type * Hotfix for merlin server unable to write config files. (#394) * Hotfix for merlin server unable to write config files. Change files to modules and copy files from new file modules Signed-off-by: dependabot[bot] Co-authored-by: Luc Peterson Co-authored-by: Joseph M. Koning Co-authored-by: Joe Koning Co-authored-by: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Brian Gunnarson --- CHANGELOG.md | 7 ++++ MANIFEST.in | 1 + Makefile | 2 +- merlin/__init__.py | 4 +-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- .../security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 29 +++++++++++++++++ merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 32 ++++++++++++------- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 6 ++-- requirements/release.txt | 1 + setup.py | 2 +- tests/integration/run_tests.py | 2 +- 55 files changed, 112 insertions(+), 64 deletions(-) create mode 100644 merlin/server/__init__.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a9cde753..5e480c8d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.9.1] +### Fixed +- Added merlin/spec/merlinspec.json to MANIFEST.in so pip will actually install it when ran +- Fixed a bug where "from celery import Celery" was failing on python 3.7 +- Numpy error about numpy.str not existing from a new numpy release +- Made merlin server configurations into modules that can be loaded and written to users + ## [1.9.0] ### Added - Added support for Python 3.11 diff --git a/MANIFEST.in b/MANIFEST.in index cefbd23a5..11e403335 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ recursive-include merlin/server *.yaml *.py recursive-include merlin/examples *.py *.yaml *.c *.json *.sbatch *.bsub *.txt *.temp include requirements.txt include requirements/* +include merlin/spec/merlinspec.json \ No newline at end of file diff --git a/Makefile b/Makefile index b0b1fb0a9..570ee3e25 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index aa33bc4d2..eaf3ff668 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.9.0" +__version__ = "1.9.1" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 0b5971627..a9c6093ca 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index ebac67eec..0041523b2 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index b02f9e909..30fea7f47 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 26c64e9e2..da214275b 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 8d9a89285..274ddb062 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 1859a55df..591f9a49f 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 6f3e58e9a..4861a757b 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index e378573c4..229f00851 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 16365e32c..1ba552626 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 9820e1041..6049bb5b5 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 027bcd291..352da5550 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 7ade90e05..b460be069 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index a546591f1..e30a18ca0 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index a8c0a1ef2..7822e7086 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 5dd08ddfb..bd22ddaa3 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 335bd05f5..e435e1ba0 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index cd47be750..626b776d5 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index 3b1469f70..30140981e 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 39c5cf2e0..43573b416 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index d59fc8511..2796465ab 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 269bf6097..816546f67 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 33c6776e4..f540e6f1f 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 9e207b749..5de8423b0 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 4195ceacc..a1e4de372 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 8858dcfad..90711756c 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py new file mode 100644 index 000000000..7155d0c5f --- /dev/null +++ b/merlin/server/__init__.py @@ -0,0 +1,29 @@ +############################################################################### +# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.0. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 8a570b70d..38359938c 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 57cb5af22..ea5c8f1a8 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # @@ -32,7 +32,6 @@ import logging import os import random -import shutil import string import subprocess from io import BufferedReader @@ -40,6 +39,12 @@ import yaml + +try: + import importlib.resources as resources +except ImportError: + import importlib_resources as resources + from merlin.server.server_util import ( CONTAINER_TYPES, MERLIN_CONFIG_DIR, @@ -158,18 +163,22 @@ def create_server_config() -> bool: continue LOG.info(f"Copying file {file} to configuration directory.") try: - shutil.copy(os.path.join(os.path.dirname(os.path.abspath(__file__)), file), config_dir) + with resources.path("merlin.server", file) as config_file: + with open(file_path, "w") as outfile, open(config_file, "r") as infile: + outfile.write(infile.read()) except OSError: LOG.error(f"Destination location {config_dir} is not writable.") return False # Load Merlin Server Configuration and apply it to app.yaml - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), MERLIN_SERVER_CONFIG)) as f: - main_server_config = yaml.load(f, yaml.Loader) - filename = LOCAL_APP_YAML if os.path.exists(LOCAL_APP_YAML) else AppYaml.default_filename - merlin_app_yaml = AppYaml(filename) - merlin_app_yaml.update_data(main_server_config) - merlin_app_yaml.write(filename) + with resources.path("merlin.server", MERLIN_SERVER_CONFIG) as merlin_server_config: + with open(merlin_server_config) as f: + main_server_config = yaml.load(f, yaml.Loader) + filename = LOCAL_APP_YAML if os.path.exists(LOCAL_APP_YAML) else AppYaml.default_filename + merlin_app_yaml = AppYaml(filename) + merlin_app_yaml.update_data(main_server_config) + merlin_app_yaml.write(filename) + LOG.info("Applying merlin server configuration to app.yaml") server_config = pull_server_config() if not server_config: @@ -301,8 +310,9 @@ def pull_server_image() -> bool: if not os.path.exists(os.path.join(config_dir, config_file)): LOG.info("Copying default redis configuration file.") try: - file_dir = os.path.dirname(os.path.abspath(__file__)) - shutil.copy(os.path.join(file_dir, config_file), config_dir) + with resources.path("merlin.server", config_file) as file: + with open(os.path.join(config_dir, config_file), "w") as outfile, open(file, "r") as infile: + outfile.write(infile.read()) except OSError: LOG.error(f"Destination location {config_dir} is not writable.") return False diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index e6eb6379d..16d2c3686 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 950ceb253..46db0fd4f 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 1c0e9fa42..46f71882f 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 254ba80be..710bc2500 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index c4fcfee97..5ca2eea6b 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index a6ecdae2a..9d93197bf 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 7155d0c5f..d56e84f40 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index f395c5d80..71e89bf84 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 34393f967..97fc40289 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 838c14762..fbd7dd6d7 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 826c1d2e3..ab998140a 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 5e7d89e43..3a55df606 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index b7f990a2a..9d36e4173 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index 2959b79e2..93777d95d 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # @@ -240,9 +240,9 @@ def load_array_file(filename, ndmin=2): ) # Make sure text files load as strings with minimum number of dimensions elif protocol == "csv": - array = np.loadtxt(filename, delimiter=",", ndmin=ndmin, dtype=np.str) + array = np.loadtxt(filename, delimiter=",", ndmin=ndmin, dtype=str) elif protocol == "tab": - array = np.loadtxt(filename, ndmin=ndmin, dtype=np.str) + array = np.loadtxt(filename, ndmin=ndmin, dtype=str) else: raise TypeError( f"{protocol} is not a valid array file extension.\ diff --git a/requirements/release.txt b/requirements/release.txt index 821589c41..11fd85129 100644 --- a/requirements/release.txt +++ b/requirements/release.txt @@ -2,6 +2,7 @@ cached_property celery[redis,sqlalchemy]>=5.0.3 coloredlogs cryptography +importlib_metadata<5.0.0; python_version == '3.7' importlib_resources; python_version < '3.7' maestrowf>=1.1.9dev1 numpy diff --git a/setup.py b/setup.py index 9bf170126..06706ca57 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index bfd80eb83..913a2ab08 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.9.0. +# This file is part of Merlin, Version: 1.9.1. # # For details, see https://github.com/LLNL/merlin. # From 37661756d084169d3b9c947f72918bd916ff5180 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 15 Mar 2023 14:56:12 -0700 Subject: [PATCH 2/2] Update integration test suite and add stop-workers tests Update integration test suite and add stop-workers tests Update integration test suite and add stop-workers tests modify changelog to show test changes update copyright year and CI explanation --- .github/workflows/push-pr_workflow.yml | 8 +- .gitignore | 3 + CHANGELOG.md | 12 + LICENSE | 2 +- Makefile | 25 +- config.mk | 2 + merlin/__init__.py | 2 +- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- .../security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 98 ++- merlin/examples/__init__.py | 2 +- .../dev_workflows/multiple_workers.yaml | 56 ++ merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/README.md | 163 ++++- tests/integration/conditions.py | 83 ++- tests/integration/run_tests.py | 200 +++++-- tests/integration/test_definitions.py | 563 +++++++++++------- 61 files changed, 942 insertions(+), 371 deletions(-) create mode 100644 merlin/examples/dev_workflows/multiple_workers.yaml diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 4d7a51ccb..e2d9e164e 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -131,7 +131,7 @@ jobs: run: | python3 -m pytest tests/unit/ - - name: Run integration test suite + - name: Run integration test suite for local tests run: | python3 tests/integration/run_tests.py --verbose --local @@ -179,7 +179,7 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt - - name: Install merlin to run unit tests + - name: Install merlin and setup redis as the broker run: | pip3 install -e . merlin config --broker redis @@ -189,12 +189,12 @@ jobs: merlin example feature_demo pip3 install -r feature_demo/requirements.txt - - name: Run integration test suite for Redis + - name: Run integration test suite for distributed tests env: REDIS_HOST: redis REDIS_PORT: 6379 run: | - python3 tests/integration/run_tests.py --verbose --ids 31 32 + python3 tests/integration/run_tests.py --verbose --distributed # - name: Setup rabbitmq config # run: | diff --git a/.gitignore b/.gitignore index e427f8ad4..8b0fb8ad2 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,9 @@ ARCHIVE_DIR *_OUTPUT/ *_OUTPUT_D/ *_ensemble_*/ +studies/ +appendonlydir/ +cli_test_studies/ # Scheduler logs flux.out diff --git a/CHANGELOG.md b/CHANGELOG.md index 502d48379..3aa643c8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,15 +8,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Pip wheel wasn't including .sh files for merlin examples - The learn.py script in the openfoam_wf* examples will now create the missing Energy v Lidspeed plot + ### Added - Now loads np.arrays of dtype='object', allowing mix-type sample npy - Added a singularity container openfoam_wf example - Added flux native worker launch support - Added PBS flux launch support - Added check_for_flux, check_for_slurm, check_for_lsf, and check_for_pbs utility functions +- Tests for the `stop-workers` command +- A function in `run_tests.py` to check that an integration test definition is formatted correctly +- A new dev_workflow example `multiple_workers.yaml` that's used for testing the `stop-workers` command +- Ability to start 2 subprocesses for a single test +- Added the --distributed and --display-table flags to run_tests.py + - --distributed: only run distributed tests + - --display-tests: displays a table of all existing tests and the id associated with each test ### Changed - Changed celery_regex to celery_slurm_regex in test_definitions.py +- Reformatted how integration tests are defined and part of how they run + - Test values are now dictionaries rather than tuples + - Stopped using `subprocess.Popen()` and `subprocess.communicate()` to run tests and now instead use `subprocess.run()` for simplicity and to keep things up-to-date with the latest subprocess release (`run()` will call `Popen()` and `communicate()` under the hood so we don't have to handle that anymore) +- Rewrote the README in the integration tests folder to explain the new integration test format ## [1.9.1] ### Fixed diff --git a/LICENSE b/LICENSE index 746aac58b..3adc85cb9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 Lawrence Livermore National Laboratory +Copyright (c) 2023 Lawrence Livermore National Laboratory Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index fd6949bcf..897b44a31 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # @@ -38,6 +38,8 @@ include config.mk .PHONY : e2e-tests-diagnostic .PHONY : e2e-tests-local .PHONY : e2e-tests-local-diagnostic +.PHONY : e2e-tests-distributed +.PHONY : e2e-tests-distributed-diagnostic .PHONY : tests .PHONY : check-flake8 .PHONY : check-black @@ -109,6 +111,16 @@ e2e-tests-local-diagnostic: $(PYTHON) $(TEST)/integration/run_tests.py --local --verbose +e2e-tests-distributed: + . $(VENV)/bin/activate; \ + $(PYTHON) $(TEST)/integration/run_tests.py --distributed; \ + + +e2e-tests-distributed-diagnostic: + . $(VENV)/bin/activate; \ + $(PYTHON) $(TEST)/integration/run_tests.py --distributed --verbose + + # run unit and CLI tests tests: unit-tests e2e-tests @@ -185,6 +197,17 @@ version: find tests/ -type f -print0 | xargs -0 sed -i 's/Version: $(VSTRING)/Version: $(VER)/g' find Makefile -type f -print0 | xargs -0 sed -i 's/Version: $(VSTRING)/Version: $(VER)/g' +# Increment copyright year +year: +# do LICENSE (no comma after year) + sed -i 's/$(YEAR) Lawrence Livermore/$(NEW_YEAR) Lawrence Livermore/g' LICENSE + +# do all file headers (works on linux) + find merlin/ -type f -print0 | xargs -0 sed -i 's/$(YEAR), Lawrence Livermore/$(NEW_YEAR), Lawrence Livermore/g' + find *.py -type f -print0 | xargs -0 sed -i 's/$(YEAR), Lawrence Livermore/$(NEW_YEAR), Lawrence Livermore/g' + find tests/ -type f -print0 | xargs -0 sed -i 's/$(YEAR), Lawrence Livermore/$(NEW_YEAR), Lawrence Livermore/g' + find Makefile -type f -print0 | xargs -0 sed -i 's/$(YEAR), Lawrence Livermore/$(NEW_YEAR), Lawrence Livermore/g' + # Make a list of all dependencies/requirements reqlist: johnnydep merlin --output-format pinned diff --git a/config.mk b/config.mk index 38a87a9bf..f1cfbcea3 100644 --- a/config.mk +++ b/config.mk @@ -19,6 +19,8 @@ endif VER?=1.0.0 VSTRING=[0-9]\+\.[0-9]\+\.[0-9]\+ +YEAR=20[0-9][0-9] +NEW_YEAR?=2023 CHANGELOG_VSTRING="## \[$(VSTRING)\]" INIT_VSTRING="__version__ = \"$(VSTRING)\"" diff --git a/merlin/__init__.py b/merlin/__init__.py index eaf3ff668..1222a38f6 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index b961afe50..a521d2cc5 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/celery.py b/merlin/celery.py index 0041523b2..52d4e4589 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index 30fea7f47..f242b15fb 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index da214275b..5e82abbc2 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -1,7 +1,7 @@ #!/usr/bin/env python ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 274ddb062..c2391ca6d 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -1,7 +1,7 @@ #!/usr/bin/env python ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 591f9a49f..dd93cf5a7 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 4861a757b..45ddb2ed1 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 229f00851..6a0766427 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 1ba552626..930c67b5f 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 4ec7c5dbe..b0eca1b6c 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 352da5550..bd19ac539 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index b460be069..08a0362ae 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index e30a18ca0..ea26edc8f 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index 7822e7086..38fba1ddc 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -3,7 +3,7 @@ """ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index bd22ddaa3..bb7f79875 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index e435e1ba0..40ea19af7 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 626b776d5..3cce32883 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/display.py b/merlin/display.py index 30140981e..0e0e11e66 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # @@ -45,7 +45,27 @@ from merlin.config.configfile import default_config_info +# TODO: make these color blind compliant +# (see https://mikemol.github.io/technique/colorblind/2018/02/11/color-safe-palette.html) +ANSI_COLORS = { + "RESET": "\033[0m", + "GREY": "\033[90m", + "RED": "\033[91m", + "GREEN": "\033[92m", + "YELLOW": "\033[93m", + "BLUE": "\033[94m", + "MAGENTA": "\033[95m", + "CYAN": "\033[96m", + "WHITE": "\033[97m", +} + + class ConnProcess(Process): + """ + An extension of Multiprocessing's Process class in order + to overwrite the run and exception defintions. + """ + def __init__(self, *args, **kwargs): Process.__init__(self, *args, **kwargs) self._pconn, self._cconn = Pipe() @@ -55,19 +75,24 @@ def run(self): try: Process.run(self) self._cconn.send(None) - except Exception as e: - tb = traceback.format_exc() - self._cconn.send((e, tb)) + except Exception as e: # pylint: disable=W0718,C0103 + trace_back = traceback.format_exc() + self._cconn.send((e, trace_back)) # raise e # You can still rise this exception if you need to @property def exception(self): + """Create custom exception""" if self._pconn.poll(): self._exception = self._pconn.recv() return self._exception def check_server_access(sconf): + """ + Check if there are any issues connecting to the servers. + If there are, output the errors. + """ servers = ["broker server", "results server"] if sconf.keys(): @@ -75,25 +100,25 @@ def check_server_access(sconf): print("-" * 28) excpts = {} - for s in servers: - if s in sconf: - _examine_connection(s, sconf, excpts) + for server in servers: + if server in sconf: + _examine_connection(server, sconf, excpts) if excpts: print("\nExceptions:") - for k, v in excpts.items(): - print(f"{k}: {v}") + for key, val in excpts.items(): + print(f"{key}: {val}") -def _examine_connection(s, sconf, excpts): +def _examine_connection(server, sconf, excpts): connect_timeout = 60 try: ssl_conf = None - if "broker" in s: + if "broker" in server: ssl_conf = broker.get_ssl_config() - if "results" in s: + if "results" in server: ssl_conf = results_backend.get_ssl_config() - conn = Connection(sconf[s], ssl=ssl_conf) + conn = Connection(sconf[server], ssl=ssl_conf) conn_check = ConnProcess(target=conn.connect) conn_check.start() counter = 0 @@ -102,16 +127,16 @@ def _examine_connection(s, sconf, excpts): counter += 1 if counter > connect_timeout: conn_check.kill() - raise Exception(f"Connection was killed due to timeout ({connect_timeout}s)") + raise TimeoutError(f"Connection was killed due to timeout ({connect_timeout}server)") conn.release() if conn_check.exception: - error, traceback = conn_check.exception + error, _ = conn_check.exception raise error - except Exception as e: - print(f"{s} connection: Error") - excpts[s] = e + except Exception as e: # pylint: disable=W0718,C0103 + print(f"{server} connection: Error") + excpts[server] = e else: - print(f"{s} connection: OK") + print(f"{server} connection: OK") def display_config_info(): @@ -129,7 +154,7 @@ def display_config_info(): conf["broker server"] = broker.get_connection_string(include_password=False) sconf["broker server"] = broker.get_connection_string() conf["broker ssl"] = broker.get_ssl_config() - except Exception as e: + except Exception as e: # pylint: disable=W0718,C0103 conf["broker server"] = "Broker server error." excpts["broker server"] = e @@ -137,7 +162,7 @@ def display_config_info(): conf["results server"] = results_backend.get_connection_string(include_password=False) sconf["results server"] = results_backend.get_connection_string() conf["results ssl"] = results_backend.get_ssl_config() - except Exception as e: + except Exception as e: # pylint: disable=W0718,C0103 conf["results server"] = "No results server configured or error." excpts["results server"] = e @@ -145,8 +170,8 @@ def display_config_info(): if excpts: print("\nExceptions:") - for k, v in excpts.items(): - print(f"{k}: {v}") + for key, val in excpts.items(): + print(f"{key}: {val}") check_server_access(sconf) @@ -170,7 +195,7 @@ def display_multiple_configs(files, configs): pprint.pprint(config) -def print_info(args): +def print_info(args): # pylint: disable=W0613 """ Provide version and location information about python and pip to facilitate user troubleshooting. 'merlin info' is a CLI tool only for @@ -187,9 +212,30 @@ def print_info(args): print("") info_calls = ["which python3", "python3 --version", "which pip3", "pip3 --version"] info_str = "" - for x in info_calls: - info_str += 'echo " $ ' + x + '" && ' + x + "\n" + for cmd in info_calls: + info_str += 'echo " $ ' + cmd + '" && ' + cmd + "\n" info_str += "echo \n" info_str += r"echo \"echo \$PYTHONPATH\" && echo $PYTHONPATH" _ = subprocess.run(info_str, shell=True) print("") + + +def tabulate_info(info, headers=None, color=None): + """ + Display info in a table. Colorize the table if you'd like. + Intended for use for functions outside of this file so they don't + need to import tabulate. + :param `info`: The info you want to tabulate. + :param `headers`: A string or list stating what you'd like the headers to be. + Options: "firstrow", "keys", or List[str] + :param `color`: An ANSI color. + """ + # Adds the color at the start of the print + if color: + print(color, end="") + + # \033[0m resets color to white + if headers: + print(tabulate(info, headers=headers), ANSI_COLORS["RESET"]) + else: + print(tabulate(info), ANSI_COLORS["RESET"]) diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/examples/dev_workflows/multiple_workers.yaml b/merlin/examples/dev_workflows/multiple_workers.yaml new file mode 100644 index 000000000..f393f87d3 --- /dev/null +++ b/merlin/examples/dev_workflows/multiple_workers.yaml @@ -0,0 +1,56 @@ +description: + name: multiple_workers + description: a very simple merlin workflow with multiple workers + +global.parameters: + GREET: + values : ["hello","hola"] + label : GREET.%% + WORLD: + values : ["world","mundo"] + label : WORLD.%% + +study: + - name: step_1 + description: say hello + run: + cmd: | + echo "$(GREET), $(WORLD)!" + task_queue: hello_queue + + - name: step_2 + description: step 2 + run: + cmd: | + echo "step_2" + depends: [step_1_*] + task_queue: echo_queue + + - name: step_3 + description: stop workers + run: + cmd: | + echo "stop workers" + depends: [step_2] + task_queue: other_queue + + - name: step_4 + description: another step + run: + cmd: | + echo "another step" + depends: [step_3] + task_queue: other_queue + +merlin: + resources: + workers: + step_1_merlin_test_worker: + args: -l INFO + steps: [step_1] + step_2_merlin_test_worker: + args: -l INFO + steps: [step_2] + other_merlin_test_worker: + args: -l INFO + steps: [step_3, step_4] \ No newline at end of file diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 43573b416..7fbd502b1 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 2796465ab..c97cc2757 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 816546f67..225e69d28 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index f540e6f1f..475982d8c 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -1,7 +1,7 @@ """This module handles setting up the extensive logging system in Merlin.""" ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/main.py b/merlin/main.py index 5de8423b0..47f471dd8 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -1,7 +1,7 @@ """The top level main function for invoking Merlin.""" ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index a1e4de372..a1af0298b 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/router.py b/merlin/router.py index 90711756c..3ec9122ff 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 31115004f..5653cba21 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 38359938c..045ef22e3 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -1,7 +1,7 @@ """Main functions for instantiating and running Merlin server containers.""" ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 298a360b7..e62e12e4f 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 16d2c3686..65f0b2abb 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 46db0fd4f..ac0031823 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 46f71882f..7b54e5200 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 710bc2500..f4ea42e24 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 5ca2eea6b..4c9291693 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 9d93197bf..287fab1f6 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index d56e84f40..8f5174427 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index af6ed22b5..9a42873f1 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 97fc40289..f6d344a25 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 37a429055..f7ba3532f 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index ab998140a..027ebaff9 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 3a55df606..031302480 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/study/study.py b/merlin/study/study.py index dd108005a..6bf07653f 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/merlin/utils.py b/merlin/utils.py index 5217e1d7a..48def3a10 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/setup.py b/setup.py index 06706ca57..1a1ad94ba 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # diff --git a/tests/integration/README.md b/tests/integration/README.md index ef402c9e5..1d575e022 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,21 +1,160 @@ -# Integration test script: run_tests.py +# Integration Tests -To run command line-level tests of Merlin, follow these steps: +This directory contains 3 key files for testing: +1. `run_tests.py` - script to launch tests +2. `test_definitions.py` - test definitions +3. `conditions.py` - test conditions + +## How to Run + +To run command-line-level tests of Merlin, follow these steps: 1. activate the Merlin virtual environment 2. navigate to the top-level `merlin` directory 3. run `python tests/integration/run_tests.py` -This will run all tests found in the `define_tests` function. -A test is a python dict where the key is the test name, and the -value is a tuple holding the test shell command, and a regexp string -to search for, if any. Without a regexp string, the script will -output 'FAIL' on non-zero return codes. With a regexp string, the -script outputs 'FAIL' if the string cannot be found in the -test's stdout. +This will run all tests found in the `define_tests` function located within `test_definitions.py`. + +## How Tests are Defined + +A test is a python dict where the key is the test name and the +value is another dict. The value dict can currently have 5 keys: + +Required: + +1. `cmds` + - Type: Str or List[Str] + - Functionality: Defines the CLI commands to run for a test + - Limitations: The number of strings here should be equal to `num procs` (see `5.num procs` below) +2. `conditions` + - Type: Condition or List[Condition] + - Functionality: Defines the conditions to check against for this test + - Condition classes can be found in `conditions.py` + +Optional: + +3. `run type` + - Type: Str + - Functionality: Defines the type of run (either `local` or `distributed`) + - Default: None +4. `cleanup` + - Type: Str + - Functionality: Defines a CLI command to run that will clean the output of your test + - Default: None +5. `num procs` + - Type: int + - Functionality: Defines the number of subprocesses required for a test + - Default: 1 + - Limitations: + - Currently the value here can only be 1 or 2 + - The number of `cmds` must be equal to `num procs` (i.e. one command will be run per subprocess launched) + +## Examples + +This section will show both valid and invalid test definitions. + +### Valid Test Definitions + +The most basic test you can provide can be written 4 ways since `cmds` and `conditions` can both be 2 different types: + + "cmds as string, conditions as Condition": { + "cmds": "echo hello", + "conditions": HasReturnCode(), + } + + "cmds as list, conditions as Condition": { + "cmds": ["echo hello"], + "conditions": HasReturnCode(), + } + + "cmds as string, conditions as list": { + "cmds": "echo hello", + "conditions": [HasReturnCode()], + } + + "cmds as list, conditions as list": { + "cmds": ["echo hello"], + "conditions": [HasReturnCode()], + } + +Adding slightly more complexity, we provide a run type: + + "basic test with run type": { + "cmds": "echo hello", + "conditions": HasReturnCode(), + "run type": "local" # This could also be "distributed" + } + +Now we'll add a cleanup command: + + "basic test with cleanup": { + "cmds": "mkdir output_dir/", + "conditions": HasReturnCode(), + "run type": "local", + "cleanup": "rm -rf output_dir/" + } + +Finally we'll play with the number of processes to start: + + "test with 1 process": { + "cmds": "mkdir output_dir/", + "conditions": HasReturnCode(), + "run type": "local", + "cleanup": "rm -rf output_dir/", + "num procs": 1 + } + + "test with 2 processes": { + "cmds": ["mkdir output_dir/", "touch output_dir/new_file.txt"], + "conditions": HasReturnCode(), + "run type": "local", + "cleanup": "rm -rf output_dir/", + "num procs": 2 + } + +Similarly, the test with 2 processes can be condensed to a test with 1 process +by placing them in the same string and separating them with a semi-colon: + + "condensing test with 2 processes into 1 process": { + "cmds": ["mkdir output_dir/ ; touch output_dir/new_file.txt"], + "conditions": HasReturnCode(), + "run type": "local", + "cleanup": "rm -rf output_dir/", + "num procs": 1 + } + + +### Invalid Test Definitions + +No `cmds` provided: + + "no cmd": { + "conditions": HasReturnCode(), + } + +No `conditions` provided: + + "no conditions": { + "cmds": "echo hello", + } + +Number of `cmds` does not match `num procs`: + + "num cmds != num procs": { + "cmds": ["echo hello; echo goodbye"], + "conditions": HasReturnCode(), + "num procs": 2 + } + +Note: Technically 2 commands were provided here ("echo hello" and "echo goodbye") +but since they were placed in one string it will be viewed as one command. +Changing the `cmds` section here to be: + + "cmds": ["echo hello", "echo goodbye"] + +would fix this issue and create a valid test definition. -### Continuous integration -Currently run from [Bamboo](https://lc.llnl.gov/bamboo/chain/admin/config/defaultStages.action?buildKey=MLSI-TES). +## Continuous Integration -Our Bamboo agents make the virtual environment, staying at the `merlin/` location, then run: `python tests/integration/run_tests.py`. +Merlin's CI is currently done through [GitHub Actions](https://github.com/features/actions). If you're needing to modify this CI, you'll need to update `/.github/workflows/push-pr_workflow.yml`. diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 3448ec61a..9c44e1f5f 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -1,3 +1,33 @@ +############################################################################### +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.1. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### +"""This module defines the different conditions to test against.""" import os from abc import ABC, abstractmethod from glob import glob @@ -6,6 +36,8 @@ # TODO when moving command line tests to pytest, change Condition boolean returns to assertions class Condition(ABC): + """Abstract Condition class that other conditions will inherit from""" + def ingest_info(self, info): """ This function allows child classes of Condition @@ -14,11 +46,14 @@ def ingest_info(self, info): for key, val in info.items(): setattr(self, key, val) + @property @abstractmethod def passes(self): + """The method that will check if the test passes or not""" pass +# pylint: disable=no-member class HasReturnCode(Condition): """ A condition that some process must return 0 @@ -80,7 +115,7 @@ def is_within(self, text): @property def passes(self): if self.negate: - return not self.is_within(self.stdout) + return not self.is_within(self.stdout) and not self.is_within(self.stderr) return self.is_within(self.stdout) or self.is_within(self.stderr) @@ -99,6 +134,9 @@ def __init__(self, study_name, output_path): self.dirpath_glob = f"{self.output_path}/{self.study_name}" f"_[0-9]*-[0-9]*" def glob(self, glob_string): + """ + Returns a regex string for the glob library to recursively find files with. + """ candidates = glob(glob_string) if isinstance(candidates, list): return sorted(candidates)[-1] @@ -110,7 +148,7 @@ class StepFileExists(StudyOutputAware): A StudyOutputAware that checks for a particular file's existence. """ - def __init__(self, step, filename, study_name, output_path, params=False): + def __init__(self, step, filename, study_name, output_path, params=False): # pylint: disable=R0913 """ :param `step`: the name of a step :param `filename`: name of file to search for in step's workspace directory @@ -127,12 +165,16 @@ def __str__(self): @property def glob_string(self): + """ + Returns a regex string for the glob library to recursively find files with. + """ param_glob = "" if self.params: param_glob = "*/" return f"{self.dirpath_glob}/{self.step}/{param_glob}{self.filename}" def file_exists(self): + """Check if the file path created by glob_string exists""" glob_string = self.glob_string try: filename = self.glob(glob_string) @@ -150,7 +192,7 @@ class StepFileHasRegex(StudyOutputAware): A StudyOutputAware that checks that a particular file contains a regex. """ - def __init__(self, step, filename, study_name, output_path, regex): + def __init__(self, step, filename, study_name, output_path, regex): # pylint: disable=R0913 """ :param `step`: the name of a step :param `filename`: name of file to search for in step's workspace directory @@ -163,20 +205,25 @@ def __init__(self, step, filename, study_name, output_path, regex): self.regex = regex def __str__(self): - return f"{__class__.__name__} expected to find '{self.regex}' regex match in file '{self.glob_string}', but match was not found" + return f"""{__class__.__name__} expected to find '{self.regex}' + regex match in file '{self.glob_string}', but match was not found""" @property def glob_string(self): + """ + Returns a regex string for the glob library to recursively find files with. + """ return f"{self.dirpath_glob}/{self.step}/{self.filename}" def contains(self): + """See if the regex is within the filetext""" glob_string = self.glob_string try: filename = self.glob(glob_string) with open(filename, "r") as textfile: filetext = textfile.read() return self.is_within(filetext) - except Exception: + except Exception: # pylint: disable=W0718 return False def is_within(self, text): @@ -196,7 +243,7 @@ class ProvenanceYAMLFileHasRegex(HasRegex): MUST contain a given regular expression. """ - def __init__(self, regex, name, output_path, provenance_type, negate=False): + def __init__(self, regex, name, output_path, provenance_type, negate=False): # pylint: disable=R0913 """ :param `regex`: a string regex pattern :param `name`: the name of a study @@ -214,14 +261,19 @@ def __init__(self, regex, name, output_path, provenance_type, negate=False): def __str__(self): if self.negate: - return f"{__class__.__name__} expected to find no '{self.regex}' regex match in provenance spec '{self.glob_string}', but match was found" - return f"{__class__.__name__} expected to find '{self.regex}' regex match in provenance spec '{self.glob_string}', but match was not found" + return f"""{__class__.__name__} expected to find no '{self.regex}' + regex match in provenance spec '{self.glob_string}', but match was found""" + return f"""{__class__.__name__} expected to find '{self.regex}' + regex match in provenance spec '{self.glob_string}', but match was not found""" @property def glob_string(self): + """ + Returns a regex string for the glob library to recursively find files with. + """ return f"{self.output_path}/{self.name}" f"_[0-9]*-[0-9]*/merlin_info/{self.name}.{self.prov_type}.yaml" - def is_within(self): + def is_within(self): # pylint: disable=W0221 """ Uses glob to find the correct provenance yaml spec. Returns True if that file contains a match to this @@ -249,6 +301,7 @@ def __init__(self, pathname) -> None: self.pathname = pathname def path_exists(self) -> bool: + """Check if a path exists""" return os.path.exists(self.pathname) def __str__(self) -> str: @@ -270,18 +323,21 @@ def __init__(self, filename, regex) -> None: self.regex = regex def contains(self) -> bool: + """Checks if the regex matches anywhere in the filetext""" try: - with open(self.filename, "r") as f: + with open(self.filename, "r") as f: # pylint: disable=C0103 filetext = f.read() return self.is_within(filetext) - except Exception: + except Exception: # pylint: disable=W0718 return False def is_within(self, text): + """Check if there's a match for the regex in text""" return search(self.regex, text) is not None def __str__(self) -> str: - return f"{__class__.__name__} expected to find {self.regex} regex match within {self.filename} file but no match was found" + return f"""{__class__.__name__} expected to find {self.regex} + regex match within {self.filename} file but no match was found""" @property def passes(self): @@ -295,7 +351,8 @@ class FileHasNoRegex(FileHasRegex): """ def __str__(self) -> str: - return f"{__class__.__name__} expected to find {self.regex} regex to not match within {self.filename} file but a match was found" + return f"""{__class__.__name__} expected to find {self.regex} + regex to not match within {self.filename} file but a match was found""" @property def passes(self): diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 913a2ab08..0282f888b 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -1,5 +1,5 @@ ############################################################################### -# Copyright (c) 2022, Lawrence Livermore National Security, LLC. +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory # Written by the Merlin dev team, listed in the CONTRIBUTORS file. # @@ -37,36 +37,89 @@ import sys import time from contextlib import suppress -from subprocess import PIPE, Popen +from subprocess import TimeoutExpired, run -from test_definitions import OUTPUT_DIR, define_tests +from test_definitions import OUTPUT_DIR, define_tests # pylint: disable=E0401 -def run_single_test(name, test, test_label="", buffer_length=50): - dot_length = buffer_length - len(name) - len(str(test_label)) - print(f"TEST {test_label}: {name}{'.'*dot_length}", end="") - command = test[0] - conditions = test[1] +def get_definition_issues(test): + """ + Function to make sure the test definition was written properly. + :param `test`: The test definition we're checking + :returns: A list of errors found with the test definition + """ + errors = [] + # Check that commands were provided + try: + commands = test["cmds"] + if not isinstance(commands, list): + commands = [commands] + except KeyError: + errors.append("'cmds' flag not defined") + commands = None + + # Check that conditions were provided + if "conditions" not in test: + errors.append("'conditions' flag not defined") + + # Check that correct number of cmds were given depending on + # the number of processes we'll need to start + if commands: + if "num procs" not in test: + num_procs = 1 + else: + num_procs = test["num procs"] + + if num_procs == 1 and len(commands) != 1: + errors.append(f"Need 1 'cmds' since 'num procs' is 1 but {len(commands)} 'cmds' were given") + elif num_procs == 2 and len(commands) != 2: + errors.append(f"Need 2 'cmds' since 'num procs' is 2 but {len(commands)} 'cmds' were given") + + return errors + + +def run_single_test(test): + """ + Runs a single test and returns whether it passed or not + and information about the test for logging purposes. + :param `test`: A dictionary that defines the test + :returns: A tuple of type (bool, dict) where the bool + represents if the test passed and the dict + contains info about the test. + """ + # Parse the test definition + commands = test.pop("cmds", None) + if not isinstance(commands, list): + commands = [commands] + conditions = test.pop("conditions", None) if not isinstance(conditions, list): conditions = [conditions] + cleanup = test.pop("cleanup", None) + num_procs = test.pop("num procs", 1) start_time = time.time() - process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) - stdout, stderr = process.communicate() + # As of now the only time we need 2 processes is to test stop-workers + # Therefore we only care about the result of the second process + if num_procs == 2: + # First command should start the workers + try: + run(commands[0], timeout=8, capture_output=True, shell=True) + except TimeoutExpired: + pass + # Second command should stop the workers + result = run(commands[1], capture_output=True, text=True, shell=True) + else: + # Run the commands + result = run(commands[0], capture_output=True, text=True, shell=True) end_time = time.time() total_time = end_time - start_time - if stdout is not None: - stdout = stdout.decode("utf-8") - if stderr is not None: - stderr = stderr.decode("utf-8") - return_code = process.returncode info = { "total_time": total_time, - "command": command, - "stdout": stdout, - "stderr": stderr, - "return_code": return_code, + "command": commands, + "stdout": result.stdout, + "stderr": result.stderr, + "return_code": result.returncode, "violated_condition": None, } @@ -78,11 +131,10 @@ def run_single_test(name, test, test_label="", buffer_length=50): info["violated_condition"] = (condition, i, len(conditions)) break - if len(test) == 4: - end_process = Popen(test[3], stdout=PIPE, stderr=PIPE, shell=True) - end_stdout, end_stderr = end_process.communicate() - info["end_stdout"] = end_stdout - info["end_stderr"] = end_stderr + if cleanup: + end_process = run(cleanup, capture_output=True, text=True, shell=True) + info["end_stdout"] = end_process.stdout + info["end_stderr"] = end_process.stderr return passed, info @@ -96,7 +148,7 @@ def clear_test_studies_dir(): shutil.rmtree(f"./{OUTPUT_DIR}") -def process_test_result(passed, info, is_verbose, exit): +def process_test_result(passed, info, is_verbose, exit_on_failure): """ Process and print test results to the console. """ @@ -104,9 +156,9 @@ def process_test_result(passed, info, is_verbose, exit): if passed is False and "merlin: command not found" in info["stderr"]: print(f"\nMissing from environment:\n\t{info['stderr']}") return None - elif passed is False: + if passed is False: print("FAIL") - if exit is True: + if exit_on_failure is True: return None else: print("pass") @@ -127,11 +179,18 @@ def process_test_result(passed, info, is_verbose, exit): return passed -def run_tests(args, tests): +def filter_tests_to_run(args, tests): """ - Run all inputted tests. - :param `tests`: a dictionary of - {"test_name" : ("test_command", [conditions])} + Filter which tests to run based on args. The tests to + run will be what makes up the args.ids list. This function + will return whether we're being selective with what tests + we run and also the number of tests that match the filter. + :param `args`: CLI args given by user + :param `tests`: a dict of all the tests that exist + :returns: a tuple where the first entry is a bool on whether + we filtered the tests at all and the second entry + is an int representing the number of tests we're + going to run. """ selective = False n_to_run = len(tests) @@ -140,19 +199,27 @@ def run_tests(args, tests): raise ValueError(f"Test ids must be between 1 and {len(tests)}, inclusive.") selective = True n_to_run = len(args.ids) - elif args.local is not None: + elif args.local is not None or args.distributed is not None: args.ids = [] n_to_run = 0 selective = True for test_id, test in enumerate(tests.values()): - # Ensures that test definitions are atleast size 3. - # 'local' variable is stored in 3rd element of the test definitions, - # but an optional 4th element can be provided for an ending command - # to be ran after all checks have been made. - if len(test) >= 3 and test[2] == "local": + run_type = test.pop("run type", None) + if (args.local and run_type == "local") or (args.distributed and run_type == "distributed"): args.ids.append(test_id + 1) n_to_run += 1 + return selective, n_to_run + + +def run_tests(args, tests): # pylint: disable=R0914 + """ + Run all inputted tests. + :param `tests`: a dictionary of + {"test_name" : ("test_command", [conditions])} + """ + selective, n_to_run = filter_tests_to_run(args, tests) + print(f"Running {n_to_run} integration tests...") start_time = time.time() @@ -163,14 +230,32 @@ def run_tests(args, tests): if selective and test_label not in args.ids: total += 1 continue - try: - passed, info = run_single_test(test_name, test, test_label) - except Exception as e: - print(e) + dot_length = 50 - len(test_name) - len(str(test_label)) + print(f"TEST {test_label}: {test_name}{'.'*dot_length}", end="") + # Check the format of the test definition + definition_issues = get_definition_issues(test) + if definition_issues: + print("FAIL") + print(f"\tTest with name '{test_name}' has problems with its' test definition. Skipping...") + if args.verbose: + print(f"\tFound {len(definition_issues)} problems with the definition of '{test_name}':") + for error in definition_issues: + print(f"\t- {error}") + total += 1 passed = False - info = None + if args.exit: + result = None + else: + result = False + else: + try: + passed, info = run_single_test(test) + except Exception as e: # pylint: disable=C0103,W0718 + print(e) + passed = False + info = None + result = process_test_result(passed, info, args.verbose, args.exit) - result = process_test_result(passed, info, args.verbose, args.exit) clear_test_studies_dir() if result is None: print("Exiting early") @@ -190,6 +275,10 @@ def run_tests(args, tests): def setup_argparse(): + """ + Using ArgumentParser, define the arguments allowed for this script. + :returns: An ArgumentParser object + """ parser = argparse.ArgumentParser(description="run_tests cli parser") parser.add_argument( "--exit", @@ -198,6 +287,7 @@ def setup_argparse(): ) parser.add_argument("--verbose", action="store_true", help="Flag for more detailed output messages") parser.add_argument("--local", action="store_true", default=None, help="Run only local tests") + parser.add_argument("--distributed", action="store_true", default=None, help="Run only distributed tests") parser.add_argument( "--ids", action="store", @@ -207,9 +297,29 @@ def setup_argparse(): default=None, help="Provide space-delimited ids of tests you want to run. Example: '--ids 1 5 8 13'", ) + parser.add_argument( + "--display-tests", action="store_true", default=False, help="Display a table format of test names and ids" + ) return parser +def display_tests(tests): + """ + Helper function to display a table of tests and associated ids. + Helps choose which test to run if you're trying to debug and use + the --id flag. + :param `tests`: A dict of tests (Dict) + """ + from merlin.display import tabulate_info # pylint: disable=C0415 + + test_names = list(tests.keys()) + test_table = [(i + 1, test_names[i]) for i in range(len(test_names))] + test_table.insert(0, ("ID", "Test Name")) + print() + tabulate_info(test_table, headers="firstrow") + print() + + def main(): """ High-level CLI test operations. @@ -219,6 +329,10 @@ def main(): tests = define_tests() + if args.display_tests: + display_tests(tests) + return + clear_test_studies_dir() result = run_tests(args, tests) sys.exit(result) diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index feb5168ba..752b3650d 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -1,4 +1,46 @@ -from conditions import ( +############################################################################### +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.9.1. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### +""" +This module defines all the integration tests to be ran through run_tests.py. + +Each test looks like: +"test name": { + "cmds": the commands to run, + "conditions": the conditions to check for, + "run type": the type of test (local or distributed), + "cleanup": the command to run after your test in order to clean output, + "num procs": the number of processes you need to start for a test (1 or 2) +} +""" + +from conditions import ( # pylint: disable=E0401 FileHasNoRegex, FileHasRegex, HasRegex, @@ -14,9 +56,11 @@ OUTPUT_DIR = "cli_test_studies" CLEAN_MERLIN_SERVER = "rm -rf appendonly.aof dump.rdb merlin_server/" +# KILL_WORKERS = "pkill -9 -f '.*merlin_test_worker'" +KILL_WORKERS = "pkill -9 -f 'celery'" -def define_tests(): +def define_tests(): # pylint: disable=R0914 """ Returns a dictionary of tests, where the key is the test's name, and the value is a tuple @@ -48,62 +92,64 @@ def define_tests(): pbs_path = f"{examples}/flux/scripts/pbs_test" workers_pbs = f"""PATH="{pbs_path}:$PATH";merlin {err_lvl} run-workers""" lsf = f"{examples}/lsf/lsf_par.yaml" + mul_workers_demo = f"{dev_examples}/multiple_workers.yaml" black = "black --check --target-version py36" config_dir = "./CLI_TEST_MERLIN_CONFIG" release_dependencies = "./requirements/release.txt" basic_checks = { - "merlin": ("merlin", HasReturnCode(1), "local"), - "merlin help": ("merlin --help", HasReturnCode(), "local"), - "merlin version": ("merlin --version", HasReturnCode(), "local"), - "merlin config": ( - f"merlin config -o {config_dir}; rm -rf {config_dir}", - HasReturnCode(), - "local", - ), + "merlin": {"cmds": "merlin", "conditions": HasReturnCode(1), "run type": "local"}, + "merlin help": {"cmds": "merlin --help", "conditions": HasReturnCode(), "run type": "local"}, + "merlin version": {"cmds": "merlin --version", "conditions": HasReturnCode(), "run type": "local"}, + "merlin config": { + "cmds": f"merlin config -o {config_dir}", + "conditions": HasReturnCode(), + "run type": "local", + "cleanup": f"rm -rf {config_dir}", + }, } server_basic_tests = { - "merlin server init": ( - "merlin server init", - HasRegex(".*successful"), - "local", - CLEAN_MERLIN_SERVER, - ), - "merlin server start/stop": ( - """merlin server init; - merlin server start; - merlin server status; - merlin server stop;""", - [ + "merlin server init": { + "cmds": "merlin server init", + "conditions": HasRegex(".*successful"), + "run type": "local", + "cleanup": CLEAN_MERLIN_SERVER, + }, + "merlin server start/stop": { + "cmds": """merlin server init ; + merlin server start ; + merlin server status ; + merlin server stop""", + "conditions": [ HasRegex("Server started with PID [0-9]*"), HasRegex("Merlin server is running"), HasRegex("Merlin server terminated"), ], - "local", - CLEAN_MERLIN_SERVER, - ), - "merlin server restart": ( - """merlin server init; + "run type": "local", + "cleanup": CLEAN_MERLIN_SERVER, + }, + "merlin server restart": { + "cmds": """merlin server init; merlin server start; merlin server restart; merlin server status; merlin server stop;""", - [ + "conditions": [ HasRegex("Server started with PID [0-9]*"), HasRegex("Merlin server is running"), HasRegex("Merlin server terminated"), ], - "local", - CLEAN_MERLIN_SERVER, - ), + "run type": "local", + "cleanup": CLEAN_MERLIN_SERVER, + }, } server_config_tests = { - "merlin server change config": ( - """merlin server init; + "merlin server change config": { + "cmds": """merlin server init; merlin server config -p 8888 -pwd new_password -d ./config_dir -ss 80 -sc 8 -sf new_sf -am always -af new_af.aof; merlin server start; merlin server stop;""", - [ + "conditions": [ FileHasRegex("merlin_server/redis.conf", "port 8888"), FileHasRegex("merlin_server/redis.conf", "requirepass new_password"), FileHasRegex("merlin_server/redis.conf", "dir ./config_dir"), @@ -116,11 +162,11 @@ def define_tests(): HasRegex("Server started with PID [0-9]*"), HasRegex("Merlin server terminated"), ], - "local", - "rm -rf appendonly.aof dump.rdb merlin_server/ config_dir/", - ), - "merlin server config add/remove user": ( - """merlin server init; + "run type": "local", + "cleanup": "rm -rf appendonly.aof dump.rdb merlin_server/ config_dir/", + }, + "merlin server config add/remove user": { + "cmds": """merlin server init; merlin server start; merlin server config --add-user new_user new_password; merlin server stop; @@ -129,129 +175,132 @@ def define_tests(): merlin server config --remove-user new_user; merlin server stop; """, - [ + "conditions": [ FileHasRegex("./merlin_server/redis.users_new", "new_user"), FileHasNoRegex("./merlin_server/redis.users", "new_user"), ], - "local", - CLEAN_MERLIN_SERVER, - ), + "run type": "local", + "cleanup": CLEAN_MERLIN_SERVER, + }, } examples_check = { - "example list": ( - "merlin example list", - HasReturnCode(), - "local", - ), + "example list": { + "cmds": "merlin example list", + "conditions": HasReturnCode(), + "run type": "local", + }, } run_workers_echo_tests = { - "run-workers echo simple_chain": ( - f"{workers} {simple} --echo", - [HasReturnCode(), HasRegex(celery_slurm_regex)], - "local", - ), - "run-workers echo feature_demo": ( - f"{workers} {demo} --echo", - [HasReturnCode(), HasRegex(celery_slurm_regex)], - "local", - ), - "run-workers echo slurm_test": ( - f"{workers} {slurm} --echo", - [HasReturnCode(), HasRegex(celery_slurm_regex)], - "local", - ), - "run-workers echo flux_test": ( - f"{workers} {flux} --echo", - [HasReturnCode(), HasRegex(celery_slurm_regex)], - "local", - ), - "run-workers echo flux_native_test": ( - f"{workers_flux} {flux_native} --echo", - [HasReturnCode(), HasRegex(celery_flux_regex)], - "local", - ), - "run-workers echo pbs_test": ( - f"{workers_pbs} {flux_native} --echo", - [HasReturnCode(), HasRegex(celery_pbs_regex)], - "local", - ), - "run-workers echo override feature_demo": ( - f"{workers} {demo} --echo --vars VERIFY_QUEUE=custom_verify_queue", - [HasReturnCode(), HasRegex("custom_verify_queue")], - "local", - ), + "run-workers echo simple_chain": { + "cmds": f"{workers} {simple} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_slurm_regex)], + "run type": "local", + }, + "run-workers echo feature_demo": { + "cmds": f"{workers} {demo} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_slurm_regex)], + "run type": "local", + }, + "run-workers echo slurm_test": { + "cmds": f"{workers} {slurm} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_slurm_regex)], + "run type": "local", + }, + "run-workers echo flux_test": { + "cmds": f"{workers} {flux} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_slurm_regex)], + "run type": "local", + }, + "run-workers echo flux_native_test": { + "cmds": f"{workers_flux} {flux_native} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_flux_regex)], + "run type": "local", + }, + "run-workers echo pbs_test": { + "cmds": f"{workers_pbs} {flux_native} --echo", + "conditions": [HasReturnCode(), HasRegex(celery_pbs_regex)], + "run type": "local", + }, + "run-workers echo override feature_demo": { + "cmds": f"{workers} {demo} --echo --vars VERIFY_QUEUE=custom_verify_queue", + "conditions": [HasReturnCode(), HasRegex("custom_verify_queue")], + "run type": "local", + }, } wf_format_tests = { - "local minimum_format": ( - f"mkdir {OUTPUT_DIR} ; cd {OUTPUT_DIR} ; merlin run ../{dev_examples}/minimum_format.yaml --local", - StepFileExists( + "local minimum_format": { + "cmds": f"mkdir {OUTPUT_DIR}; cd {OUTPUT_DIR}; merlin run ../{dev_examples}/minimum_format.yaml --local", + "conditions": StepFileExists( "step1", "MERLIN_FINISHED", "minimum_format", OUTPUT_DIR, params=False, ), - "local", - ), - "local no_description": ( - f"mkdir {OUTPUT_DIR} ; cd {OUTPUT_DIR} ; merlin run ../merlin/examples/dev_workflows/no_description.yaml --local", - HasReturnCode(1), - "local", - ), - "local no_steps": ( - f"mkdir {OUTPUT_DIR} ; cd {OUTPUT_DIR} ; merlin run ../merlin/examples/dev_workflows/no_steps.yaml --local", - HasReturnCode(1), - "local", - ), - "local no_study": ( - f"mkdir {OUTPUT_DIR} ; cd {OUTPUT_DIR} ; merlin run ../merlin/examples/dev_workflows/no_study.yaml --local", - HasReturnCode(1), - "local", - ), + "run type": "local", + }, + "local no_description": { + "cmds": f"""mkdir {OUTPUT_DIR}; cd {OUTPUT_DIR}; + merlin run ../merlin/examples/dev_workflows/no_description.yaml --local""", + "conditions": HasReturnCode(1), + "run type": "local", + }, + "local no_steps": { + "cmds": f"mkdir {OUTPUT_DIR}; cd {OUTPUT_DIR}; merlin run ../merlin/examples/dev_workflows/no_steps.yaml --local", + "conditions": HasReturnCode(1), + "run type": "local", + }, + "local no_study": { + "cmds": f"mkdir {OUTPUT_DIR}; cd {OUTPUT_DIR}; merlin run ../merlin/examples/dev_workflows/no_study.yaml --local", + "conditions": HasReturnCode(1), + "run type": "local", + }, } example_tests = { - "example failure": ("merlin example failure", HasRegex("not found"), "local"), - "example simple_chain": ( - f"merlin example simple_chain ; {run} simple_chain.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR} ; rm simple_chain.yaml", - HasReturnCode(), - "local", - ), + "example failure": {"cmds": "merlin example failure", "conditions": HasRegex("not found"), "run type": "local"}, + "example simple_chain": { + "cmds": f"""merlin example simple_chain; + {run} simple_chain.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}; rm simple_chain.yaml""", + "conditions": HasReturnCode(), + "run type": "local", + }, } restart_step_tests = { - "local restart_shell": ( - f"{run} {dev_examples}/restart_shell.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileExists( + "local restart_shell": { + "cmds": f"{run} {dev_examples}/restart_shell.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileExists( "step2", "MERLIN_FINISHED", "restart_shell", OUTPUT_DIR, params=False, ), - "local", - ), - "local restart": ( - f"{run} {dev_examples}/restart.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileExists( + "run type": "local", + }, + "local restart": { + "cmds": f"{run} {dev_examples}/restart.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileExists( "final_check_for_no_hard_fails", "MERLIN_FINISHED", "restart", OUTPUT_DIR, params=False, ), - "local", - ), + "run type": "local", + }, } restart_wf_tests = { - "restart local simple_chain": ( - f"{run} {simple} --local --vars OUTPUT_PATH=./{OUTPUT_DIR} ; {restart} $(find ./{OUTPUT_DIR} -type d -name 'simple_chain_*') --local", - HasReturnCode(), - "local", - ), + "restart local simple_chain": { + "cmds": f"""{run} {simple} --local --vars OUTPUT_PATH=./{OUTPUT_DIR}; + {restart} $(find ./{OUTPUT_DIR} -type d -name 'simple_chain_*') --local""", + "conditions": HasReturnCode(), + "run type": "local", + }, } dry_run_tests = { - "dry feature_demo": ( - f"{run} {demo} --local --dry --vars OUTPUT_PATH=./{OUTPUT_DIR}", - [ + "dry feature_demo": { + "cmds": f"{run} {demo} --local --dry --vars OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": [ StepFileExists( "verify", "verify_*.sh", @@ -261,61 +310,61 @@ def define_tests(): ), HasReturnCode(), ], - "local", - ), - "dry launch slurm": ( - f"{run} {slurm} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex("runs", "*/runs.slurm.sh", "slurm_test", OUTPUT_DIR, "srun "), - "local", - ), - "dry launch flux": ( - f"{run} {flux} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex( + "run type": "local", + }, + "dry launch slurm": { + "cmds": f"{run} {slurm} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileHasRegex("runs", "*/runs.slurm.sh", "slurm_test", OUTPUT_DIR, "srun "), + "run type": "local", + }, + "dry launch flux": { + "cmds": f"{run} {flux} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileHasRegex( "runs", "*/runs.slurm.sh", "flux_test", OUTPUT_DIR, get_flux_cmd("flux", no_errors=True), ), - "local", - ), - "dry launch lsf": ( - f"{run} {lsf} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex("runs", "*/runs.slurm.sh", "lsf_par", OUTPUT_DIR, "jsrun "), - "local", - ), - "dry launch slurm restart": ( - f"{run} {slurm_restart} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex( + "run type": "local", + }, + "dry launch lsf": { + "cmds": f"{run} {lsf} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileHasRegex("runs", "*/runs.slurm.sh", "lsf_par", OUTPUT_DIR, "jsrun "), + "run type": "local", + }, + "dry launch slurm restart": { + "cmds": f"{run} {slurm_restart} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileHasRegex( "runs", "*/runs.restart.slurm.sh", "slurm_par_restart", OUTPUT_DIR, "srun ", ), - "local", - ), - "dry launch flux restart": ( - f"{run} {flux_restart} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileHasRegex( + "run type": "local", + }, + "dry launch flux restart": { + "cmds": f"{run} {flux_restart} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": StepFileHasRegex( "runs_rs", "*/runs_rs.restart.slurm.sh", "flux_par_restart", OUTPUT_DIR, get_flux_cmd("flux", no_errors=True), ), - "local", - ), + "run type": "local", + }, } other_local_tests = { - "local simple_chain": ( - f"{run} {simple} --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", - HasReturnCode(), - "local", - ), - "local override feature_demo": ( - f"{run} {demo} --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR} --local", - [ + "local simple_chain": { + "cmds": f"{run} {simple} --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", + "conditions": HasReturnCode(), + "run type": "local", + }, + "local override feature_demo": { + "cmds": f"{run} {demo} --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR} --local", + "conditions": [ HasReturnCode(), ProvenanceYAMLFileHasRegex( regex=r"HELLO: \$\(SCRIPTS\)/hello_world.py", @@ -356,10 +405,11 @@ def define_tests(): params=True, ), ], - "local", - ), + "run type": "local", + }, # "local restart expand name": ( - # f"{run} {demo} --local --vars OUTPUT_PATH=./{OUTPUT_DIR} NAME=test_demo ; {restart} $(find ./{OUTPUT_DIR} -type d -name 'test_demo_*') --local", + # f"""{run} {demo} --local --vars OUTPUT_PATH=./{OUTPUT_DIR} NAME=test_demo; + # {restart} $(find ./{OUTPUT_DIR} -type d -name 'test_demo_*') --local""", # [ # HasReturnCode(), # ProvenanceYAMLFileHasRegex( @@ -374,19 +424,23 @@ def define_tests(): # ], # "local", # ), - "local csv feature_demo": ( - f"echo 42.0,47.0 > foo_testing_temp.csv; merlin run {demo} --samplesfile foo_testing_temp.csv --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; rm -f foo_testing_temp.csv", - [HasRegex("1 sample loaded."), HasReturnCode()], - "local", - ), - "local tab feature_demo": ( - f"echo '42.0\t47.0\n7.0 5.3' > foo_testing_temp.tab; merlin run {demo} --samplesfile foo_testing_temp.tab --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; rm -f foo_testing_temp.tab", - [HasRegex("2 samples loaded."), HasReturnCode()], - "local", - ), - "local pgen feature_demo": ( - f"{run} {demo} --pgen {demo_pgen} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local", - [ + "local csv feature_demo": { + "cmds": f"""echo 42.0,47.0 > foo_testing_temp.csv; + merlin run {demo} --samplesfile foo_testing_temp.csv --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; + rm -f foo_testing_temp.csv""", + "conditions": [HasRegex("1 sample loaded."), HasReturnCode()], + "run type": "local", + }, + "local tab feature_demo": { + "cmds": f"""echo '42.0\t47.0\n7.0 5.3' > foo_testing_temp.tab; + merlin run {demo} --samplesfile foo_testing_temp.tab --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; + rm -f foo_testing_temp.tab""", + "conditions": [HasRegex("2 samples loaded."), HasReturnCode()], + "run type": "local", + }, + "local pgen feature_demo": { + "cmds": f"{run} {demo} --pgen {demo_pgen} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local", + "conditions": [ ProvenanceYAMLFileHasRegex( regex=r"\[0.3333333", name="feature_demo", @@ -402,35 +456,118 @@ def define_tests(): ), HasReturnCode(), ], - "local", - ), + "run type": "local", + }, } - provenence_equality_checks = { # noqa: F841 - "local provenance spec equality": ( - f"{run} {simple} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local ; cp $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml') ./{OUTPUT_DIR}/FILE1 ; rm -rf ./{OUTPUT_DIR}/simple_chain_* ; {run} ./{OUTPUT_DIR}/FILE1 --vars OUTPUT_PATH=./{OUTPUT_DIR} --local ; cmp ./{OUTPUT_DIR}/FILE1 $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml')", - HasReturnCode(), - "local", - ), + provenence_equality_checks = { # noqa: F841 pylint: disable=W0612 + "local provenance spec equality": { + "cmds": f"""{run} {simple} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; + cp $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml') ./{OUTPUT_DIR}/FILE1; + rm -rf ./{OUTPUT_DIR}/simple_chain_*; + {run} ./{OUTPUT_DIR}/FILE1 --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; + cmp ./{OUTPUT_DIR}/FILE1 $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml')""", # pylint: disable=C0301 + "conditions": HasReturnCode(), + "run type": "local", + }, } - style_checks = { # noqa: F841 - "black check merlin": (f"{black} merlin/", HasReturnCode(), "local"), - "black check tests": (f"{black} tests/", HasReturnCode(), "local"), + style_checks = { # noqa: F841 pylint: disable=W0612 + "black check merlin": {"cmds": f"{black} merlin/", "conditions": HasReturnCode(), "run type": "local"}, + "black check tests": {"cmds": f"{black} tests/", "conditions": HasReturnCode(), "run type": "local"}, } dependency_checks = { - "deplic no GNU": ( - f"deplic {release_dependencies}", - [HasRegex("GNU", negate=True), HasRegex("GPL", negate=True)], - "local", - ), + "deplic no GNU": { + "cmds": f"deplic {release_dependencies}", + "conditions": [HasRegex("GNU", negate=True), HasRegex("GPL", negate=True)], + "run type": "local", + }, + } + stop_workers_tests = { + "stop workers no workers": { + "cmds": "merlin stop-workers", + "conditions": [ + HasReturnCode(), + HasRegex("No workers found to stop"), + HasRegex("step_1_merlin_test_worker", negate=True), + HasRegex("step_2_merlin_test_worker", negate=True), + HasRegex("other_merlin_test_worker", negate=True), + ], + "run type": "distributed", + }, + "stop workers no flags": { + "cmds": [ + f"{workers} {mul_workers_demo}", + "merlin stop-workers", + ], + "conditions": [ + HasReturnCode(), + HasRegex("No workers found to stop", negate=True), + HasRegex("step_1_merlin_test_worker"), + HasRegex("step_2_merlin_test_worker"), + HasRegex("other_merlin_test_worker"), + ], + "run type": "distributed", + "cleanup": KILL_WORKERS, + "num procs": 2, + }, + "stop workers with spec flag": { + "cmds": [ + f"{workers} {mul_workers_demo}", + f"merlin stop-workers --spec {mul_workers_demo}", + ], + "conditions": [ + HasReturnCode(), + HasRegex("No workers found to stop", negate=True), + HasRegex("step_1_merlin_test_worker"), + HasRegex("step_2_merlin_test_worker"), + HasRegex("other_merlin_test_worker"), + ], + "run type": "distributed", + "cleanup": KILL_WORKERS, + "num procs": 2, + }, + "stop workers with workers flag": { + "cmds": [ + f"{workers} {mul_workers_demo}", + "merlin stop-workers --workers step_1_merlin_test_worker step_2_merlin_test_worker", + ], + "conditions": [ + HasReturnCode(), + HasRegex("No workers found to stop", negate=True), + HasRegex("step_1_merlin_test_worker"), + HasRegex("step_2_merlin_test_worker"), + HasRegex("other_merlin_test_worker", negate=True), + ], + "run type": "distributed", + "cleanup": KILL_WORKERS, + "num procs": 2, + }, + "stop workers with queues flag": { + "cmds": [ + f"{workers} {mul_workers_demo}", + "merlin stop-workers --queues hello_queue", + ], + "conditions": [ + HasReturnCode(), + HasRegex("No workers found to stop", negate=True), + HasRegex("step_1_merlin_test_worker"), + HasRegex("step_2_merlin_test_worker", negate=True), + HasRegex("other_merlin_test_worker", negate=True), + ], + "run type": "distributed", + "cleanup": KILL_WORKERS, + "num procs": 2, + }, } distributed_tests = { # noqa: F841 - "run and purge feature_demo": ( - f"{run} {demo} ; {purge} {demo} -f", - HasReturnCode(), - ), - "remote feature_demo": ( - f"{run} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; {workers} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers", - [ + "run and purge feature_demo": { + "cmds": f"{run} {demo}; {purge} {demo} -f", + "conditions": HasReturnCode(), + "run type": "distributed", + }, + "remote feature_demo": { + "cmds": f"""{run} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers; + {workers} {remote_demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers""", + "conditions": [ HasReturnCode(), ProvenanceYAMLFileHasRegex( regex="cli_test_demo_workers:", @@ -446,27 +583,8 @@ def define_tests(): params=True, ), ], - ), - # this test is deactivated until the --spec option for stop-workers is active again - # "stop workers for distributed feature_demo": ( - # f"{run} {demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; {workers} {demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; sleep 20 ; merlin stop-workers --spec {demo}", - # [ - # HasReturnCode(), - # ProvenanceYAMLFileHasRegex( - # regex="cli_test_demo_workers:", - # name="feature_demo", - # output_path=OUTPUT_DIR, - # provenance_type="expanded", - # ), - # StepFileExists( - # "verify", - # "MERLIN_FINISHED", - # "feature_demo", - # OUTPUT_DIR, - # params=True, - # ), - # ], - # ), + "run type": "distributed", + }, } # combine and return test dictionaries @@ -486,6 +604,7 @@ def define_tests(): # provenence_equality_checks, # omitting provenance equality check because it is broken # style_checks, # omitting style checks due to different results on different machines dependency_checks, + stop_workers_tests, distributed_tests, ]: all_tests.update(test_dict)