refactor/integration-tests (#404)

Update integration test suite and add stop-workers tests. Also update the copyright year in every file.

.github/workflows/push-pr_workflow.yml

@@ -131,7 +131,7 @@ jobs:
       run: |
         python3 -m pytest tests/unit/
 
-    - name: Run integration test suite
+    - name: Run integration test suite for local tests
       run: |
         python3 tests/integration/run_tests.py --verbose --local
 
@@ -179,7 +179,7 @@ jobs:
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         pip3 install -r requirements/dev.txt
 
-    - name: Install merlin to run unit tests
+    - name: Install merlin and setup redis as the broker
       run: |
         pip3 install -e .
         merlin config --broker redis
@@ -189,12 +189,12 @@ jobs:
         merlin example feature_demo
         pip3 install -r feature_demo/requirements.txt
 
-    - name: Run integration test suite for Redis
+    - name: Run integration test suite for distributed tests
       env:
         REDIS_HOST: redis
         REDIS_PORT: 6379
       run: |
-        python3 tests/integration/run_tests.py --verbose --ids 31 32
+        python3 tests/integration/run_tests.py --verbose --distributed
 
     # - name: Setup rabbitmq config
     #   run: |

.gitignore

@@ -30,6 +30,9 @@ ARCHIVE_DIR
 *_OUTPUT/
 *_OUTPUT_D/
 *_ensemble_*/
+studies/
+appendonlydir/
+cli_test_studies/
 
 # Scheduler logs
 flux.out

CHANGELOG.md

@@ -8,15 +8,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - Pip wheel wasn't including .sh files for merlin examples
 - The learn.py script in the openfoam_wf* examples will now create the missing Energy v Lidspeed plot
+
 ### Added
 - Now loads np.arrays of dtype='object', allowing mix-type sample npy
 - Added a singularity container openfoam_wf example
 - Added flux native worker launch support
 - Added PBS flux launch support
 - Added check_for_flux, check_for_slurm, check_for_lsf, and check_for_pbs utility functions
+- Tests for the `stop-workers` command
+- A function in `run_tests.py` to check that an integration test definition is formatted correctly
+- A new dev_workflow example `multiple_workers.yaml` that's used for testing the `stop-workers` command
+- Ability to start 2 subprocesses for a single test
+- Added the --distributed and --display-table flags to run_tests.py
+  - --distributed: only run distributed tests
+  - --display-tests: displays a table of all existing tests and the id associated with each test
 
 ### Changed
 - Changed celery_regex to celery_slurm_regex in test_definitions.py
+- Reformatted how integration tests are defined and part of how they run
+  - Test values are now dictionaries rather than tuples
+  - Stopped using `subprocess.Popen()` and `subprocess.communicate()` to run tests and now instead use `subprocess.run()` for simplicity and to keep things up-to-date with the latest subprocess release (`run()` will call `Popen()` and `communicate()` under the hood so we don't have to handle that anymore)
+- Rewrote the README in the integration tests folder to explain the new integration test format
 
 ## [1.9.1]
 ### Fixed

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 Lawrence Livermore National Laboratory
+Copyright (c) 2023 Lawrence Livermore National Laboratory
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>
@@ -38,6 +38,8 @@ include config.mk
 .PHONY : e2e-tests-diagnostic
 .PHONY : e2e-tests-local
 .PHONY : e2e-tests-local-diagnostic
+.PHONY : e2e-tests-distributed
+.PHONY : e2e-tests-distributed-diagnostic
 .PHONY : tests
 .PHONY : check-flake8
 .PHONY : check-black
@@ -109,6 +111,16 @@ e2e-tests-local-diagnostic:
 	$(PYTHON) $(TEST)/integration/run_tests.py --local --verbose
 
 
+e2e-tests-distributed:
+	. $(VENV)/bin/activate; \
+	$(PYTHON) $(TEST)/integration/run_tests.py --distributed; \
+
+
+e2e-tests-distributed-diagnostic:
+	. $(VENV)/bin/activate; \
+	$(PYTHON) $(TEST)/integration/run_tests.py --distributed --verbose
+
+
 # run unit and CLI tests
 tests: unit-tests e2e-tests
 
@@ -185,6 +197,17 @@ version:
 	find tests/ -type f -print0 | xargs -0 sed -i 's/Version: $(VSTRING)/Version: $(VER)/g'
 	find Makefile -type f -print0 | xargs -0 sed -i 's/Version: $(VSTRING)/Version: $(VER)/g'
 
+# Increment copyright year
+year:
+# do LICENSE (no comma after year)
+	sed -i 's/$(YEAR) Lawrence Livermore/$(NEW_YEAR) Lawrence Livermore/g' LICENSE
+
+# do all file headers (works on linux)
+	find merlin/ -type f -print0 | xargs -0 sed -i 's/$(YEAR), Lawrence Livermore/$(NEW_YEAR), Lawrence Livermore/g'
+	find *.py -type f -print0 | xargs -0 sed -i 's/$(YEAR), Lawrence Livermore/$(NEW_YEAR), Lawrence Livermore/g'
+	find tests/ -type f -print0 | xargs -0 sed -i 's/$(YEAR), Lawrence Livermore/$(NEW_YEAR), Lawrence Livermore/g'
+	find Makefile -type f -print0 | xargs -0 sed -i 's/$(YEAR), Lawrence Livermore/$(NEW_YEAR), Lawrence Livermore/g'
+
 # Make a list of all dependencies/requirements
 reqlist:
 	johnnydep merlin --output-format pinned

config.mk

@@ -19,6 +19,8 @@ endif
 
 VER?=1.0.0
 VSTRING=[0-9]\+\.[0-9]\+\.[0-9]\+
+YEAR=20[0-9][0-9]
+NEW_YEAR?=2023
 CHANGELOG_VSTRING="## \[$(VSTRING)\]"
 INIT_VSTRING="__version__ = \"$(VSTRING)\""
 

merlin/__init__.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/ascii_art.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/celery.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/common/__init__.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/common/abstracts/__init__.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/common/abstracts/enums/__init__.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/common/openfilelist.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/common/opennpylib.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/common/sample_index.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/common/sample_index_factory.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/common/security/__init__.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/common/security/encrypt.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/common/security/encrypt_backend_traffic.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/common/tasks.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/common/util_sampling.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/config/__init__.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/config/broker.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/config/celeryconfig.py

@@ -3,7 +3,7 @@
 """
 
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/config/configfile.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/config/results_backend.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/config/utils.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>

merlin/data/celery/__init__.py

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC.
+# Copyright (c) 2023, Lawrence Livermore National Security, LLC.
 # Produced at the Lawrence Livermore National Laboratory
 # Written by the Merlin dev team, listed in the CONTRIBUTORS file.
 # <[email protected]>