Have integration tests run with both queue and scheduler

This commits adds the pytest.mark.scheduler mark and scheduler fixture to some of the integration tests, so that they will be ran with both the scheduler and job queue.

tests/integration_tests/analysis/test_adaptive_localization.py

@@ -43,9 +43,10 @@ def run_cli_ES_with_case(poly_config):
     return prior_sample, posterior_sample
 
 
+@pytest.mark.scheduler
 @pytest.mark.integration_test
 def test_that_adaptive_localization_with_cutoff_1_equals_ensemble_prior(
-    copy_poly_case
+    copy_poly_case, try_queue_and_scheduler
 ):
     set_adaptive_localization_1 = dedent(
         """
@@ -67,9 +68,10 @@ def test_that_adaptive_localization_with_cutoff_1_equals_ensemble_prior(
     assert np.allclose(posterior_sample, prior_sample)
 
 
+@pytest.mark.scheduler
 @pytest.mark.integration_test
 def test_that_adaptive_localization_with_cutoff_0_equals_ESupdate(
-    copy_poly_case
+    copy_poly_case, try_queue_and_scheduler
 ):
     """
     Note that "RANDOM_SEED" in both ert configs needs to be the same to obtain
@@ -105,9 +107,10 @@ def test_that_adaptive_localization_with_cutoff_0_equals_ESupdate(
     assert np.allclose(posterior_sample_loc0, posterior_sample_noloc)
 
 
+@pytest.mark.scheduler
 @pytest.mark.integration_test
 def test_that_posterior_generalized_variance_increases_in_cutoff(
-    copy_poly_case
+    copy_poly_case, try_queue_and_scheduler
 ):
     rng = np.random.default_rng(42)
     cutoff1 = rng.uniform(0, 1)

tests/integration_tests/analysis/test_es_update.py

@@ -64,9 +64,10 @@ def obs():
     )
 
 
+@pytest.mark.scheduler
 @pytest.mark.integration_test
 def test_that_posterior_has_lower_variance_than_prior(
-    copy_case
+    copy_case, try_queue_and_scheduler
 ):
     copy_case("poly_example")
 
@@ -102,9 +103,10 @@ def test_that_posterior_has_lower_variance_than_prior(
     )
 
 
+@pytest.mark.scheduler
 @pytest.mark.integration_test
 def test_that_surfaces_retain_their_order_when_loaded_and_saved_by_ert(
-    copy_case
+    copy_case, try_queue_and_scheduler
 ):
     """This is a regression test to make sure ert does not use the wrong order
     (row-major / column-major) when working with surfaces.
@@ -187,8 +189,9 @@ def sample_prior(nx, ny):
         )
 
 
+@pytest.mark.scheduler
 @pytest.mark.integration_test
-def test_update_multiple_param(copy_case):
+def test_update_multiple_param(copy_case, try_queue_and_scheduler):
     """
     Note that this is now a snapshot test, so there is no guarantee that the
     snapshots are correct, they are just documenting the current behavior.

tests/integration_tests/cli/test_integration_cli.py

@@ -41,8 +41,9 @@ def fixture_mock_cli_run(monkeypatch):
     yield mocked_monitor, mocked_thread_join, mocked_thread_start
 
 
+@pytest.mark.scheduler
 @pytest.mark.integration_test
-def test_runpath_file(tmpdir, source_root):
+def test_runpath_file(tmpdir, source_root, try_queue_and_scheduler):
     shutil.copytree(
         os.path.join(source_root, "test-data", "poly_example"),
         os.path.join(str(tmpdir), "poly_example"),
@@ -77,8 +78,9 @@ def test_runpath_file(tmpdir, source_root):
         assert os.path.isfile("RUNPATH_WORKFLOW_1.OK")
 
 
+@pytest.mark.scheduler
 @pytest.mark.integration_test
-def test_ensemble_evaluator(tmpdir, source_root):
+def test_ensemble_evaluator(tmpdir, source_root, try_queue_and_scheduler):
     shutil.copytree(
         os.path.join(source_root, "test-data", "poly_example"),
         os.path.join(str(tmpdir), "poly_example"),
@@ -103,8 +105,9 @@ def test_ensemble_evaluator(tmpdir, source_root):
         FeatureToggling.reset()
 
 
+@pytest.mark.scheduler
 @pytest.mark.integration_test
-def test_es_mda(tmpdir, source_root, snapshot):
+def test_es_mda(tmpdir, source_root, snapshot, try_queue_and_scheduler):
     shutil.copytree(
         os.path.join(source_root, "test-data", "poly_example"),
         os.path.join(str(tmpdir), "poly_example"),
@@ -195,9 +198,10 @@ def remove_linestartswith(file_name: str, startswith: str):
             run_cli(parsed)
 
 
+@pytest.mark.scheduler
 @pytest.mark.integration_test
 def test_ensemble_evaluator_disable_monitoring(
-    tmpdir, source_root
+    tmpdir, source_root, try_queue_and_scheduler
 ):
     shutil.copytree(
         os.path.join(source_root, "test-data", "poly_example"),
@@ -224,8 +228,9 @@ def test_ensemble_evaluator_disable_monitoring(
         FeatureToggling.reset()
 
 
+@pytest.mark.scheduler
 @pytest.mark.integration_test
-def test_cli_test_run(tmpdir, source_root, mock_cli_run):
+def test_cli_test_run(tmpdir, source_root, mock_cli_run, try_queue_and_scheduler):
     shutil.copytree(
         os.path.join(source_root, "test-data", "poly_example"),
         os.path.join(str(tmpdir), "poly_example"),
@@ -242,8 +247,9 @@ def test_cli_test_run(tmpdir, source_root, mock_cli_run):
     thread_start_mock.assert_has_calls([[call(), call()]])
 
 
+@pytest.mark.scheduler
 @pytest.mark.integration_test
-def test_ies(tmpdir, source_root):
+def test_ies(tmpdir, source_root, try_queue_and_scheduler):
     shutil.copytree(
         os.path.join(source_root, "test-data", "poly_example"),
         os.path.join(str(tmpdir), "poly_example"),
@@ -268,6 +274,7 @@ def test_ies(tmpdir, source_root):
         FeatureToggling.reset()
 
 
+@pytest.mark.scheduler
 @pytest.mark.integration_test
 def test_that_running_ies_with_different_steplength_produces_different_result(
     tmpdir, source_root
@@ -348,6 +355,7 @@ def _run(target):
         assert not np.isclose(result_1.loc["iter-1"], result_2.loc["iter-1"]).all()
 
 
+@pytest.mark.scheduler
 @pytest.mark.integration_test
 @pytest.mark.parametrize(
     "prior_mask,reals_rerun_option,should_resample",
@@ -374,7 +382,8 @@ def test_that_prior_is_not_overwritten_in_ensemble_experiment(
     reals_rerun_option,
     should_resample,
     tmpdir,
-    source_root
+    source_root,
+    try_queue_and_scheduler,
 ):
     shutil.copytree(
         os.path.join(source_root, "test-data", "poly_example"),
@@ -420,8 +429,9 @@ def test_that_prior_is_not_overwritten_in_ensemble_experiment(
         storage.close()
 
 
+@pytest.mark.scheduler(skip=True)
 @pytest.mark.integration_test
-@pytest.mark.usefixtures("copy_poly_case")
+@pytest.mark.usefixtures("copy_poly_case", "try_queue_and_scheduler")
 def test_failing_job_cli_error_message():
     # modify poly_eval.py
     with open("poly_eval.py", mode="a", encoding="utf-8") as poly_script:

tests/integration_tests/job_queue/test_lsf_driver.py

@@ -176,7 +176,9 @@ def copy_lsf_poly_case(copy_poly_case, tmp_path):
     "mock_bsub",
     "mock_bjobs",
     "mock_start_server",
+    "try_queue_and_scheduler",
 )
+@pytest.mark.scheduler(skip=True)
 @pytest.mark.integration_test
 def test_run_mocked_lsf_queue():
     run_cli(

tests/integration_tests/shared/share/test_shell.py

@@ -5,8 +5,9 @@
 import pytest
 
 
+@pytest.mark.scheduler
 @pytest.mark.integration_test
-def test_shell_scripts_integration(tmpdir):
+def test_shell_scripts_integration(tmpdir, try_queue_and_scheduler):
     """
     The following test is a regression test that
     checks that the scripts under src/ert/shared/share/ert/shell_scripts

tests/integration_tests/status/test_tracking_integration.py

@@ -47,6 +47,7 @@ def check_expression(original, path_expression, expected, msg_start):
     assert match_found, f"{msg_start} Nothing matched {path_expression}"
 
 
+@pytest.mark.scheduler(skip=True)
 @pytest.mark.integration_test
 @pytest.mark.parametrize(
     (
@@ -151,6 +152,7 @@ def test_tracking(
     tmpdir,
     source_root,
     storage,
+    try_queue_and_scheduler,
 ):
     experiment_folder = "poly_example"
     shutil.copytree(
@@ -258,6 +260,7 @@ def test_tracking(
     FeatureToggling.reset()
 
 
+@pytest.mark.scheduler(skip=True)
 @pytest.mark.integration_test
 @pytest.mark.parametrize(
     ("mode, cmd_line_arguments"),
@@ -281,7 +284,7 @@ def test_tracking(
     ],
 )
 def test_setting_env_context_during_run(
-    mode, cmd_line_arguments, tmpdir, source_root, storage
+    mode, cmd_line_arguments, tmpdir, source_root, storage, try_queue_and_scheduler
 ):
     experiment_folder = "poly_example"
     shutil.copytree(
@@ -359,9 +362,10 @@ def run_sim(start_date):
     summary.fwrite()
 
 
+@pytest.mark.scheduler(skip=True)
 @pytest.mark.integration_test
 def test_tracking_missing_ecl(
-    tmpdir, source_root, caplog, storage
+    tmpdir, source_root, caplog, storage, try_queue_and_scheduler
 ):
     with tmpdir.as_cwd():
         config = dedent(

tests/integration_tests/storage/test_field_parameter.py

@@ -144,9 +144,7 @@ def test_field_param_update(tmpdir):
 
 
 @pytest.mark.integration_test
-def test_parameter_update_with_inactive_cells_xtgeo_grdecl(
-    tmpdir
-):
+def test_parameter_update_with_inactive_cells_xtgeo_grdecl(tmpdir):
     """
     This replicates the poly example, only it uses FIELD parameter
     """