From 7c8c21830082f69fe9036556b868385b3c30dfe3 Mon Sep 17 00:00:00 2001 From: "Frode Aarstad (IT SI SIB)" Date: Fri, 16 Oct 2020 08:30:11 +0200 Subject: [PATCH] Rewrite to use argparser --- python/job_runner/cli.py | 32 +++-- python/tests/job_runner/test_job_dispatch.py | 127 ++++++++++++++++++- 2 files changed, 144 insertions(+), 15 deletions(-) diff --git a/python/job_runner/cli.py b/python/job_runner/cli.py index 17cbe29c7a..8523359613 100644 --- a/python/job_runner/cli.py +++ b/python/job_runner/cli.py @@ -1,3 +1,4 @@ +import argparse import os import signal import sys @@ -8,21 +9,28 @@ def main(args): - # If run_path is defined, enter into that directory - if len(args) > 1: - run_path = args[1] - if not os.path.exists(run_path): - sys.exit("No such directory: {}".format(run_path)) - os.chdir(run_path) + parser = argparse.ArgumentParser( + description="Run all the jobs specified in jobs.json, or specify the names of the jobs to run." + ) + parser.add_argument("run_path", nargs="?", help="Path where jobs.json is located") + parser.add_argument( + "job", + nargs="*", + help="One or more jobs to be executed from the jobs.json file. If no jobs are specified, all jobs will be executed.", + ) + + parsed_args = parser.parse_args(args[1:]) - jobs_to_run = [] - if len(args) > 2: - jobs_to_run = args[2:] + # If run_path is defined, enter into that directory + if parsed_args.run_path is not None: + if not os.path.exists(parsed_args.run_path): + sys.exit("No such directory: {}".format(parsed_args.run_path)) + os.chdir(parsed_args.run_path) reporters = [] - is_interactive_run = len(args) > 2 - if is_interactive_run: + + if len(parsed_args.job) > 0: reporters.append(reporting.Interactive()) else: reporters.append(reporting.File()) @@ -30,7 +38,7 @@ def main(args): job_runner = JobRunner() - for job_status in job_runner.run(jobs_to_run): + for job_status in job_runner.run(parsed_args.job): for reporter in reporters: reporter.report(job_status) diff --git a/python/tests/job_runner/test_job_dispatch.py b/python/tests/job_runner/test_job_dispatch.py index 7e9aa8203f..55b718abca 100644 --- a/python/tests/job_runner/test_job_dispatch.py +++ b/python/tests/job_runner/test_job_dispatch.py @@ -5,14 +5,12 @@ import sys import time import unittest -from subprocess import Popen - import psutil +from subprocess import Popen from job_runner.cli import main from job_runner.reporting.message import Finish from tests.utils import tmpdir, wait_until - from unittest.mock import patch @@ -123,3 +121,126 @@ def test_job_dispatch_kills_itself_after_unsuccessful_job(self): main(["script.py", "/foo/bar/baz"]) mock_os.killpg.assert_called_with(17, signal.SIGKILL) + + @tmpdir(None) + def test_job_dispatch_run_subset_specified_as_parmeter(self): + + with open("dummy_executable", "w") as f: + f.write( + "#!/usr/bin/env python\n" + "import sys, os\n" + 'filename = "job_{}.out".format(sys.argv[1])\n' + 'f = open(filename, "w")\n' + "f.close()\n" + ) + + executable = os.path.realpath("dummy_executable") + os.chmod("dummy_executable", stat.S_IRWXU | stat.S_IRWXO | stat.S_IRWXG) + + self.job_list = { + "umask": "0002", + "DATA_ROOT": "", + "global_environment": {}, + "global_update_path": {}, + "jobList": [ + { + "name": "job_A", + "executable": executable, + "target_file": None, + "error_file": None, + "start_file": None, + "stdout": "dummy.stdout", + "stderr": "dummy.stderr", + "stdin": None, + "argList": ["A"], + "environment": None, + "exec_env": None, + "license_path": None, + "max_running_minutes": None, + "max_running": None, + "min_arg": 1, + "arg_types": [], + "max_arg": None, + }, + { + "name": "job_B", + "executable": executable, + "target_file": None, + "error_file": None, + "start_file": None, + "stdout": "dummy.stdout", + "stderr": "dummy.stderr", + "stdin": None, + "argList": ["B"], + "environment": None, + "exec_env": None, + "license_path": None, + "max_running_minutes": None, + "max_running": None, + "min_arg": 1, + "arg_types": [], + "max_arg": None, + }, + { + "name": "job_C", + "executable": executable, + "target_file": None, + "error_file": None, + "start_file": None, + "stdout": "dummy.stdout", + "stderr": "dummy.stderr", + "stdin": None, + "argList": ["C"], + "environment": None, + "exec_env": None, + "license_path": None, + "max_running_minutes": None, + "max_running": None, + "min_arg": 1, + "arg_types": [], + "max_arg": None, + }, + ], + "run_id": "", + "ert_pid": "", + } + + with open("jobs.json", "w") as f: + f.write(json.dumps(self.job_list)) + + current_dir = os.path.realpath(os.curdir) + + # Required to execute job_dispatch in separate process group by + # os.setsid moves the current process out of the current group + with open("job_dispatch_executer", "w") as f: + f.write( + "#!/usr/bin/env python\n" + "import os, sys\n" + "os.setsid()\n" + "os.execv(sys.argv[1], sys.argv[1:])\n" + "\n" + ) + os.chmod("job_dispatch_executer", stat.S_IRWXU | stat.S_IRWXO | stat.S_IRWXG) + + job_dispatch_script = os.path.realpath( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "../../../bin/job_dispatch.py", + ) + ) + + job_dispatch_process = Popen( + [ + os.path.realpath("job_dispatch_executer"), + job_dispatch_script, + current_dir, + "job_B", + "job_C", + ] + ) + + job_dispatch_process.wait() + + assert not os.path.isfile("{}/job_A.out".format(current_dir)) + assert os.path.isfile("{}/job_B.out".format(current_dir)) + assert os.path.isfile("{}/job_C.out".format(current_dir))