From bb68600ddc25b2e7a2e64370310569cc331fc7d6 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Wed, 9 Jun 2021 18:25:54 +0100 Subject: [PATCH 1/5] Allow pointing to a directory containing tests and emulator --- run_tests.py | 48 ++++++++++++++++++++------------------ run_utils.py | 10 ++++---- utils/calibration_utils.py | 6 ++--- utils/channel_access.py | 6 +++-- utils/emulator_launcher.py | 23 ++++++++++-------- 5 files changed, 50 insertions(+), 43 deletions(-) diff --git a/run_tests.py b/run_tests.py index 72c15de30..31e194280 100644 --- a/run_tests.py +++ b/run_tests.py @@ -8,6 +8,7 @@ import traceback import unittest from typing import List, Any +import importlib import six import xmlrunner @@ -17,7 +18,8 @@ from run_utils import ModuleTests from utils.device_launcher import device_launcher, device_collection_launcher -from utils.emulator_launcher import LewisLauncher, NullEmulatorLauncher, MultiLewisLauncher, Emulator, TestEmulatorData +from utils.emulator_launcher import LewisLauncher, NullEmulatorLauncher, MultiLewisLauncher, Emulator, TestEmulatorData, \ + DEVICE_EMULATOR_PATH from utils.ioc_launcher import IocLauncher, EPICS_TOP, IOCS_DIR from utils.free_ports import get_free_ports from utils.test_modes import TestModes @@ -98,10 +100,11 @@ def make_device_launchers_from_module(test_module, mode): if "emulator" in ioc and mode != TestModes.RECSIM: emulator_launcher_class = ioc.get("emulator_launcher_class", LewisLauncher) - emulator_launcher = emulator_launcher_class(test_module.__name__, ioc["emulator"], var_dir, + emulator_launcher = emulator_launcher_class(test_module.__name__, ioc["emulator"], emulator_path, var_dir, emmulator_port, ioc) elif "emulator" in ioc: - emulator_launcher = NullEmulatorLauncher(test_module.__name__, ioc["emulator"], var_dir, None, ioc) + emulator_launcher = NullEmulatorLauncher(test_module.__name__, ioc["emulator"], emulator_path, var_dir, + None, ioc) elif "emulators" in ioc and mode != TestModes.RECSIM: emulator_launcher_class = ioc.get("emulators_launcher_class", MultiLewisLauncher) test_emulator_data: List[TestEmulatorData] = ioc.get("emulators", []) @@ -273,7 +276,7 @@ def run_tests(prefix, module_name, tests_to_run, device_launchers, failfast_swit 'EPICS_CA_ADDR_LIST': "127.255.255.255" } - test_names = ["{}.{}".format(arguments.tests_path, test) for test in tests_to_run] + test_names = [f"tests.{test}" for test in tests_to_run] runner = xmlrunner.XMLTestRunner(output='test-reports', stream=sys.stdout, failfast=failfast_switch) test_suite = unittest.TestLoader().loadTestsFromNames(test_names) @@ -296,11 +299,6 @@ def run_tests(prefix, module_name, tests_to_run, device_launchers, failfast_swit pythondir = os.environ.get("PYTHON3DIR", None) - if pythondir is not None: - emulator_path = os.path.join(pythondir, "scripts") - else: - emulator_path = None - parser = argparse.ArgumentParser( description='Test an IOC under emulation by running tests against it') parser.add_argument('-l', '--list-devices', @@ -309,10 +307,6 @@ def run_tests(prefix, module_name, tests_to_run, device_launchers, failfast_swit help='Report devices that have not been tested.', action="store_true") parser.add_argument('-pf', '--prefix', default=os.environ.get("MYPVPREFIX", None), help='The instrument prefix; e.g. TE:NDW1373') - parser.add_argument('-e', '--emulator-path', default=emulator_path, - help="The path of the lewis.py file") - parser.add_argument('-py', '--python-path', default="C:\Instrument\Apps\Python3\python.exe", - help="The path of python.exe") parser.add_argument('--var-dir', default=None, help="Directory in which to create a log dir to write log file to and directory in which to " "create tmp dir which contains environments variables for the IOC. Defaults to " @@ -322,7 +316,7 @@ def run_tests(prefix, module_name, tests_to_run, device_launchers, failfast_swit Module just runs the tests in a module. Module.class runs the the test class in Module. Module.class.method runs a specific test.""") - parser.add_argument('-tp', '--tests-path', default="tests", + parser.add_argument('-tp', '--tests-path', default=f"{os.path.dirname(os.path.realpath(__file__))}\\tests", help="""Path to find the tests in, this must be a valid python module. Default is in the tests folder of this repo""") parser.add_argument('-f', '--failfast', action='store_true', @@ -332,17 +326,29 @@ def run_tests(prefix, module_name, tests_to_run, device_launchers, failfast_swit emulator/IOC or attach debugger for tests""") parser.add_argument('-tm', '--tests-mode', default=None, choices=['DEVSIM', 'RECSIM'], help="""Tests mode to run e.g. DEVSIM or RECSIM (default: both).""") + parser.add_argument('--test_and_emulator', default=None, + help="""Specify a folder that holds both the tests (in a folder called tests) and a lewis + emulator (in a folder called lewis_emulators).""") arguments = parser.parse_args() + if arguments.test_and_emulator: + arguments.tests_path = os.path.join(arguments.test_and_emulator, "tests") + emulator_path = arguments.test_and_emulator + else: + emulator_path = DEVICE_EMULATOR_PATH + if os.path.dirname(arguments.tests_path): full_path = os.path.abspath(arguments.tests_path) - if not os.path.isdir(full_path): - print("Test path {} not found".format(full_path)) + init_file_path = os.path.join(full_path, "__init__.py") + if not os.path.isfile(init_file_path): + print(f"Test path {full_path} not found") sys.exit(-1) - tests_module_path = os.path.dirname(full_path) - sys.path.insert(0, tests_module_path) - arguments.tests_path = os.path.basename(arguments.tests_path) + # Import the specified path as the tests module + spec = importlib.util.spec_from_file_location("tests", init_file_path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) if arguments.list_devices: print("Available tests:") @@ -356,10 +362,6 @@ def run_tests(prefix, module_name, tests_to_run, device_launchers, failfast_swit print("Cannot run without instrument prefix, you may need to run this using an EPICS terminal") sys.exit(-1) - if arguments.emulator_path is None: - print("Cannot run without emulator path, you may need to run this using an EPICS terminal") - sys.exit(-1) - tests = arguments.tests if arguments.tests is not None else package_contents(arguments.tests_path) failfast = arguments.failfast report_coverage = arguments.report_coverage diff --git a/run_utils.py b/run_utils.py index 36ce12f74..cca1da81c 100644 --- a/run_utils.py +++ b/run_utils.py @@ -3,15 +3,15 @@ from contextlib import contextmanager -def package_contents(package_name): +def package_contents(package_path): """ - Finds all the modules in a package. + Finds all the files in a package. - :param package_name: the name of the package + :param package_path: the name of the package :return: a set containing all the module names """ - pathname = importlib.util.find_spec(package_name).name - return set([os.path.splitext(module)[0] for module in os.listdir(pathname) + print(f"Test path is {package_path}") + return set([os.path.splitext(module)[0] for module in os.listdir(package_path) if module.endswith('.py') and not module.startswith("__init__")]) diff --git a/utils/calibration_utils.py b/utils/calibration_utils.py index 18473579f..2d1ca4d63 100644 --- a/utils/calibration_utils.py +++ b/utils/calibration_utils.py @@ -11,12 +11,12 @@ def set_calibration_file(channel_access, filename): max_retries = 10 for _ in range(max_retries): - channel_access.set_pv_value(CAL_SEL_PV, filename) - channel_access.assert_that_pv_alarm_is(CAL_SEL_PV, channel_access.Alarms.NONE) - time.sleep(3) channel_access.assert_that_pv_alarm_is("{}:RBV".format(CAL_SEL_PV), channel_access.Alarms.NONE) if channel_access.get_pv_value("CAL:RBV") == filename: break + channel_access.set_pv_value(CAL_SEL_PV, filename) + channel_access.assert_that_pv_alarm_is(CAL_SEL_PV, channel_access.Alarms.NONE) + time.sleep(3) else: raise Exception("Couldn't set calibration file to '{}' after {} tries".format(filename, max_retries)) diff --git a/utils/channel_access.py b/utils/channel_access.py index d1b731304..977e302ed 100644 --- a/utils/channel_access.py +++ b/utils/channel_access.py @@ -242,9 +242,11 @@ def _wrapper(message): if message is None: message = "Expected function '{}' to evaluate to True when reading PV '{}'." \ .format(func.__name__, self.create_pv_with_prefix(pv)) - + + start_time = time.time() err = self._wait_for_pv_lambda(partial(_wrapper, message), timeout) - + end_time = time.time() + print(f"Waiting on {pv} took {end_time-start_time}") if err is not None: raise AssertionError(err) diff --git a/utils/emulator_launcher.py b/utils/emulator_launcher.py index 0f3c6e45d..1817142d4 100644 --- a/utils/emulator_launcher.py +++ b/utils/emulator_launcher.py @@ -64,11 +64,12 @@ def remove_emulator(cls, name): @six.add_metaclass(abc.ABCMeta) class EmulatorLauncher(object): - def __init__(self, test_name, device, var_dir, port, options): + def __init__(self, test_name, device, emulator_path, var_dir, port, options): """ Args: test_name: The name of the test we are creating a device emulator for device: The name of the device to emulate + emulator_path: The path where the emulator can be found var_dir: The directory in which to store logs port: The TCP port to listen on for connections options: Dictionary of any additional options required by specific launchers @@ -79,6 +80,7 @@ def __init__(self, test_name, device, var_dir, port, options): self._port = port self._options = options self._test_name = test_name + self._emulator_path = emulator_path def __enter__(self): self._open() @@ -354,22 +356,23 @@ class LewisLauncher(EmulatorLauncher): _DEFAULT_LEWIS_PATH = os.path.join(DEFAULT_PY_PATH, "scripts") - def __init__(self, test_name, device, var_dir, port, options): + def __init__(self, test_name, device, emulator_path, var_dir, port, options): """ Constructor that also launches Lewis. Args: test_name: name of test we are creating device emulator for device: device to start + emulator_path: The path where the emulator can be found var_dir: location of directory to write log file and macros directories port: the port to use """ - super(LewisLauncher, self).__init__(test_name, device, var_dir, port, options) + super(LewisLauncher, self).__init__(test_name, device, emulator_path, var_dir, port, options) self._lewis_path = options.get("lewis_path", LewisLauncher._DEFAULT_LEWIS_PATH) self._python_path = options.get("python_path", os.path.join(DEFAULT_PY_PATH, "python.exe")) self._lewis_protocol = options.get("lewis_protocol", "stream") - self._lewis_additional_path = options.get("lewis_additional_path", DEVICE_EMULATOR_PATH) + self._lewis_additional_path = options.get("lewis_additional_path", emulator_path) self._lewis_package = options.get("lewis_package", "lewis_emulators") self._default_timeout = options.get("default_timeout", 5) self._speed = options.get("speed", 100) @@ -624,8 +627,8 @@ def backdoor_run_function_on_device(self, launcher_address, function_name, argum class CommandLineEmulatorLauncher(EmulatorLauncher): - def __init__(self, test_name, device, var_dir, port, options): - super(CommandLineEmulatorLauncher, self).__init__(test_name, device, var_dir, port, options) + def __init__(self, test_name, device, emulator_path, var_dir, port, options): + super(CommandLineEmulatorLauncher, self).__init__(test_name, device, emulator_path, var_dir, port, options) try: self.command_line = options["emulator_command_line"] except KeyError: @@ -677,7 +680,7 @@ def backdoor_run_function_on_device(self, *args, **kwargs): class BeckhoffEmulatorLauncher(CommandLineEmulatorLauncher): - def __init__(self, test_name, device, var_dir, port, options): + def __init__(self, test_name, device, emulator_path, var_dir, port, options): try: self.beckhoff_root = options["beckhoff_root"] self.solution_path = options["solution_path"] @@ -695,20 +698,20 @@ def __init__(self, test_name, device, var_dir, port, options): options["emulator_command_line"] = self.startup_command options["emulator_wait_to_finish"] = True - super(BeckhoffEmulatorLauncher, self).__init__(test_name, device, var_dir, port, options) + super(BeckhoffEmulatorLauncher, self).__init__(test_name, device, emulator_path, var_dir, port, options) else: raise IOError("Unable to find AutomationTools.exe. Hint: You must build the solution located at:" " {} \n".format(automation_tools_dir)) class DAQMxEmulatorLauncher(CommandLineEmulatorLauncher): - def __init__(self, test_name, device, var_dir, port, options): + def __init__(self, test_name, device, emulator_path, var_dir, port, options): labview_scripts_dir = os.path.join(DEVICE_EMULATOR_PATH, "other_emulators", "DAQmx") self.start_command = os.path.join(labview_scripts_dir, "start_sim.bat") self.stop_command = os.path.join(labview_scripts_dir, "stop_sim.bat") options["emulator_command_line"] = self.start_command options["emulator_wait_to_finish"] = True - super(DAQMxEmulatorLauncher, self).__init__(test_name, device, var_dir, port, options) + super(DAQMxEmulatorLauncher, self).__init__(test_name, device, emulator_path, var_dir, port, options) def _close(self): self.disconnect_device() From 273a863901092f77b3e59a3e7c78393fa719d2a9 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Wed, 9 Jun 2021 18:26:48 +0100 Subject: [PATCH 2/5] Remove ccd100 as now in it's support folder --- tests/ccd100.py | 113 ------------------------------------------------ 1 file changed, 113 deletions(-) delete mode 100644 tests/ccd100.py diff --git a/tests/ccd100.py b/tests/ccd100.py deleted file mode 100644 index 92e034cd1..000000000 --- a/tests/ccd100.py +++ /dev/null @@ -1,113 +0,0 @@ -import unittest -from time import sleep - -from utils.test_modes import TestModes -from utils.channel_access import ChannelAccess -from utils.ioc_launcher import get_default_ioc_dir -from utils.testing import get_running_lewis_and_ioc, assert_log_messages, skip_if_recsim, unstable_test -from parameterized import parameterized - -# Device prefix -DEVICE_A_PREFIX = "CCD100_01" -DEVICE_E_PREFIX = "CCD100_02" - -EMULATOR_DEVICE = "CCD100" - -IOCS = [ - { - "name": DEVICE_A_PREFIX, - "directory": get_default_ioc_dir("CCD100"), - "emulator": EMULATOR_DEVICE, - "emulator_id": DEVICE_A_PREFIX, - }, - - { - "name": DEVICE_E_PREFIX, - "directory": get_default_ioc_dir("CCD100", iocnum=2), - "emulator": EMULATOR_DEVICE, - "emulator_id": DEVICE_E_PREFIX, - "macros": {"ADDRESS": "e"}, - }, -] - - -TEST_MODES = [TestModes.RECSIM, TestModes.DEVSIM] - - -def set_up_connections(device): - _lewis, _ioc = get_running_lewis_and_ioc(device, device) - - _lewis.backdoor_set_on_device('connected', True) - _lewis.backdoor_set_on_device("is_giving_errors", False) - - return _lewis, _ioc, ChannelAccess(device_prefix=device) - - -class CCD100Tests(unittest.TestCase): - """ - General tests for the CCD100. - """ - def setUp(self): - self._lewis, self._ioc, self.ca = set_up_connections(DEVICE_A_PREFIX) - - @parameterized.expand([("0", 0), ("1.23", 1.23), ("10", 10)]) - def test_GIVEN_setpoint_set_WHEN_readback_THEN_readback_is_same_as_setpoint(self, _, point): - self.ca.set_pv_value("READING:SP", point) - self.ca.assert_that_pv_is("READING:SP:RBV", point) - - @skip_if_recsim("Can not test disconnection in rec sim") - def test_GIVEN_device_not_connected_WHEN_get_status_THEN_alarm(self): - self._lewis.backdoor_set_on_device('connected', False) - self.ca.assert_that_pv_alarm_is('READING', ChannelAccess.Alarms.INVALID) - - -class CCD100SecondDeviceTests(CCD100Tests): - """ - Tests for the second CCD100 device. - """ - def setUp(self): - self._lewis, self._ioc, self.ca = set_up_connections(DEVICE_E_PREFIX) - self._lewis.backdoor_set_on_device("address", "e") - - -class CCD100LogTests(unittest.TestCase): - """ - Tests for the log messages produced by CCD100. - - In general all we want to test here is that we're not producing excessive messages. Unfortunately some of the - messages outputted by autosave etc. are outside our control so we're just testing messages are less than some value. - """ - NUM_OF_PVS = 3 - - def setUp(self): - self._lewis, self._ioc, self.ca = set_up_connections(DEVICE_A_PREFIX) - - def _set_error_state(self, state): - self._lewis.backdoor_set_on_device("is_giving_errors", state) - sleep(2) # Wait for previous errors to get logged - - @skip_if_recsim("Cannot check log messages in rec sim") - def test_GIVEN_not_in_error_WHEN_put_in_error_THEN_less_than_four_log_messages_per_pv_logged_in_five_secs(self): - self._set_error_state(False) # Should already be out of error state but doing this again to ingest logs - # Actually expect 3 messages but checking for 4 as a buffer see comment above - with assert_log_messages(self._ioc, self.NUM_OF_PVS*4, 5) as log: - self._set_error_state(True) - - @skip_if_recsim("Cannot check log messages in rec sim") - def test_GIVEN_in_error_WHEN_error_cleared_THEN_less_than_two_log_message_per_pv_logged(self): - self._set_error_state(True) - # Actually expect 1 message but checking for 2 as a buffer see comment above - with assert_log_messages(self._ioc, self.NUM_OF_PVS*2): - self._set_error_state(False) - - @skip_if_recsim("Cannot check log messages in rec sim") - def test_GIVEN_in_error_WHEN_error_string_changed_THEN_less_than_four_log_message_per_pv_logged_in_five_secs(self): - self._lewis.backdoor_set_on_device("out_error", "OLD_ERROR") - self._set_error_state(True) - - new_error = "A_NEW_ERROR" - # Actually expect 3 messages but checking for 4 as a buffer see comment above - with assert_log_messages(self._ioc, self.NUM_OF_PVS*4, 5) as log: - self._lewis.backdoor_set_on_device("out_error", new_error) - - self.assertTrue(any([new_error in _ for _ in log.messages[-3:]])) From 0db9a389e6bea53541cbf0c8c3571600f0199026 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Wed, 9 Jun 2021 18:35:53 +0100 Subject: [PATCH 3/5] Added makefile so tests can be run using make ioctests --- Makefile | 2 ++ run_tests.py | 2 -- run_utils.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..fcab4dc6b --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +ioctests: + $(PYTHON3) run_tests.py diff --git a/run_tests.py b/run_tests.py index 31e194280..92c35ef22 100644 --- a/run_tests.py +++ b/run_tests.py @@ -297,8 +297,6 @@ def run_tests(prefix, module_name, tests_to_run, device_launchers, failfast_swit print("IOC system tests should now be run under python 3. Aborting.") sys.exit(-1) - pythondir = os.environ.get("PYTHON3DIR", None) - parser = argparse.ArgumentParser( description='Test an IOC under emulation by running tests against it') parser.add_argument('-l', '--list-devices', diff --git a/run_utils.py b/run_utils.py index cca1da81c..8ed9fb12c 100644 --- a/run_utils.py +++ b/run_utils.py @@ -10,7 +10,6 @@ def package_contents(package_path): :param package_path: the name of the package :return: a set containing all the module names """ - print(f"Test path is {package_path}") return set([os.path.splitext(module)[0] for module in os.listdir(package_path) if module.endswith('.py') and not module.startswith("__init__")]) From b6e01bd8e91eec6d004b6a57636ea9293393c0a9 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Thu, 10 Jun 2021 18:04:11 +0100 Subject: [PATCH 4/5] Remove accidental commits --- utils/calibration_utils.py | 6 +++--- utils/channel_access.py | 3 --- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/utils/calibration_utils.py b/utils/calibration_utils.py index 2d1ca4d63..18473579f 100644 --- a/utils/calibration_utils.py +++ b/utils/calibration_utils.py @@ -11,12 +11,12 @@ def set_calibration_file(channel_access, filename): max_retries = 10 for _ in range(max_retries): - channel_access.assert_that_pv_alarm_is("{}:RBV".format(CAL_SEL_PV), channel_access.Alarms.NONE) - if channel_access.get_pv_value("CAL:RBV") == filename: - break channel_access.set_pv_value(CAL_SEL_PV, filename) channel_access.assert_that_pv_alarm_is(CAL_SEL_PV, channel_access.Alarms.NONE) time.sleep(3) + channel_access.assert_that_pv_alarm_is("{}:RBV".format(CAL_SEL_PV), channel_access.Alarms.NONE) + if channel_access.get_pv_value("CAL:RBV") == filename: + break else: raise Exception("Couldn't set calibration file to '{}' after {} tries".format(filename, max_retries)) diff --git a/utils/channel_access.py b/utils/channel_access.py index 977e302ed..6a7c26e03 100644 --- a/utils/channel_access.py +++ b/utils/channel_access.py @@ -243,10 +243,7 @@ def _wrapper(message): message = "Expected function '{}' to evaluate to True when reading PV '{}'." \ .format(func.__name__, self.create_pv_with_prefix(pv)) - start_time = time.time() err = self._wait_for_pv_lambda(partial(_wrapper, message), timeout) - end_time = time.time() - print(f"Waiting on {pv} took {end_time-start_time}") if err is not None: raise AssertionError(err) From 2e83b5828053c843f9f15c808ec0bf30bc1901e7 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Mon, 14 Jun 2021 12:24:46 +0100 Subject: [PATCH 5/5] Update documentation --- README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ab3ea1fc1..f031c237d 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ It recommended that you don't have the server side of IBEX running when testing ### Running all tests -To run all the tests in the test framework, use: +To run all the tests that are bundled in the test framework, use: ``` C:\Instrument\Apps\EPICS\config_env.bat @@ -23,7 +23,9 @@ Then `cd` to `C:\Instrument\Apps\EPICS\support\IocTestFramework\master` and use: python run_tests.py ``` -There is a batch file which does this for you, called `run_all_tests.bat` +There is a batch file which does this for you, called `run_all_tests.bat`. Or, if you are already in an EPICS +terminal you can call `make ioctests` (note that if your MAKEFLAGS env variable contains `-Otarget` you will +not see the test output until all tests are complete. ### Running tests in modules @@ -83,6 +85,13 @@ Sometimes you might want to run all the tests only in RECSIM or only in DEVSIM. > `python run_tests.py -tm RECSIM` +### Run test and emulator from specific directory + +For newer IOCs the emulator and tests live in the support folder of the IOC. To specify this to the test framework +you can use: + +> `python run_tests.py --test_and_emulator C:\Instrument\Apps\EPICS\support\CCD100\master\system_tests` + ## Troubleshooting If all tests are failing then it is likely that the PV prefix is incorrect.