From 6858181c71559e9cf89c0c0a46de7960f728e2f9 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Mon, 15 Jul 2024 15:11:13 +0100 Subject: [PATCH 1/5] messages: enforce unique task messages * Closes https://github.com/cylc/cylc-flow/issues/6056 --- cylc/flow/config.py | 9 ++++++ tests/integration/validate/test_outputs.py | 34 ++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/cylc/flow/config.py b/cylc/flow/config.py index 5b7738f3e6c..d29537ce7ab 100644 --- a/cylc/flow/config.py +++ b/cylc/flow/config.py @@ -2434,9 +2434,18 @@ def get_taskdef( raise WorkflowConfigError(str(exc)) else: # Record custom message outputs from [runtime]. + messages = set(self.cfg['runtime'][name]['outputs'].values()) for output, message in ( self.cfg['runtime'][name]['outputs'].items() ): + try: + messages.remove(message) + except KeyError: + raise WorkflowConfigError( + 'Duplicate task message in' + f' "[runtime][{name}][outputs]' + f'{output} = {message}" - messages must be unique' + ) valid, msg = TaskOutputValidator.validate(output) if not valid: raise WorkflowConfigError( diff --git a/tests/integration/validate/test_outputs.py b/tests/integration/validate/test_outputs.py index b26bce529fb..1a8177cd35b 100644 --- a/tests/integration/validate/test_outputs.py +++ b/tests/integration/validate/test_outputs.py @@ -339,3 +339,37 @@ def test_completion_expression_cylc7_compat( match="completion cannot be used in Cylc 7 compatibility mode." ): validate(id_) + + +def test_unique_messages( + flow, + validate +): + """Task messages must be unique in the [outputs] section. + + See: https://github.com/cylc/cylc-flow/issues/6056 + """ + id_ = flow({ + 'scheduling': { + 'graph': {'R1': 'foo'} + }, + 'runtime': { + 'foo': { + 'outputs': { + 'a': 'foo', + 'b': 'bar', + 'c': 'baz', + 'd': 'foo', + } + }, + } + }) + + with pytest.raises( + WorkflowConfigError, + match=( + r'"\[runtime\]\[foo\]\[outputs\]d = foo"' + ' - messages must be unique' + ), + ): + validate(id_) From 5e6e741f9c135b09a1bec7761b3d0cbc1917fabc Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Tue, 14 May 2024 13:54:34 +0100 Subject: [PATCH 2/5] data store: include absolute graph edges * Closes #5845 --- 6103.fix.md | 2 ++ cylc/flow/taskdef.py | 6 +---- tests/integration/test_data_store_mgr.py | 34 ++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 6103.fix.md diff --git a/6103.fix.md b/6103.fix.md new file mode 100644 index 00000000000..e969f9ae2bf --- /dev/null +++ b/6103.fix.md @@ -0,0 +1,2 @@ +Absolute dependencies (dependenies on tasks in a specified cycle rather than at a specified offset) are now visible in the GUI beyond the specified cycle. + diff --git a/cylc/flow/taskdef.py b/cylc/flow/taskdef.py index 68f754277d8..448844c8cc3 100644 --- a/cylc/flow/taskdef.py +++ b/cylc/flow/taskdef.py @@ -101,11 +101,7 @@ def generate_graph_parents(tdef, point, taskdefs): # where (point -Px) does not land on a valid point for woo. # TODO ideally validation would flag this as an error. continue - is_abs = (trigger.offset_is_absolute or - trigger.offset_is_from_icp) - if is_abs and parent_point != point: - # If 'foo[^] => bar' only spawn off of '^'. - continue + is_abs = trigger.offset_is_absolute or trigger.offset_is_from_icp graph_parents[seq].append((parent_name, parent_point, is_abs)) if tdef.sequential: diff --git a/tests/integration/test_data_store_mgr.py b/tests/integration/test_data_store_mgr.py index 4d808bacc0a..906b1ac052d 100644 --- a/tests/integration/test_data_store_mgr.py +++ b/tests/integration/test_data_store_mgr.py @@ -18,6 +18,7 @@ from typing import TYPE_CHECKING from cylc.flow.data_store_mgr import ( + EDGES, FAMILY_PROXIES, JOBS, TASKS, @@ -316,3 +317,36 @@ def test_delta_task_prerequisite(harness): p.satisfied for t in schd.data_store_mgr.updated[TASK_PROXIES].values() for p in t.prerequisites}) + + +async def test_absolute_graph_edges(flow, scheduler, start): + """It should add absolute graph edges to the store. + + See: https://github.com/cylc/cylc-flow/issues/5845 + """ + runahead_cycles = 1 + id_ = flow({ + 'scheduling': { + 'initial cycle point': '1', + 'cycling mode': 'integer', + 'runahead limit': f'P{runahead_cycles}', + 'graph': { + 'R1': 'build', + 'P1': 'build[^] => run', + }, + }, + }) + schd = scheduler(id_) + + async with start(schd): + await schd.update_data_structure() + + assert { + (Tokens(edge.source).relative_id, Tokens(edge.target).relative_id) + for edge in schd.data_store_mgr.data[schd.id][EDGES].values() + } == { + ('1/build', f'{cycle}/run') + # +1 for Python's range() + # +2 for Cylc's runahead + for cycle in range(1, runahead_cycles + 3) + } From 0c6314b0004559e7a8ad3cd0625b835627b6b113 Mon Sep 17 00:00:00 2001 From: David Sutherland Date: Wed, 17 Jul 2024 17:26:44 +1200 Subject: [PATCH 3/5] add share/bin to env path --- cylc/flow/config.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/cylc/flow/config.py b/cylc/flow/config.py index 5b7738f3e6c..b03a8f5358a 100644 --- a/cylc/flow/config.py +++ b/cylc/flow/config.py @@ -1723,8 +1723,21 @@ def process_config_env(self): os.environ['CYLC_CYCLING_MODE'] = self.cfg['scheduling'][ 'cycling mode'] # Add workflow bin directory to PATH for workflow and event handlers - os.environ['PATH'] = os.pathsep.join([ - os.path.join(self.fdir, 'bin'), os.environ['PATH']]) + if self.share_dir is not None: + os.environ['PATH'] = os.pathsep.join( + [ + os.path.join(self.share_dir, 'bin'), + os.path.join(self.fdir, 'bin'), + os.environ['PATH'] + ] + ) + else: + os.environ['PATH'] = os.pathsep.join( + [ + os.path.join(self.fdir, 'bin'), + os.environ['PATH'] + ] + ) def run_mode(self) -> str: """Return the run mode.""" From d97c1421ecbb481cc117fa37ea845ef1961aeb75 Mon Sep 17 00:00:00 2001 From: David Sutherland Date: Thu, 18 Jul 2024 16:07:09 +1200 Subject: [PATCH 4/5] change log entry added --- changes.d/6242.fix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes.d/6242.fix.md diff --git a/changes.d/6242.fix.md b/changes.d/6242.fix.md new file mode 100644 index 00000000000..87edde1efab --- /dev/null +++ b/changes.d/6242.fix.md @@ -0,0 +1 @@ +Put `share/bin` in the `PATH` of scheduler environment, event handlers therein will now be found. \ No newline at end of file From 48fed325cc03ebe3c086b3b587d67bcbb4955e00 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Thu, 18 Jul 2024 16:21:41 +0100 Subject: [PATCH 5/5] Update 6103.fix.md Co-authored-by: Tim Pillinger <26465611+wxtim@users.noreply.github.com> --- 6103.fix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/6103.fix.md b/6103.fix.md index e969f9ae2bf..47844cd1ea6 100644 --- a/6103.fix.md +++ b/6103.fix.md @@ -1,2 +1,2 @@ -Absolute dependencies (dependenies on tasks in a specified cycle rather than at a specified offset) are now visible in the GUI beyond the specified cycle. +Absolute dependencies (dependencies on tasks in a specified cycle rather than at a specified offset) are now visible in the GUI beyond the specified cycle.