From 0550c43591ad0c7e13fa79f1e0085a8cbd55e070 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Tue, 18 Apr 2023 10:18:21 -0400 Subject: [PATCH] feat: create transforms for adding dependencies These transforms will implement part of the work that the multi_dep loader is doing in other projects. Issue: #227 --- docs/reference/index.rst | 1 + .../reference/source/taskgraph.transforms.rst | 8 + docs/reference/transforms/after_deps.rst | 148 ++++++++++++++++ docs/reference/transforms/index.rst | 11 ++ src/taskgraph/transforms/after_deps.py | 160 ++++++++++++++++++ src/taskgraph/util/attributes.py | 28 ++- test/fixtures/gen.py | 4 +- test/test_optimize.py | 56 +++--- test/test_transforms_after_deps.py | 143 ++++++++++++++++ 9 files changed, 524 insertions(+), 35 deletions(-) create mode 100644 docs/reference/transforms/after_deps.rst create mode 100644 docs/reference/transforms/index.rst create mode 100644 src/taskgraph/transforms/after_deps.py create mode 100644 test/test_transforms_after_deps.py diff --git a/docs/reference/index.rst b/docs/reference/index.rst index 4e7179bf5..ab6e87e5d 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -6,6 +6,7 @@ Reference cli parameters + transforms/index optimization-strategies source/modules migrations diff --git a/docs/reference/source/taskgraph.transforms.rst b/docs/reference/source/taskgraph.transforms.rst index 0374fc57d..1b70be851 100644 --- a/docs/reference/source/taskgraph.transforms.rst +++ b/docs/reference/source/taskgraph.transforms.rst @@ -20,6 +20,14 @@ taskgraph.transforms.base module :undoc-members: :show-inheritance: +taskgraph.transforms.after\_deps module +----------------------------------------- + +.. automodule:: taskgraph.transforms.after_deps + :members: + :undoc-members: + :show-inheritance: + taskgraph.transforms.cached\_tasks module ----------------------------------------- diff --git a/docs/reference/transforms/after_deps.rst b/docs/reference/transforms/after_deps.rst new file mode 100644 index 000000000..a6f70a839 --- /dev/null +++ b/docs/reference/transforms/after_deps.rst @@ -0,0 +1,148 @@ +.. _after_deps: + +After Dependencies +================== + +The :mod:`taskgraph.transforms.after_deps` transforms can be used to create +tasks based on the kind dependencies, filtering on common attributes like the +``build-type``. + +These transforms are useful when you want to create follow-up tasks for some +indeterminate subset of existing tasks. For example, maybe you want to run +a signing task after each build task. + + +Usage +----- + +Add the transform to the ``transforms`` key in your ``kind.yml`` file: + +.. code-block:: yaml + + transforms: + - taskgraph.transforms.add_deps + # ... + +Then create an ``after-deps`` section in your task definition, e.g: + +.. code-block:: yaml + + kind-dependencies: + - build + - toolchain + + tasks: + signing: + after-deps: + kinds: [build] + +This example will split the ``signing`` task into many, one for each build. If +``kinds`` is unspecified, then it defaults to all kinds listed in the +``kind-dependencies`` key. So the following is valid: + +.. code-block:: yaml + + kind-dependencies: + - build + - toolchain + + tasks: + signing: + after-deps: {} + +In this example, a task will be created for each build *and* each toolchain. + +Limiting Dependencies by Attribute +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It may not be desirable to create a new task for *all* tasks in the given kind. +It's possible to limit the tasks given some attribute: + +.. code-block:: yaml + + kind-dependencies: + - build + + tasks: + signing: + after-deps: + with-attributes: + platform: linux + +In the above example, follow-up tasks will only be created for builds whose +``platform`` attribute equals "linux". Multiple attributes can be specified, +these are resolved using the :func:`~taskgraph.util.attributes.attrmatch` +utility function. + +Grouping Dependencies +~~~~~~~~~~~~~~~~~~~~~ + +Sometimes, it may be desirable to run a task after a group of related tasks +rather than an individual task. One example of this, is a task to notify when a +release is finished for each platform we're shipping. We want it to depend on +every task from that particular release phase. + +To accomplish this, we specify the ``group-by`` key: + +.. code-block:: yaml + + kind-dependencies: + - build + - signing + - publish + + tasks: + notify: + after-deps: + group-by: + attribute: platform + +In this example, tasks across the ``build``, ``signing`` and ``publish`` kinds will +be scanned for an attribute called "platform" and sorted into corresponding groups. +Assuming we're shipping on Windows, Mac and Linux, it might create the following +groups: + +.. code-block:: + + - build-windows, signing-windows, publish-windows + - build-mac, signing-mac, publish-mac + - build-linux, signing-linux, publish-linux + +Then the ``notify`` task will be duplicated into three, one for each group. The +notify tasks will depend on each task in its associated group. + +Custom Grouping +~~~~~~~~~~~~~~~ + +Only the default ``single`` and the ``attribute`` group-by functions are +built-in. But if more complex grouping is needed, custom functions can be +implemented as well: + +.. code-block:: python + + from typing import List + + from taskgraph.task import Task + from taskgraph.transforms.after_deps import group_by + from taskgraph.transforms.base import TransformConfig + + @group_by("custom-name") + def group_by(config: TransformConfig, tasks: List[Task]) -> List[List[Task]]: + pass + +It's also possible to specify a schema for your custom group-by function, which +allows tasks to pass down additional context (such as with the built-in +``attribute`` function): + +.. code-block:: python + + from typing import List + + from taskgraph.task import Task + from taskgraph.transforms.after_deps import group_by + from taskgraph.transforms.base import TransformConfig + from taskgraph.util.schema import Schema + + @group_by("custom-name", schema=Schema(str)) + def group_by(config: TransformConfig, tasks: List[Task], ctx: str) -> List[List[Task]]: + pass diff --git a/docs/reference/transforms/index.rst b/docs/reference/transforms/index.rst new file mode 100644 index 000000000..690b09aed --- /dev/null +++ b/docs/reference/transforms/index.rst @@ -0,0 +1,11 @@ +Transforms +========== + +Taskgraph includes several transforms out of the box. These can be used to +accomplish common tasks such as adding dependencies programmatically or setting +up notifications. This section contains a reference of the transforms Taskgraph +provides, read below to learn how to use them. + +.. toctree:: + + add_deps diff --git a/src/taskgraph/transforms/after_deps.py b/src/taskgraph/transforms/after_deps.py new file mode 100644 index 000000000..ede1e89fc --- /dev/null +++ b/src/taskgraph/transforms/after_deps.py @@ -0,0 +1,160 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Transforms used to create tasks based on the kind dependencies, filtering on +common attributes like the ``build-type``. + +These transforms are useful when follow-up tasks are needed for some +indeterminate subset of existing tasks. For example, running a signing task +after each build task, whatever builds may exist. +""" +from copy import deepcopy +from textwrap import dedent + +from voluptuous import ALLOW_EXTRA, Any, Optional, Required + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.attributes import attrmatch +from taskgraph.util.schema import Schema + +# Define a collection of group_by functions +GROUP_BY_MAP = {} + + +def group_by(name, schema=None): + def wrapper(func): + GROUP_BY_MAP[name] = func + func.schema = schema + return func + + return wrapper + + +@group_by("single") +def group_by_single(config, tasks): + for task in tasks: + yield [task] + + +@group_by("attribute", schema=Schema(str)) +def group_by_attribute(config, tasks, attr): + groups = {} + for task in tasks: + val = task.attributes.get(attr) + if not val: + continue + groups.setdefault(val, []).append(task) + + return groups.values() + + +AFTER_DEPS_SCHEMA = Schema( + { + Required("after-deps"): { + Optional( + "kinds", + description=dedent( + """ + Limit dependencies to specified kinds (defaults to all kinds in + `kind-dependencies`). + """.lstrip() + ), + ): list, + Optional( + "with-attributes", + description=dedent( + """ + Limit dependencies to tasks whose attributes match + using :func:`~taskgraph.util.attributes.attrmatch`. + """.lstrip() + ), + ): {str: Any(list, str)}, + Optional( + "group-by", + description=dedent( + """ + Group cross-kind dependencies using the given group-by + function. One task will be created for each group. If not + specified, the 'single' function will be used which creates + a new task for each individual dependency. + """.lstrip() + ), + ): Any(None, *GROUP_BY_MAP.keys()), + } + }, + extra=ALLOW_EXTRA, +) +"""Schema for after_deps transforms.""" + +transforms = TransformSequence() +transforms.add_validate(AFTER_DEPS_SCHEMA) + + +@transforms.add +def after_deps(config, tasks): + for task in tasks: + # Setup and error handling. + after_deps = task.pop("after-deps") + kind_deps = set(config.config.get("kind-dependencies", [])) + kinds = set(after_deps.get("kinds", kind_deps)) + + if kinds - kind_deps: + invalid = "\n".join(sorted(kinds - kind_deps)) + raise Exception( + dedent( + f""" + The `after-deps.kinds` key contains the following kinds + that are not defined in `kind-dependencies`: + {invalid} + """.lstrip() + ) + ) + + if not kinds: + raise Exception( + dedent( + """ + The `after_deps` transforms require at least one kind defined + in `kind-dependencies`! + """.lstrip() + ) + ) + + # Resolve desired dependencies. + with_attributes = after_deps.get("with-attributes") + deps = [ + task + for task in config.kind_dependencies_tasks.values() + if task.kind in kinds + if not with_attributes or attrmatch(task.attributes, **with_attributes) + ] + + # Resolve groups. + group_by = after_deps.get("group-by", "single") + groups = set() + + if isinstance(group_by, dict): + assert len(group_by) == 1 + group_by, arg = group_by.popitem() + func = GROUP_BY_MAP[group_by] + if func.schema: + func.schema(arg) + groups = func(config, deps, arg) + else: + func = GROUP_BY_MAP[group_by] + groups = func(config, deps) + + # Split the task, one per group. + for group in groups: + # Verify there is only one task per kind in each group. + kinds = {t.kind for t in group} + if len(kinds) < len(group): + raise Exception( + "The after_deps transforms only allow a single task per kind in a group!" + ) + + new_task = deepcopy(task) + new_task["dependencies"] = {dep.kind: dep.label for dep in group} + yield new_task diff --git a/src/taskgraph/util/attributes.py b/src/taskgraph/util/attributes.py index cf6f11c57..74d699662 100644 --- a/src/taskgraph/util/attributes.py +++ b/src/taskgraph/util/attributes.py @@ -7,18 +7,30 @@ def attrmatch(attributes, **kwargs): - """Determine whether the given set of task attributes matches. The - conditions are given as keyword arguments, where each keyword names an - attribute. The keyword value can be a literal, a set, or a callable. A - literal must match the attribute exactly. Given a set, the attribute value - must be in the set. A callable is called with the attribute value. If an - attribute is specified as a keyword argument but not present in the - attributes, the result is False.""" + """Determine whether the given set of task attributes matches. + + The conditions are given as keyword arguments, where each keyword names an + attribute. The keyword value can be a literal, a set, or a callable: + + * A literal must match the attribute exactly. + * Given a set or list, the attribute value must be contained within it. + * A callable is called with the attribute value and returns a boolean. + + If an attribute is specified as a keyword argument but not present in the + task's attributes, the result is False. + + Args: + attributes (dict): The task's attributes object. + kwargs (dict): The conditions the task's attributes must satisfy in + order to match. + Returns: + bool: Whether the task's attributes match the conditions or not. + """ for kwkey, kwval in kwargs.items(): if kwkey not in attributes: return False attval = attributes[kwkey] - if isinstance(kwval, set): + if isinstance(kwval, (set, list)): if attval not in kwval: return False elif callable(kwval): diff --git a/test/fixtures/gen.py b/test/fixtures/gen.py index 044a1e801..3edcdef86 100644 --- a/test/fixtures/gen.py +++ b/test/fixtures/gen.py @@ -217,8 +217,8 @@ def inner(func, tasks, config=None): def make_task( label, + kind="test", optimization=None, - optimized=None, task_def=None, task_id=None, dependencies=None, @@ -232,7 +232,7 @@ def make_task( task = Task( attributes=attributes or {}, if_dependencies=if_dependencies or [], - kind="test", + kind=kind, label=label, task=task_def, ) diff --git a/test/test_optimize.py b/test/test_optimize.py index 399a84779..1ad54d910 100644 --- a/test/test_optimize.py +++ b/test/test_optimize.py @@ -51,9 +51,9 @@ def make_triangle(deps=True, **opts): `---- t2 --' """ return make_graph( - make_task("t1", opts.get("t1")), - make_task("t2", opts.get("t2")), - make_task("t3", opts.get("t3")), + make_task("t1", optimization=opts.get("t1")), + make_task("t2", optimization=opts.get("t2")), + make_task("t3", optimization=opts.get("t3")), ("t3", "t2", "dep"), ("t3", "t1", "dep2"), ("t2", "t1", "dep"), @@ -111,8 +111,8 @@ def make_triangle(deps=True, **opts): # Tasks with the 'not' composite strategy are removed when the substrategy says not to pytest.param( make_graph( - make_task("t1", {"not-never": None}), - make_task("t2", {"not-remove": None}), + make_task("t1", optimization={"not-never": None}), + make_task("t2", optimization={"not-remove": None}), ), { "strategies": lambda: { @@ -150,10 +150,12 @@ def make_triangle(deps=True, **opts): # Tasks with 'if_dependencies' are removed when deps are not run pytest.param( make_graph( - make_task("t1", {"remove": None}), - make_task("t2", {"remove": None}), - make_task("t3", {"never": None}, if_dependencies=["t1", "t2"]), - make_task("t4", {"never": None}, if_dependencies=["t1"]), + make_task("t1", optimization={"remove": None}), + make_task("t2", optimization={"remove": None}), + make_task( + "t3", optimization={"never": None}, if_dependencies=["t1", "t2"] + ), + make_task("t4", optimization={"never": None}, if_dependencies=["t1"]), ("t3", "t2", "dep"), ("t3", "t1", "dep2"), ("t2", "t1", "dep"), @@ -167,10 +169,12 @@ def make_triangle(deps=True, **opts): # Parents of tasks with 'if_dependencies' are also removed even if requested pytest.param( make_graph( - make_task("t1", {"remove": None}), - make_task("t2", {"remove": None}), - make_task("t3", {"never": None}, if_dependencies=["t1", "t2"]), - make_task("t4", {"never": None}, if_dependencies=["t1"]), + make_task("t1", optimization={"remove": None}), + make_task("t2", optimization={"remove": None}), + make_task( + "t3", optimization={"never": None}, if_dependencies=["t1", "t2"] + ), + make_task("t4", optimization={"never": None}, if_dependencies=["t1"]), ("t3", "t2", "dep"), ("t3", "t1", "dep2"), ("t2", "t1", "dep"), @@ -184,10 +188,12 @@ def make_triangle(deps=True, **opts): # Tasks with 'if_dependencies' are kept if at least one of said dependencies are kept pytest.param( make_graph( - make_task("t1", {"never": None}), - make_task("t2", {"remove": None}), - make_task("t3", {"never": None}, if_dependencies=["t1", "t2"]), - make_task("t4", {"never": None}, if_dependencies=["t1"]), + make_task("t1", optimization={"never": None}), + make_task("t2", optimization={"remove": None}), + make_task( + "t3", optimization={"never": None}, if_dependencies=["t1", "t2"] + ), + make_task("t4", optimization={"never": None}, if_dependencies=["t1"]), ("t3", "t2", "dep"), ("t3", "t1", "dep2"), ("t2", "t1", "dep"), @@ -201,9 +207,9 @@ def make_triangle(deps=True, **opts): # Ancestor of task with 'if_dependencies' does not cause it to be kept pytest.param( make_graph( - make_task("t1", {"never": None}), - make_task("t2", {"remove": None}), - make_task("t3", {"never": None}, if_dependencies=["t2"]), + make_task("t1", optimization={"never": None}), + make_task("t2", optimization={"remove": None}), + make_task("t3", optimization={"never": None}, if_dependencies=["t2"]), ("t3", "t2", "dep"), ("t2", "t1", "dep2"), ), @@ -216,10 +222,10 @@ def make_triangle(deps=True, **opts): # don't have any dependents and are not in 'requested_tasks' pytest.param( make_graph( - make_task("t1", {"never": None}), - make_task("t2", {"never": None}, if_dependencies=["t1"]), - make_task("t3", {"remove": None}), - make_task("t4", {"never": None}, if_dependencies=["t3"]), + make_task("t1", optimization={"never": None}), + make_task("t2", optimization={"never": None}, if_dependencies=["t1"]), + make_task("t3", optimization={"remove": None}), + make_task("t4", optimization={"never": None}, if_dependencies=["t3"]), ("t2", "t1", "e1"), ("t4", "t2", "e2"), ("t4", "t3", "e3"), @@ -333,7 +339,7 @@ def test_remove_tasks(monkeypatch, graph, kwargs, exp_removed): # A task which expires before a dependents deadline is not a valid replacement. pytest.param( make_graph( - make_task("t1", {"replace": "e1"}), + make_task("t1", optimization={"replace": "e1"}), make_task( "t2", task_def={"deadline": {"relative-datestamp": "2 days"}} ), diff --git a/test/test_transforms_after_deps.py b/test/test_transforms_after_deps.py new file mode 100644 index 000000000..bca301815 --- /dev/null +++ b/test/test_transforms_after_deps.py @@ -0,0 +1,143 @@ +""" +Tests for the 'after_deps' transforms. +""" + +from pprint import pprint + +import pytest + +from taskgraph.transforms import after_deps + +from .conftest import make_task + + +def handle_exception(obj, exc=None): + if exc: + assert isinstance(obj, exc) + elif isinstance(obj, Exception): + raise obj + + +def assert_no_kind_dependencies(e): + handle_exception(e, exc=Exception) + + +def assert_invalid_only_kinds(e): + handle_exception(e, exc=Exception) + + +def assert_defaults(tasks): + handle_exception(tasks) + assert len(tasks) == 2 + assert tasks[0]["dependencies"] == {"foo": "foo"} + assert tasks[1]["dependencies"] == {"bar": "bar"} + + +assert_group_by_single = assert_defaults + + +def assert_group_by_attribute(tasks): + handle_exception(tasks) + assert len(tasks) == 2 + assert tasks[0]["dependencies"] == {"foo": "a"} + assert tasks[1]["dependencies"] == {"foo": "b", "bar": "c"} + + +def assert_group_by_attribute_dupe(e): + handle_exception(e, exc=Exception) + + +@pytest.mark.parametrize( + "task, kind_config, deps", + ( + pytest.param( + # task + {"after-deps": {}}, + # kind config + {}, + # deps + None, + id="no_kind_dependencies", + ), + pytest.param( + # task + {"after-deps": {"only-kinds": ["foo", "baz"]}}, + # kind config + None, + # deps + None, + id="invalid_only_kinds", + ), + pytest.param( + # task + {"after-deps": {}}, + # kind config + None, + # deps + None, + id="defaults", + ), + pytest.param( + # task + {"after-deps": {}}, + # kind config + None, + # deps + None, + id="group_by_single", + ), + pytest.param( + # task + {"after-deps": {"group-by": {"attribute": "build-type"}}}, + # kind config + None, + # deps + { + "a": make_task("a", attributes={"build-type": "linux"}, kind="foo"), + "b": make_task("b", attributes={"build-type": "win"}, kind="foo"), + "c": make_task("c", attributes={"build-type": "win"}, kind="bar"), + }, + id="group_by_attribute", + ), + pytest.param( + # task + {"after-deps": {"group-by": {"attribute": "build-type"}}}, + # kind config + None, + # deps + { + "a": make_task("a", attributes={"build-type": "linux"}, kind="foo"), + "b": make_task("b", attributes={"build-type": "win"}, kind="foo"), + "c": make_task("c", attributes={"build-type": "win"}, kind="foo"), + }, + id="group_by_attribute_dupe", + ), + ), +) +def test_transforms( + request, make_transform_config, run_transform, task, kind_config, deps +): + task.setdefault("name", "task") + task.setdefault("description", "description") + + if kind_config is None: + kind_config = {"kind-dependencies": ["foo", "bar"]} + + if deps is None: + deps = { + "foo": make_task("foo", kind="foo"), + "bar": make_task("bar", kind="bar"), + } + config = make_transform_config(kind_config, deps) + + try: + result = run_transform(after_deps.transforms, task, config) + except Exception as e: + result = e + + print("Dumping result:") + pprint(result, indent=2) + + param_id = request.node.callspec.id + assert_func = globals()[f"assert_{param_id}"] + assert_func(result)