diff --git a/src/mozilla_taskgraph/transforms/scriptworker/signing.py b/src/mozilla_taskgraph/transforms/scriptworker/signing.py new file mode 100644 index 0000000..e9dc38b --- /dev/null +++ b/src/mozilla_taskgraph/transforms/scriptworker/signing.py @@ -0,0 +1,158 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import re + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import ( + Schema, + optionally_keyed_by, + resolve_keyed_by, +) +from voluptuous import ALLOW_EXTRA, Any, Optional, Required + +SIGNING_FORMATS = ["autograph_gpg"] +SIGNING_TYPES = ["dep", "release"] +DETACHED_SIGNATURE_EXTENSION = ".asc" + +signing_schema = Schema( + { + Required("attributes"): { + Optional("artifacts"): dict, + Required("build-type"): str, + }, + Required("signing"): optionally_keyed_by( + "build-type", + "level", + { + Required("format"): optionally_keyed_by( + "build-type", "level", Any(*SIGNING_FORMATS) + ), + Optional("type"): optionally_keyed_by( + "build-type", "level", Any(*SIGNING_TYPES) + ), + Optional("ignore-artifacts"): list, + }, + ), + Required("worker"): { + Required("upstream-artifacts"): [ + { + # Paths to the artifacts to sign + Required("paths"): [str], + } + ], + }, + }, + extra=ALLOW_EXTRA, +) + +transforms = TransformSequence() +transforms.add_validate(signing_schema) + + +@transforms.add +def resolve_signing_keys(config, tasks): + for task in tasks: + for key in ( + "signing", + "signing.format", + "signing.type", + ): + resolve_keyed_by( + task, + key, + item_name=task["name"], + **{ + "build-type": task["attributes"]["build-type"], + "level": config.params["level"], + }, + ) + yield task + + +@transforms.add +def set_signing_attributes(_, tasks): + for task in tasks: + task["attributes"]["signed"] = True + yield task + + +@transforms.add +def set_signing_format(_, tasks): + for task in tasks: + for upstream_artifact in task["worker"]["upstream-artifacts"]: + upstream_artifact["formats"] = [task["signing"]["format"]] + yield task + + +@transforms.add +def set_signing_and_worker_type(config, tasks): + for task in tasks: + signing_type = task["signing"].get("type") + if not signing_type: + signing_type = "release" if config.params["level"] == "3" else "dep" + + task.setdefault("worker", {})["signing-type"] = f"{signing_type}-signing" + + if "worker-type" not in task: + worker_type = "signing" + build_type = task["attributes"]["build-type"] + + if signing_type == "dep": + worker_type = f"dep-{worker_type}" + if build_type == "macos": + worker_type = f"{build_type}-{worker_type}" + task["worker-type"] = worker_type + + yield task + + +@transforms.add +def filter_out_ignored_artifacts(_, tasks): + for task in tasks: + ignore = task["signing"].get("ignore-artifacts") + if not ignore: + yield task + continue + + def is_ignored(artifact): + return not any(re.search(i, artifact) for i in ignore) + + if task["attributes"].get("artifacts"): + task["attributes"]["artifacts"] = { + extension: path + for extension, path in task["attributes"]["artifacts"].items() + if is_ignored(path) + } + + for upstream_artifact in task["worker"]["upstream-artifacts"]: + upstream_artifact["paths"] = [ + path for path in upstream_artifact["paths"] if is_ignored(path) + ] + + yield task + + +@transforms.add +def set_gpg_detached_signature_artifacts(_, tasks): + for task in tasks: + if task["signing"]["format"] != "autograph_gpg": + yield task + continue + + task["attributes"]["artifacts"] = { + extension + + DETACHED_SIGNATURE_EXTENSION: path + + DETACHED_SIGNATURE_EXTENSION + for extension, path in task["attributes"]["artifacts"].items() + } + + yield task + + +@transforms.add +def remove_signing_config(_, tasks): + for task in tasks: + del task["signing"] + yield task diff --git a/taskcluster/test/params/main-push.yml b/taskcluster/test/params/main-push.yml new file mode 100755 index 0000000..c93089a --- /dev/null +++ b/taskcluster/test/params/main-push.yml @@ -0,0 +1,28 @@ +--- +base_ref: refs/heads/main +base_repository: https://github.com/mozilla-releng/mozilla-taskgraph +base_rev: a76ea4308313211a99e8e501c5a97a5ce2c08cc1 +build_date: 1681151087 +build_number: 1 +do_not_optimize: [] +enable_always_target: true +existing_tasks: {} +filters: + - target_tasks_method +head_ref: refs/heads/main +head_repository: https://github.com/mozilla-releng/mozilla-taskgraph +head_rev: a0785edae4a841b6119925280c744000f59b903e +head_tag: '' +level: '1' +moz_build_date: '20230410182447' +next_version: null +optimize_strategies: null +optimize_target_tasks: true +owner: ahal@pm.me +project: mozilla-taskgraph +pushdate: 0 +pushlog_id: '0' +repository_type: git +target_tasks_method: default +tasks_for: github-push +version: null diff --git a/taskcluster/test/params/pull-request.yml b/taskcluster/test/params/pull-request.yml new file mode 100755 index 0000000..a0634d6 --- /dev/null +++ b/taskcluster/test/params/pull-request.yml @@ -0,0 +1,28 @@ +--- +base_ref: main +base_repository: https://github.com/mozilla-releng/mozilla-taskgraph +base_rev: a0785edae4a841b6119925280c744000f59b903e +build_date: 1681154438 +build_number: 1 +do_not_optimize: [] +enable_always_target: true +existing_tasks: {} +filters: + - target_tasks_method +head_ref: codecov +head_repository: https://github.com/user/mozilla-taskgraph +head_rev: 06c766e8e9d558eed5ccf8029164120a27af5fb1 +head_tag: '' +level: '1' +moz_build_date: '20230410192038' +next_version: null +optimize_strategies: null +optimize_target_tasks: true +owner: user@example.com +project: mozilla-taskgraph +pushdate: 0 +pushlog_id: '0' +repository_type: git +target_tasks_method: default +tasks_for: github-pull-request +version: null diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..27534fb --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,118 @@ +from pathlib import Path + +import pytest +from taskgraph.config import GraphConfig +from taskgraph.transforms.base import TransformConfig + +here = Path(__file__).parent + + +@pytest.fixture(scope="session") +def datadir(): + return here / "data" + + +def fake_load_graph_config(root_dir): + graph_config = GraphConfig( + { + "trust-domain": "test-domain", + "taskgraph": { + "repositories": { + "ci": {"name": "Taskgraph"}, + } + }, + "workers": { + "aliases": { + "b-linux": { + "provisioner": "taskgraph-b", + "implementation": "docker-worker", + "os": "linux", + "worker-type": "linux", + }, + "t-linux": { + "provisioner": "taskgraph-t", + "implementation": "docker-worker", + "os": "linux", + "worker-type": "linux", + }, + } + }, + "task-priority": "low", + "treeherder": {"group-names": {"T": "tests"}}, + }, + root_dir, + ) + graph_config.__dict__["register"] = lambda: None + return graph_config + + +@pytest.fixture +def graph_config(datadir): + return fake_load_graph_config(str(datadir / "taskcluster" / "ci")) + + +class FakeParameters(dict): + strict = True + + def is_try(self): + return False + + def file_url(self, path, pretty=False): + return path + + +@pytest.fixture +def parameters(): + return FakeParameters( + { + "base_repository": "http://hg.example.com", + "build_date": 0, + "build_number": 1, + "enable_always_target": True, + "head_repository": "http://hg.example.com", + "head_rev": "abcdef", + "head_ref": "default", + "level": "1", + "moz_build_date": 0, + "next_version": "1.0.1", + "owner": "some-owner", + "project": "some-project", + "pushlog_id": 1, + "repository_type": "hg", + "target_tasks_method": "test_method", + "tasks_for": "hg-push", + "try_mode": None, + "version": "1.0.0", + } + ) + + +@pytest.fixture +def make_transform_config(parameters, graph_config): + def inner(kind_config=None, kind_dependencies_tasks=None): + kind_config = kind_config or {} + kind_dependencies_tasks = kind_dependencies_tasks or {} + return TransformConfig( + "test", + str(here), + kind_config, + parameters, + kind_dependencies_tasks, + graph_config, + write_artifacts=False, + ) + + return inner + + +@pytest.fixture +def run_transform(make_transform_config): + def inner(func, tasks, config=None): + if not isinstance(tasks, list): + tasks = [tasks] + + if not config: + config = make_transform_config() + return list(func(config, tasks)) + + return inner diff --git a/test/test_transforms_signing.py b/test/test_transforms_signing.py new file mode 100644 index 0000000..e65c4cf --- /dev/null +++ b/test/test_transforms_signing.py @@ -0,0 +1,135 @@ +import pytest +from taskgraph.util.templates import merge + +from mozilla_taskgraph.transforms.scriptworker.signing import ( + transforms as signing_transforms, +) + +DEFAULT_EXPECTED = { + "attributes": {"artifacts": {}, "build-type": "linux", "signed": True}, + "name": "task", + "worker": { + "signing-type": "dep-signing", + "upstream-artifacts": [], + }, + "worker-type": "dep-signing", +} + + +def assert_level_3(task): + expected = merge( + DEFAULT_EXPECTED, + { + "worker-type": "signing", + "worker": { + "signing-type": "release-signing", + "upstream-artifacts": [ + {"paths": ["build.zip"], "formats": ["autograph_gpg"]} + ], + }, + }, + ) + assert task == expected + + +def assert_level_1(task): + expected = merge( + DEFAULT_EXPECTED, + { + "worker": { + "upstream-artifacts": [ + {"paths": ["build.zip"], "formats": ["autograph_gpg"]} + ], + } + }, + ) + assert task == expected + + +def assert_macos_worker_type(task): + assert task["worker-type"] == "macos-dep-signing" + + +def assert_ignore_artifacts(task): + assert task["worker"]["upstream-artifacts"] == [ + { + "formats": ["autograph_gpg"], + "paths": ["build.zip"], + } + ] + + +@pytest.mark.parametrize( + "params,task", + ( + pytest.param( + # params + {"level": "3"}, + # task + { + "signing": { + "format": "autograph_gpg", + }, + "worker": {"upstream-artifacts": [{"paths": ["build.zip"]}]}, + }, + id="level_3", + ), + pytest.param( + # params + {"level": "1"}, + # task + { + "signing": { + "format": "autograph_gpg", + }, + "worker": {"upstream-artifacts": [{"paths": ["build.zip"]}]}, + }, + id="level_1", + ), + pytest.param( + # params + {"level": "1"}, + # task + { + "attributes": { + "build-type": "macos", + }, + "signing": { + "format": "autograph_gpg", + }, + }, + id="macos_worker_type", + ), + pytest.param( + # params + {"level": "1"}, + # task + { + "signing": { + "format": "autograph_gpg", + "ignore-artifacts": [r".*\.txt"], + }, + "worker": {"upstream-artifacts": [{"paths": ["build.zip", "log.txt"]}]}, + }, + id="ignore_artifacts", + ), + ), +) +def test_signing_transforms( + request, make_transform_config, run_transform, params, task +): + task.setdefault("name", "task") + task.setdefault("worker", {}).setdefault("upstream-artifacts", []) + attributes = task.setdefault("attributes", {}) + attributes.setdefault("artifacts", {}) + attributes.setdefault("build-type", "linux") + + config = make_transform_config() + config.params.update(params) + + tasks = run_transform(signing_transforms, task, config) + assert len(tasks) == 1 + + param_id = request.node.callspec.id + assert_func = globals()[f"assert_{param_id}"] + assert_func(tasks[0])