From 61632629d066db649be494f86cd2e98c69e37785 Mon Sep 17 00:00:00 2001 From: Heitor Neiva Date: Fri, 26 Apr 2024 14:02:29 -0700 Subject: [PATCH] feat(bitrisescript): Add global and workflow params --- src/mozilla_taskgraph/worker_types.py | 132 ++++++++++---------------- test/test_worker_types.py | 71 ++++++++++---- 2 files changed, 101 insertions(+), 102 deletions(-) diff --git a/src/mozilla_taskgraph/worker_types.py b/src/mozilla_taskgraph/worker_types.py index a030a7c..3c6ba35 100644 --- a/src/mozilla_taskgraph/worker_types.py +++ b/src/mozilla_taskgraph/worker_types.py @@ -1,5 +1,5 @@ from taskgraph.transforms.task import payload_builder, taskref_or_string -from voluptuous import Extra, Optional, Required +from voluptuous import Any, Extra, Optional, Required @payload_builder( @@ -11,65 +11,16 @@ ): str, Required( "workflows", - description="List of workflows to trigger on specified app.", - ): [str], - Optional( - "env", description="Environment variables to pass into the build" - ): {str: taskref_or_string}, - Optional( - "build_params", - description="Parameters describing the build context to pass " - "onto Bitrise. All keys are optional but specific workflows " - "may depend on particular keys being set.", - ): { - Optional( - "branch", - description="The branch running the build. For pull " - "requests, this should be the head branch.", - ): str, - Optional( - "branch_dest", - description="The destination branch where the branch " - "running the build will merge into. Only valid for pull " - "requests.", - ): str, - Optional( - "branch_dest_repo_owner", - description="The repository owning the destination branch. " - "Only valid for pull requests.", - ): str, - Optional( - "branch_repo_owner", description="The repository owning the branch." - ): str, - Optional( - "commit_hash", - description="The hash of the commit running the build.", - ): str, - Optional( - "commit_message", - description="The commit message of the commit running the build.", - ): str, - Optional( - "environments", - description="Environment variables to pass into the build.", - ): [{Required("mapped_to"): str, Optional("value"): taskref_or_string}], - Optional( - "pull_request_author", - description="The author of the pull request running the build.", - ): str, - Optional( - "pull_request_id", - description="The id of the pull request running the build.", - ): int, - Optional( - "skip_git_status_report", - description="Whether Bitrise should send a status report to " - "Github (default False).", - ): bool, - Optional( - "tag", description="The tag of the commit running the build." - ): str, - }, + description="List of workflows to trigger on specified app. " + "Can also be an object that maps workflow_ids to environment variables.", + ): [ + Any( + # Workflow id - no special environment variable + str, + # Map of workflow id to environment variables + {str: {str: taskref_or_string}}, + ) + ], Optional( "artifact_prefix", description="Directory prefix to store artifacts. Set this to 'public' " @@ -81,27 +32,37 @@ ) def build_bitrise_payload(config, task, task_def): bitrise = task["worker"]["bitrise"] - build_params = bitrise.get("build_params") or {} task_def["tags"]["worker-implementation"] = "scriptworker" + # Normalize environment variables to bitrise's format. + workflow_params = {} + for workflow in bitrise["workflows"]: + if isinstance(workflow, str): + # Empty environments + continue + for workflow_id, envs in workflow.items(): + workflow_params[workflow_id] = {"environments": []} + for k, v in envs.items(): + workflow_params[workflow_id]["environments"].append( + {"mapped_to": k, "value": v}, + ) + + def get_workflow_ids(): + ids = [] + for w in bitrise["workflows"]: + if isinstance(w, str): + ids.append(w) + else: + ids.extend(w.keys()) + return ids + scope_prefix = config.graph_config["scriptworker"]["scope-prefix"] scopes = task_def.setdefault("scopes", []) scopes.append(f"{scope_prefix}:bitrise:app:{bitrise['app']}") scopes.extend( - [f"{scope_prefix}:bitrise:workflow:{wf}" for wf in bitrise["workflows"]] + [f"{scope_prefix}:bitrise:workflow:{wf}" for wf in get_workflow_ids()] ) - # Normalize environment variables to bitrise's format. - env = bitrise.get("env", {}) - if env: - build_params.setdefault("environments", []) - for k, v in env.items(): - build_params["environments"].append({"mapped_to": k, "value": v}) - - # Set some build_params implicitly from Taskcluster params. - build_params.setdefault("commit_hash", config.params["head_rev"]) - build_params.setdefault("branch_repo_owner", config.params["head_repository"]) - def normref(ref, type="heads"): if ref: prefix = f"refs/{type}/" @@ -113,30 +74,37 @@ def normref(ref, type="heads"): return None return ref + # Set some global_params implicitly from Taskcluster params. + global_params = { + "commit_hash": config.params["head_rev"], + "branch_repo_owner": config.params["head_repository"], + } + head_ref = normref(config.params["head_ref"]) head_tag = normref(config.params["head_tag"], type="tags") base_ref = normref(config.params["base_ref"]) if head_ref: - build_params.setdefault("branch", head_ref) + global_params["branch"] = head_ref if head_tag: - build_params.setdefault("tag", head_tag) + global_params["tag"] = head_tag if config.params["tasks_for"] == "github-pull-request": - build_params.setdefault("pull_request_author", config.params["owner"]) + global_params["pull_request_author"] = config.params["owner"] if base_ref: - build_params.setdefault("branch_dest", base_ref) + global_params["branch_dest"] = base_ref if config.params["base_repository"]: - build_params.setdefault( - "branch_dest_repo_owner", config.params["base_repository"] - ) + global_params["branch_dest_repo_owner"] = config.params["base_repository"] + + task_def["payload"] = {"global_params": global_params} + if workflow_params: + task_def["payload"]["workflow_params"] = workflow_params - payload = task_def["payload"] = {"build_params": build_params} if bitrise.get("artifact_prefix"): - payload["artifact_prefix"] = bitrise["artifact_prefix"] + task_def["payload"]["artifact_prefix"] = bitrise["artifact_prefix"] @payload_builder( diff --git a/test/test_worker_types.py b/test/test_worker_types.py index 3d74ab0..7fc3b1c 100644 --- a/test/test_worker_types.py +++ b/test/test_worker_types.py @@ -12,16 +12,36 @@ "worker,extra_params,expected", ( pytest.param({}, {}, Exception, id="missing bitrise"), - pytest.param({"bitrise": {"workflows": []}}, {}, Exception, id="missing app"), + pytest.param( + {"bitrise": {"workflows": ["foo"]}}, {}, Exception, id="missing app" + ), pytest.param( {"bitrise": {"app": "foo"}}, {}, Exception, id="missing workflows" ), + pytest.param( + {"bitrise": {"app": "foo", "workflows": {"err": "nope"}}}, + {}, + Exception, + id="wrong workflows format", + ), + pytest.param( + {"bitrise": {"app": "foo", "workflows": [{"nope": ["wut"]}]}}, + {}, + Exception, + id="wrong workflows format2", + ), + pytest.param( + {"bitrise": {"app": "foo", "workflows": [{"foo": {"bar": ["oops"]}}]}}, + {}, + Exception, + id="wrong workflows format3", + ), pytest.param( {"bitrise": {"app": "some-app", "workflows": ["bar", "baz"]}}, {}, { "payload": { - "build_params": { + "global_params": { "branch": "default", "branch_repo_owner": "http://example.com/head/repo", "commit_hash": "abcdef", @@ -41,7 +61,7 @@ {"tasks_for": "github-pull-request"}, { "payload": { - "build_params": { + "global_params": { "branch": "default", "branch_dest": "123456", "branch_dest_repo_owner": "http://example.com/base/repo", @@ -66,7 +86,7 @@ }, { "payload": { - "build_params": { + "global_params": { "branch_repo_owner": "http://example.com/head/repo", "commit_hash": "abcdef", "pull_request_author": "some-owner", @@ -89,7 +109,7 @@ }, { "payload": { - "build_params": { + "global_params": { "branch": "bar", "branch_dest": "foo", "branch_repo_owner": "http://example.com/head/repo", @@ -114,7 +134,7 @@ }, { "payload": { - "build_params": { + "global_params": { "branch_repo_owner": "http://example.com/head/repo", "commit_hash": "abcdef", "pull_request_author": "some-owner", @@ -129,31 +149,42 @@ { "bitrise": { "app": "some-app", - "env": { - "FOO": "bar", - "PATH": {"artifact-reference": ""}, - }, - "workflows": ["bar"], + "workflows": [ + "foo", + { + "bar": { + "FOO": "bar", + "PATH": {"artifact-reference": ""}, + } + }, + ], } }, {}, { "payload": { - "build_params": { + "global_params": { "branch": "default", "branch_repo_owner": "http://example.com/head/repo", "commit_hash": "abcdef", - "environments": [ - {"mapped_to": "FOO", "value": "bar"}, - { - "mapped_to": "PATH", - "value": {"artifact-reference": ""}, - }, - ], + }, + "workflow_params": { + "bar": { + "environments": [ + {"mapped_to": "FOO", "value": "bar"}, + { + "mapped_to": "PATH", + "value": { + "artifact-reference": "" + }, + }, + ] + } }, }, "scopes": [ "foo:bitrise:app:some-app", + "foo:bitrise:workflow:foo", "foo:bitrise:workflow:bar", ], "tags": {"worker-implementation": "scriptworker"}, @@ -172,7 +203,7 @@ { "payload": { "artifact_prefix": "public", - "build_params": { + "global_params": { "branch": "default", "branch_repo_owner": "http://example.com/head/repo", "commit_hash": "abcdef",