Skip to content

Commit

Permalink
Merge pull request #64 from mozilla-releng/hneiva/bitrise-params
Browse files Browse the repository at this point in the history 
feat(bitrisescript): Make workflows accept a dict representing the env variables
  • Loading branch information
@hneiva
hneiva authored May 8, 2024
2 parents facc39b + 6163262 commit e2ae827
Show file tree
Hide file tree
Showing 2 changed files with 101 additions and 102 deletions.
132 changes: 50 additions & 82 deletions src/mozilla_taskgraph/worker_types.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from taskgraph.transforms.task import payload_builder, taskref_or_string
from voluptuous import Extra, Optional, Required
from voluptuous import Any, Extra, Optional, Required


@payload_builder(
Expand All @@ -11,65 +11,16 @@
): str,
Required(
"workflows",
description="List of workflows to trigger on specified app.",
): [str],
Optional(
"env", description="Environment variables to pass into the build"
): {str: taskref_or_string},
Optional(
"build_params",
description="Parameters describing the build context to pass "
"onto Bitrise. All keys are optional but specific workflows "
"may depend on particular keys being set.",
): {
Optional(
"branch",
description="The branch running the build. For pull "
"requests, this should be the head branch.",
): str,
Optional(
"branch_dest",
description="The destination branch where the branch "
"running the build will merge into. Only valid for pull "
"requests.",
): str,
Optional(
"branch_dest_repo_owner",
description="The repository owning the destination branch. "
"Only valid for pull requests.",
): str,
Optional(
"branch_repo_owner", description="The repository owning the branch."
): str,
Optional(
"commit_hash",
description="The hash of the commit running the build.",
): str,
Optional(
"commit_message",
description="The commit message of the commit running the build.",
): str,
Optional(
"environments",
description="Environment variables to pass into the build.",
): [{Required("mapped_to"): str, Optional("value"): taskref_or_string}],
Optional(
"pull_request_author",
description="The author of the pull request running the build.",
): str,
Optional(
"pull_request_id",
description="The id of the pull request running the build.",
): int,
Optional(
"skip_git_status_report",
description="Whether Bitrise should send a status report to "
"Github (default False).",
): bool,
Optional(
"tag", description="The tag of the commit running the build."
): str,
},
description="List of workflows to trigger on specified app. "
"Can also be an object that maps workflow_ids to environment variables.",
): [
Any(
# Workflow id - no special environment variable
str,
# Map of workflow id to environment variables
{str: {str: taskref_or_string}},
)
],
Optional(
"artifact_prefix",
description="Directory prefix to store artifacts. Set this to 'public' "
Expand All @@ -81,27 +32,37 @@
)
def build_bitrise_payload(config, task, task_def):
bitrise = task["worker"]["bitrise"]
build_params = bitrise.get("build_params") or {}
task_def["tags"]["worker-implementation"] = "scriptworker"

# Normalize environment variables to bitrise's format.
workflow_params = {}
for workflow in bitrise["workflows"]:
if isinstance(workflow, str):
# Empty environments
continue
for workflow_id, envs in workflow.items():
workflow_params[workflow_id] = {"environments": []}
for k, v in envs.items():
workflow_params[workflow_id]["environments"].append(
{"mapped_to": k, "value": v},
)

def get_workflow_ids():
ids = []
for w in bitrise["workflows"]:
if isinstance(w, str):
ids.append(w)
else:
ids.extend(w.keys())
return ids

scope_prefix = config.graph_config["scriptworker"]["scope-prefix"]
scopes = task_def.setdefault("scopes", [])
scopes.append(f"{scope_prefix}:bitrise:app:{bitrise['app']}")
scopes.extend(
[f"{scope_prefix}:bitrise:workflow:{wf}" for wf in bitrise["workflows"]]
[f"{scope_prefix}:bitrise:workflow:{wf}" for wf in get_workflow_ids()]
)

# Normalize environment variables to bitrise's format.
env = bitrise.get("env", {})
if env:
build_params.setdefault("environments", [])
for k, v in env.items():
build_params["environments"].append({"mapped_to": k, "value": v})

# Set some build_params implicitly from Taskcluster params.
build_params.setdefault("commit_hash", config.params["head_rev"])
build_params.setdefault("branch_repo_owner", config.params["head_repository"])

def normref(ref, type="heads"):
if ref:
prefix = f"refs/{type}/"
Expand All @@ -113,30 +74,37 @@ def normref(ref, type="heads"):
return None
return ref

# Set some global_params implicitly from Taskcluster params.
global_params = {
"commit_hash": config.params["head_rev"],
"branch_repo_owner": config.params["head_repository"],
}

head_ref = normref(config.params["head_ref"])
head_tag = normref(config.params["head_tag"], type="tags")
base_ref = normref(config.params["base_ref"])

if head_ref:
build_params.setdefault("branch", head_ref)
global_params["branch"] = head_ref

if head_tag:
build_params.setdefault("tag", head_tag)
global_params["tag"] = head_tag

if config.params["tasks_for"] == "github-pull-request":
build_params.setdefault("pull_request_author", config.params["owner"])
global_params["pull_request_author"] = config.params["owner"]

if base_ref:
build_params.setdefault("branch_dest", base_ref)
global_params["branch_dest"] = base_ref

if config.params["base_repository"]:
build_params.setdefault(
"branch_dest_repo_owner", config.params["base_repository"]
)
global_params["branch_dest_repo_owner"] = config.params["base_repository"]

task_def["payload"] = {"global_params": global_params}
if workflow_params:
task_def["payload"]["workflow_params"] = workflow_params

payload = task_def["payload"] = {"build_params": build_params}
if bitrise.get("artifact_prefix"):
payload["artifact_prefix"] = bitrise["artifact_prefix"]
task_def["payload"]["artifact_prefix"] = bitrise["artifact_prefix"]


@payload_builder(
Expand Down
71 changes: 51 additions & 20 deletions test/test_worker_types.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,36 @@
"worker,extra_params,expected",
(
pytest.param({}, {}, Exception, id="missing bitrise"),
pytest.param({"bitrise": {"workflows": []}}, {}, Exception, id="missing app"),
pytest.param(
{"bitrise": {"workflows": ["foo"]}}, {}, Exception, id="missing app"
),
pytest.param(
{"bitrise": {"app": "foo"}}, {}, Exception, id="missing workflows"
),
pytest.param(
{"bitrise": {"app": "foo", "workflows": {"err": "nope"}}},
{},
Exception,
id="wrong workflows format",
),
pytest.param(
{"bitrise": {"app": "foo", "workflows": [{"nope": ["wut"]}]}},
{},
Exception,
id="wrong workflows format2",
),
pytest.param(
{"bitrise": {"app": "foo", "workflows": [{"foo": {"bar": ["oops"]}}]}},
{},
Exception,
id="wrong workflows format3",
),
pytest.param(
{"bitrise": {"app": "some-app", "workflows": ["bar", "baz"]}},
{},
{
"payload": {
"build_params": {
"global_params": {
"branch": "default",
"branch_repo_owner": "http://example.com/head/repo",
"commit_hash": "abcdef",
Expand All @@ -41,7 +61,7 @@
{"tasks_for": "github-pull-request"},
{
"payload": {
"build_params": {
"global_params": {
"branch": "default",
"branch_dest": "123456",
"branch_dest_repo_owner": "http://example.com/base/repo",
Expand All @@ -66,7 +86,7 @@
},
{
"payload": {
"build_params": {
"global_params": {
"branch_repo_owner": "http://example.com/head/repo",
"commit_hash": "abcdef",
"pull_request_author": "some-owner",
Expand All @@ -89,7 +109,7 @@
},
{
"payload": {
"build_params": {
"global_params": {
"branch": "bar",
"branch_dest": "foo",
"branch_repo_owner": "http://example.com/head/repo",
Expand All @@ -114,7 +134,7 @@
},
{
"payload": {
"build_params": {
"global_params": {
"branch_repo_owner": "http://example.com/head/repo",
"commit_hash": "abcdef",
"pull_request_author": "some-owner",
Expand All @@ -129,31 +149,42 @@
{
"bitrise": {
"app": "some-app",
"env": {
"FOO": "bar",
"PATH": {"artifact-reference": "<build/target.zip>"},
},
"workflows": ["bar"],
"workflows": [
"foo",
{
"bar": {
"FOO": "bar",
"PATH": {"artifact-reference": "<build/target.zip>"},
}
},
],
}
},
{},
{
"payload": {
"build_params": {
"global_params": {
"branch": "default",
"branch_repo_owner": "http://example.com/head/repo",
"commit_hash": "abcdef",
"environments": [
{"mapped_to": "FOO", "value": "bar"},
{
"mapped_to": "PATH",
"value": {"artifact-reference": "<build/target.zip>"},
},
],
},
"workflow_params": {
"bar": {
"environments": [
{"mapped_to": "FOO", "value": "bar"},
{
"mapped_to": "PATH",
"value": {
"artifact-reference": "<build/target.zip>"
},
},
]
}
},
},
"scopes": [
"foo:bitrise:app:some-app",
"foo:bitrise:workflow:foo",
"foo:bitrise:workflow:bar",
],
"tags": {"worker-implementation": "scriptworker"},
Expand All @@ -172,7 +203,7 @@
{
"payload": {
"artifact_prefix": "public",
"build_params": {
"global_params": {
"branch": "default",
"branch_repo_owner": "http://example.com/head/repo",
"commit_hash": "abcdef",
Expand Down

0 comments on commit e2ae827

Please sign in to comment.