From 19fd8128aea8777c970e58836496dc8f0ef98152 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Fri, 22 May 2020 16:27:18 -0700 Subject: [PATCH 1/4] fix xpi private cot verification Previously, we assumed that we would list any and all private github repos in constants.py. However, with the xpi project, we'll get many private repos that we will want to verify without needing to land a change in scriptworker. Let's allow for ssh://github.com source urls to imply a private repository. As ridealongs, I added `--verbose` and `--no-check-task` to `verify-cot`. I also cleaned up some extraneous output at the end of `verify_cot`. --- src/scriptworker/cot/verify.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/scriptworker/cot/verify.py b/src/scriptworker/cot/verify.py index ec8acd58..e50dc395 100644 --- a/src/scriptworker/cot/verify.py +++ b/src/scriptworker/cot/verify.py @@ -1303,7 +1303,9 @@ async def get_in_tree_template(link): raise CoTError("{} source url {} doesn't end in .yml or .yaml!".format(link.name, source_url)) auth = None - if (any(vcs_rule.get("require_secret") for vcs_rule in context.config["trusted_vcs_rules"])) and "github.com" in source_url: + if ( + source_url.startswith("ssh://") or any(vcs_rule.get("require_secret") for vcs_rule in context.config["trusted_vcs_rules"]) + ) and "github.com" in source_url: newurl = re.sub( r"^(?:ssh://|https?://)(?:[^@/\:]*(?:\:[^@/\:]*)?@)?github.com(?:\:\d*)?/(?P.*)/raw/(?P[a-zA-Z0-9]*)/(?P.*)$", r"https://raw.githubusercontent.com/\g/\g/\g", @@ -1969,11 +1971,8 @@ async def _async_verify_cot_cmdln(opts, tmp): if os.environ.get("SCRIPTWORKER_GITHUB_OAUTH_TOKEN"): context.config["github_oauth_token"] = os.environ.get("SCRIPTWORKER_GITHUB_OAUTH_TOKEN") cot = ChainOfTrust(context, opts.task_type, task_id=opts.task_id) - await verify_chain_of_trust(cot, check_task=True) - log.info(format_json(cot.dependent_task_ids())) - log.info("{} : {}".format(cot.name, cot.task_id)) - for link in cot.links: - log.info("{} : {}".format(link.name, link.task_id)) + check_task = opts.no_check_task is False + await verify_chain_of_trust(cot, check_task=check_task) def verify_cot_cmdln(args=None, event_loop=None): @@ -2008,11 +2007,14 @@ def verify_cot_cmdln(args=None, event_loop=None): parser.add_argument("--cleanup", help="clean up the temp dir afterwards", dest="cleanup", action="store_true", default=False) parser.add_argument("--cot-product", help="the product type to test", default="firefox") parser.add_argument("--verify-sigs", help="enable signature verification", action="store_true", default=False) + parser.add_argument("--verbose", "-v", help="enable debug logging", action="store_true", default=False) + parser.add_argument("--no-check-task", help="skip verifying the taskId's cot status", action="store_true", default=False) opts = parser.parse_args(args) tmp = tempfile.mkdtemp() log = logging.getLogger("scriptworker") - log.setLevel(logging.DEBUG) - logging.basicConfig() + level = logging.DEBUG if opts.verbose else logging.INFO + log.setLevel(level) + logging.basicConfig(level=level) event_loop = event_loop or asyncio.get_event_loop() try: event_loop.run_until_complete(_async_verify_cot_cmdln(opts, tmp)) From ea85bcd710d2626d12f04cb4915c33d437c32032 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Fri, 22 May 2020 16:47:28 -0700 Subject: [PATCH 2/4] stop creating `...` directories The base_*_dir and task_log_dir_template config entries were from a historic, unmerged PR. Let's set the actual config paths inside our tempdir. Bonus: we won't have weird bustage related to hardcoded directory paths if and when we run two concurrent sets of tests. --- tests/test_production.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_production.py b/tests/test_production.py index e1989d43..7a89c8dc 100644 --- a/tests/test_production.py +++ b/tests/test_production.py @@ -34,9 +34,10 @@ def build_config(override, basedir): config.update( { "log_dir": os.path.join(basedir, "log"), - "base_artifact_dir": os.path.join(basedir, "artifact"), - "task_log_dir_template": os.path.join(basedir, "artifact", "public", "logs"), - "base_work_dir": os.path.join(basedir, "work"), + "artifact_dir": os.path.join(basedir, "artifact"), + "task_log_dir": os.path.join(basedir, "artifact", "public", "logs"), + "work_dir": os.path.join(basedir, "work"), + "ed25519_private_key_path": "", } ) del config["credentials"] @@ -45,6 +46,10 @@ def build_config(override, basedir): with open(os.path.join(basedir, "config.json"), "w") as fh: json.dump(config, fh, indent=2, sort_keys=True) config = apply_product_config(config) + # Avoid creating a `...` directory + for k,v in config.items(): + if v == '...': + raise Exception(f"Let's not keep any '...' config values. {k} is {v}!") return config From 8ae27fec6be312e9d1b06915a48e72953459219e Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Tue, 26 May 2020 10:56:44 -0700 Subject: [PATCH 3/4] fix black --- tests/test_production.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_production.py b/tests/test_production.py index 7a89c8dc..7ab1926a 100644 --- a/tests/test_production.py +++ b/tests/test_production.py @@ -47,8 +47,8 @@ def build_config(override, basedir): json.dump(config, fh, indent=2, sort_keys=True) config = apply_product_config(config) # Avoid creating a `...` directory - for k,v in config.items(): - if v == '...': + for k, v in config.items(): + if v == "...": raise Exception(f"Let's not keep any '...' config values. {k} is {v}!") return config From 2b076073d5904237fe2c60c43fd1562b0f47fcab Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Tue, 26 May 2020 11:10:53 -0700 Subject: [PATCH 4/4] 34.2.0 --- HISTORY.rst | 12 ++++++++++++ src/scriptworker/version.py | 2 +- version.json | 4 ++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e8278c91..17bcf774 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,18 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to `Semantic Versioning `__. +[34.2.0] - 2020-05-26 +--------------------- + +Changed +~~~~~~~ +- Github source urls starting with ``ssh://`` are now treated as private repositories. +- ``verify_cot`` now takes ``--verbose`` and ``--no-check-task`` options. + +Fixed +~~~~~ +- ``test_production`` should no longer leave behind temp ``...`` directories. + [34.1.0] - 2020-05-04 --------------------- diff --git a/src/scriptworker/version.py b/src/scriptworker/version.py index 8a97d8f2..3804a6b0 100755 --- a/src/scriptworker/version.py +++ b/src/scriptworker/version.py @@ -54,7 +54,7 @@ def get_version_string(version: Union[ShortVerType, LongVerType]) -> str: # 1}}} # Semantic versioning 2.0.0 http://semver.org/ -__version__ = (34, 1, 0) +__version__ = (34, 2, 0) __version_string__ = get_version_string(__version__) diff --git a/version.json b/version.json index 3da46ca0..aaec9817 100644 --- a/version.json +++ b/version.json @@ -1,8 +1,8 @@ { "version":[ 34, - 1, + 2, 0 ], - "version_string":"34.1.0" + "version_string":"34.2.0" }