fix xpi private cot verification (#455)

* fix xpi private cot verification

Previously, we assumed that we would list any and all private github
repos in constants.py. However, with the xpi project, we'll get many
private repos that we will want to verify without needing to land a
change in scriptworker.

Let's allow for ssh://github.com source urls to imply a private
repository.

As ridealongs, I added `--verbose` and `--no-check-task` to
`verify-cot`. I also cleaned up some extraneous output at the end of
`verify_cot`.

* stop creating `...` directories

The base_*_dir and task_log_dir_template config entries were from a
historic, unmerged PR. Let's set the actual config paths inside our tempdir.
Bonus: we won't have weird bustage related to hardcoded directory paths if and
when we run two concurrent sets of tests.

* fix black

* 34.2.0

HISTORY.rst

@@ -4,6 +4,18 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to `Semantic Versioning <http://semver.org/>`__.
 
+[34.2.0] - 2020-05-26
+---------------------
+
+Changed
+~~~~~~~
+- Github source urls starting with ``ssh://`` are now treated as private repositories.
+- ``verify_cot`` now takes ``--verbose`` and ``--no-check-task`` options.
+
+Fixed
+~~~~~
+- ``test_production`` should no longer leave behind temp ``...`` directories.
+
 [34.1.0] - 2020-05-04
 ---------------------
 

src/scriptworker/cot/verify.py

@@ -1303,7 +1303,9 @@ async def get_in_tree_template(link):
         raise CoTError("{} source url {} doesn't end in .yml or .yaml!".format(link.name, source_url))
 
     auth = None
-    if (any(vcs_rule.get("require_secret") for vcs_rule in context.config["trusted_vcs_rules"])) and "github.com" in source_url:
+    if (
+        source_url.startswith("ssh://") or any(vcs_rule.get("require_secret") for vcs_rule in context.config["trusted_vcs_rules"])
+    ) and "github.com" in source_url:
         newurl = re.sub(
             r"^(?:ssh://|https?://)(?:[^@/\:]*(?:\:[^@/\:]*)?@)?github.com(?:\:\d*)?/(?P<repopath>.*)/raw/(?P<sha>[a-zA-Z0-9]*)/(?P<filepath>.*)$",
             r"https://raw.githubusercontent.com/\g<repopath>/\g<sha>/\g<filepath>",
@@ -1969,11 +1971,8 @@ async def _async_verify_cot_cmdln(opts, tmp):
         if os.environ.get("SCRIPTWORKER_GITHUB_OAUTH_TOKEN"):
             context.config["github_oauth_token"] = os.environ.get("SCRIPTWORKER_GITHUB_OAUTH_TOKEN")
         cot = ChainOfTrust(context, opts.task_type, task_id=opts.task_id)
-        await verify_chain_of_trust(cot, check_task=True)
-        log.info(format_json(cot.dependent_task_ids()))
-        log.info("{} : {}".format(cot.name, cot.task_id))
-        for link in cot.links:
-            log.info("{} : {}".format(link.name, link.task_id))
+        check_task = opts.no_check_task is False
+        await verify_chain_of_trust(cot, check_task=check_task)
 
 
 def verify_cot_cmdln(args=None, event_loop=None):
@@ -2008,11 +2007,14 @@ def verify_cot_cmdln(args=None, event_loop=None):
     parser.add_argument("--cleanup", help="clean up the temp dir afterwards", dest="cleanup", action="store_true", default=False)
     parser.add_argument("--cot-product", help="the product type to test", default="firefox")
     parser.add_argument("--verify-sigs", help="enable signature verification", action="store_true", default=False)
+    parser.add_argument("--verbose", "-v", help="enable debug logging", action="store_true", default=False)
+    parser.add_argument("--no-check-task", help="skip verifying the taskId's cot status", action="store_true", default=False)
     opts = parser.parse_args(args)
     tmp = tempfile.mkdtemp()
     log = logging.getLogger("scriptworker")
-    log.setLevel(logging.DEBUG)
-    logging.basicConfig()
+    level = logging.DEBUG if opts.verbose else logging.INFO
+    log.setLevel(level)
+    logging.basicConfig(level=level)
     event_loop = event_loop or asyncio.get_event_loop()
     try:
         event_loop.run_until_complete(_async_verify_cot_cmdln(opts, tmp))

src/scriptworker/version.py

@@ -54,7 +54,7 @@ def get_version_string(version: Union[ShortVerType, LongVerType]) -> str:
 
 # 1}}}
 # Semantic versioning 2.0.0  http://semver.org/
-__version__ = (34, 1, 0)
+__version__ = (34, 2, 0)
 __version_string__ = get_version_string(__version__)
 
 

tests/test_production.py

@@ -34,9 +34,10 @@ def build_config(override, basedir):
     config.update(
         {
             "log_dir": os.path.join(basedir, "log"),
-            "base_artifact_dir": os.path.join(basedir, "artifact"),
-            "task_log_dir_template": os.path.join(basedir, "artifact", "public", "logs"),
-            "base_work_dir": os.path.join(basedir, "work"),
+            "artifact_dir": os.path.join(basedir, "artifact"),
+            "task_log_dir": os.path.join(basedir, "artifact", "public", "logs"),
+            "work_dir": os.path.join(basedir, "work"),
+            "ed25519_private_key_path": "",
         }
     )
     del config["credentials"]
@@ -45,6 +46,10 @@ def build_config(override, basedir):
     with open(os.path.join(basedir, "config.json"), "w") as fh:
         json.dump(config, fh, indent=2, sort_keys=True)
     config = apply_product_config(config)
+    # Avoid creating a `...` directory
+    for k, v in config.items():
+        if v == "...":
+            raise Exception(f"Let's not keep any '...' config values. {k} is {v}!")
     return config
 
 

version.json

@@ -1,8 +1,8 @@
 {
     "version":[
         34,
-        1,
+        2,
         0
     ],
-    "version_string":"34.1.0"
+    "version_string":"34.2.0"
 }