feat: add support for in tree actions from all configured branches

.github/workflows/deploy.yml

@@ -17,6 +17,7 @@ jobs:
         environment: ["staging", "firefoxci"]
     env:
       TASKCLUSTER_ROOT_URL: ${{vars.TASKCLUSTER_ROOT_URL}}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
@@ -48,6 +49,7 @@ jobs:
       TASKCLUSTER_ROOT_URL: ${{vars.TASKCLUSTER_ROOT_URL}}
       TASKCLUSTER_CLIENT_ID: ${{vars.TASKCLUSTER_CLIENT_ID}}
       TASKCLUSTER_ACCESS_TOKEN: ${{secrets.TASKCLUSTER_ACCESS_TOKEN}}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python

src/ciadmin/boot.py

@@ -47,4 +47,8 @@
 
 
 def boot():
+    if not os.environ.get("GITHUB_TOKEN"):
+        print(
+            "WARNING: GITHUB_TOKEN is not present in the environment; you may run into rate limits querying for GitHub branches"
+        )
     main(appconfig)

src/ciadmin/generate/branches.py

@@ -0,0 +1,72 @@
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+from asyncio import Lock
+
+import aiohttp
+from aiohttp_retry import ExponentialRetry, RetryClient
+from tcadmin.util.sessions import aiohttp_session
+
+_cache = {}
+_lock = {}
+
+
+# TODO: support private repositories. this will most likely require querying
+# GitHub as an app.
+async def get(repo_path, repo_type="git"):
+    """Get the list of branches present in `repo_path`. Only supported for
+    public GitHub repositories."""
+    if repo_type != "git":
+        raise Exception("Branches can only be fetched for git repositories!")
+
+    if repo_path.endswith("/"):
+        repo_path = repo_path[:-1]
+    branches_url = f"https://api.github.com/repos/{repo_path}/branches"
+
+    # 100 is the maximum allowed
+    # https://docs.github.com/en/rest/branches/branches?apiVersion=2022-11-28#list-branches
+    params = {"per_page": 100}
+    headers = {}
+    if "GITHUB_TOKEN" in os.environ:
+        github_token = os.environ["GITHUB_TOKEN"]
+        headers["Authorization"] = f"Bearer {github_token}"
+
+    async with _lock.setdefault(repo_path, Lock()):
+        if repo_path in _cache:
+            return _cache[repo_path]
+
+        branches = []
+        client = RetryClient(
+            client_session=aiohttp_session(),
+            retry_options=ExponentialRetry(attempts=5),
+        )
+        while branches_url:
+            async with client.get(
+                branches_url, headers=headers, params=params
+            ) as response:
+                try:
+                    response.raise_for_status()
+                    result = await response.json()
+                    branches.extend([b["name"] for b in result])
+                    # If `link` is present in the response it will contain
+                    # pagination information. We need to examine it to see
+                    # if there are additional pages of results to fetch.
+                    # See https://docs.github.com/en/rest/using-the-rest-api/using-pagination-in-the-rest-api?apiVersion=2022-11-28#using-link-headers
+                    # for a full description of the responses.
+                    # This icky parsing can probably go away when we switch
+                    # to a GitHub app, as we'll likely be using a proper
+                    # client at that point.
+                    for l in response.headers.get("link", "").split(","):
+                        if 'rel="next"' in l:
+                            branches_url = l.split(">")[0].split("<")[1]
+                            break
+                    else:
+                        branches_url = None
+                except aiohttp.ClientResponseError as e:
+                    print(f"Got error when querying {branches_url}: {e}")
+                    raise e
+
+        _cache[repo_path] = branches
+        return _cache[repo_path]

src/ciadmin/generate/in_tree_actions.py

@@ -19,7 +19,9 @@
 from tcadmin.util.scopes import normalizeScopes
 from tcadmin.util.sessions import aiohttp_session
 
-from . import tcyml
+from ciadmin.util.matching import glob_match
+
+from . import branches, tcyml
 from .ciconfig.actions import Action
 from .ciconfig.projects import Project
 
@@ -30,34 +32,44 @@
 HOOK_RETENTION_TIME = datetime.timedelta(days=60)
 
 
+def should_hash(project):
+    if project.feature("gecko-actions"):
+        if not project.feature("hg-push") and not project.feature("gecko-cron"):
+            return False
+        if project.is_try:
+            return False
+        return True
+    elif project.feature("taskgraph-actions"):
+        # globbed projects (those with "*" in p.repo) could theoretically
+        # support in tree actions if we looked up the full repository
+        # list matching the glob. it's probably not worth doing though.
+        if "*" in project.repo:
+            return False
+        # At this time, we don't support fetching tcymls from private
+        # repos, so we can't generate action hooks for them.
+        if project.feature("github-private-repo"):
+            return False
+        return True
+    else:
+        return False
+
+
+async def get_project_branches(project):
+    if project.repo_type == "git":
+        return await branches.get(project.repo_path)
+    elif project.repo_type == "hg":
+        return ["default"]
+
+    raise Exception(f"unsupported repo type {project.repo_type} for {project.alias}")
+
+
 async def hash_taskcluster_ymls():
     """
     Download and hash .taskcluster.yml from every project repository.  Returns
     {alias: (parsed content, hash)}.
     """
     projects = await Project.fetch_all()
 
-    def should_hash(project):
-        if project.feature("gecko-actions"):
-            if not project.feature("hg-push") and not project.feature("gecko-cron"):
-                return False
-            if project.is_try:
-                return False
-            return True
-        elif project.feature("taskgraph-actions"):
-            # globbed projects (those with "*" in p.repo) could theoretically
-            # support in tree actions if we looked up the full repository
-            # list matching the glob. it's probably not worth doing though.
-            if "*" in project.repo:
-                return False
-            # At this time, we don't support fetching tcymls from private
-            # repos, so we can't generate action hooks for them.
-            if project.feature("github-private-repo"):
-                return False
-            return True
-        else:
-            return False
-
     # hash the value of this .taskcluster.yml.  Note that this must match the
     # hashing in taskgraph/actions/registry.py
     def hash(val):
@@ -69,12 +81,18 @@ def hash(val):
     for p in tcyml_projects:
         rv[p.alias] = {}
 
-        for b in set([b.name for b in p.branches] + [p.default_branch]):
-            # Can't fetch a .taskcluster.yml for a globbed branch
-            # TODO: perhaps we should do this for partly globbed branches,
-            # eg: release* ?
-            # we'd have to fetch that list from the server to do that
-            if "*" not in b and "*" not in p.repo:
+        # If "*" is a configured branch we explicitly ignore it; otherwise
+        # we could end up fetching 100s or 1000s of tcymls and generating
+        # hooks for them. Substring globs may still exist, and are
+        # supported.
+        configured_branches = [b.name for b in p.branches if b.name != "*"]
+        # The default branch is considered to be _always_ configured, even if
+        # not explicitly named in `branches`. This is primarily to ensure that
+        # cases where `*` is the only branch explicitly listed, that we still
+        # generate actions for the default branch.
+        configured_branches.append(p.default_branch)
+        for b in await get_project_branches(p):
+            if glob_match(configured_branches, b):
 
                 def process(project, branch_name, task):
                     try:
@@ -140,9 +158,7 @@ def make_hook(action, tcyml_content, tcyml_hash, projects, pr=False):
             if project[branch]["hash"] == tcyml_hash and str(action.level) == str(
                 project[branch]["level"]
             ):
-                matching_projects.append(
-                    f"{project[branch]['alias']}, branch: '{branch}'"
-                )
+                matching_projects.append(f"{key}, branch: '{branch}'")
 
     # schema-generation utilities
 
@@ -347,22 +363,19 @@ async def update_resources(resources):
         # gather the hashes at the action's level or higher
         hooks_to_make = {}  # {hash: content}, for uniqueness
         for project in projects:
-            for branch_name in set(
-                [b.name for b in project.branches] + [project.default_branch]
-            ):
-                # We don't have taskcluster.ymls from globbed branches;
-                # see comment in hash_taskcluster_ymls
-                if "*" in branch_name:
-                    continue
+            if not should_hash(project):
+                continue
+
+            for branch_name in await get_project_branches(project):
                 if project.alias not in hashed_tcymls:
                     continue
+                if branch_name not in hashed_tcymls[project.alias]:
+                    # Branch didn't exist, or doesn't have a parseable tcyml
+                    continue
                 if project.get_level(branch_name) < action.level:
                     continue
                 if project.trust_domain != action.trust_domain:
                     continue
-                if branch_name not in hashed_tcymls[project.alias]:
-                    # Branch didn't exist, or doesn't have a parseable tcyml
-                    continue
 
                 content, hash = (
                     hashed_tcymls[project.alias][branch_name]["parsed"],