From c4ed70cf4e25a1c16b6945e2b73af47d82d31000 Mon Sep 17 00:00:00 2001 From: Gabor Boros Date: Mon, 4 Dec 2023 13:39:52 +0400 Subject: [PATCH] fix: rate limiting caused by git repos cache check See the extended conversation here: https://github.com/overhangio/tutor-mfe/pull/163 Close #161 --- README.rst | 6 +-- ...231208_193911_regis_mfe_refs_for_master.md | 3 ++ tutormfe/hooks.py | 2 +- tutormfe/plugin.py | 45 +++++-------------- tutormfe/templates/mfe/build/mfe/Dockerfile | 6 +-- 5 files changed, 17 insertions(+), 45 deletions(-) create mode 100644 changelog.d/20231208_193911_regis_mfe_refs_for_master.md diff --git a/README.rst b/README.rst index 3e6b8c2c..424f9a59 100644 --- a/README.rst +++ b/README.rst @@ -138,14 +138,10 @@ Other MFE developers can take advantage of this plugin to deploy their own MFEs. "repository": "https://github.com/myorg/mymfe", "port": 2001, "version": "me/my-custom-branch", # optional, will default to the Open edX current tag. - "refs": https://api.github.com/repos/myorg/mymfe/git/refs/heads", # optional } return mfes -The MFE assets will then be bundled in the "mfe" Docker image whenever it is rebuilt with ``tutor images build mfe``. Providing a ``refs`` URL will ensure the build cache for that MFE is invalidated whenever a change is detected upstream at the git version in question. You can use the `GitHub references API`_ or the `GitLab branches API`_ for this. - -.. _GitHub references API: https://docs.github.com/en/rest/git/refs?apiVersion=2022-11-28#get-a-reference -.. _GitLab branches API: https://docs.gitlab.com/ee/api/branches.html#get-single-repository-branch +The MFE assets will then be bundled in the "mfe" Docker image whenever it is rebuilt with ``tutor images build mfe``. Assets will be served at ``http(s)://{{ MFE_HOST }}/mymfe``. Developers are free to add extra template patches to their plugins, as usual: for instance LMS setting patches to make sure that the LMS correctly connects to the MFEs. diff --git a/changelog.d/20231208_193911_regis_mfe_refs_for_master.md b/changelog.d/20231208_193911_regis_mfe_refs_for_master.md new file mode 100644 index 00000000..9a5f70d8 --- /dev/null +++ b/changelog.d/20231208_193911_regis_mfe_refs_for_master.md @@ -0,0 +1,3 @@ +- [Bugfix] Support MFE remotes that are not on GitHub. (by @gabor-boros and @regisb) +- 💥[Bugfix] Bypass rate-limiting when checking for upstream git changes. To achieve that, we use the `ADD --keep-git-dir` option, which is only compatible with BuildKit 0.11, from January 2023. Also, we get rid of the `get_github_refs_path` function. (by @gabor-boros and @regisb) + diff --git a/tutormfe/hooks.py b/tutormfe/hooks.py index c7a6b5e2..e62c2eb2 100644 --- a/tutormfe/hooks.py +++ b/tutormfe/hooks.py @@ -9,6 +9,6 @@ from tutor.core.hooks import Filter -MFE_ATTRS_TYPE = t.Dict[t.Literal["repository", "refs", "port"], t.Union["str", int]] +MFE_ATTRS_TYPE = t.Dict[t.Literal["repository", "port", "version"], t.Union["str", int]] MFE_APPS: Filter[dict[str, MFE_ATTRS_TYPE], []] = Filter() diff --git a/tutormfe/plugin.py b/tutormfe/plugin.py index c8705630..77ba5b26 100644 --- a/tutormfe/plugin.py +++ b/tutormfe/plugin.py @@ -29,70 +29,45 @@ }, } - -def get_github_refs_path(name: str) -> str: - """ - Generate a URL to access refs in heads (nightly) or tags (stable) via Github API. - Args: - name (str): Consisted of the repository owner and the repository name, as a string in 'owner/repo' format. - - Returns: - str: A string URL to the Github API, pointing to heads if version_suffix is set, tags otherwise. - - """ - - return f"https://api.github.com/repos/{name}/git/refs/{'heads' if __version_suffix__ else 'tags'}" - - CORE_MFE_APPS: dict[str, MFE_ATTRS_TYPE] = { "authn": { - "repository": "https://github.com/openedx/frontend-app-authn", - "refs": get_github_refs_path("openedx/frontend-app-authn"), + "repository": "https://github.com/openedx/frontend-app-authn.git", "port": 1999, }, "account": { - "repository": "https://github.com/openedx/frontend-app-account", - "refs": get_github_refs_path("openedx/frontend-app-account"), + "repository": "https://github.com/openedx/frontend-app-account.git", "port": 1997, }, "communications": { - "repository": "https://github.com/openedx/frontend-app-communications", - "refs": get_github_refs_path("openedx/frontend-app-communications"), + "repository": "https://github.com/openedx/frontend-app-communications.git", "port": 1984, }, "course-authoring": { - "repository": "https://github.com/openedx/frontend-app-course-authoring", - "refs": get_github_refs_path("openedx/frontend-app-course-authoring"), + "repository": "https://github.com/openedx/frontend-app-course-authoring.git", "port": 2001, }, "discussions": { - "repository": "https://github.com/openedx/frontend-app-discussions", - "refs": get_github_refs_path("openedx/frontend-app-discussions"), + "repository": "https://github.com/openedx/frontend-app-discussions.git", "port": 2002, }, "gradebook": { - "repository": "https://github.com/openedx/frontend-app-gradebook", - "refs": get_github_refs_path("openedx/frontend-app-gradebook"), + "repository": "https://github.com/openedx/frontend-app-gradebook.git", "port": 1994, }, "learner-dashboard": { - "repository": "https://github.com/openedx/frontend-app-learner-dashboard", - "refs": get_github_refs_path("openedx/frontend-app-learner-dashboard"), + "repository": "https://github.com/openedx/frontend-app-learner-dashboard.git", "port": 1996, }, "learning": { - "repository": "https://github.com/openedx/frontend-app-learning", - "refs": get_github_refs_path("openedx/frontend-app-learning"), + "repository": "https://github.com/openedx/frontend-app-learning.git", "port": 2000, }, "ora-grading": { - "repository": "https://github.com/openedx/frontend-app-ora-grading", - "refs": get_github_refs_path("openedx/frontend-app-ora-grading"), + "repository": "https://github.com/openedx/frontend-app-ora-grading.git", "port": 1993, }, "profile": { - "repository": "https://github.com/openedx/frontend-app-profile", - "refs": get_github_refs_path("openedx/frontend-app-profile"), + "repository": "https://github.com/openedx/frontend-app-profile.git", "port": 1995, }, } diff --git a/tutormfe/templates/mfe/build/mfe/Dockerfile b/tutormfe/templates/mfe/build/mfe/Dockerfile index 0a1c43e9..21617c6e 100644 --- a/tutormfe/templates/mfe/build/mfe/Dockerfile +++ b/tutormfe/templates/mfe/build/mfe/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 # https://hub.docker.com/_/node/tags FROM docker.io/node:18.16.0-bullseye-slim AS base @@ -34,10 +35,7 @@ RUN echo "copying i18n data" \ ######## {{ app_name }} (git) FROM base AS {{ app_name }}-git {#- Invalidate the build cache if a change is detected upstream #} -{%- if app.get("refs") %} -ADD {{ app["refs"] }}/{{ app.get("version", MFE_COMMON_VERSION) }} /tmp/gitref-{{ app_name }} -{%- endif %} -RUN git clone {{ app["repository"] }} --branch {{ app.get("version", MFE_COMMON_VERSION) }} --depth 1 . +ADD --keep-git-dir=true {{ app["repository"] }}#{{ app.get("version", MFE_COMMON_VERSION) }} . ######## {{ app_name }} (src) # Empty layer with just the repo at the root, for build-time bind-mounts