From be1e16984c7a5c873529ac407dc0c855e55329b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 15 Dec 2022 10:23:55 +0100 Subject: [PATCH 01/16] fix: tutor nightly version suffix --- tutor/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutor/__about__.py b/tutor/__about__.py index 14c03be9b0..a8254bd4b3 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -10,7 +10,7 @@ # the nightly branch. # The suffix is cleanly separated from the __version__ in this module to avoid # conflicts when merging branches. -__version_suffix__ = "" +__version_suffix__ = "nightly" # The app name will be used to define the name of the default tutor root and # plugin directory. To avoid conflicts between multiple locally-installed From c0d7e262b0348f0630344ca48160a0e2e5561ece Mon Sep 17 00:00:00 2001 From: Abderraouf Mehdi Bouhali Date: Wed, 28 Dec 2022 10:12:19 +0100 Subject: [PATCH 02/16] fix(config): use master branch instead of olive.1 --- tutor/templates/config/defaults.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 819a476b75..1070a7f5c2 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -51,7 +51,7 @@ OPENEDX_CMS_UWSGI_WORKERS: 2 OPENEDX_LMS_UWSGI_WORKERS: 2 OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_MYSQL_USERNAME: "openedx" -OPENEDX_COMMON_VERSION: "open-release/olive.1" +OPENEDX_COMMON_VERSION: "master" OPENEDX_EXTRA_PIP_REQUIREMENTS: - "openedx-scorm-xblock>=15.0.0,<16.0.0" MYSQL_HOST: "mysql" From 5a3b7628578e1fdfddf0cf672c1f1a5b0a95b755 Mon Sep 17 00:00:00 2001 From: Abderraouf Mehdi Bouhali Date: Wed, 28 Dec 2022 10:26:33 +0100 Subject: [PATCH 03/16] fix(patches): remove openedx backport from nightly This removes an openedx/edx-platform commit backported as a patch to tutor to olive.1 release Since the commit is already merged into edx-platform:master branch used by tutor nightly, there is no further need for it. --- tutor/templates/build/openedx/Dockerfile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index b78e6b77fb..a1ba493803 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -42,15 +42,8 @@ WORKDIR /openedx/edx-platform RUN git config --global user.email "tutor@overhang.io" \ && git config --global user.name "Tutor" -{%- if patch("openedx-dockerfile-git-patches-default") %} # Custom edx-platform patches {{ patch("openedx-dockerfile-git-patches-default") }} -{%- else %} -# Patch edx-platform -# Fix broken Circuit Schematic Builder problem template -# https://github.com/openedx/edx-platform/pull/31365 -RUN curl -fsSL https://github.com/openedx/edx-platform/commit/20b93b8b01276edadddfbbb67f15714fddd81c31.patch | git am -{%- endif %} {# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/ | git am #} {{ patch("openedx-dockerfile-post-git-checkout") }} From 4da32ab84e449f3190c5105593ace720f11b2130 Mon Sep 17 00:00:00 2001 From: Carlos Muniz Date: Tue, 17 Jan 2023 13:57:23 -0500 Subject: [PATCH 04/16] refactor: annotation with __future__.annotations Adds `from __future__ import annotations` to the top of every module, right below the module's docstring. Replaces any usages of t.List, t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents: list, dict, set, tuple, and type. Ensures that make test still passes under Python 3.7, 3.8 and 3.9. --- ...10_130740_cmuniz_built_in_generic_types.md | 1 + docs/conf.py | 6 ++ tests/commands/base.py | 6 +- tests/commands/test_compose.py | 15 +++-- tests/hooks/test_filters.py | 5 +- tests/test_plugins_v0.py | 5 +- tutor/commands/cli.py | 3 +- tutor/commands/compose.py | 57 +++++++++---------- tutor/commands/config.py | 11 ++-- tutor/commands/dev.py | 4 +- tutor/commands/images.py | 31 +++++----- tutor/commands/jobs.py | 15 ++--- tutor/commands/local.py | 3 +- tutor/commands/plugins.py | 11 ++-- tutor/config.py | 6 +- tutor/env.py | 3 +- tutor/hooks/actions.py | 6 +- tutor/hooks/contexts.py | 4 +- tutor/hooks/filters.py | 20 +++---- tutor/hooks/priorities.py | 3 +- tutor/plugins/__init__.py | 7 ++- tutor/serialize.py | 3 +- tutor/types.py | 6 +- 23 files changed, 126 insertions(+), 105 deletions(-) create mode 100644 changelog.d/20230110_130740_cmuniz_built_in_generic_types.md diff --git a/changelog.d/20230110_130740_cmuniz_built_in_generic_types.md b/changelog.d/20230110_130740_cmuniz_built_in_generic_types.md new file mode 100644 index 0000000000..25b31a8ecd --- /dev/null +++ b/changelog.d/20230110_130740_cmuniz_built_in_generic_types.md @@ -0,0 +1 @@ +- [Improvement] Changes annotations from `typing` to use built-in generic types from `__future__.annotations` (by @Carlos-Muniz) diff --git a/docs/conf.py b/docs/conf.py index 068538f186..1f17b4af85 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -38,6 +38,12 @@ ("py:class", "tutor.hooks.filters.P"), ("py:class", "tutor.hooks.filters.T"), ("py:class", "tutor.hooks.actions.P"), + ("py:class", "P"), + ("py:class", "P.args"), + ("py:class", "P.kwargs"), + ("py:class", "T"), + ("py:class", "t.Any"), + ("py:class", "t.Optional"), ] # -- Sphinx-Click configuration diff --git a/tests/commands/base.py b/tests/commands/base.py index e68dc2932d..868e00d75b 100644 --- a/tests/commands/base.py +++ b/tests/commands/base.py @@ -1,4 +1,4 @@ -import typing as t +from __future__ import annotations import click.testing @@ -12,13 +12,13 @@ class TestCommandMixin: """ @staticmethod - def invoke(args: t.List[str]) -> click.testing.Result: + def invoke(args: list[str]) -> click.testing.Result: with temporary_root() as root: return TestCommandMixin.invoke_in_root(root, args) @staticmethod def invoke_in_root( - root: str, args: t.List[str], catch_exceptions: bool = True + root: str, args: list[str], catch_exceptions: bool = True ) -> click.testing.Result: """ Use this method for commands that all need to run in the same root: diff --git a/tests/commands/test_compose.py b/tests/commands/test_compose.py index aeb91e0888..9eb505e967 100644 --- a/tests/commands/test_compose.py +++ b/tests/commands/test_compose.py @@ -1,3 +1,4 @@ +from __future__ import annotations import typing as t import unittest from io import StringIO @@ -62,9 +63,9 @@ def test_compose_local_tmp_generation(self, _mock_stdout: StringIO) -> None: # Mount volumes compose.mount_tmp_volumes(mount_args, LocalContext("")) - compose_file: t.Dict[str, t.Any] = hooks.Filters.COMPOSE_LOCAL_TMP.apply({}) - actual_services: t.Dict[str, t.Any] = compose_file["services"] - expected_services: t.Dict[str, t.Any] = { + compose_file: dict[str, t.Any] = hooks.Filters.COMPOSE_LOCAL_TMP.apply({}) + actual_services: dict[str, t.Any] = compose_file["services"] + expected_services: dict[str, t.Any] = { "cms": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, "cms-worker": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, "lms": { @@ -78,11 +79,9 @@ def test_compose_local_tmp_generation(self, _mock_stdout: StringIO) -> None: } self.assertEqual(actual_services, expected_services) - compose_jobs_file: t.Dict[ - str, t.Any - ] = hooks.Filters.COMPOSE_LOCAL_JOBS_TMP.apply({}) - actual_jobs_services: t.Dict[str, t.Any] = compose_jobs_file["services"] - expected_jobs_services: t.Dict[str, t.Any] = { + compose_jobs_file = hooks.Filters.COMPOSE_LOCAL_JOBS_TMP.apply({}) + actual_jobs_services = compose_jobs_file["services"] + expected_jobs_services: dict[str, t.Any] = { "cms-job": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, "lms-job": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, } diff --git a/tests/hooks/test_filters.py b/tests/hooks/test_filters.py index 796ff72191..4c23310d30 100644 --- a/tests/hooks/test_filters.py +++ b/tests/hooks/test_filters.py @@ -1,3 +1,4 @@ +from __future__ import annotations import typing as t import unittest @@ -23,14 +24,14 @@ def filter1(value: int) -> int: def test_add_items(self) -> None: @hooks.filters.add("tests:add-sheeps") - def filter1(sheeps: t.List[int]) -> t.List[int]: + def filter1(sheeps: list[int]) -> list[int]: return sheeps + [0] hooks.filters.add_item("tests:add-sheeps", 1) hooks.filters.add_item("tests:add-sheeps", 2) hooks.filters.add_items("tests:add-sheeps", [3, 4]) - sheeps: t.List[int] = hooks.filters.apply("tests:add-sheeps", []) + sheeps: list[int] = hooks.filters.apply("tests:add-sheeps", []) self.assertEqual([0, 1, 2, 3, 4], sheeps) def test_filter_callbacks(self) -> None: diff --git a/tests/test_plugins_v0.py b/tests/test_plugins_v0.py index 85dddb9c10..311cac6477 100644 --- a/tests/test_plugins_v0.py +++ b/tests/test_plugins_v0.py @@ -1,3 +1,4 @@ +from __future__ import annotations import typing as t from unittest.mock import patch @@ -197,9 +198,7 @@ def test_dict_plugin(self) -> None: {"name": "myplugin", "config": {"set": {"KEY": "value"}}, "version": "0.1"} ) plugins.load("myplugin") - overriden_items: t.List[ - t.Tuple[str, t.Any] - ] = hooks.Filters.CONFIG_OVERRIDES.apply([]) + overriden_items = hooks.Filters.CONFIG_OVERRIDES.apply([]) versions = list(plugins.iter_info()) self.assertEqual("myplugin", plugin.name) self.assertEqual([("myplugin", "0.1")], versions) diff --git a/tutor/commands/cli.py b/tutor/commands/cli.py index 758ff1ab0b..98da421013 100755 --- a/tutor/commands/cli.py +++ b/tutor/commands/cli.py @@ -1,3 +1,4 @@ +from __future__ import annotations import sys import typing as t @@ -61,7 +62,7 @@ def ensure_plugins_enabled(cls, ctx: click.Context) -> None: hooks.Actions.PROJECT_ROOT_READY.do(ctx.params["root"]) cls.IS_ROOT_READY = True - def list_commands(self, ctx: click.Context) -> t.List[str]: + def list_commands(self, ctx: click.Context) -> list[str]: """ This is run in the following cases: - shell autocompletion: tutor diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index a1e24f7bd0..921ff3cef7 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -1,3 +1,4 @@ +from __future__ import annotations import os import re import typing as t @@ -16,15 +17,15 @@ from tutor.tasks import BaseComposeTaskRunner from tutor.types import Config -COMPOSE_FILTER_TYPE: TypeAlias = "hooks.filters.Filter[t.Dict[str, t.Any], []]" +COMPOSE_FILTER_TYPE: TypeAlias = "hooks.filters.Filter[dict[str, t.Any], []]" class ComposeTaskRunner(BaseComposeTaskRunner): def __init__(self, root: str, config: Config): super().__init__(root, config) self.project_name = "" - self.docker_compose_files: t.List[str] = [] - self.docker_compose_job_files: t.List[str] = [] + self.docker_compose_files: list[str] = [] + self.docker_compose_job_files: list[str] = [] def docker_compose(self, *command: str) -> int: """ @@ -55,7 +56,7 @@ def update_docker_compose_tmp( Update the contents of the docker-compose.tmp.yml and docker-compose.jobs.tmp.yml files, which are generated at runtime. """ - compose_base: t.Dict[str, t.Any] = { + compose_base: dict[str, t.Any] = { "version": "{{ DOCKER_COMPOSE_VERSION }}", "services": {}, } @@ -134,11 +135,11 @@ def convert( value: str, param: t.Optional["click.Parameter"], ctx: t.Optional[click.Context], - ) -> t.List["MountType"]: + ) -> list["MountType"]: mounts = self.convert_explicit_form(value) or self.convert_implicit_form(value) return mounts - def convert_explicit_form(self, value: str) -> t.List["MountParam.MountType"]: + def convert_explicit_form(self, value: str) -> list["MountParam.MountType"]: """ Argument is of the form "containers:/host/path:/container/path". """ @@ -146,8 +147,8 @@ def convert_explicit_form(self, value: str) -> t.List["MountParam.MountType"]: if not match: return [] - mounts: t.List["MountParam.MountType"] = [] - services: t.List[str] = [ + mounts: list["MountParam.MountType"] = [] + services: list[str] = [ service.strip() for service in match["services"].split(",") ] host_path = os.path.abspath(os.path.expanduser(match["host_path"])) @@ -159,11 +160,11 @@ def convert_explicit_form(self, value: str) -> t.List["MountParam.MountType"]: mounts.append((service, host_path, container_path)) return mounts - def convert_implicit_form(self, value: str) -> t.List["MountParam.MountType"]: + def convert_implicit_form(self, value: str) -> list["MountParam.MountType"]: """ Argument is of the form "/host/path" """ - mounts: t.List["MountParam.MountType"] = [] + mounts: list["MountParam.MountType"] = [] host_path = os.path.abspath(os.path.expanduser(value)) for service, container_path in hooks.Filters.COMPOSE_MOUNTS.iterate( os.path.basename(host_path) @@ -175,7 +176,7 @@ def convert_implicit_form(self, value: str) -> t.List["MountParam.MountType"]: def shell_complete( self, ctx: click.Context, param: click.Parameter, incomplete: str - ) -> t.List[CompletionItem]: + ) -> list[CompletionItem]: """ Mount argument completion works only for the single path (implicit) form. The reason is that colons break words in bash completion: @@ -197,7 +198,7 @@ def shell_complete( def mount_tmp_volumes( - all_mounts: t.Tuple[t.List[MountParam.MountType], ...], + all_mounts: tuple[list[MountParam.MountType], ...], context: BaseComposeContext, ) -> None: for mounts in all_mounts: @@ -230,8 +231,8 @@ def mount_tmp_volume( @compose_tmp_filter.add() def _add_mounts_to_docker_compose_tmp( - docker_compose: t.Dict[str, t.Any], - ) -> t.Dict[str, t.Any]: + docker_compose: dict[str, t.Any], + ) -> dict[str, t.Any]: services = docker_compose.setdefault("services", {}) services.setdefault(service, {"volumes": []}) services[service]["volumes"].append(f"{host_path}:{container_path}") @@ -251,8 +252,8 @@ def start( context: BaseComposeContext, skip_build: bool, detach: bool, - mounts: t.Tuple[t.List[MountParam.MountType]], - services: t.List[str], + mounts: tuple[list[MountParam.MountType]], + services: list[str], ) -> None: command = ["up", "--remove-orphans"] if not skip_build: @@ -269,7 +270,7 @@ def start( @click.command(help="Stop a running platform") @click.argument("services", metavar="service", nargs=-1) @click.pass_obj -def stop(context: BaseComposeContext, services: t.List[str]) -> None: +def stop(context: BaseComposeContext, services: list[str]) -> None: config = tutor_config.load(context.root) context.job_runner(config).docker_compose("stop", *services) @@ -281,7 +282,7 @@ def stop(context: BaseComposeContext, services: t.List[str]) -> None: @click.option("-d", "--detach", is_flag=True, help="Start in daemon mode") @click.argument("services", metavar="service", nargs=-1) @click.pass_context -def reboot(context: click.Context, detach: bool, services: t.List[str]) -> None: +def reboot(context: click.Context, detach: bool, services: list[str]) -> None: context.invoke(stop, services=services) context.invoke(start, detach=detach, services=services) @@ -295,7 +296,7 @@ def reboot(context: click.Context, detach: bool, services: t.List[str]) -> None: ) @click.argument("services", metavar="service", nargs=-1) @click.pass_obj -def restart(context: BaseComposeContext, services: t.List[str]) -> None: +def restart(context: BaseComposeContext, services: list[str]) -> None: config = tutor_config.load(context.root) command = ["restart"] if "all" in services: @@ -315,9 +316,7 @@ def restart(context: BaseComposeContext, services: t.List[str]) -> None: @jobs.do_group @mount_option @click.pass_obj -def do( - context: BaseComposeContext, mounts: t.Tuple[t.List[MountParam.MountType]] -) -> None: +def do(context: BaseComposeContext, mounts: tuple[list[MountParam.MountType]]) -> None: """ Run a custom job in the right container(s). """ @@ -345,8 +344,8 @@ def _mount_tmp_volumes(_job_name: str, *_args: t.Any, **_kwargs: t.Any) -> None: @click.pass_context def run( context: click.Context, - mounts: t.Tuple[t.List[MountParam.MountType]], - args: t.List[str], + mounts: tuple[list[MountParam.MountType]], + args: list[str], ) -> None: extra_args = ["--rm"] if not utils.is_a_tty(): @@ -411,7 +410,7 @@ def copyfrom( ) @click.argument("args", nargs=-1, required=True) @click.pass_context -def execute(context: click.Context, args: t.List[str]) -> None: +def execute(context: click.Context, args: list[str]) -> None: context.invoke(dc_command, command="exec", args=args) @@ -454,9 +453,9 @@ def status(context: click.Context) -> None: @click.pass_obj def dc_command( context: BaseComposeContext, - mounts: t.Tuple[t.List[MountParam.MountType]], + mounts: tuple[list[MountParam.MountType]], command: str, - args: t.List[str], + args: list[str], ) -> None: mount_tmp_volumes(mounts, context) config = tutor_config.load(context.root) @@ -465,8 +464,8 @@ def dc_command( @hooks.Filters.COMPOSE_MOUNTS.add() def _mount_edx_platform( - volumes: t.List[t.Tuple[str, str]], name: str -) -> t.List[t.Tuple[str, str]]: + volumes: list[tuple[str, str]], name: str +) -> list[tuple[str, str]]: """ When mounting edx-platform with `--mount=/path/to/edx-platform`, bind-mount the host repo in the lms/cms containers. diff --git a/tutor/commands/config.py b/tutor/commands/config.py index f48bf85aad..3d65801539 100644 --- a/tutor/commands/config.py +++ b/tutor/commands/config.py @@ -1,3 +1,4 @@ +from __future__ import annotations import json import typing as t @@ -27,7 +28,7 @@ class ConfigKeyParamType(click.ParamType): def shell_complete( self, ctx: click.Context, param: click.Parameter, incomplete: str - ) -> t.List[click.shell_completion.CompletionItem]: + ) -> list[click.shell_completion.CompletionItem]: return [ click.shell_completion.CompletionItem(key) for key, _value in self._shell_complete_config_items(ctx, incomplete) @@ -36,7 +37,7 @@ def shell_complete( @staticmethod def _shell_complete_config_items( ctx: click.Context, incomplete: str - ) -> t.List[t.Tuple[str, ConfigValue]]: + ) -> list[tuple[str, ConfigValue]]: # Here we want to auto-complete the name of the config key. For that we need to # figure out the list of enabled plugins, and for that we need the project root. # The project root would ordinarily be stored in ctx.obj.root, but during @@ -58,7 +59,7 @@ class ConfigKeyValParamType(ConfigKeyParamType): name = "configkeyval" - def convert(self, value: str, param: t.Any, ctx: t.Any) -> t.Tuple[str, t.Any]: + def convert(self, value: str, param: t.Any, ctx: t.Any) -> tuple[str, t.Any]: result = serialize.parse_key_value(value) if result is None: self.fail(f"'{value}' is not of the form 'key=value'.", param, ctx) @@ -66,7 +67,7 @@ def convert(self, value: str, param: t.Any, ctx: t.Any) -> t.Tuple[str, t.Any]: def shell_complete( self, ctx: click.Context, param: click.Parameter, incomplete: str - ) -> t.List[click.shell_completion.CompletionItem]: + ) -> list[click.shell_completion.CompletionItem]: """ Nice and friendly = auto-completion. """ @@ -117,7 +118,7 @@ def save( context: Context, interactive: bool, set_vars: Config, - unset_vars: t.List[str], + unset_vars: list[str], env_only: bool, ) -> None: config = tutor_config.load_minimal(context.root) diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index 908b75bf93..0fcb58df3d 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -1,4 +1,4 @@ -import typing as t +from __future__ import annotations import click @@ -70,7 +70,7 @@ def launch( context: click.Context, non_interactive: bool, pullimages: bool, - mounts: t.Tuple[t.List[compose.MountParam.MountType]], + mounts: tuple[list[compose.MountParam.MountType]], ) -> None: compose.mount_tmp_volumes(mounts, context.obj) try: diff --git a/tutor/commands/images.py b/tutor/commands/images.py index e6e78958bd..abc9207095 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -1,3 +1,4 @@ +from __future__ import annotations import typing as t import click @@ -21,9 +22,9 @@ @hooks.Filters.IMAGES_BUILD.add() def _add_core_images_to_build( - build_images: t.List[t.Tuple[str, t.Tuple[str, ...], str, t.Tuple[str, ...]]], + build_images: list[tuple[str, tuple[str, ...], str, tuple[str, ...]]], config: Config, -) -> t.List[t.Tuple[str, t.Tuple[str, ...], str, t.Tuple[str, ...]]]: +) -> list[tuple[str, tuple[str, ...], str, tuple[str, ...]]]: """ Add base images to the list of Docker images to build on `tutor build all`. """ @@ -35,8 +36,8 @@ def _add_core_images_to_build( @hooks.Filters.IMAGES_PULL.add() def _add_images_to_pull( - remote_images: t.List[t.Tuple[str, str]], config: Config -) -> t.List[t.Tuple[str, str]]: + remote_images: list[tuple[str, str]], config: Config +) -> list[tuple[str, str]]: """ Add base and vendor images to the list of Docker images to pull on `tutor pull all`. """ @@ -50,8 +51,8 @@ def _add_images_to_pull( @hooks.Filters.IMAGES_PUSH.add() def _add_core_images_to_push( - remote_images: t.List[t.Tuple[str, str]], config: Config -) -> t.List[t.Tuple[str, str]]: + remote_images: list[tuple[str, str]], config: Config +) -> list[tuple[str, str]]: """ Add base images to the list of Docker images to push on `tutor push all`. """ @@ -100,12 +101,12 @@ def images_command() -> None: @click.pass_obj def build( context: Context, - image_names: t.List[str], + image_names: list[str], no_cache: bool, - build_args: t.List[str], - add_hosts: t.List[str], + build_args: list[str], + add_hosts: list[str], target: str, - docker_args: t.List[str], + docker_args: list[str], ) -> None: config = tutor_config.load(context.root) command_args = [] @@ -132,7 +133,7 @@ def build( @click.command(short_help="Pull images from the Docker registry") @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj -def pull(context: Context, image_names: t.List[str]) -> None: +def pull(context: Context, image_names: list[str]) -> None: config = tutor_config.load_full(context.root) for image in image_names: for tag in find_remote_image_tags(config, hooks.Filters.IMAGES_PULL, image): @@ -142,7 +143,7 @@ def pull(context: Context, image_names: t.List[str]) -> None: @click.command(short_help="Push images to the Docker registry") @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj -def push(context: Context, image_names: t.List[str]) -> None: +def push(context: Context, image_names: list[str]) -> None: config = tutor_config.load_full(context.root) for image in image_names: for tag in find_remote_image_tags(config, hooks.Filters.IMAGES_PUSH, image): @@ -152,7 +153,7 @@ def push(context: Context, image_names: t.List[str]) -> None: @click.command(short_help="Print tag associated to a Docker image") @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj -def printtag(context: Context, image_names: t.List[str]) -> None: +def printtag(context: Context, image_names: list[str]) -> None: config = tutor_config.load_full(context.root) for image in image_names: for _name, _path, tag, _args in find_images_to_build(config, image): @@ -161,7 +162,7 @@ def printtag(context: Context, image_names: t.List[str]) -> None: def find_images_to_build( config: Config, image: str -) -> t.Iterator[t.Tuple[str, t.Tuple[str, ...], str, t.Tuple[str, ...]]]: +) -> t.Iterator[tuple[str, tuple[str, ...], str, tuple[str, ...]]]: """ Iterate over all images to build. @@ -182,7 +183,7 @@ def find_images_to_build( def find_remote_image_tags( config: Config, - filtre: "hooks.filters.Filter[t.List[t.Tuple[str, str]], [Config]]", + filtre: "hooks.filters.Filter[list[tuple[str, str]], [Config]]", image: str, ) -> t.Iterator[str]: """ diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 984030740c..de80725679 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -1,6 +1,7 @@ """ Common jobs that must be added both to local, dev and k8s commands. """ +from __future__ import annotations import functools import typing as t @@ -49,7 +50,7 @@ def _add_core_init_tasks() -> None: @click.command("init", help="Initialise all applications") @click.option("-l", "--limit", help="Limit initialisation to this service or plugin") -def initialise(limit: t.Optional[str]) -> t.Iterator[t.Tuple[str, str]]: +def initialise(limit: t.Optional[str]) -> t.Iterator[tuple[str, str]]: fmt.echo_info("Initialising all services...") filter_context = hooks.Contexts.APP(limit).name if limit else None @@ -99,7 +100,7 @@ def createuser( password: str, name: str, email: str, -) -> t.Iterable[t.Tuple[str, str]]: +) -> t.Iterable[tuple[str, str]]: """ Create an Open edX user @@ -127,7 +128,7 @@ def create_user_template( @click.command(help="Import the demo course") -def importdemocourse() -> t.Iterable[t.Tuple[str, str]]: +def importdemocourse() -> t.Iterable[tuple[str, str]]: template = """ # Import demo course git clone https://github.com/openedx/edx-demo-course --branch {{ OPENEDX_COMMON_VERSION }} --depth 1 ../edx-demo-course @@ -150,7 +151,7 @@ def importdemocourse() -> t.Iterable[t.Tuple[str, str]]: ), ) @click.argument("theme_name") -def settheme(domains: t.List[str], theme_name: str) -> t.Iterable[t.Tuple[str, str]]: +def settheme(domains: list[str], theme_name: str) -> t.Iterable[tuple[str, str]]: """ Assign a theme to the LMS and the CMS. @@ -159,7 +160,7 @@ def settheme(domains: t.List[str], theme_name: str) -> t.Iterable[t.Tuple[str, s yield ("lms", set_theme_template(theme_name, domains)) -def set_theme_template(theme_name: str, domain_names: t.List[str]) -> str: +def set_theme_template(theme_name: str, domain_names: list[str]) -> str: """ For each domain, get or create a Site object and assign the selected theme. """ @@ -231,7 +232,7 @@ def _patch_do_commands_callbacks() -> None: def _patch_callback( - job_name: str, func: t.Callable[P, t.Iterable[t.Tuple[str, str]]] + job_name: str, func: t.Callable[P, t.Iterable[tuple[str, str]]] ) -> t.Callable[P, None]: """ Modify a subcommand callback function such that its results are processed by `do_callback`. @@ -247,7 +248,7 @@ def new_callback(*args: P.args, **kwargs: P.kwargs) -> None: return new_callback -def do_callback(service_commands: t.Iterable[t.Tuple[str, str]]) -> None: +def do_callback(service_commands: t.Iterable[tuple[str, str]]) -> None: """ This function must be added as a callback to all `do` subcommands. diff --git a/tutor/commands/local.py b/tutor/commands/local.py index 409e16e8b0..140edc0d21 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -1,3 +1,4 @@ +from __future__ import annotations import typing as t import click @@ -70,7 +71,7 @@ def local(context: click.Context) -> None: @click.pass_context def launch( context: click.Context, - mounts: t.Tuple[t.List[compose.MountParam.MountType]], + mounts: tuple[list[compose.MountParam.MountType]], non_interactive: bool, pullimages: bool, ) -> None: diff --git a/tutor/commands/plugins.py b/tutor/commands/plugins.py index ed671494fd..225be11d5b 100644 --- a/tutor/commands/plugins.py +++ b/tutor/commands/plugins.py @@ -1,3 +1,4 @@ +from __future__ import annotations import os import typing as t import urllib.request @@ -22,13 +23,13 @@ def __init__(self, allow_all: bool = False): def shell_complete( self, ctx: click.Context, param: click.Parameter, incomplete: str - ) -> t.List[click.shell_completion.CompletionItem]: + ) -> list[click.shell_completion.CompletionItem]: return [ click.shell_completion.CompletionItem(name) for name in self.get_names(incomplete) ] - def get_names(self, incomplete: str) -> t.List[str]: + def get_names(self, incomplete: str) -> list[str]: candidates = [] if self.allow_all: candidates.append("all") @@ -67,7 +68,7 @@ def list_command() -> None: @click.command(help="Enable a plugin") @click.argument("plugin_names", metavar="plugin", nargs=-1, type=PluginName()) @click.pass_obj -def enable(context: Context, plugin_names: t.List[str]) -> None: +def enable(context: Context, plugin_names: list[str]) -> None: config = tutor_config.load_minimal(context.root) for plugin in plugin_names: plugins.load(plugin) @@ -87,10 +88,10 @@ def enable(context: Context, plugin_names: t.List[str]) -> None: "plugin_names", metavar="plugin", nargs=-1, type=PluginName(allow_all=True) ) @click.pass_obj -def disable(context: Context, plugin_names: t.List[str]) -> None: +def disable(context: Context, plugin_names: list[str]) -> None: config = tutor_config.load_minimal(context.root) disable_all = "all" in plugin_names - disabled: t.List[str] = [] + disabled: list[str] = [] for plugin in tutor_config.get_enabled_plugins(config): if disable_all or plugin in plugin_names: fmt.echo_info(f"Disabling plugin {plugin}...") diff --git a/tutor/config.py b/tutor/config.py index b77a494811..5512b8441e 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -1,5 +1,5 @@ +from __future__ import annotations import os -import typing as t from tutor import env, exceptions, fmt, hooks, plugins, serialize, utils from tutor.types import Config, ConfigValue, cast_config, get_typed @@ -108,7 +108,7 @@ def get_base() -> Config: Entries in this configuration are unrendered. """ base = get_template("base.yml") - extra_config: t.List[t.Tuple[str, ConfigValue]] = [] + extra_config: list[tuple[str, ConfigValue]] = [] extra_config = hooks.Filters.CONFIG_UNIQUE.apply(extra_config) extra_config = hooks.Filters.CONFIG_OVERRIDES.apply(extra_config) for name, value in extra_config: @@ -269,7 +269,7 @@ def enable_plugins(config: Config) -> None: plugins.load_all(get_enabled_plugins(config)) -def get_enabled_plugins(config: Config) -> t.List[str]: +def get_enabled_plugins(config: Config) -> list[str]: """ Return the list of plugins that are enabled, as per the configuration. Note that this may differ from the list of loaded plugins. For instance when a plugin is diff --git a/tutor/env.py b/tutor/env.py index b3e2299507..908a38e055 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -1,3 +1,4 @@ +from __future__ import annotations import os import re import shutil @@ -111,7 +112,7 @@ def iter_templates_in(self, *prefix: str) -> t.Iterable[str]: The elements of `prefix` must contain only "/", and not os.sep. """ full_prefix = "/".join(prefix) - env_templates: t.List[str] = self.environment.loader.list_templates() + env_templates: list[str] = self.environment.loader.list_templates() for template in env_templates: if template.startswith(full_prefix): # Exclude templates that match certain patterns diff --git a/tutor/hooks/actions.py b/tutor/hooks/actions.py index 1ef6db44c3..ccaf46709b 100644 --- a/tutor/hooks/actions.py +++ b/tutor/hooks/actions.py @@ -1,3 +1,5 @@ +from __future__ import annotations + # The Tutor plugin system is licensed under the terms of the Apache 2.0 license. __license__ = "Apache 2.0" @@ -53,11 +55,11 @@ class Action(t.Generic[P]): This strong typing makes it easier for plugin developers to quickly check whether they are adding and calling action callbacks correctly. """ - INDEX: t.Dict[str, "Action[t.Any]"] = {} + INDEX: dict[str, "Action[t.Any]"] = {} def __init__(self, name: str) -> None: self.name = name - self.callbacks: t.List[ActionCallback[P]] = [] + self.callbacks: list[ActionCallback[P]] = [] def __repr__(self) -> str: return f"{self.__class__.__name__}('{self.name}')" diff --git a/tutor/hooks/contexts.py b/tutor/hooks/contexts.py index 0ac98216e0..75a3182b71 100644 --- a/tutor/hooks/contexts.py +++ b/tutor/hooks/contexts.py @@ -1,3 +1,5 @@ +from __future__ import annotations + # The Tutor plugin system is licensed under the terms of the Apache 2.0 license. __license__ = "Apache 2.0" @@ -6,7 +8,7 @@ class Context: - CURRENT: t.List[str] = [] + CURRENT: list[str] = [] def __init__(self, name: str): self.name = name diff --git a/tutor/hooks/filters.py b/tutor/hooks/filters.py index 85fa29423b..4aa10f9fd2 100644 --- a/tutor/hooks/filters.py +++ b/tutor/hooks/filters.py @@ -1,3 +1,5 @@ +from __future__ import annotations + # The Tutor plugin system is licensed under the terms of the Apache 2.0 license. __license__ = "Apache 2.0" @@ -58,11 +60,11 @@ class Filter(t.Generic[T, P]): they are adding and calling filter callbacks correctly. """ - INDEX: t.Dict[str, "Filter[t.Any, t.Any]"] = {} + INDEX: dict[str, "Filter[t.Any, t.Any]"] = {} def __init__(self, name: str) -> None: self.name = name - self.callbacks: t.List[FilterCallback[T, P]] = [] + self.callbacks: list[FilterCallback[T, P]] = [] def __repr__(self) -> str: return f"{self.__class__.__name__}('{self.name}')" @@ -143,12 +145,12 @@ def clear(self, context: t.Optional[str] = None) -> None: # The methods below are specific to filters which take lists as first arguments def add_item( - self: "Filter[t.List[E], P]", item: E, priority: t.Optional[int] = None + self: "Filter[list[E], P]", item: E, priority: t.Optional[int] = None ) -> None: self.add_items([item], priority=priority) def add_items( - self: "Filter[t.List[E], P]", items: t.List[E], priority: t.Optional[int] = None + self: "Filter[list[E], P]", items: list[E], priority: t.Optional[int] = None ) -> None: # Unfortunately we have to type-ignore this line. If not, mypy complains with: # @@ -158,18 +160,16 @@ def add_items( # But we are unable to mark arguments positional-only (by adding / after values arg) in Python 3.7. # Get rid of this statement after Python 3.7 EOL. @self.add(priority=priority) # type: ignore - def callback( - values: t.List[E], *_args: P.args, **_kwargs: P.kwargs - ) -> t.List[E]: + def callback(values: list[E], *_args: P.args, **_kwargs: P.kwargs) -> list[E]: return values + items def iterate( - self: "Filter[t.List[E], P]", *args: P.args, **kwargs: P.kwargs + self: "Filter[list[E], P]", *args: P.args, **kwargs: P.kwargs ) -> t.Iterator[E]: yield from self.iterate_from_context(None, *args, **kwargs) def iterate_from_context( - self: "Filter[t.List[E], P]", + self: "Filter[list[E], P]", context: t.Optional[str], *args: P.args, **kwargs: P.kwargs, @@ -268,7 +268,7 @@ def add_item(name: str, item: T, priority: t.Optional[int] = None) -> None: get(name).add_item(item, priority=priority) -def add_items(name: str, items: t.List[T], priority: t.Optional[int] = None) -> None: +def add_items(name: str, items: list[T], priority: t.Optional[int] = None) -> None: """ Convenience function to add multiple item to a filter that returns a list of items. diff --git a/tutor/hooks/priorities.py b/tutor/hooks/priorities.py index 3c43d536a1..c493bdb54c 100644 --- a/tutor/hooks/priorities.py +++ b/tutor/hooks/priorities.py @@ -1,3 +1,4 @@ +from __future__ import annotations import typing as t from typing_extensions import Protocol @@ -14,7 +15,7 @@ class PrioritizedCallback(Protocol): TPrioritized = t.TypeVar("TPrioritized", bound=PrioritizedCallback) -def insert_callback(callback: TPrioritized, callbacks: t.List[TPrioritized]) -> None: +def insert_callback(callback: TPrioritized, callbacks: list[TPrioritized]) -> None: # I wish we could use bisect.insort_right here but the `key=` parameter # is unsupported in Python 3.9 position = 0 diff --git a/tutor/plugins/__init__.py b/tutor/plugins/__init__.py index cc5c5d250f..fd137bee96 100644 --- a/tutor/plugins/__init__.py +++ b/tutor/plugins/__init__.py @@ -1,6 +1,7 @@ """ Provide API for plugin features. """ +from __future__ import annotations import typing as t from copy import deepcopy @@ -20,7 +21,7 @@ def _convert_plugin_patches() -> None: This action is run after plugins have been loaded. """ - patches: t.Iterable[t.Tuple[str, str]] = hooks.Filters.ENV_PATCHES.iterate() + patches: t.Iterable[tuple[str, str]] = hooks.Filters.ENV_PATCHES.iterate() for name, content in patches: hooks.Filters.ENV_PATCH(name).add_item(content) @@ -44,14 +45,14 @@ def iter_installed() -> t.Iterator[str]: yield from sorted(hooks.Filters.PLUGINS_INSTALLED.iterate()) -def iter_info() -> t.Iterator[t.Tuple[str, t.Optional[str]]]: +def iter_info() -> t.Iterator[tuple[str, t.Optional[str]]]: """ Iterate on the information of all installed plugins. Yields (, ) tuples. """ - def plugin_info_name(info: t.Tuple[str, t.Optional[str]]) -> str: + def plugin_info_name(info: tuple[str, t.Optional[str]]) -> str: return info[0] yield from sorted(hooks.Filters.PLUGINS_INFO.iterate(), key=plugin_info_name) diff --git a/tutor/serialize.py b/tutor/serialize.py index 415f75cf46..a838d40ab9 100644 --- a/tutor/serialize.py +++ b/tutor/serialize.py @@ -1,3 +1,4 @@ +from __future__ import annotations import re import typing as t @@ -36,7 +37,7 @@ def parse(v: t.Union[str, t.IO[str]]) -> t.Any: return v -def parse_key_value(text: str) -> t.Optional[t.Tuple[str, t.Any]]: +def parse_key_value(text: str) -> t.Optional[tuple[str, t.Any]]: """ Parse = command line arguments. diff --git a/tutor/types.py b/tutor/types.py index 4f772053f0..c6156b3173 100644 --- a/tutor/types.py +++ b/tutor/types.py @@ -1,3 +1,5 @@ +from __future__ import annotations + # The Tutor plugin system is licensed under the terms of the Apache 2.0 license. __license__ = "Apache 2.0" @@ -38,9 +40,9 @@ def cast_config(config: t.Any) -> Config: def get_typed( - config: t.Dict[str, t.Any], + config: dict[str, t.Any], key: str, - expected_type: t.Type[T], + expected_type: type[T], default: t.Optional[T] = None, ) -> T: value = config.get(key, default) From 86bf655842149a3781ac2ed3c2f8980f3f9b80e3 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Tue, 31 Jan 2023 12:13:20 -0500 Subject: [PATCH 05/16] docs: fix typos in hooks changelog entry * Remove redundant changelog line * The change should NOT affect most developers ;) --- changelog.d/20230106_190620_regis_hooks_api.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/changelog.d/20230106_190620_regis_hooks_api.md b/changelog.d/20230106_190620_regis_hooks_api.md index b71d379ee0..b3e9e4b1a9 100644 --- a/changelog.d/20230106_190620_regis_hooks_api.md +++ b/changelog.d/20230106_190620_regis_hooks_api.md @@ -1,5 +1,4 @@ -- 💥[Feature] Simplify the hooks API. Plugin developers who were previously using `hooks.actions`, `hooks.filters` or `hooks.contexts` should now import these modules explicitely. (by @regisb) -- 💥[Feature] Simplify the hooks API. The modules `tutor.hooks.actions`, `tutor.hooks.filters`, and `tutor.hooks.contexts` are no longer part of the API. This change should affect mosst developers, who only use the `Actions` and `Filters` classes (notice the plural) from `tutor.hooks`. (by @regisb) +- 💥[Feature] Simplify the hooks API. The modules `tutor.hooks.actions`, `tutor.hooks.filters`, and `tutor.hooks.contexts` are no longer part of the API. This change should not affect most developers, who only use the `Actions` and `Filters` classes (notice the plural) from `tutor.hooks`. (by @regisb) - Instead of `tutor.hooks.actions.get("some:action")`, use `tutor.hooks.Actions.SOME_ACTION`. - Instead of `tutor.hooks.filters.get("some:filter")`, use `tutor.hooks.Filters.SOME_FILTER`. - Instead of `tutor.hooks.actions.add("some:action")`, use `tutor.hooks.Actions.SOME_ACTION.add()`. The same applies to the `do` method. From 1d3a215c00eaa15c1fb4c89def53771d00bd6888 Mon Sep 17 00:00:00 2001 From: Emad Rad Date: Thu, 1 Jun 2023 16:36:44 +0330 Subject: [PATCH 06/16] feat: Add support for loading in-cluster config when running inside a pod in some cases, tutor might run inside a pod, which that pod has access to a cluster via role binding and a service account. this way, there's no ./kube/config file, but kubectl commands run with no issue. Close #843 --- tutor/commands/k8s.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index db601f4cf6..ce8c0c41b8 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -1,3 +1,4 @@ +import os from datetime import datetime from time import sleep from typing import Any, Iterable, List, Optional, Type @@ -26,7 +27,20 @@ def __init__(self) -> None: # pylint: disable=import-outside-toplevel from kubernetes import client, config - config.load_kube_config() + if os.path.exists(config.kube_config.KUBE_CONFIG_DEFAULT_LOCATION): + # found the kubeconfig file, let's load it! + config.load_kube_config() + elif ( + config.incluster_config.SERVICE_HOST_ENV_NAME in os.environ + and config.incluster_config.SERVICE_PORT_ENV_NAME in os.environ + ): + # We are running inside a cluster, let's load the in-cluster configuration. + config.load_incluster_config() + else: + raise exceptions.TutorError( + f"there is no Kubernetes configuration file located in {config.kube_config.KUBE_CONFIG_DEFAULT_LOCATION}, and the service environment variables {config.incluster_config.SERVICE_HOST_ENV_NAME} and {config.incluster_config.SERVICE_PORT_ENV_NAME} have not been defined. To connect to a cluster, please configure your host appropriately." + ) + self._batch_api = None self._core_api = None self._client = client From 2d9a09f792eb447850eaf5bd5463573ba639dada Mon Sep 17 00:00:00 2001 From: Emad Rad Date: Thu, 1 Jun 2023 16:38:39 +0330 Subject: [PATCH 07/16] doc: changelog entry created --- changelog.d/20230527_174319_codewithemad_kubeconfig.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/20230527_174319_codewithemad_kubeconfig.md diff --git a/changelog.d/20230527_174319_codewithemad_kubeconfig.md b/changelog.d/20230527_174319_codewithemad_kubeconfig.md new file mode 100644 index 0000000000..3a3222615b --- /dev/null +++ b/changelog.d/20230527_174319_codewithemad_kubeconfig.md @@ -0,0 +1 @@ + - [Feature] Add support for loading in-cluster config when running inside a pod. In certain scenarios, Tutor may operate within a pod that has access to a cluster through role binding and a service account. In these instances, the ./kube/config file may not be present, but kubectl commands can still execute without any problems. (by @CodeWithEmad) \ No newline at end of file From e4ddee2604986d3bf043d698a9588a05c01effea Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Mon, 17 Jul 2023 14:26:02 -0400 Subject: [PATCH 08/16] build: copy in copy-node-modules.sh for npm post-install hook This post-install hook replaces `openedx-assets npm`. Part of: https://github.com/openedx/edx-platform/issues/31604 --- tutor/templates/build/openedx/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index abfe968fd2..7231792a53 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -123,6 +123,7 @@ RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt # Install nodejs requirements ARG NPM_REGISTRY={{ NPM_REGISTRY }} WORKDIR /openedx/edx-platform +COPY --from=edx-platform scripts/copy-node-modules.sh scripts RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \ --mount=type=bind,from=edx-platform,source=/package-lock.json,target=/openedx/edx-platform/package-lock.json \ --mount=type=cache,target=/root/.npm,sharing=shared {% endif %}npm clean-install --no-audit --registry=$NPM_REGISTRY From 98d7532d0d8cd3897c4c38d9976430d897e96a29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 21 Jul 2023 12:00:25 +0200 Subject: [PATCH 09/16] fix: copy-node-modules error with buildx --- tutor/templates/build/openedx/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 7231792a53..198dda6943 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -123,9 +123,9 @@ RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt # Install nodejs requirements ARG NPM_REGISTRY={{ NPM_REGISTRY }} WORKDIR /openedx/edx-platform -COPY --from=edx-platform scripts/copy-node-modules.sh scripts RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \ --mount=type=bind,from=edx-platform,source=/package-lock.json,target=/openedx/edx-platform/package-lock.json \ + --mount=type=bind,from=edx-platform,source=/scripts/copy-node-modules.sh,target=/openedx/edx-platform/scripts/copy-node-modules.sh \ --mount=type=cache,target=/root/.npm,sharing=shared {% endif %}npm clean-install --no-audit --registry=$NPM_REGISTRY ###### Production image with system and python requirements From 78ba4ea7cfeb0a98ec41d8b6e9f441e16b6f70ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 28 Jul 2023 21:58:10 +0200 Subject: [PATCH 10/16] fix: don't apply edx-platform patches in nightly --- tutor/templates/build/openedx/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index eeec492e3b..198dda6943 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -50,9 +50,6 @@ RUN git config --global user.email "tutor@overhang.io" \ {{ patch("openedx-dockerfile-git-patches-default") }} {%- else %} # Patch edx-platform -# Security advisory: https://github.com/openedx/edx-platform/security/advisories/GHSA-3q74-3rfh-g37j -# https://github.com/openedx/edx-platform/pull/32838 -RUN curl -fsSL https://github.com/openedx/edx-platform/commit/163259779297a7dccb28e1f8c3dfa4d2cbdb9655.patch | git am {%- endif %} {# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/.patch | git am #} From 7f47c3c8affd265e89d21786c57aa6e9d39b687d Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Fri, 20 Oct 2023 14:36:00 -0400 Subject: [PATCH 11/16] fix: include full-complement paramters in ``JWT_PRIVATE_SIGNING_JWK`` Addresses the breaking upstream change in this commit: https://github.com/openedx/edx-platform/commit/92731be0dc69d82cc4ec5b9c511e86df9aa9e632 See Changelog entry for details. --- changelog.d/20231020_143112_kyle_jwk_full_complement.md | 1 + tutor/templates/apps/openedx/settings/partials/common_all.py | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 changelog.d/20231020_143112_kyle_jwk_full_complement.md diff --git a/changelog.d/20231020_143112_kyle_jwk_full_complement.md b/changelog.d/20231020_143112_kyle_jwk_full_complement.md new file mode 100644 index 0000000000..162b89db40 --- /dev/null +++ b/changelog.d/20231020_143112_kyle_jwk_full_complement.md @@ -0,0 +1 @@ +- [Bugfix] Updated how the Tutor setting ``JWT_RSA_PRIVATE_KEY`` is rendered into the LMS Django setting ``JWT_AUTH['JWT_PRIVATE_SIGNING_JWK']`` as required by a recent breaking upstream change. The new representation of the ``JWT_PRIVATE_SIGNING_JWK`` simply adds the ``dq``, ``dp``, and ``qi`` parameters. Without this fix, LMS would encounter an ``InvalidKeyError`` on all logins (by @kdmccormick). diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index b1d5a9f82a..c9b747a89f 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -189,6 +189,9 @@ "n": "{{ jwt_rsa_key.n|long_to_base64 }}", "p": "{{ jwt_rsa_key.p|long_to_base64 }}", "q": "{{ jwt_rsa_key.q|long_to_base64 }}", + "dq": "{{ jwt_rsa_key.dq|long_to_base64 }}", + "dp": "{{ jwt_rsa_key.dp|long_to_base64 }}", + "qi": "{{ jwt_rsa_key.invq|long_to_base64 }}", } ) JWT_AUTH["JWT_PUBLIC_SIGNING_JWK_SET"] = json.dumps( From c273e7f1050d860939925e0ffcd0c2a1ae0757b4 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 27 Oct 2023 04:54:23 -0400 Subject: [PATCH 12/16] feat!: assume BuildKit is available --- changelog.d/20230818_112124_kyle_buildkit.md | 2 + docs/tutor.rst | 7 +++ tests/commands/test_images.py | 8 +-- tutor/commands/images.py | 32 ++++------ tutor/env.py | 4 +- tutor/hooks/catalog.py | 5 +- tutor/images.py | 5 +- tutor/templates/build/openedx/Dockerfile | 64 +++++++++++--------- tutor/utils.py | 19 ------ 9 files changed, 68 insertions(+), 78 deletions(-) create mode 100644 changelog.d/20230818_112124_kyle_buildkit.md diff --git a/changelog.d/20230818_112124_kyle_buildkit.md b/changelog.d/20230818_112124_kyle_buildkit.md new file mode 100644 index 0000000000..7084b29041 --- /dev/null +++ b/changelog.d/20230818_112124_kyle_buildkit.md @@ -0,0 +1,2 @@ +- [Deprecation] The template variable ``is_buildkit_enabled``, which now always returns True, is deprecated. Plugin authors should assume BuildKit is enabled and remove the variable from their templates (by @kdmccormick). +- 💥[Deprecation] Tutor no longer supports the legacy Docker builder, which was previously available by setting ``DOCKER_BUILDKIT=0`` in the host environment. Going forward, Tutor will always use BuildKit (a.k.a. ``docker buildx`` in Docker v19-v22, or just ``docker build`` in Docker v23). This transition will improve build performance and should be seamless for Tutor users who are running a supported Docker version (by @kdmccormick). diff --git a/docs/tutor.rst b/docs/tutor.rst index 4036ff8678..25ec6279da 100644 --- a/docs/tutor.rst +++ b/docs/tutor.rst @@ -93,6 +93,13 @@ An optional BRANCH suffix may be appended to the release name to indicate that e `Officially-supported plugins `__ follow the same versioning pattern. As a third-party plugin developer, you are encouraged to use the same pattern to make it immediately clear to your end-users which Open edX versions are supported. +In Tutor and its officially-supported plugins, certain features, API endpoints, and older depenency versions are periodically deprecated. Generally, warnings are added to the Changelogs and/or the command-line interface one major release before support for any behavior is removed. In order to keep track of pending removals in the source code, comments containing the string ``REMOVE-AFTER-VXX`` should be used, where ```` is the last major version that must support the behavior. For example:: + + # This has been replaced with SOME_NEW_HOOK (REMOVE-AFTER-V25). + SOME_OLD_HOOK = Filter() + +indicates that this filter definition can be removed as soon as Tutor v26.0.0. + .. _contributing: Contributing to Tutor diff --git a/tests/commands/test_images.py b/tests/commands/test_images.py index 7b0957790d..32ddba885f 100644 --- a/tests/commands/test_images.py +++ b/tests/commands/test_images.py @@ -128,10 +128,8 @@ def test_images_build_plugin_with_args(self, image_build: Mock) -> None: "service1", ] with temporary_root() as root: - utils.is_buildkit_enabled.cache_clear() - with patch.object(utils, "is_buildkit_enabled", return_value=False): - self.invoke_in_root(root, ["config", "save"]) - result = self.invoke_in_root(root, build_args) + self.invoke_in_root(root, ["config", "save"]) + result = self.invoke_in_root(root, build_args) self.assertIsNone(result.exception) self.assertEqual(0, result.exit_code) image_build.assert_called() @@ -146,7 +144,9 @@ def test_images_build_plugin_with_args(self, image_build: Mock) -> None: "host", "--target", "target", + "--output=type=docker", "docker_args", + "--cache-from=type=registry,ref=service1:1.0.0-cache", ], list(image_build.call_args[0][1:]), ) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index d21e2801ca..5dfe2b95d4 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -156,7 +156,7 @@ def images_command() -> None: # Export image to docker. This is necessary to make the image available to docker-compose. # The `--load` option is a shorthand for `--output=type=docker`. default="type=docker", - help="Same as `docker build --output=...`. This option will only be used when BuildKit is enabled.", + help="Same as `docker build --output=...`.", ) @click.option( "-a", @@ -211,7 +211,7 @@ def build( command_args += ["--add-host", add_host] if target: command_args += ["--target", target] - if utils.is_buildkit_enabled() and docker_output: + if docker_output: command_args.append(f"--output={docker_output}") if docker_args: command_args += docker_args @@ -223,27 +223,19 @@ def build( image_build_args = [*command_args, *custom_args] # Registry cache - if utils.is_buildkit_enabled(): - if not no_registry_cache: - image_build_args.append( - f"--cache-from=type=registry,ref={tag}-cache" - ) - if cache_to_registry: - image_build_args.append( - f"--cache-to=type=registry,mode=max,ref={tag}-cache" - ) + if not no_registry_cache: + image_build_args.append(f"--cache-from=type=registry,ref={tag}-cache") + if cache_to_registry: + image_build_args.append( + f"--cache-to=type=registry,mode=max,ref={tag}-cache" + ) # Build contexts for host_path, stage_name in build_contexts.get(name, []): - if utils.is_buildkit_enabled(): - fmt.echo_info( - f"Adding {host_path} to the build context '{stage_name}' of image '{image}'" - ) - image_build_args.append(f"--build-context={stage_name}={host_path}") - else: - fmt.echo_alert( - f"Unable to add {host_path} to the build context '{stage_name}' of image '{host_path}' because BuildKit is disabled." - ) + fmt.echo_info( + f"Adding {host_path} to the build context '{stage_name}' of image '{image}'" + ) + image_build_args.append(f"--build-context={stage_name}={host_path}") # Build images.build( diff --git a/tutor/env.py b/tutor/env.py index 9ced7d5b8c..1baf54b508 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -54,7 +54,9 @@ def _prepare_environment() -> None: ("HOST_USER_ID", utils.get_user_id()), ("TUTOR_APP", __app__.replace("-", "_")), ("TUTOR_VERSION", __version__), - ("is_buildkit_enabled", utils.is_buildkit_enabled), + # BuildKit used to be optional. Now, it's always enabled. + # This constant is just for temporary backwards compatibility (REMOVE-AFTER-V16). + ("is_buildkit_enabled", lambda: True), ], ) diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 66ed97a4ca..fa89015bfd 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -251,8 +251,7 @@ def your_filter_callback(some_data): #: names must be prefixed with the plugin name in all-caps. CONFIG_UNIQUE: Filter[list[tuple[str, Any]], []] = Filter() - #: Use this filter to modify the ``docker build`` command. For instance, to replace - #: the ``build`` subcommand by ``buildx build``. + #: Use this filter to modify the ``docker build`` command. #: #: :parameter list[str] command: the full build command, including options and #: arguments. Note that these arguments do not include the leading ``docker`` command. @@ -335,7 +334,7 @@ def your_filter_callback(some_data): #: - ``HOST_USER_ID``: the numerical ID of the user on the host. #: - ``TUTOR_APP``: the app name ("tutor" by default), used to determine the dev/local project names. #: - ``TUTOR_VERSION``: the current version of Tutor. - #: - ``is_buildkit_enabled``: a boolean function that indicates whether BuildKit is available on the host. + #: - ``is_buildkit_enabled``: a deprecated function which always returns ``True`` now. Will be removed after Quince. #: - ``iter_values_named``: a function to iterate on variables that start or end with a given string. #: - ``iter_mounts``: a function that yields compose-compatible bind-mounts for any given service. #: - ``patch``: a function to incorporate extra content into a template. diff --git a/tutor/images.py b/tutor/images.py index 26f80b1824..ebab4ec505 100644 --- a/tutor/images.py +++ b/tutor/images.py @@ -4,8 +4,9 @@ def build(path: str, tag: str, *args: str) -> None: fmt.echo_info(f"Building image {tag}") build_command = ["build", f"--tag={tag}", *args, path] - if utils.is_buildkit_enabled(): - build_command.insert(0, "buildx") + # `buildx` can be removed once Tutor requires Docker v23+. At that point, BuildKit will be + # enabled by default for all Docker users. + build_command.insert(0, "buildx") command = hooks.Filters.DOCKER_BUILD_COMMAND.apply(build_command) utils.docker(*command) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 8b775b5c6a..0f0aa66f4e 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -1,11 +1,11 @@ -{% if is_buildkit_enabled() %}# syntax=docker/dockerfile:1.4{% endif %} +# syntax=docker/dockerfile:1.4 ###### Minimal image with base system requirements for most stages FROM docker.io/ubuntu:20.04 as minimal LABEL maintainer="Overhang.io " ENV DEBIAN_FRONTEND=noninteractive -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked{% endif %} \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt update && \ apt install -y build-essential curl git language-pack-en ENV LC_ALL en_US.UTF-8 @@ -14,8 +14,9 @@ ENV LC_ALL en_US.UTF-8 ###### Install python with pyenv in /opt/pyenv and create virtualenv in /openedx/venv FROM minimal as python # https://github.com/pyenv/pyenv/wiki/Common-build-problems#prerequisites -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt update && \ apt install -y libssl-dev zlib1g-dev libbz2-dev \ libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ xz-utils tk-dev libffi-dev liblzma-dev python-openssl git @@ -77,12 +78,14 @@ ENV PATH /openedx/venv/bin:${PATH} ENV VIRTUAL_ENV /openedx/venv/ ENV XDG_CACHE_HOME /openedx/.cache -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt update \ && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev # Install the right version of pip/setuptools -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \ +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install \ # https://pypi.org/project/setuptools/ # https://pypi.org/project/pip/ # https://pypi.org/project/wheel/ @@ -92,14 +95,13 @@ RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip, RUN pip install https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz # Install base requirements -{% if not is_buildkit_enabled() %} -COPY --from=edx-platform /requirements/edx/base.txt /openedx/edx-platform/requirements/edx/base.txt -{% endif %} -RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \ - --mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r /openedx/edx-platform/requirements/edx/base.txt +RUN --mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \ + --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install -r /openedx/edx-platform/requirements/edx/base.txt # Install extra requirements -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \ +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install \ # Use redis as a django cache https://pypi.org/project/django-redis/ django-redis==5.2.0 \ # uwsgi server https://pypi.org/project/uWSGI/ @@ -109,11 +111,14 @@ RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip, # Install private requirements: this is useful for installing custom xblocks. COPY ./requirements/ /openedx/requirements -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}cd /openedx/requirements/ \ +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + cd /openedx/requirements/ \ && touch ./private.txt \ && pip install -r ./private.txt -{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install '{{ extra_requirements }}' +{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %} +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install '{{ extra_requirements }}' {% endfor %} ###### Install nodejs with nodeenv in /openedx/nodeenv @@ -129,21 +134,19 @@ RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt # Install nodejs requirements ARG NPM_REGISTRY={{ NPM_REGISTRY }} WORKDIR /openedx/edx-platform -{% if not is_buildkit_enabled() %} -COPY --from=edx-platform /package.json /openedx/edx-platform/package.json -COPY --from=edx-platform /package-lock.json /openedx/edx-platform/package-lock.json -{% endif %} -RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \ +RUN --mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \ --mount=type=bind,from=edx-platform,source=/package-lock.json,target=/openedx/edx-platform/package-lock.json \ --mount=type=bind,from=edx-platform,source=/scripts/copy-node-modules.sh,target=/openedx/edx-platform/scripts/copy-node-modules.sh \ - --mount=type=cache,target=/root/.npm,sharing=shared {% endif %}npm clean-install --no-audit --registry=$NPM_REGISTRY + --mount=type=cache,target=/root/.npm,sharing=shared \ + npm clean-install --no-audit --registry=$NPM_REGISTRY ###### Production image with system and python requirements FROM minimal as production # Install system requirements -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt update \ && apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx mysql-client ntp pkg-config rdfind # From then on, run as unprivileged "app" user @@ -154,7 +157,7 @@ RUN useradd --no-log-init --home-dir /openedx --create-home --shell /bin/bash -- USER ${APP_USER_ID} # https://hub.docker.com/r/powerman/dockerize/tags -COPY {% if is_buildkit_enabled() %}--link {% endif %}--from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize +COPY --link --from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize COPY --chown=app:app --from=edx-platform / /openedx/edx-platform COPY --chown=app:app --from=locales /openedx/locale /openedx/locale COPY --chown=app:app --from=python /opt/pyenv /opt/pyenv @@ -248,16 +251,19 @@ FROM production as development # Install useful system requirements (as root) USER root -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt update && \ apt install -y vim iputils-ping dnsutils telnet USER app # Install dev python requirements -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r requirements/edx/development.txt +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install -r requirements/edx/development.txt # https://pypi.org/project/ipdb/ # https://pypi.org/project/ipython -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install ipdb==0.13.13 ipython==8.12.0 +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install ipdb==0.13.13 ipython==8.12.0 # Add ipdb as default PYTHONBREAKPOINT ENV PYTHONBREAKPOINT=ipdb.set_trace diff --git a/tutor/utils.py b/tutor/utils.py index 59adee4377..2d8d3e561e 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -173,25 +173,6 @@ def docker(*command: str) -> int: return execute("docker", *command) -@lru_cache(maxsize=None) -def is_buildkit_enabled() -> bool: - """ - A helper function to determine whether we can run `docker buildx` with BuildKit. - """ - # First, we respect the DOCKER_BUILDKIT environment variable - enabled_by_env = { - "1": True, - "0": False, - }.get(os.environ.get("DOCKER_BUILDKIT", "")) - if enabled_by_env is not None: - return enabled_by_env - try: - subprocess.run(["docker", "buildx", "version"], capture_output=True, check=True) - return True - except subprocess.CalledProcessError: - return False - - def docker_compose(*command: str) -> int: return execute("docker", "compose", *command) From 61d37b436dc3ad5b3d58dbae150e5a47d1e4ee7f Mon Sep 17 00:00:00 2001 From: Emad Rad Date: Mon, 6 Nov 2023 20:05:01 +0330 Subject: [PATCH 13/16] fix: save configs by enable/disable plugins. before this, after enabling/disabling any plugins we should re-generate all files with tutor config save. --- ...231011_000434_codewithemad_save_configs.md | 1 + tests/test_utils.py | 2 +- tutor/commands/plugins.py | 28 +++++++++---------- tutor/config.py | 4 +-- tutor/plugins/__init__.py | 11 ++++++++ 5 files changed, 28 insertions(+), 18 deletions(-) create mode 100644 changelog.d/20231011_000434_codewithemad_save_configs.md diff --git a/changelog.d/20231011_000434_codewithemad_save_configs.md b/changelog.d/20231011_000434_codewithemad_save_configs.md new file mode 100644 index 0000000000..0afbad42ea --- /dev/null +++ b/changelog.d/20231011_000434_codewithemad_save_configs.md @@ -0,0 +1 @@ +- [Improvement] You don't have to run `tutor config save` every time you enable or disable a plugin anymore. (by @CodeWithEmad) \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py index 21af475d3b..5d5a6428ee 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,9 +1,9 @@ import base64 import os +import subprocess import tempfile import unittest from io import StringIO -import subprocess from typing import List, Tuple from unittest.mock import MagicMock, mock_open, patch diff --git a/tutor/commands/plugins.py b/tutor/commands/plugins.py index e80b6fc33e..45d8bd677f 100644 --- a/tutor/commands/plugins.py +++ b/tutor/commands/plugins.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import sys import tempfile import typing as t @@ -9,6 +10,7 @@ from tutor import config as tutor_config from tutor import exceptions, fmt, hooks, plugins, utils +from tutor.commands.config import save as config_save_command from tutor.plugins import indexes from tutor.plugins.base import PLUGINS_ROOT, PLUGINS_ROOT_ENV_VAR_NAME from tutor.types import Config @@ -133,17 +135,15 @@ def list_command(show_enabled_only: bool) -> None: @click.command(help="Enable a plugin") @click.argument("plugin_names", metavar="plugin", nargs=-1, type=PluginName()) -@click.pass_obj -def enable(context: Context, plugin_names: list[str]) -> None: - config = tutor_config.load_minimal(context.root) +@click.pass_context +def enable(context: click.Context, plugin_names: list[str]) -> None: + config = tutor_config.load_minimal(context.obj.root) for plugin in plugin_names: plugins.load(plugin) fmt.echo_info(f"Plugin {plugin} enabled") tutor_config.save_enabled_plugins(config) - tutor_config.save_config_file(context.root, config) - fmt.echo_info( - "You should now re-generate your environment with `tutor config save`." - ) + tutor_config.save_config_file(context.obj.root, config) + context.invoke(config_save_command, env_only=True) @click.command( @@ -153,22 +153,20 @@ def enable(context: Context, plugin_names: list[str]) -> None: @click.argument( "plugin_names", metavar="plugin", nargs=-1, type=PluginName(allow_all=True) ) -@click.pass_obj -def disable(context: Context, plugin_names: list[str]) -> None: - config = tutor_config.load_minimal(context.root) +@click.pass_context +def disable(context: click.Context, plugin_names: list[str]) -> None: + config = tutor_config.load_minimal(context.obj.root) disable_all = "all" in plugin_names disabled: list[str] = [] for plugin in tutor_config.get_enabled_plugins(config): if disable_all or plugin in plugin_names: fmt.echo_info(f"Disabling plugin {plugin}...") - hooks.Actions.PLUGIN_UNLOADED.do(plugin, context.root, config) + hooks.Actions.PLUGIN_UNLOADED.do(plugin, context.obj.root, config) disabled.append(plugin) fmt.echo_info(f"Plugin {plugin} disabled") if disabled: - tutor_config.save_config_file(context.root, config) - fmt.echo_info( - "You should now re-generate your environment with `tutor config save`." - ) + tutor_config.save_config_file(context.obj.root, config) + context.invoke(config_save_command, env_only=True) @click.command(name="update") diff --git a/tutor/config.py b/tutor/config.py index 2a0ee73731..35e35c1ce0 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -1,8 +1,8 @@ from __future__ import annotations -from copy import deepcopy -import typing as t import os +import typing as t +from copy import deepcopy from tutor import env, exceptions, fmt, hooks, plugins, serialize, utils from tutor.types import Config, ConfigValue, cast_config, get_typed diff --git a/tutor/plugins/__init__.py b/tutor/plugins/__init__.py index c5e88d20b5..f96afab99a 100644 --- a/tutor/plugins/__init__.py +++ b/tutor/plugins/__init__.py @@ -32,6 +32,17 @@ def _convert_plugin_patches() -> None: ENV_PATCHES_DICT[name].append(content) +@hooks.Actions.PLUGIN_UNLOADED.add() +def _clear_plugin_patches(plugin: str, root: str, _config: Config) -> None: + """ + After disabling a plugin, ENV_PATCHES_DICT should be cleared + and re-calculated. + + This action is run after plugins have been unloaded. + """ + ENV_PATCHES_DICT.clear() + + def is_installed(name: str) -> bool: """ Return true if the plugin is installed. From 6582e3a9e016561c2fa5978c47cf342a81ff8e7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 8 Dec 2023 11:55:55 +0100 Subject: [PATCH 14/16] fix: fill patch catch on plugin load/unload Also, update docs on `tutor config save`. Note that we had to fix an issue where the plugin unload callback was being called too late. --- docs/plugins/intro.rst | 4 ---- docs/plugins/v0/gettingstarted.rst | 4 ---- docs/tutorials/plugin.rst | 4 ++-- tutor/config.py | 6 ++++-- tutor/plugins/__init__.py | 32 ++++++++++++++++-------------- 5 files changed, 23 insertions(+), 27 deletions(-) diff --git a/docs/plugins/intro.rst b/docs/plugins/intro.rst index cb75f63660..57fd8d46ae 100644 --- a/docs/plugins/intro.rst +++ b/docs/plugins/intro.rst @@ -29,10 +29,6 @@ Enable/disable a plugin:: tutor plugins enable myplugin tutor plugins disable myplugin -After enabling or disabling a plugin, the environment should be re-generated with:: - - tutor config save - The full plugins CLI is described in the :ref:`reference documentation `. .. _existing_plugins: diff --git a/docs/plugins/v0/gettingstarted.rst b/docs/plugins/v0/gettingstarted.rst index 8fdaaf42a9..1f597e1946 100644 --- a/docs/plugins/v0/gettingstarted.rst +++ b/docs/plugins/v0/gettingstarted.rst @@ -41,10 +41,6 @@ You can then enable your newly-created plugin:: tutor plugins enable googleanalytics -Update your environment to apply changes from your plugin:: - - tutor config save - You should be able to view your changes in every LMS and CMS settings file:: grep -r googleanalytics "$(tutor config printroot)/env/apps/openedx/settings/" diff --git a/docs/tutorials/plugin.rst b/docs/tutorials/plugin.rst index 55d91bdefa..6469f49f66 100644 --- a/docs/tutorials/plugin.rst +++ b/docs/tutorials/plugin.rst @@ -38,9 +38,9 @@ Our plugin is disabled, for now. To enable it, we run:: $ tutor plugins enable myplugin Plugin myplugin enabled Configuration saved to /home/yourusername/.local/share/tutor/config.yml - You should now re-generate your environment with `tutor config save`. + Environment generated in /home/yourusername/.local/share/tutor/env -At this point you could re-generate your environment with ``tutor config save``, but there would not be any change to your environment... because the plugin does not do anything. So let's get started and make some changes. +At this point your environment was updated, but there would not be any change there... because the plugin does not do anything. So let's get started and make some changes. Modifying existing files with patches ------------------------------------- diff --git a/tutor/config.py b/tutor/config.py index 35e35c1ce0..35ee80149d 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -328,7 +328,9 @@ def _enable_plugins(root: str) -> None: enable_plugins(config) -@hooks.Actions.PLUGIN_UNLOADED.add() +# This is run with a very high priority such that it is called before the plugin hooks +# are actually cleared. +@hooks.Actions.PLUGIN_UNLOADED.add(priority=hooks.priorities.HIGH - 1) def _remove_plugin_config_overrides_on_unload( plugin: str, _root: str, config: Config ) -> None: @@ -342,7 +344,7 @@ def _remove_plugin_config_overrides_on_unload( fmt.echo_info(f" config - removing entry: {key}={value}") -@hooks.Actions.PLUGIN_UNLOADED.add(priority=100) +@hooks.Actions.PLUGIN_UNLOADED.add(priority=hooks.priorities.LOW) def _update_enabled_plugins_on_unload(_plugin: str, _root: str, config: Config) -> None: """ Update the list of enabled plugins. diff --git a/tutor/plugins/__init__.py b/tutor/plugins/__init__.py index f96afab99a..d9dcc21f53 100644 --- a/tutor/plugins/__init__.py +++ b/tutor/plugins/__init__.py @@ -17,30 +17,32 @@ @hooks.Actions.PLUGINS_LOADED.add() -def _convert_plugin_patches() -> None: +def _fill_patch_cache_on_load() -> None: """ - Some patches are added as (name, content) tuples with the ENV_PATCHES - filter. We convert these patches to add them to ENV_PATCHES_DICT. This makes it - easier for end-user to declare patches, and it's more performant. - This action is run after plugins have been loaded. """ - ENV_PATCHES_DICT.clear() - patches: t.Iterable[tuple[str, str]] = hooks.Filters.ENV_PATCHES.iterate() - for name, content in patches: - ENV_PATCHES_DICT.setdefault(name, []) - ENV_PATCHES_DICT[name].append(content) + _fill_patches_cache() @hooks.Actions.PLUGIN_UNLOADED.add() -def _clear_plugin_patches(plugin: str, root: str, _config: Config) -> None: +def _fill_patch_cache_on_unload(plugin: str, root: str, _config: Config) -> None: """ - After disabling a plugin, ENV_PATCHES_DICT should be cleared - and re-calculated. - This action is run after plugins have been unloaded. """ + _fill_patches_cache() + + +def _fill_patches_cache() -> None: + """ + Some patches are added as (name, content) tuples with the ENV_PATCHES + filter. We convert these patches to add them to ENV_PATCHES_DICT. This makes it + easier for end-user to declare patches, and it's more performant. + """ ENV_PATCHES_DICT.clear() + patches: t.Iterable[tuple[str, str]] = hooks.Filters.ENV_PATCHES.iterate() + for name, content in patches: + ENV_PATCHES_DICT.setdefault(name, []) + ENV_PATCHES_DICT[name].append(content) def is_installed(name: str) -> bool: @@ -135,6 +137,6 @@ def unload(plugin: str) -> None: hooks.clear_all(context=hooks.Contexts.app(plugin).name) -@hooks.Actions.PLUGIN_UNLOADED.add(priority=50) +@hooks.Actions.PLUGIN_UNLOADED.add(priority=hooks.priorities.HIGH) def _unload_on_disable(plugin: str, _root: str, _config: Config) -> None: unload(plugin) From 6da97d22f4b1ed49c57d3f031f2cd8a7125988b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 5 Dec 2023 11:35:48 +0100 Subject: [PATCH 15/16] feat: local.overhang.io -> local.edly.io The new domain name points to 127.0.0.1, just like the previous one. We keep the local.overhang.io domain names for backward compatibility. In the future, we hope to migrate to "*.openedx.io" but that will not happen before Redwood. Close #945 --- changelog.d/20231205_113430_regis_local_edly_io.md | 1 + docs/dev.rst | 6 +++--- docs/tutorials/theming.rst | 4 ++-- tutor/interactive.py | 9 ++++++--- tutor/templates/local/docker-compose.prod.yml | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) create mode 100644 changelog.d/20231205_113430_regis_local_edly_io.md diff --git a/changelog.d/20231205_113430_regis_local_edly_io.md b/changelog.d/20231205_113430_regis_local_edly_io.md new file mode 100644 index 0000000000..dec279fb6b --- /dev/null +++ b/changelog.d/20231205_113430_regis_local_edly_io.md @@ -0,0 +1 @@ +- 💥[Feature] Replace "*.local.overhang.io" hostnames by "*.local.edly.io". (by @regisb) diff --git a/docs/dev.rst b/docs/dev.rst index 3f951a04a9..d4224270a7 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -26,7 +26,7 @@ This will perform several tasks. It will: * build the "openedx-dev" Docker image, which is based on the "openedx" production image but is `specialized for developer usage`_ (eventually with your fork), * stop any existing locally-running Tutor containers, * disable HTTPS, -* set ``LMS_HOST`` to `local.overhang.io `_ (a convenience domain that simply `points at 127.0.0.1 `_), +* set ``LMS_HOST`` to `local.edly.io `_ (a convenience domain that simply `points at 127.0.0.1 `_), * prompt for a platform details (with suitable defaults), * build an ``openedx-dev`` image, * start LMS, CMS, supporting services, and any plugged-in services, @@ -41,8 +41,8 @@ Additionally, when a local clone of edx-platform is bind-mounted, it will: Once setup is complete, the platform will be running in the background: -* LMS will be accessible at `http://local.overhang.io:8000 `_. -* CMS will be accessible at `http://studio.local.overhang.io:8001 `_. +* LMS will be accessible at `http://local.edly.io:8000 `_. +* CMS will be accessible at `http://studio.local.edly.io:8001 `_. * Plugged-in services should be accessible at their documented URLs. Now, use the ``tutor dev ...`` command-line interface to manage the development environment. Some common commands are described below. diff --git a/docs/tutorials/theming.rst b/docs/tutorials/theming.rst index 224a5595a2..6c98384612 100644 --- a/docs/tutorials/theming.rst +++ b/docs/tutorials/theming.rst @@ -46,7 +46,7 @@ Then, run a local webserver:: tutor dev start lms -The LMS can then be accessed at http://local.overhang.io:8000. You will then have to :ref:`enable that theme `:: +The LMS can then be accessed at http://local.edly.io:8000. You will then have to :ref:`enable that theme `:: tutor dev do settheme mythemename @@ -54,4 +54,4 @@ Watch the themes folders for changes (in a different terminal):: tutor dev run watchthemes -Make changes to some of the files inside the theme directory: the theme assets should be automatically recompiled and visible at http://local.overhang.io:8000. +Make changes to some of the files inside the theme directory: the theme assets should be automatically recompiled and visible at http://local.edly.io:8000. diff --git a/tutor/interactive.py b/tutor/interactive.py index 2e9e2f512f..820268a93a 100644 --- a/tutor/interactive.py +++ b/tutor/interactive.py @@ -18,7 +18,10 @@ def ask_questions(config: Config, run_for_prod: Optional[bool] = None) -> None: """ defaults = tutor_config.get_defaults() if run_for_prod is None: - run_for_prod = config.get("LMS_HOST") != "local.overhang.io" + run_for_prod = not config.get("LMS_HOST") in [ + "local.edly.io", + "local.overhang.io", + ] run_for_prod = click.confirm( fmt.question( "Are you configuring a production platform? " @@ -29,8 +32,8 @@ def ask_questions(config: Config, run_for_prod: Optional[bool] = None) -> None: ) if not run_for_prod: dev_values: Config = { - "LMS_HOST": "local.overhang.io", - "CMS_HOST": "studio.local.overhang.io", + "LMS_HOST": "local.edly.io", + "CMS_HOST": "studio.local.edly.io", "ENABLE_HTTPS": False, } fmt.echo_info( diff --git a/tutor/templates/local/docker-compose.prod.yml b/tutor/templates/local/docker-compose.prod.yml index 552fa44dd0..f782a9e68a 100644 --- a/tutor/templates/local/docker-compose.prod.yml +++ b/tutor/templates/local/docker-compose.prod.yml @@ -20,7 +20,7 @@ services: networks: default: # These aliases are for internal communication between containers when running locally - # with *.local.overhang.io hostnames. + # with *.local.edly.io hostnames. aliases: - "{{ LMS_HOST }}" {{ patch("local-docker-compose-caddy-aliases")|indent(10) }} From 0ccf48690a69e8a0568d6e1c5d6b8f6f7a582f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 11 Oct 2023 09:39:55 +0200 Subject: [PATCH 16/16] feat: upgrade to quince --- CHANGELOG.md | 12 +++ changelog.d/20230818_112124_kyle_buildkit.md | 2 - ...231011_000434_codewithemad_save_configs.md | 1 - ...0231020_143112_kyle_jwk_full_complement.md | 1 - .../20231205_113430_regis_local_edly_io.md | 1 - docs/configuration.rst | 38 +++------- docs/dev.rst | 2 +- docs/faq.rst | 2 +- docs/install.rst | 6 +- docs/quickstart.rst | 2 +- docs/reference/indexes.rst | 10 +-- docs/tutorials/edx-platform.rst | 2 +- requirements/base.txt | 18 ++--- requirements/dev.in | 6 ++ requirements/dev.txt | 76 +++++++++---------- requirements/docs.txt | 22 +++--- requirements/plugins.txt | 25 +++--- tests/commands/test_config.py | 3 +- tests/test_env.py | 2 +- tutor/__about__.py | 4 +- tutor/commands/upgrade/__init__.py | 4 +- tutor/commands/upgrade/compose.py | 3 + tutor/commands/upgrade/k8s.py | 3 + tutor/config.py | 12 --- tutor/env.py | 4 +- tutor/hooks/catalog.py | 1 - tutor/plugins/openedx.py | 10 +++ .../openedx/settings/partials/common_all.py | 8 +- tutor/templates/build/openedx/Dockerfile | 41 ++++------ .../openedx/requirements/private-sample.txt | 6 -- tutor/templates/build/openedx/revisions.yml | 2 +- tutor/templates/config/base.yml | 3 +- tutor/templates/config/defaults.yml | 19 +++-- tutor/templates/dev/docker-compose.jobs.yml | 2 - tutor/templates/dev/docker-compose.yml | 2 - tutor/templates/jobs/init/cms.sh | 6 +- 36 files changed, 172 insertions(+), 189 deletions(-) delete mode 100644 changelog.d/20230818_112124_kyle_buildkit.md delete mode 100644 changelog.d/20231011_000434_codewithemad_save_configs.md delete mode 100644 changelog.d/20231020_143112_kyle_jwk_full_complement.md delete mode 100644 changelog.d/20231205_113430_regis_local_edly_io.md delete mode 100644 tutor/templates/build/openedx/requirements/private-sample.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 34f5d467ea..1fb3cb5150 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,18 @@ instructions, because git commits are used to generate release notes: + +## v17.0.0 (2023-12-09) + +- 💥[Feature] Upgrade to Quince. (by @regisb) +- 💥[Feature] Replace "*.local.overhang.io" hostnames by "*.local.edly.io". (by @regisb) +- 💥[Feature] Enable the Indigo theme by default, if no other theme is set. (by @regisb) +- 💥[Deprecation] Tutor no longer supports the legacy Docker builder, which was previously available by setting `DOCKER_BUILDKIT=0` in the host environment. Going forward, Tutor will always use BuildKit (a.k.a. `docker buildx` in Docker v19-v22, or just `docker build` in Docker v23). This transition will improve build performance and should be seamless for Tutor users who are running a supported Docker version (by @kdmccormick). +- 💥[Deprecation] The template variable `is_buildkit_enabled`, which now always returns True, is deprecated. Plugin authors should assume BuildKit is enabled and remove the variable from their templates (by @kdmccormick). +- 💥[Deprecation] Adding Python packages to edx-platform via `private.txt` is no longer supported. Instead, users should bind-mount their requirement directories with `tutor mounts add ...`. (by @regisb) +- [Bugfix] Updated how the Tutor setting `JWT_RSA_PRIVATE_KEY` is rendered into the LMS Django setting `JWT_AUTH['JWT_PRIVATE_SIGNING_JWK']` as required by a recent breaking upstream change. The new representation of the `JWT_PRIVATE_SIGNING_JWK` simply adds the `dq`, `dp`, and `qi` parameters. Without this fix, LMS would encounter an `InvalidKeyError` on all logins. (by @kdmccormick) +- [Improvement] You don't have to run `tutor config save` every time you enable or disable a plugin anymore. (by @CodeWithEmad) + ## v16.1.8 (2023-12-10) diff --git a/changelog.d/20230818_112124_kyle_buildkit.md b/changelog.d/20230818_112124_kyle_buildkit.md deleted file mode 100644 index 7084b29041..0000000000 --- a/changelog.d/20230818_112124_kyle_buildkit.md +++ /dev/null @@ -1,2 +0,0 @@ -- [Deprecation] The template variable ``is_buildkit_enabled``, which now always returns True, is deprecated. Plugin authors should assume BuildKit is enabled and remove the variable from their templates (by @kdmccormick). -- 💥[Deprecation] Tutor no longer supports the legacy Docker builder, which was previously available by setting ``DOCKER_BUILDKIT=0`` in the host environment. Going forward, Tutor will always use BuildKit (a.k.a. ``docker buildx`` in Docker v19-v22, or just ``docker build`` in Docker v23). This transition will improve build performance and should be seamless for Tutor users who are running a supported Docker version (by @kdmccormick). diff --git a/changelog.d/20231011_000434_codewithemad_save_configs.md b/changelog.d/20231011_000434_codewithemad_save_configs.md deleted file mode 100644 index 0afbad42ea..0000000000 --- a/changelog.d/20231011_000434_codewithemad_save_configs.md +++ /dev/null @@ -1 +0,0 @@ -- [Improvement] You don't have to run `tutor config save` every time you enable or disable a plugin anymore. (by @CodeWithEmad) \ No newline at end of file diff --git a/changelog.d/20231020_143112_kyle_jwk_full_complement.md b/changelog.d/20231020_143112_kyle_jwk_full_complement.md deleted file mode 100644 index 162b89db40..0000000000 --- a/changelog.d/20231020_143112_kyle_jwk_full_complement.md +++ /dev/null @@ -1 +0,0 @@ -- [Bugfix] Updated how the Tutor setting ``JWT_RSA_PRIVATE_KEY`` is rendered into the LMS Django setting ``JWT_AUTH['JWT_PRIVATE_SIGNING_JWK']`` as required by a recent breaking upstream change. The new representation of the ``JWT_PRIVATE_SIGNING_JWK`` simply adds the ``dq``, ``dp``, and ``qi`` parameters. Without this fix, LMS would encounter an ``InvalidKeyError`` on all logins (by @kdmccormick). diff --git a/changelog.d/20231205_113430_regis_local_edly_io.md b/changelog.d/20231205_113430_regis_local_edly_io.md deleted file mode 100644 index dec279fb6b..0000000000 --- a/changelog.d/20231205_113430_regis_local_edly_io.md +++ /dev/null @@ -1 +0,0 @@ -- 💥[Feature] Replace "*.local.overhang.io" hostnames by "*.local.edly.io". (by @regisb) diff --git a/docs/configuration.rst b/docs/configuration.rst index 2efd18054f..0f88d79e19 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -136,7 +136,7 @@ Open edX customisation This defines the git repository from which you install Open edX platform code. If you run an Open edX fork with custom patches, set this to your own git repository. You may also override this configuration parameter at build time, by providing a ``--build-arg`` option. -- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/palm.4"``, or ``master`` in :ref:`nightly `) +- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/quince.1"``, or ``master`` in :ref:`nightly `) This defines the default version that will be pulled from all Open edX git repositories. @@ -156,9 +156,9 @@ These two configuration parameters define which Redis database to use for Open e .. _openedx_extra_pip_requirements: -- ``OPENEDX_EXTRA_PIP_REQUIREMENTS`` (default: ``["openedx-scorm-xblock>=16.0.0,<17.0.0"]``) +- ``OPENEDX_EXTRA_PIP_REQUIREMENTS`` (default: ``[]``) -This defines extra pip packages that are going to be installed for Open edX. +Define extra pip packages that are going to be installed for edx-platform. - ``NPM_REGISTRY`` (default: ``"https://registry.npmjs.org/"``) @@ -354,36 +354,16 @@ See :ref:`the corresponding tutorial `. .. _custom_extra_xblocks: -Installing extra xblocks and requirements ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Would you like to include custom xblocks, or extra requirements to your Open edX platform? Additional requirements can be added to the ``OPENEDX_EXTRA_PIP_REQUIREMENTS`` parameter in the :ref:`config file ` or to the ``env/build/openedx/requirements/private.txt`` file. The difference between them, is that ``private.txt`` file, even though it could be used for both, :ref:`should be used for installing extra xblocks or requirements from private repositories `. For instance, to include the `polling xblock from Opencraft `_: +Would you like to include custom xblocks, or extra requirements to your Open edX platform? Additional requirements can be added to the ``OPENEDX_EXTRA_PIP_REQUIREMENTS`` parameter in the :ref:`config file `. For instance, to include the `polling xblock from Opencraft `_: tutor config save --append OPENEDX_EXTRA_PIP_REQUIREMENTS=git+https://github.com/open-craft/xblock-poll.git -Alternatively, add the dependency to ``private.txt``:: - - echo "git+https://github.com/open-craft/xblock-poll.git" >> "$(tutor config printroot)/env/build/openedx/requirements/private.txt" - Then, the ``openedx`` docker image must be rebuilt:: tutor images build openedx -.. _extra_private_xblocks: - -Installing extra requirements from private repositories -******************************************************* - -When installing extra xblock or requirements from private repositories, ``private.txt`` file should be used, because it allows installing dependencies without adding git credentials to the Docker image. By adding your git credentials to the Docker image, you're risking leaking your git credentials, if you were to publish (intentionally or unintentionally) the Docker image in a public place. - -To install xblocks from a private repository that requires authentication, you must first clone the repository inside the ``openedx/requirements`` folder on the host:: - - git clone git@github.com:me/myprivaterepo.git "$(tutor config printroot)/env/build/openedx/requirements/myprivaterepo" - -Then, declare your extra requirements with the ``-e`` flag in ``openedx/requirements/private.txt``:: - - echo "-e ./myprivaterepo" >> "$(tutor config printroot)/env/build/openedx/requirements/private.txt" - .. _edx_platform_fork: Running a fork of ``edx-platform`` @@ -401,16 +381,16 @@ Note that your edx-platform version must be a fork of the latest release **tag** If you don't create your fork from this tag, you *will* have important compatibility issues with other services. In particular: -- Do not try to run a fork from an older (pre-Palm) version of edx-platform: this will simply not work. +- Do not try to run a fork from an older (pre-Quince) version of edx-platform: this will simply not work. - Do not try to run a fork from the edx-platform master branch: there is a 99% probability that it will fail. -- Do not try to run a fork from the open-release/palm.master branch: Tutor will attempt to apply security and bug fix patches that might already be included in the open-release/palm.master but which were not yet applied to the latest release tag. Patch application will thus fail if you base your fork from the open-release/palm.master branch. +- Do not try to run a fork from the open-release/quince.master branch: Tutor will attempt to apply security and bug fix patches that might already be included in the open-release/quince.master but which were not yet applied to the latest release tag. Patch application will thus fail if you base your fork from the open-release/quince.master branch. .. _i18n: Adding custom translations ~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you are not running Open edX in English (``LANGUAGE_CODE`` default: ``"en"``), chances are that some strings will not be properly translated. In most cases, this is because not enough contributors have helped translate Open edX into your language. It happens! With Tutor, available translated languages include those that come bundled with `edx-platform `__ as well as those from `openedx-i18n `__. +If you are not running Open edX in English (``LANGUAGE_CODE`` default: ``"en"``), chances are that some strings will not be properly translated. In most cases, this is because not enough contributors have helped translate Open edX into your language. It happens! With Tutor, available translated languages include those that come bundled with `edx-platform `__ as well as those from `openedx-i18n `__. Tutor offers a relatively simple mechanism to add custom translations to the openedx Docker image. You should create a folder that corresponds to your language code in the "build/openedx/locale" folder of the Tutor environment. This folder should contain a "LC_MESSAGES" folder. For instance:: @@ -431,9 +411,9 @@ Then, add a "django.po" file there that will contain your custom translations:: .. warning:: Don't forget to specify the file ``Content-Type`` when adding message strings with non-ASCII characters; otherwise a ``UnicodeDecodeError`` will be raised during compilation. -The "String to translate" part should match *exactly* the string that you would like to translate. You cannot make it up! The best way to find this string is to copy-paste it from the `upstream django.po file for the English language `__. +The "String to translate" part should match *exactly* the string that you would like to translate. You cannot make it up! The best way to find this string is to copy-paste it from the `upstream django.po file for the English language `__. -If you cannot find the string to translate in this file, then it means that you are trying to translate a string that is used in some piece of javascript code. Those strings are stored in a different file named "djangojs.po". You can check it out `in the edx-platform repo as well `__. Your custom javascript strings should also be stored in a "djangojs.po" file that should be placed in the same directory. +If you cannot find the string to translate in this file, then it means that you are trying to translate a string that is used in some piece of javascript code. Those strings are stored in a different file named "djangojs.po". You can check it out `in the edx-platform repo as well `__. Your custom javascript strings should also be stored in a "djangojs.po" file that should be placed in the same directory. To recap, here is an example. To translate a few strings in French, both from django.po and djangojs.po, we would have the following file hierarchy:: diff --git a/docs/dev.rst b/docs/dev.rst index 278975e4f6..6b1f1975bf 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -113,7 +113,7 @@ The ``openedx-dev`` Docker image is based on the same ``openedx`` image used by - The user that runs inside the container has the same UID as the user on the host, to avoid permission problems inside mounted volumes (and in particular in the edx-platform repository). - Additional Python and system requirements are installed for convenient debugging: `ipython `__, `ipdb `__, vim, telnet. -- The edx-platform `development requirements `__ are installed. +- The edx-platform `development requirements `__ are installed. If you are using a custom ``openedx`` image, then you will need to rebuild ``openedx-dev`` every time you modify ``openedx``. To so, run:: diff --git a/docs/faq.rst b/docs/faq.rst index f4907d854c..6467ecc4ac 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -38,7 +38,7 @@ The `devstack `_ is meant for development o Is Tutor officially supported by edX? ------------------------------------- -Yes: as of the Open edX Maple release (December 9th 2021), Tutor is the only officially supported installation method for Open edX: see the `official installation instructions `__. +Yes: as of the Open edX Maple release (December 9th 2021), Tutor is the only officially supported installation method for Open edX: see the `official installation instructions `__. What features are missing from Tutor? ------------------------------------- diff --git a/docs/install.rst b/docs/install.rst index df4069261c..b058ddced5 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -12,7 +12,7 @@ Requirements * Architecture: Both AMD64 and ARM64 are supported. * Required software: - - `Docker `__: v20.10.15+ + - `Docker `__: v24.0.5+ (with BuildKit 0.11+) - `Docker Compose `__: v2.0.0+ .. warning:: @@ -123,11 +123,11 @@ Major Open edX releases are published twice a year, in June and December, by the 4. Test the new release in a sandboxed environment. 5. If you are running edx-platform, or some other repository from a custom branch, then you should rebase (and test) your changes on top of the latest release tag (see :ref:`edx_platform_fork`). -The process for upgrading from one major release to the next works similarly to any other upgrade, with the ``launch`` command (see above). The single difference is that if the ``launch`` command detects that your tutor environment was generated with an older release, it will perform a few release-specific upgrade steps. These extra upgrade steps will be performed just once. But they will be ignored if you updated your local environment (for instance: with ``tutor config save``) before running ``launch``. This situation typically occurs if you need to re-build some Docker images (see above). In such a case, you should make use of the ``upgrade`` command. For instance, to upgrade a local installation from Olive to Palm and rebuild some Docker images, run:: +The process for upgrading from one major release to the next works similarly to any other upgrade, with the ``launch`` command (see above). The single difference is that if the ``launch`` command detects that your tutor environment was generated with an older release, it will perform a few release-specific upgrade steps. These extra upgrade steps will be performed just once. But they will be ignored if you updated your local environment (for instance: with ``tutor config save``) before running ``launch``. This situation typically occurs if you need to re-build some Docker images (see above). In such a case, you should make use of the ``upgrade`` command. For instance, to upgrade a local installation from Palm to Quince and rebuild some Docker images, run:: tutor config save tutor images build all # list the images that should be rebuilt here - tutor local upgrade --from=olive + tutor local upgrade --from=palm tutor local launch .. _autocomplete: diff --git a/docs/quickstart.rst b/docs/quickstart.rst index cad6dd2077..58cf4d4e32 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -22,7 +22,7 @@ Yes :) This is what happens when you run ``tutor local launch``: 2. Configuration files are generated from templates. 3. Docker images are downloaded. 4. Docker containers are provisioned. -5. A full, production-ready Open edX platform (`Palm `__ release) is run with docker-compose. +5. A full, production-ready Open edX platform (`Quince `__ release) is run with docker-compose. The whole procedure should require less than 10 minutes, on a server with good bandwidth. Note that your host environment will not be affected in any way, since everything runs inside docker containers. Root access is not even necessary. diff --git a/docs/reference/indexes.rst b/docs/reference/indexes.rst index db06fcfadb..e9c337ea33 100644 --- a/docs/reference/indexes.rst +++ b/docs/reference/indexes.rst @@ -7,10 +7,10 @@ Plugin indexes are a great way to have your plugins discovered by other users. P Index file paths ================ -A plugin index is a yaml-formatted file. It can be stored on the web or on your computer. In both cases, the index file location must end with "/plugins.yml". For instance, the following are valid index locations if you run the Open edX "Palm" release: +A plugin index is a yaml-formatted file. It can be stored on the web or on your computer. In both cases, the index file location must end with "/plugins.yml". For instance, the following are valid index locations if you run the Open edX "Quince" release: -- https://overhang.io/tutor/main/palm/plugins.yml -- ``/path/to/your/local/index/palm/plugins.yml`` +- https://overhang.io/tutor/main/quince/plugins.yml +- ``/path/to/your/local/index/quince/plugins.yml`` To add either indexes, run the ``tutor plugins index add`` command without the suffix. For instance:: @@ -106,9 +106,9 @@ Manage plugins in development Plugin developers and maintainers often want to install local versions of their plugins. They usually achieve this with ``pip install -e /path/to/tutor-plugin``. We can improve that workflow by creating an index for local plugins:: # Create the plugin index directory - mkdir -p ~/localindex/palm/ + mkdir -p ~/localindex/quince/ # Edit the index - vim ~/localindex/palm/plugins.yml + vim ~/localindex/quince/plugins.yml Add the following to the index:: diff --git a/docs/tutorials/edx-platform.rst b/docs/tutorials/edx-platform.rst index fde1ea9737..7d60e6b74d 100644 --- a/docs/tutorials/edx-platform.rst +++ b/docs/tutorials/edx-platform.rst @@ -69,7 +69,7 @@ Quite often, developers don't want to work on edx-platform directly, but on a de cd /my/workspace/edx-ora2 git clone https://github.com/openedx/edx-ora2 . -Then, check out the right version of the package. This is the version that is indicated in the `edx-platform/requirements/edx/base.txt `__. Be careful that the version that is currently in use in your version of edx-platform is **not necessarily the head of the master branch**:: +Then, check out the right version of the package. This is the version that is indicated in the `edx-platform/requirements/edx/base.txt `__. Be careful that the version that is currently in use in your version of edx-platform is **not necessarily the head of the master branch**:: git checkout diff --git a/requirements/base.txt b/requirements/base.txt index e79b5bdcc2..53ea28465e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,27 +6,27 @@ # appdirs==1.4.4 # via -r requirements/base.in -cachetools==5.3.1 +cachetools==5.3.2 # via google-auth certifi==2023.7.22 # via # kubernetes # requests -charset-normalizer==3.2.0 +charset-normalizer==3.3.1 # via requests click==8.1.7 # via -r requirements/base.in -google-auth==2.22.0 +google-auth==2.23.3 # via kubernetes idna==3.4 # via requests jinja2==3.1.2 # via -r requirements/base.in -kubernetes==27.2.0 +kubernetes==28.1.0 # via -r requirements/base.in markupsafe==2.1.3 # via jinja2 -mypy==1.5.1 +mypy==1.6.1 # via -r requirements/base.in mypy-extensions==1.0.0 # via mypy @@ -40,7 +40,7 @@ pyasn1==0.5.0 # rsa pyasn1-modules==0.3.0 # via google-auth -pycryptodome==3.18.0 +pycryptodome==3.19.0 # via -r requirements/base.in python-dateutil==2.8.2 # via kubernetes @@ -58,19 +58,17 @@ rsa==4.9 # via google-auth six==1.16.0 # via - # google-auth # kubernetes # python-dateutil tomli==2.0.1 # via mypy -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via # -r requirements/base.in # mypy urllib3==1.26.18 # via - # google-auth # kubernetes # requests -websocket-client==1.6.2 +websocket-client==1.6.4 # via kubernetes diff --git a/requirements/dev.in b/requirements/dev.in index c53771c419..ef0d7ada7b 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -7,6 +7,12 @@ pyinstaller scriv twine +# Constraints +# for compatibility with sphinx-rtd-theme +# drop this constraint once sphinx-rtd-theme 2.0.0 comes out +# https://github.com/readthedocs/sphinx_rtd_theme/tags +docutils<0.19.0 + # Types packages types-docutils types-PyYAML diff --git a/requirements/dev.txt b/requirements/dev.txt index 8d8b265839..fd7ba3e747 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,21 +4,19 @@ # # pip-compile requirements/dev.in # -altgraph==0.17.3 +altgraph==0.17.4 # via pyinstaller appdirs==1.4.4 # via -r requirements/base.txt -astroid==2.15.6 +astroid==3.0.1 # via pylint attrs==23.1.0 # via scriv -black==23.7.0 +black==23.10.1 # via -r requirements/dev.in -bleach==6.0.0 - # via readme-renderer -build==0.10.0 +build==1.0.3 # via pip-tools -cachetools==5.3.1 +cachetools==5.3.2 # via # -r requirements/base.txt # google-auth @@ -27,9 +25,9 @@ certifi==2023.7.22 # -r requirements/base.txt # kubernetes # requests -cffi==1.15.1 +cffi==1.16.0 # via cryptography -charset-normalizer==3.2.0 +charset-normalizer==3.3.1 # via # -r requirements/base.txt # requests @@ -42,15 +40,17 @@ click==8.1.7 # scriv click-log==0.4.0 # via scriv -coverage==7.3.0 +coverage==7.3.2 # via -r requirements/dev.in cryptography==41.0.7 # via secretstorage dill==0.3.7 # via pylint -docutils==0.20.1 - # via readme-renderer -google-auth==2.22.0 +docutils==0.18.1 + # via + # -r requirements/dev.in + # readme-renderer +google-auth==2.23.3 # via # -r requirements/base.txt # kubernetes @@ -60,7 +60,9 @@ idna==3.4 # requests importlib-metadata==6.8.0 # via + # build # keyring + # pyinstaller # twine importlib-resources==6.1.1 # via keyring @@ -78,12 +80,12 @@ jinja2==3.1.2 # scriv keyring==24.2.0 # via twine -kubernetes==27.2.0 +kubernetes==28.1.0 # via -r requirements/base.txt -lazy-object-proxy==1.9.0 - # via astroid markdown-it-py==3.0.0 - # via rich + # via + # rich + # scriv markupsafe==2.1.3 # via # -r requirements/base.txt @@ -94,29 +96,32 @@ mdurl==0.1.2 # via markdown-it-py more-itertools==10.1.0 # via jaraco-classes -mypy==1.5.1 +mypy==1.6.1 # via -r requirements/base.txt mypy-extensions==1.0.0 # via # -r requirements/base.txt # black # mypy +nh3==0.2.14 + # via readme-renderer oauthlib==3.2.2 # via # -r requirements/base.txt # kubernetes # requests-oauthlib -packaging==23.1 +packaging==23.2 # via # black # build + # pyinstaller pathspec==0.11.2 # via black pip-tools==7.3.0 # via -r requirements/dev.in pkginfo==1.9.6 # via twine -platformdirs==3.10.0 +platformdirs==3.11.0 # via # black # pylint @@ -131,17 +136,17 @@ pyasn1-modules==0.3.0 # google-auth pycparser==2.21 # via cffi -pycryptodome==3.18.0 +pycryptodome==3.19.0 # via -r requirements/base.txt pygments==2.16.1 # via # readme-renderer # rich -pyinstaller==5.13.1 +pyinstaller==6.1.0 # via -r requirements/dev.in -pyinstaller-hooks-contrib==2023.7 +pyinstaller-hooks-contrib==2023.10 # via pyinstaller -pylint==2.17.5 +pylint==3.0.2 # via -r requirements/dev.in pyproject-hooks==1.0.0 # via build @@ -153,7 +158,7 @@ pyyaml==6.0.1 # via # -r requirements/base.txt # kubernetes -readme-renderer==41.0 +readme-renderer==42.0 # via twine requests==2.31.0 # via @@ -171,21 +176,19 @@ requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine -rich==13.5.2 +rich==13.6.0 # via twine rsa==4.9 # via # -r requirements/base.txt # google-auth -scriv==1.3.1 +scriv==1.5.0 # via -r requirements/dev.in secretstorage==3.3.3 # via keyring six==1.16.0 # via # -r requirements/base.txt - # bleach - # google-auth # kubernetes # python-dateutil tomli==2.0.1 @@ -203,11 +206,11 @@ twine==4.0.2 # via -r requirements/dev.in types-docutils==0.20.0.3 # via -r requirements/dev.in -types-pyyaml==6.0.12.11 +types-pyyaml==6.0.12.12 # via -r requirements/dev.in -types-setuptools==68.1.0.0 +types-setuptools==68.2.0.0 # via -r requirements/dev.in -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via # -r requirements/base.txt # astroid @@ -218,21 +221,16 @@ typing-extensions==4.7.1 urllib3==1.26.18 # via # -r requirements/base.txt - # google-auth # kubernetes # requests # twine -webencodings==0.5.1 - # via bleach -websocket-client==1.6.2 +websocket-client==1.6.4 # via # -r requirements/base.txt # kubernetes wheel==0.41.2 # via pip-tools -wrapt==1.15.0 - # via astroid -zipp==3.16.2 +zipp==3.17.0 # via # importlib-metadata # importlib-resources diff --git a/requirements/docs.txt b/requirements/docs.txt index 80bb030110..c4f5c6e9c2 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -8,9 +8,9 @@ alabaster==0.7.13 # via sphinx appdirs==1.4.4 # via -r requirements/base.txt -babel==2.12.1 +babel==2.13.1 # via sphinx -cachetools==5.3.1 +cachetools==5.3.2 # via # -r requirements/base.txt # google-auth @@ -19,7 +19,7 @@ certifi==2023.7.22 # -r requirements/base.txt # kubernetes # requests -charset-normalizer==3.2.0 +charset-normalizer==3.3.1 # via # -r requirements/base.txt # requests @@ -32,7 +32,7 @@ docutils==0.18.1 # sphinx # sphinx-click # sphinx-rtd-theme -google-auth==2.22.0 +google-auth==2.23.3 # via # -r requirements/base.txt # kubernetes @@ -48,13 +48,13 @@ jinja2==3.1.2 # via # -r requirements/base.txt # sphinx -kubernetes==27.2.0 +kubernetes==28.1.0 # via -r requirements/base.txt markupsafe==2.1.3 # via # -r requirements/base.txt # jinja2 -mypy==1.5.1 +mypy==1.6.1 # via -r requirements/base.txt mypy-extensions==1.0.0 # via @@ -65,7 +65,7 @@ oauthlib==3.2.2 # -r requirements/base.txt # kubernetes # requests-oauthlib -packaging==23.1 +packaging==23.2 # via sphinx pyasn1==0.5.0 # via @@ -76,7 +76,7 @@ pyasn1-modules==0.3.0 # via # -r requirements/base.txt # google-auth -pycryptodome==3.18.0 +pycryptodome==3.19.0 # via -r requirements/base.txt pygments==2.16.1 # via sphinx @@ -107,7 +107,6 @@ rsa==4.9 six==1.16.0 # via # -r requirements/base.txt - # google-auth # kubernetes # python-dateutil snowballstemmer==2.2.0 @@ -140,17 +139,16 @@ tomli==2.0.1 # via # -r requirements/base.txt # mypy -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via # -r requirements/base.txt # mypy urllib3==1.26.18 # via # -r requirements/base.txt - # google-auth # kubernetes # requests -websocket-client==1.6.2 +websocket-client==1.6.4 # via # -r requirements/base.txt # kubernetes diff --git a/requirements/plugins.txt b/requirements/plugins.txt index e67affd113..c78ef5773f 100644 --- a/requirements/plugins.txt +++ b/requirements/plugins.txt @@ -1,12 +1,13 @@ -# change version ranges when upgrading from palm -tutor-android>=16.0.0,<17.0.0 -tutor-cairn>=16.0.0,<17.0.0 -tutor-discovery>=16.0.0,<17.0.0 -tutor-ecommerce>=16.0.0,<17.0.0 -tutor-forum>=16.0.0,<17.0.0 -tutor-license>=16.0.0,<17.0.0 -tutor-mfe>=16.0.0,<17.0.0 -tutor-minio>=16.0.0,<17.0.0 -tutor-notes>=16.0.0,<17.0.0 -tutor-webui>=16.0.0,<17.0.0 -tutor-xqueue>=16.0.0,<17.0.0 +# change version ranges when upgrading from quince +tutor-android>=17.0.0,<18.0.0 +tutor-cairn>=17.0.0,<18.0.0 +tutor-credentials>=17.0.0,<18.0.0 +tutor-discovery>=17.0.0,<18.0.0 +tutor-ecommerce>=17.0.0,<18.0.0 +tutor-forum>=17.0.0,<18.0.0 +tutor-jupyter>=17.0.0,<18.0.0 +tutor-mfe>=17.0.0,<18.0.0 +tutor-minio>=17.0.0,<18.0.0 +tutor-notes>=17.0.0,<18.0.0 +tutor-webui>=17.0.0,<18.0.0 +tutor-xqueue>=17.0.0,<18.0.0 diff --git a/tests/commands/test_config.py b/tests/commands/test_config.py index 5c9dcea307..58b15edd93 100644 --- a/tests/commands/test_config.py +++ b/tests/commands/test_config.py @@ -94,9 +94,8 @@ def test_config_append_with_existing_default(self) -> None: ) config = tutor_config.load(root) assert isinstance(config["OPENEDX_EXTRA_PIP_REQUIREMENTS"], list) - self.assertEqual(2, len(config["OPENEDX_EXTRA_PIP_REQUIREMENTS"])) self.assertEqual( - "my-package==1.0.0", config["OPENEDX_EXTRA_PIP_REQUIREMENTS"][1] + ["my-package==1.0.0"], config["OPENEDX_EXTRA_PIP_REQUIREMENTS"] ) diff --git a/tests/test_env.py b/tests/test_env.py index a79060991a..27cff9678b 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -260,7 +260,7 @@ def test_current_version_in_latest_env(self) -> None: ) as f: f.write(__version__) self.assertEqual(__version__, env.current_version(root)) - self.assertEqual("palm", env.get_env_release(root)) + self.assertEqual("quince", env.get_env_release(root)) self.assertIsNone(env.should_upgrade_from_release(root)) self.assertTrue(env.is_up_to_date(root)) diff --git a/tutor/__about__.py b/tutor/__about__.py index 2599ad87d1..33297dd3a4 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "16.1.8" +__version__ = "17.0.0" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and @@ -10,7 +10,7 @@ # the nightly branch. # The suffix is cleanly separated from the __version__ in this module to avoid # conflicts when merging branches. -__version_suffix__ = "nightly" +__version_suffix__ = "" # The app name will be used to define the name of the default tutor root and # plugin directory. To avoid conflicts between multiple locally-installed diff --git a/tutor/commands/upgrade/__init__.py b/tutor/commands/upgrade/__init__.py index 9c1d203424..b7a4e028de 100644 --- a/tutor/commands/upgrade/__init__.py +++ b/tutor/commands/upgrade/__init__.py @@ -1,4 +1,4 @@ -# Note: don't forget to change this when we upgrade from palm +# Note: don't forget to change this when we upgrade from quince OPENEDX_RELEASE_NAMES = [ "ironwood", "juniper", @@ -7,4 +7,6 @@ "maple", "nutmeg", "olive", + "palm", + "quince", ] diff --git a/tutor/commands/upgrade/compose.py b/tutor/commands/upgrade/compose.py index ea0eaaf9dd..7aef353bda 100644 --- a/tutor/commands/upgrade/compose.py +++ b/tutor/commands/upgrade/compose.py @@ -43,6 +43,9 @@ def upgrade_from(context: click.Context, from_release: str) -> None: upgrade_from_olive(context, config) running_release = "palm" + if running_release == "palm": + running_release = "quince" + def upgrade_from_ironwood(context: click.Context, config: Config) -> None: click.echo(fmt.title("Upgrading from Ironwood")) diff --git a/tutor/commands/upgrade/k8s.py b/tutor/commands/upgrade/k8s.py index b84ac4f300..64cebaf354 100644 --- a/tutor/commands/upgrade/k8s.py +++ b/tutor/commands/upgrade/k8s.py @@ -42,6 +42,9 @@ def upgrade_from(context: click.Context, from_release: str) -> None: upgrade_from_olive(context.obj, config) running_release = "palm" + if running_release == "palm": + running_release = "quince" + def upgrade_from_ironwood(config: Config) -> None: upgrade_mongodb(config, "3.4.24", "3.4") diff --git a/tutor/config.py b/tutor/config.py index 35ee80149d..b5f48f30c3 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -153,18 +153,6 @@ def _load_config_defaults_yml( return items -@hooks.Filters.CONFIG_DEFAULTS.add() -def _set_openedx_common_version_in_nightly( - items: list[tuple[str, t.Any]] -) -> list[tuple[str, t.Any]]: - # REMOVE-AFTER-v16 move this callback to the dedicated openedx/plugin.py module - from tutor.__about__ import __version_suffix__ - - if __version_suffix__ == "nightly": - items.append(("OPENEDX_COMMON_VERSION", "master")) - return items - - def get_template(filename: str) -> Config: """ Get one of the configuration templates. diff --git a/tutor/env.py b/tutor/env.py index dd29b0c603..cccce94d0d 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -54,9 +54,6 @@ def _prepare_environment() -> None: ("HOST_USER_ID", utils.get_user_id()), ("TUTOR_APP", __app__.replace("-", "_")), ("TUTOR_VERSION", __version__), - # BuildKit used to be optional. Now, it's always enabled. - # This constant is just for temporary backwards compatibility (REMOVE-AFTER-V16). - ("is_buildkit_enabled", lambda: True), ("is_docker_rootless", utils.is_docker_rootless), ], ) @@ -460,6 +457,7 @@ def get_release(version: str) -> str: "14": "nutmeg", "15": "olive", "16": "palm", + "17": "quince", }[version.split(".", maxsplit=1)[0]] diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index e6dec94c52..56deed4e5e 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -334,7 +334,6 @@ def your_filter_callback(some_data): #: - ``HOST_USER_ID``: the numerical ID of the user on the host. #: - ``TUTOR_APP``: the app name ("tutor" by default), used to determine the dev/local project names. #: - ``TUTOR_VERSION``: the current version of Tutor. - #: - ``is_buildkit_enabled``: a deprecated function which always returns ``True`` now. Will be removed after Quince. #: - ``iter_values_named``: a function to iterate on variables that start or end with a given string. #: - ``iter_mounts``: a function that yields compose-compatible bind-mounts for any given service. #: - ``iter_mounted_directories``: iterate on bind-mounted directory names. diff --git a/tutor/plugins/openedx.py b/tutor/plugins/openedx.py index 0bf2c041bc..1ef4eff822 100644 --- a/tutor/plugins/openedx.py +++ b/tutor/plugins/openedx.py @@ -6,6 +6,16 @@ from tutor import bindmount from tutor import hooks +from tutor.__about__ import __version_suffix__ + + +@hooks.Filters.CONFIG_DEFAULTS.add() +def _set_openedx_common_version_in_nightly( + items: list[tuple[str, t.Any]] +) -> list[tuple[str, t.Any]]: + if __version_suffix__ == "nightly": + items.append(("OPENEDX_COMMON_VERSION", "master")) + return items @hooks.Filters.APP_PUBLIC_HOSTS.add() diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index c9b747a89f..c088b97b17 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -149,9 +149,11 @@ # These warnings are visible in simple commands and init tasks import warnings -from django.utils.deprecation import RemovedInDjango40Warning, RemovedInDjango41Warning -warnings.filterwarnings("ignore", category=RemovedInDjango40Warning) -warnings.filterwarnings("ignore", category=RemovedInDjango41Warning) + +from django.utils.deprecation import RemovedInDjango50Warning, RemovedInDjango51Warning +warnings.filterwarnings("ignore", category=RemovedInDjango50Warning) +warnings.filterwarnings("ignore", category=RemovedInDjango51Warning) + warnings.filterwarnings("ignore", category=DeprecationWarning, module="wiki.plugins.links.wiki_plugin") warnings.filterwarnings("ignore", category=DeprecationWarning, module="boto.plugin") warnings.filterwarnings("ignore", category=DeprecationWarning, module="botocore.vendored.requests.packages.urllib3._collections") diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index a5b1452387..0aef8e81b8 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.4 +# syntax=docker/dockerfile:1 ###### Minimal image with base system requirements for most stages FROM docker.io/ubuntu:20.04 as minimal LABEL maintainer="Overhang.io " @@ -24,9 +24,9 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ # Install pyenv # https://www.python.org/downloads/ # https://github.com/pyenv/pyenv/releases -ARG PYTHON_VERSION=3.8.15 +ARG PYTHON_VERSION=3.8.18 ENV PYENV_ROOT /opt/pyenv -RUN git clone https://github.com/pyenv/pyenv $PYENV_ROOT --branch v2.3.17 --depth 1 +RUN git clone https://github.com/pyenv/pyenv $PYENV_ROOT --branch v2.3.29 --depth 1 # Install Python RUN $PYENV_ROOT/bin/pyenv install $PYTHON_VERSION @@ -69,13 +69,12 @@ FROM scratch as mnt-{{ name }} ###### Download extra locales to /openedx/locale/contrib/locale FROM minimal as locales -ARG OPENEDX_I18N_VERSION={{ OPENEDX_COMMON_VERSION }} -RUN cd /tmp \ - && curl -L -o openedx-i18n.tar.gz https://github.com/openedx/openedx-i18n/archive/$OPENEDX_I18N_VERSION.tar.gz \ - && tar xzf /tmp/openedx-i18n.tar.gz \ - && mkdir -p /openedx/locale/contrib \ - && mv openedx-i18n-*/edx-platform/locale /openedx/locale/contrib \ - && rm -rf openedx-i18n* +ARG OPENEDX_I18N_REPOSITORY=https://github.com/openedx/openedx-i18n.git +ARG OPENEDX_I18N_VERSION=release/quince +ADD --keep-git-dir=true $OPENEDX_I18N_REPOSITORY#$OPENEDX_I18N_VERSION /tmp/openedx-i18n +RUN mkdir --parents /openedx/locale && \ + mv /tmp/openedx-i18n/edx-platform/locale /openedx/locale/contrib && \ + rm -rf /tmp/openedx-i18n/ ###### Install python requirements in virtualenv FROM python as python-requirements @@ -94,10 +93,7 @@ RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ # https://pypi.org/project/setuptools/ # https://pypi.org/project/pip/ # https://pypi.org/project/wheel/ - setuptools==67.6.1 pip==23.0.1. wheel==0.40.0 - -# Install missing py2neo package that was abruptly trimmed from pypi -RUN pip install https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz + setuptools==68.2.2 pip==23.2.1. wheel==0.41.2 # Install base requirements RUN --mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \ @@ -108,18 +104,14 @@ RUN --mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ pip install \ # Use redis as a django cache https://pypi.org/project/django-redis/ - django-redis==5.2.0 \ + django-redis==5.4.0 \ # uwsgi server https://pypi.org/project/uWSGI/ - uwsgi==2.0.21 + uwsgi==2.0.22 {{ patch("openedx-dockerfile-post-python-requirements") }} -# Install private requirements: this is useful for installing custom xblocks. -COPY ./requirements/ /openedx/requirements -RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ - cd /openedx/requirements/ \ - && touch ./private.txt \ - && pip install -r ./private.txt +# Install scorm xblock +RUN pip install "openedx-scorm-xblock>=17.0.0,<18.0.0" {% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %} RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ @@ -167,7 +159,6 @@ COPY --chown=app:app --from=edx-platform / /openedx/edx-platform COPY --chown=app:app --from=locales /openedx/locale /openedx/locale COPY --chown=app:app --from=python /opt/pyenv /opt/pyenv COPY --chown=app:app --from=python-requirements /openedx/venv /openedx/venv -COPY --chown=app:app --from=python-requirements /openedx/requirements /openedx/requirements COPY --chown=app:app --from=python-requirements /mnt /mnt COPY --chown=app:app --from=nodejs-requirements /openedx/nodeenv /openedx/nodeenv COPY --chown=app:app --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/node_modules @@ -273,9 +264,9 @@ USER app RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ pip install -r requirements/edx/development.txt # https://pypi.org/project/ipdb/ -# https://pypi.org/project/ipython +# https://pypi.org/project/ipython (8.12.x for Python 3.8) RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ - pip install ipdb==0.13.13 ipython==8.12.0 + pip install ipdb==0.13.13 ipython==8.12.3 {# Re-install mounted requirements, otherwise they will be superseded by upstream reqs #} {% for name in iter_mounted_directories(MOUNTS, "openedx") %} diff --git a/tutor/templates/build/openedx/requirements/private-sample.txt b/tutor/templates/build/openedx/requirements/private-sample.txt deleted file mode 100644 index a0ea63a612..0000000000 --- a/tutor/templates/build/openedx/requirements/private-sample.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Add your additional requirements, such as xblocks, to this file. For -# requirements coming from private repositories, clone the repository to this -# folder and then add your requirement with the `-e` flag. Ex: -# -# git clone git@myserver:myprivaterepo.git -# echo "-e ./myprivaterepo/" >> private.txt diff --git a/tutor/templates/build/openedx/revisions.yml b/tutor/templates/build/openedx/revisions.yml index 34724a9707..c01e271e53 100644 --- a/tutor/templates/build/openedx/revisions.yml +++ b/tutor/templates/build/openedx/revisions.yml @@ -1 +1 @@ -EDX_PLATFORM_REVISION: palm +EDX_PLATFORM_REVISION: quince diff --git a/tutor/templates/config/base.yml b/tutor/templates/config/base.yml index e34095a61b..b1d6a14afa 100644 --- a/tutor/templates/config/base.yml +++ b/tutor/templates/config/base.yml @@ -6,8 +6,9 @@ MYSQL_ROOT_PASSWORD: "{{ 8|random_string }}" OPENEDX_MYSQL_PASSWORD: "{{ 8|random_string }}" OPENEDX_SECRET_KEY: "{{ 24|random_string }}" PLUGINS: - # The MFE plugin is required + # enabled by default - mfe + - indigo PLUGIN_INDEXES: # Indexes in this list will be suffixed with the Open edX named version and # "plugins.yml". E.g: https://overhang.io/tutor/main/olive/plugins.yml diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index c75332812a..9ab1967ada 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -12,12 +12,18 @@ DOCKER_COMPOSE_VERSION: "3.7" DOCKER_REGISTRY: "docker.io/" DOCKER_IMAGE_OPENEDX: "{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION }}" DOCKER_IMAGE_OPENEDX_DEV: "openedx-dev:{{ TUTOR_VERSION }}" -DOCKER_IMAGE_CADDY: "docker.io/caddy:2.6.4" -DOCKER_IMAGE_ELASTICSEARCH: "docker.io/elasticsearch:7.17.9" -DOCKER_IMAGE_MONGODB: "docker.io/mongo:4.4.22" +# https://hub.docker.com/_/caddy/tags +DOCKER_IMAGE_CADDY: "docker.io/caddy:2.7.4" +# https://hub.docker.com/_/elasticsearch/tags +DOCKER_IMAGE_ELASTICSEARCH: "docker.io/elasticsearch:7.17.13" +# https://hub.docker.com/_/mongo/tags +DOCKER_IMAGE_MONGODB: "docker.io/mongo:4.4.25" +# https://hub.docker.com/_/mysql/tags DOCKER_IMAGE_MYSQL: "docker.io/mysql:8.1.0" DOCKER_IMAGE_PERMISSIONS: "{{ DOCKER_REGISTRY }}overhangio/openedx-permissions:{{ TUTOR_VERSION }}" -DOCKER_IMAGE_REDIS: "docker.io/redis:7.0.11" +# https://hub.docker.com/_/redis/tags +DOCKER_IMAGE_REDIS: "docker.io/redis:7.2.1" +# https://hub.docker.com/r/devture/exim-relay/tags DOCKER_IMAGE_SMTP: "docker.io/devture/exim-relay:4.96-r1-0" EDX_PLATFORM_REPOSITORY: "https://github.com/openedx/edx-platform.git" EDX_PLATFORM_VERSION: "{{ OPENEDX_COMMON_VERSION }}" @@ -53,9 +59,8 @@ OPENEDX_LMS_UWSGI_WORKERS: 2 OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_MYSQL_USERNAME: "openedx" # the common version will be automatically set to "master" in the nightly branch -OPENEDX_COMMON_VERSION: "open-release/palm.4" -OPENEDX_EXTRA_PIP_REQUIREMENTS: - - "openedx-scorm-xblock>=16.0.0,<17.0.0" +OPENEDX_COMMON_VERSION: "open-release/quince.1" +OPENEDX_EXTRA_PIP_REQUIREMENTS: [] MYSQL_HOST: "mysql" MYSQL_PORT: 3306 MYSQL_ROOT_USERNAME: "root" diff --git a/tutor/templates/dev/docker-compose.jobs.yml b/tutor/templates/dev/docker-compose.jobs.yml index c9df4e2a12..5e82a455aa 100644 --- a/tutor/templates/dev/docker-compose.jobs.yml +++ b/tutor/templates/dev/docker-compose.jobs.yml @@ -16,8 +16,6 @@ x-openedx-job-service: - ../apps/openedx/config:/openedx/config:ro # theme files - ../build/openedx/themes:/openedx/themes - # editable requirements - - ../build/openedx/requirements:/openedx/requirements services: diff --git a/tutor/templates/dev/docker-compose.yml b/tutor/templates/dev/docker-compose.yml index 7c864358fc..90fbe842bd 100644 --- a/tutor/templates/dev/docker-compose.yml +++ b/tutor/templates/dev/docker-compose.yml @@ -12,8 +12,6 @@ x-openedx-service: - ../apps/openedx/config:/openedx/config:ro # theme files - ../build/openedx/themes:/openedx/themes - # editable requirements - - ../build/openedx/requirements:/openedx/requirements services: permissions: diff --git a/tutor/templates/jobs/init/cms.sh b/tutor/templates/jobs/init/cms.sh index 2f732da22f..1d420a48ab 100644 --- a/tutor/templates/jobs/init/cms.sh +++ b/tutor/templates/jobs/init/cms.sh @@ -11,4 +11,8 @@ if [ -d /openedx/data/uploads/ ]; then mv /openedx/data/uploads/* /openedx/media/ rm -rf /openedx/data/uploads/ fi -fi \ No newline at end of file +fi + +# Create waffle switches to enable some features, if they have not been explicitly defined before +# Copy-paste of units in Studio (highly requested new feature, but defaults to off in Quince) +(./manage.py cms waffle_flag --list | grep contentstore.enable_copy_paste_units) || ./manage.py lms waffle_flag --create contentstore.enable_copy_paste_units --everyone