diff --git a/backend/workers/manage_extension.py b/backend/workers/manage_extension.py new file mode 100644 index 00000000..7a605c12 --- /dev/null +++ b/backend/workers/manage_extension.py @@ -0,0 +1,266 @@ +""" +Manage a 4CAT extension +""" +import subprocess +import requests +import logging +import zipfile +import shutil +import shlex +import json +import ural +import os +import re + +from logging.handlers import RotatingFileHandler +from pathlib import Path + +from backend.lib.worker import BasicWorker +from common.config_manager import config + + +class ExtensionManipulator(BasicWorker): + """ + Manage 4CAT extensions + + 4CAT extensions are essentially git repositories. This worker can clone the + relevant git repository or delete it and clean up after it. + + This is done in a worker instead of in the front-end code because cloning + a large git repository can take some time so it is best to do it + asynchronously. This is also future-proof in that it is easy to add support + for installation code etc here later. + + Results are logged to a separate log file that can then be inspected in the + web interface. + """ + type = "manage-extension" + max_workers = 1 + + def work(self): + """ + Do something with extensions + """ + extension_reference = self.job.data["remote_id"] + task = self.job.details.get("task") + + # note that this is a databaseless config reader + # since we only need it for file paths + self.config = config + + # this worker uses its own log file instead of the main 4CAT log + # this is so that it is easier to monitor error messages about failed + # installations etc and display those separately in e.g. the web + # interface + + log_file = Path(self.config.get("PATH_ROOT")).joinpath(self.config.get("PATH_LOGS")).joinpath("extensions.log") + logger = logging.getLogger(self.type) + if not logger.handlers: + handler = RotatingFileHandler(log_file, backupCount=1, maxBytes=50000) + handler.level = logging.INFO + handler.setFormatter(logging.Formatter("%(asctime)-15s | %(levelname)s: %(message)s", + "%d-%m-%Y %H:%M:%S")) + logger.addHandler(handler) + logger.level = logging.INFO + self.extension_log = logger + + if task == "install": + self.install_extension(extension_reference) + elif task == "uninstall": + self.uninstall_extension(extension_reference) + + self.job.finish() + + def uninstall_extension(self, extension_name): + """ + Remove extension + + Currently as simple as deleting the folder, but could add further + cleaning up code later. + + While an extension can define configuration settings, we do not + explicitly remove these here. 4CAT has general cleanup code for + unreferenced settings and it may be beneficial to keep them in case + the extension is re-installed later. + + :param str extension_name: ID of the extension (i.e. name of the + folder it is in) + """ + extensions_root = self.config.get("PATH_ROOT").joinpath("extensions") + target_folder = extensions_root.joinpath(extension_name) + + if not target_folder.exists(): + return self.extension_log.error(f"Extension {extension_name} does not exist - cannot remove it.") + + try: + shutil.rmtree(target_folder) + self.extension_log.info(f"Finished uninstalling extension {extension_name}.") + except OSError as e: + self.extension_log.error(f"Could not uninstall extension {extension_name}. There may be an issue with " + f"file privileges, or the extension is installed via a symbolic link which 4CAT " + f"cannot manipulate. The system error message was: '{e}'") + + def install_extension(self, repository_reference, overwrite=False): + """ + Install a 4CAT extension + + 4CAT extensions can be installed from a git URL or a zip archive. In + either case, the files are first put into a temporary folder, after + which the manifest in that folder is read to complete installation. + + :param str repository_reference: Git repository URL, or zip archive + path. + :param bool overwrite: Overwrite extension if one exists? Set to + `true` to upgrade existing extensions (for example) + """ + if self.job.details.get("source") == "remote": + extension_folder, extension_name = self.clone_from_url(repository_reference) + else: + extension_folder, extension_name = self.unpack_from_zip(repository_reference) + + if not extension_name: + return self.extension_log.error("The 4CAT extension could not be installed.") + + # read manifest file + manifest_file = extension_folder.joinpath("metadata.json") + if not manifest_file.exists(): + shutil.rmtree(extension_folder) + return self.extension_log.error(f"Manifest file of newly cloned 4CAT extension {repository_reference} does " + f"not exist. Cannot install as a 4CAT extension.") + else: + try: + with manifest_file.open() as infile: + manifest_data = json.load(infile) + except json.JSONDecodeError: + shutil.rmtree(extension_folder) + return self.extension_log.error(f"Manifest file of newly cloned 4CAT extension {repository_reference} " + f"could not be parsed. Cannot install as a 4CAT extension.") + + canonical_name = manifest_data.get("name", extension_name) + canonical_id = manifest_data.get("id", extension_name) + + canonical_folder = extension_folder.with_name(canonical_id) + existing_name = canonical_id + existing_version = "unknown" + + if canonical_folder.exists(): + if canonical_folder.joinpath("metadata.json").exists(): + with canonical_folder.joinpath("metadata.json").open() as infile: + try: + existing_manifest = json.load(infile) + existing_name = existing_manifest.get("name", canonical_id) + existing_version = existing_manifest.get("version", "unknown") + except json.JSONDecodeError: + pass + + shutil.rmtree(canonical_folder) + if overwrite: + self.extension_log.warning(f"Uninstalling existing 4CAT extension {existing_name} (version " + f"{existing_version}.") + else: + return self.extension_log.error(f"An extension with ID {canonical_id} is already installed " + f"({extension_name}, version {existing_version}). Cannot install " + f"another one with the same ID - uninstall it first.") + + extension_folder.rename(canonical_folder) + version = f"version {manifest_data.get('version', 'unknown')}" + self.extension_log.info(f"Finished installing extension {canonical_name} (version {version}) with ID " + f"{canonical_id}.") + + + def unpack_from_zip(self, archive_path): + """ + Unpack extension files from a zip archive + + Pretty straightforward - Make a temporary folder and extract the zip + archive's contents into it. + + :param str archive_path: Path to the zip file to extract + :return tuple: Tuple of folder and extension name, or `None, None` on + failure. + """ + archive_path = Path(archive_path) + if not archive_path.exists(): + return self.extension_log.error(f"Extension file does not exist at {archive_path} - cannot install."), None + + extension_name = archive_path.stem + extensions_root = self.config.get("PATH_ROOT").joinpath("extensions") + temp_name = self.get_temporary_folder(extensions_root) + try: + with zipfile.ZipFile(archive_path, "r") as archive_file: + archive_file.extractall(temp_name) + except Exception as e: + return self.extension_log.error(f"Could not extract extension zip archive {archive_path.name}: {e}. Cannot " + f"install."), None + finally: + archive_path.unlink() + + return temp_name, extension_name + + + def clone_from_url(self, repository_url): + """ + Clone the extension files from a git repository URL + + :param str repository_url: Git repository URL to clone extension from + :return tuple: Tuple of folder and extension name, or `None, None` on + failure. + """ + # we only know how to install extensions from URLs for now + if not ural.is_url(repository_url): + return self.extension_log.error(f"Cannot install 4CAT extension - invalid repository url: " + f"{repository_url}"), None + + # normalize URL and extract name + repository_url = repository_url.strip().split("#")[-1] + if repository_url.endswith("/"): + repository_url = repository_url[:-1] + repository_url_name = re.sub(r"\.git$", "", repository_url.split("/")[-1].split("?")[0].lower()) + + try: + test_url = requests.head(repository_url) + if test_url.status_code >= 400: + return self.extension_log.error( + f"Cannot install 4CAT extension - the repository URL is unreachable (status code " + f"{test_url.status_code})"), None + except requests.RequestException as e: + return self.extension_log.error( + f"Cannot install 4CAT extension - the repository URL seems invalid or unreachable ({e})"), None + + # ok, we have a valid URL that is reachable - try cloning from it + extensions_root = self.config.get("PATH_ROOT").joinpath("extensions") + os.chdir(extensions_root) + + temp_name = self.get_temporary_folder(extensions_root) + + extension_folder = extensions_root.joinpath(temp_name) + clone_command = f"git clone {shlex.quote(repository_url)} {temp_name}" + clone_outcome = subprocess.run(shlex.split(clone_command), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + cloned_correctly = True + if clone_outcome.returncode != 0: + cloned_correctly = False + self.extension_log.info(clone_outcome.stdout.decode("utf-8")) + self.extension_log.error(f"Could not clone 4CAT extension repository from {repository_url} - see log for " + f"details.") + + if not cloned_correctly: + if extension_folder.exists(): + shutil.rmtree(extension_folder) + return self.extension_log.error(f"4CAT extension {repository_url} was not installed."), None + + return extension_folder, repository_url_name + + + def get_temporary_folder(self, extensions_root): + # clone into a temporary folder, which we will rename as needed + # this is because the repository name is not necessarily the extension + # name + temp_base = "new-extension" + temp_name = temp_base + temp_index = 0 + while extensions_root.joinpath(temp_name).exists(): + temp_index += 1 + temp_name = f"{temp_base}-{temp_index}" + + return extensions_root.joinpath(temp_name) diff --git a/common/lib/config_definition.py b/common/lib/config_definition.py index ee38ce70..45357965 100644 --- a/common/lib/config_definition.py +++ b/common/lib/config_definition.py @@ -37,6 +37,18 @@ "setting.", "indirect": True }, + # Extensions + "extensions._intro": { + "type": UserInput.OPTION_INFO, + "help": "4CAT extensions can be disabled and disabled via the control below. When enabled, extensions may " + "define further settings that can typically be configured via the extension's tab on the left side of " + "this page. **Note that 4CAT needs to be restarted for this to take effect!**" + }, + "extensions.enabled": { + "type": UserInput.OPTION_EXTENSIONS, + "default": {}, + "help": "Extensions" + }, # Configure how the tool is to be named in its web interface. The backend will # always refer to "4CAT" - the name of the software, and a "powered by 4CAT" # notice may also show up in the web interface regardless of the value entered here. @@ -149,6 +161,12 @@ "help": "Can restart/upgrade", "tooltip": "Controls whether users can restart, upgrade, and manage extensions 4CAT via the Control Panel" }, + "privileges.admin.can_manage_extensions": { + "type": UserInput.OPTION_TOGGLE, + "default": False, + "help": "Can manage extensions", + "tooltip": "Controls whether users can install and uninstall 4CAT extensions via the Control Panel" + }, "privileges.can_upgrade_to_dev": { # this is NOT an admin privilege, because all admins automatically # get all admin privileges! users still need the above privilege @@ -554,4 +572,5 @@ "dmi-service-manager": "DMI Service Manager", "ui": "User interface", "image-visuals": "Image visualization", + "extensions": "Extensions" } diff --git a/common/lib/module_loader.py b/common/lib/module_loader.py index a036bd24..cc70f842 100644 --- a/common/lib/module_loader.py +++ b/common/lib/module_loader.py @@ -99,6 +99,7 @@ def load_modules(self): # look for workers and processors in pre-defined folders and datasources extension_path = Path(config.get('PATH_ROOT'), "extensions") + enabled_extensions = [e for e, s in config.get("extensions.enabled").items() if s["enabled"]] paths = [Path(config.get('PATH_ROOT'), "processors"), Path(config.get('PATH_ROOT'), "backend", "workers"), @@ -124,6 +125,9 @@ def load_modules(self): # This skips processors/datasources that were loaded by others and may not yet be captured pass + if is_extension and len(module_name.split(".")) > 1 and module_name.split(".")[1] not in enabled_extensions: + continue + # try importing try: module = importlib.import_module(module_name) @@ -221,7 +225,13 @@ def _load_datasource(subdirectory): # Load extension datasources # os.walk is used to allow for the possibility of multiple extensions, with nested "datasources" folders - for root, dirs, files in os.walk(Path(config.get('PATH_ROOT'), "extensions"), followlinks=True): + enabled_extensions = [e for e, s in config.get("extensions.enabled").items() if s["enabled"]] + extensions_root = Path(config.get('PATH_ROOT'), "extensions") + for root, dirs, files in os.walk(extensions_root, followlinks=True): + relative_root = Path(root).relative_to(extensions_root) + if relative_root.parts and relative_root.parts[0] not in enabled_extensions: + continue + if "datasources" in dirs: for subdirectory in Path(root, "datasources").iterdir(): if subdirectory.is_dir(): diff --git a/common/lib/user_input.py b/common/lib/user_input.py index 63999083..6e10a19b 100644 --- a/common/lib/user_input.py +++ b/common/lib/user_input.py @@ -35,6 +35,7 @@ class UserInput: OPTION_FILE = "file" # file upload OPTION_HUE = "hue" # colour hue OPTION_DATASOURCES = "datasources" # data source toggling + OPTION_EXTENSIONS = "extensions" # extension toggling OPTIONS_COSMETIC = (OPTION_INFO, OPTION_DIVIDER) @@ -143,6 +144,12 @@ def parse_all(options, input, silently_correct=True): parsed_input[option] = [datasource for datasource, v in datasources.items() if v["enabled"]] parsed_input[option.split(".")[0] + ".expiration"] = datasources + elif settings.get("type") == UserInput.OPTION_EXTENSIONS: + # also a special case + parsed_input[option] = {extension: { + "enabled": f"{option}-enable-{extension}" in input + } for extension in input[option].split(",")} + elif option not in input: # not provided? use default parsed_input[option] = settings.get("default", None) diff --git a/webtool/static/css/control-panel.css b/webtool/static/css/control-panel.css index dce8cd1a..15ddfa80 100644 --- a/webtool/static/css/control-panel.css +++ b/webtool/static/css/control-panel.css @@ -46,6 +46,10 @@ table.cp-table .actions { vertical-align: middle; } +.extensions-table tr.disabled { + color: var(--gray-dark); +} + table td[colspan] { text-align: center; } @@ -89,14 +93,14 @@ table td[colspan] { max-width: 25em; } -.settings .datasource-toggle-form table { +.settings .broad-toggle-form table { position: relative; left: -15em; background: var(--always-white); width: calc(100% + 12em); } -.settings .datasource-toggle-form input[type=text] { +.settings .broad-toggle-form input[type=text] { width: 5em; } @@ -260,6 +264,10 @@ article .stats-container h3:not(.blocktitle) { margin-right: 0.5em; } +.log-display.wrapped-log { + white-space: pre-line; +} + /** ** Bulk dataset management */ diff --git a/webtool/static/css/stylesheet.css b/webtool/static/css/stylesheet.css index acc4409b..175edbab 100644 --- a/webtool/static/css/stylesheet.css +++ b/webtool/static/css/stylesheet.css @@ -192,6 +192,10 @@ fieldset legend { width: 100%; } +.form-notices-wrapper { + padding-bottom: 1em; +} + p.form-notice { text-align: center; color: var(--accent-alternate); diff --git a/webtool/templates/components/datasource-option.html b/webtool/templates/components/datasource-option.html index 4ee4ba16..d46f4b61 100644 --- a/webtool/templates/components/datasource-option.html +++ b/webtool/templates/components/datasource-option.html @@ -108,8 +108,31 @@ data-value="{% if settings.value %}{{ settings.value }}{% else %}46{% endif %}" data-update-background="#colour-example-{{ hue_id }}"> + {% elif settings.type == "extensions" %} +
+ + + + + + + + + + + + + {% for extension, extension_config in extensions_config.items() %} + + + + + {% endfor %} + +
Extension
(change all)
{{ extension_config.name }}
+
{% elif settings.type == "datasources" %} -
+
diff --git a/webtool/templates/controlpanel/extensions-list.html b/webtool/templates/controlpanel/extensions-list.html index bd7243fd..1665102d 100644 --- a/webtool/templates/controlpanel/extensions-list.html +++ b/webtool/templates/controlpanel/extensions-list.html @@ -8,15 +8,21 @@

4CAT Extensions

- {% for notice in flashes %} -

{{ notice|safe }}

- {% endfor %} -

4CAT extensions can be installed in the extensions folder in the 4CAT root. For more + {% if flashes %} +

+ {% for notice in flashes %} +

{{ notice|safe }}

+ {% endfor %} +
+ {% endif %} +

4CAT extensions can be installed in the extensions folder in the 4CAT root. For more information, see the README file in that folder. This page lists all currently installed extensions; currently, to manage extensions you will need to access the filesystem and move files into the correct location manually.

+

Extensions need to be enabled after installation to run. You can do so via the + 4CAT settings panel.

-
+
@@ -26,11 +32,16 @@

4CAT Extensions

+ {% if extensions %} {% for extension_id, extension in extensions.items() %} - - + {% endfor %} {% else %} - + {% endif %}
Extension Version LinksActions
{{ extension_id }}{% if extension_id != extension.name %} + + + {% if not extension.enabled %} + This extension is currently disabled.{% endif %} + + {{ extension_id }}{% if extension_id != extension.name %} {{ extension.name }}{% endif %} {% if extension.version %}{{ extension.version }}{% else %}unknown{% endif %} @@ -41,15 +52,61 @@

4CAT Extensions

aria-hidden="true">Remote git repository{% endif %}
+
+ + +
+
No 4CAT extensions are installed.No 4CAT extensions are installed.
+ + + +
+ +
+

Install new extension

+

Install a new extension by providing either a Git repository URL or a zip archive with + the extension files in it below. Note that extension code can basically do anything on the + system 4CAT runs on - make sure to only install code you trust.

+

After installing, the extension will initially be disabled. You can enable and disable extensions via the + 4CAT settings panel.

+ +
+
+ + +
+
+ + +
+
+ +
+
+
+ +
+

Extension installation log

+

Displaying last 150 lines of the log file.

+
+                Loading log file...
+            
+
{% endblock %} \ No newline at end of file diff --git a/webtool/views/views_admin.py b/webtool/views/views_admin.py index 9e09c9f0..65ad3684 100644 --- a/webtool/views/views_admin.py +++ b/webtool/views/views_admin.py @@ -26,7 +26,7 @@ from webtool.lib.helpers import error, Pagination, generate_css_colours, setting_required from common.lib.user import User from common.lib.dataset import DataSet -from common.lib.helpers import call_api, send_email, UserInput, folder_size, get_git_branch +from common.lib.helpers import call_api, send_email, UserInput, folder_size, get_git_branch, find_extensions from common.lib.exceptions import QueryParametersException import common.lib.config_definition as config_definition @@ -584,7 +584,7 @@ def manipulate_settings(): option_owner = option.split(".")[0] submenu = "other" if option_owner in ("4cat", "datasources", "privileges", "path", "mail", "explorer", "flask", - "logging", "ui"): + "logging", "ui", "extensions"): submenu = "core" elif option_owner.endswith("-search"): submenu = "datasources" @@ -630,10 +630,19 @@ def manipulate_settings(): } for datasource, info in fourcat_modules.datasources.items()} + # similar deal for extensions + extension_config = { + extension: { + **info, + "enabled": config.get("extensions.enabled").get(extension, {}).get("enabled") + } + for extension, info in find_extensions()[0].items() + } + return render_template("controlpanel/config.html", options=options, flashes=get_flashed_messages(), categories=categories, modules=modules, tag=tag, current_tab=tab, datasources_config=datasources, changed=changed_categories, - expire_override=expire_override) + expire_override=expire_override, extensions_config=extension_config) @app.route("/manage-notifications/", methods=["GET", "POST"]) @@ -735,7 +744,7 @@ def get_log(logfile): :param str logfile: 'backend' or 'stderr' :return: """ - if logfile not in ("stderr", "backend", "import"): + if logfile not in ("stderr", "backend", "import", "extensions"): return "Not Found", 404 if logfile == "backend": diff --git a/webtool/views/views_extensions.py b/webtool/views/views_extensions.py index 2f120e2a..d6ca9149 100644 --- a/webtool/views/views_extensions.py +++ b/webtool/views/views_extensions.py @@ -1,11 +1,13 @@ """ 4CAT extension views - routes to manipulate 4CAT extensions """ +import re -from flask import render_template, request, flash, get_flashed_messages +from flask import render_template, request, flash, get_flashed_messages, redirect, url_for from flask_login import current_user, login_required -from webtool import app, config +from webtool import app, config, queue +from webtool.lib.helpers import setting_required from common.lib.helpers import find_extensions from common.config_manager import ConfigWrapper @@ -13,16 +15,65 @@ config = ConfigWrapper(config, user=current_user, request=request) -@app.route("/admin/extensions/") +@app.route("/admin/extensions/", methods=["GET", "POST"]) @login_required +@setting_required("privileges.admin.can_manage_extensions") def extensions_panel(): extensions, load_errors = find_extensions() if extensions is None: return render_template("error.html", message="No extensions folder is available - cannot " - "list or manipulate extensions in this 4CAT server."), 500 + "list or manipulate extensions in this 4CAT server."), 500 + + incomplete = [] + if request.method == "POST": + install_started = True + + if request.files["extension-file"].filename: + uploaded_file = request.files["extension-file"].filename + stem = re.sub(r"[^a-zA-Z0-9_-]", "", uploaded_file.filename.replace(" ", "_")).strip() + temporary_path = config.get("PATH_ROOT").joinpath("extensions").joinpath(f"temp-{stem}.zip") + uploaded_file.save(temporary_path) + queue.add_job("manage-extension", details={"task": "install", "source": "local"}, + remote_id=str(temporary_path)) + extension_reference = uploaded_file.filename + + else: + extension_reference = request.form.get("extension-url") + if extension_reference: + queue.add_job("manage-extension", details={"task": "install", "source": "remote"}, + remote_id=extension_reference) + else: + install_started = False + flash("You need to provide either a repository URL or zip file to install an extension.") + incomplete.append("extension-url") + + if install_started: + flash(f"Initiated extension install from {extension_reference}. Find its status in the panel at the bottom " + f"of the page. You may need to refresh the page after installation completes.") for error in load_errors: flash(error) - return render_template("controlpanel/extensions-list.html", extensions=extensions, flashes=get_flashed_messages()) + return render_template("controlpanel/extensions-list.html", extensions=extensions, + flashes=get_flashed_messages(), incomplete=incomplete) + + +@app.route("/admin/uninstall-extension", methods=["POST"]) +@login_required +@setting_required("privileges.admin.can_manage_extensions") +def uninstall_extension(): + extensions, load_errors = find_extensions() + + extension_reference = request.form.get("extension-name") + + if not extensions or not extension_reference or extension_reference not in extensions: + flash(f"Extension {extension_reference} unknown - cannot uninstall extension.") + else: + queue.add_job("manage-extension", details={"task": "uninstall"}, + remote_id=extension_reference) + + flash(f"Initiated uninstall of extension '{extension_reference}'. Find its status in the panel at the bottom " + f"of the page. You may need to refresh the page afterwards.") + + return redirect(url_for("extensions_panel")) \ No newline at end of file