diff --git a/backend/workers/manage_extension.py b/backend/workers/manage_extension.py
new file mode 100644
index 00000000..7a605c12
--- /dev/null
+++ b/backend/workers/manage_extension.py
@@ -0,0 +1,266 @@
+"""
+Manage a 4CAT extension
+"""
+import subprocess
+import requests
+import logging
+import zipfile
+import shutil
+import shlex
+import json
+import ural
+import os
+import re
+
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+
+from backend.lib.worker import BasicWorker
+from common.config_manager import config
+
+
+class ExtensionManipulator(BasicWorker):
+ """
+ Manage 4CAT extensions
+
+ 4CAT extensions are essentially git repositories. This worker can clone the
+ relevant git repository or delete it and clean up after it.
+
+ This is done in a worker instead of in the front-end code because cloning
+ a large git repository can take some time so it is best to do it
+ asynchronously. This is also future-proof in that it is easy to add support
+ for installation code etc here later.
+
+ Results are logged to a separate log file that can then be inspected in the
+ web interface.
+ """
+ type = "manage-extension"
+ max_workers = 1
+
+ def work(self):
+ """
+ Do something with extensions
+ """
+ extension_reference = self.job.data["remote_id"]
+ task = self.job.details.get("task")
+
+ # note that this is a databaseless config reader
+ # since we only need it for file paths
+ self.config = config
+
+ # this worker uses its own log file instead of the main 4CAT log
+ # this is so that it is easier to monitor error messages about failed
+ # installations etc and display those separately in e.g. the web
+ # interface
+
+ log_file = Path(self.config.get("PATH_ROOT")).joinpath(self.config.get("PATH_LOGS")).joinpath("extensions.log")
+ logger = logging.getLogger(self.type)
+ if not logger.handlers:
+ handler = RotatingFileHandler(log_file, backupCount=1, maxBytes=50000)
+ handler.level = logging.INFO
+ handler.setFormatter(logging.Formatter("%(asctime)-15s | %(levelname)s: %(message)s",
+ "%d-%m-%Y %H:%M:%S"))
+ logger.addHandler(handler)
+ logger.level = logging.INFO
+ self.extension_log = logger
+
+ if task == "install":
+ self.install_extension(extension_reference)
+ elif task == "uninstall":
+ self.uninstall_extension(extension_reference)
+
+ self.job.finish()
+
+ def uninstall_extension(self, extension_name):
+ """
+ Remove extension
+
+ Currently as simple as deleting the folder, but could add further
+ cleaning up code later.
+
+ While an extension can define configuration settings, we do not
+ explicitly remove these here. 4CAT has general cleanup code for
+ unreferenced settings and it may be beneficial to keep them in case
+ the extension is re-installed later.
+
+ :param str extension_name: ID of the extension (i.e. name of the
+ folder it is in)
+ """
+ extensions_root = self.config.get("PATH_ROOT").joinpath("extensions")
+ target_folder = extensions_root.joinpath(extension_name)
+
+ if not target_folder.exists():
+ return self.extension_log.error(f"Extension {extension_name} does not exist - cannot remove it.")
+
+ try:
+ shutil.rmtree(target_folder)
+ self.extension_log.info(f"Finished uninstalling extension {extension_name}.")
+ except OSError as e:
+ self.extension_log.error(f"Could not uninstall extension {extension_name}. There may be an issue with "
+ f"file privileges, or the extension is installed via a symbolic link which 4CAT "
+ f"cannot manipulate. The system error message was: '{e}'")
+
+ def install_extension(self, repository_reference, overwrite=False):
+ """
+ Install a 4CAT extension
+
+ 4CAT extensions can be installed from a git URL or a zip archive. In
+ either case, the files are first put into a temporary folder, after
+ which the manifest in that folder is read to complete installation.
+
+ :param str repository_reference: Git repository URL, or zip archive
+ path.
+ :param bool overwrite: Overwrite extension if one exists? Set to
+ `true` to upgrade existing extensions (for example)
+ """
+ if self.job.details.get("source") == "remote":
+ extension_folder, extension_name = self.clone_from_url(repository_reference)
+ else:
+ extension_folder, extension_name = self.unpack_from_zip(repository_reference)
+
+ if not extension_name:
+ return self.extension_log.error("The 4CAT extension could not be installed.")
+
+ # read manifest file
+ manifest_file = extension_folder.joinpath("metadata.json")
+ if not manifest_file.exists():
+ shutil.rmtree(extension_folder)
+ return self.extension_log.error(f"Manifest file of newly cloned 4CAT extension {repository_reference} does "
+ f"not exist. Cannot install as a 4CAT extension.")
+ else:
+ try:
+ with manifest_file.open() as infile:
+ manifest_data = json.load(infile)
+ except json.JSONDecodeError:
+ shutil.rmtree(extension_folder)
+ return self.extension_log.error(f"Manifest file of newly cloned 4CAT extension {repository_reference} "
+ f"could not be parsed. Cannot install as a 4CAT extension.")
+
+ canonical_name = manifest_data.get("name", extension_name)
+ canonical_id = manifest_data.get("id", extension_name)
+
+ canonical_folder = extension_folder.with_name(canonical_id)
+ existing_name = canonical_id
+ existing_version = "unknown"
+
+ if canonical_folder.exists():
+ if canonical_folder.joinpath("metadata.json").exists():
+ with canonical_folder.joinpath("metadata.json").open() as infile:
+ try:
+ existing_manifest = json.load(infile)
+ existing_name = existing_manifest.get("name", canonical_id)
+ existing_version = existing_manifest.get("version", "unknown")
+ except json.JSONDecodeError:
+ pass
+
+ shutil.rmtree(canonical_folder)
+ if overwrite:
+ self.extension_log.warning(f"Uninstalling existing 4CAT extension {existing_name} (version "
+ f"{existing_version}.")
+ else:
+ return self.extension_log.error(f"An extension with ID {canonical_id} is already installed "
+ f"({extension_name}, version {existing_version}). Cannot install "
+ f"another one with the same ID - uninstall it first.")
+
+ extension_folder.rename(canonical_folder)
+ version = f"version {manifest_data.get('version', 'unknown')}"
+ self.extension_log.info(f"Finished installing extension {canonical_name} (version {version}) with ID "
+ f"{canonical_id}.")
+
+
+ def unpack_from_zip(self, archive_path):
+ """
+ Unpack extension files from a zip archive
+
+ Pretty straightforward - Make a temporary folder and extract the zip
+ archive's contents into it.
+
+ :param str archive_path: Path to the zip file to extract
+ :return tuple: Tuple of folder and extension name, or `None, None` on
+ failure.
+ """
+ archive_path = Path(archive_path)
+ if not archive_path.exists():
+ return self.extension_log.error(f"Extension file does not exist at {archive_path} - cannot install."), None
+
+ extension_name = archive_path.stem
+ extensions_root = self.config.get("PATH_ROOT").joinpath("extensions")
+ temp_name = self.get_temporary_folder(extensions_root)
+ try:
+ with zipfile.ZipFile(archive_path, "r") as archive_file:
+ archive_file.extractall(temp_name)
+ except Exception as e:
+ return self.extension_log.error(f"Could not extract extension zip archive {archive_path.name}: {e}. Cannot "
+ f"install."), None
+ finally:
+ archive_path.unlink()
+
+ return temp_name, extension_name
+
+
+ def clone_from_url(self, repository_url):
+ """
+ Clone the extension files from a git repository URL
+
+ :param str repository_url: Git repository URL to clone extension from
+ :return tuple: Tuple of folder and extension name, or `None, None` on
+ failure.
+ """
+ # we only know how to install extensions from URLs for now
+ if not ural.is_url(repository_url):
+ return self.extension_log.error(f"Cannot install 4CAT extension - invalid repository url: "
+ f"{repository_url}"), None
+
+ # normalize URL and extract name
+ repository_url = repository_url.strip().split("#")[-1]
+ if repository_url.endswith("/"):
+ repository_url = repository_url[:-1]
+ repository_url_name = re.sub(r"\.git$", "", repository_url.split("/")[-1].split("?")[0].lower())
+
+ try:
+ test_url = requests.head(repository_url)
+ if test_url.status_code >= 400:
+ return self.extension_log.error(
+ f"Cannot install 4CAT extension - the repository URL is unreachable (status code "
+ f"{test_url.status_code})"), None
+ except requests.RequestException as e:
+ return self.extension_log.error(
+ f"Cannot install 4CAT extension - the repository URL seems invalid or unreachable ({e})"), None
+
+ # ok, we have a valid URL that is reachable - try cloning from it
+ extensions_root = self.config.get("PATH_ROOT").joinpath("extensions")
+ os.chdir(extensions_root)
+
+ temp_name = self.get_temporary_folder(extensions_root)
+
+ extension_folder = extensions_root.joinpath(temp_name)
+ clone_command = f"git clone {shlex.quote(repository_url)} {temp_name}"
+ clone_outcome = subprocess.run(shlex.split(clone_command), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+ cloned_correctly = True
+ if clone_outcome.returncode != 0:
+ cloned_correctly = False
+ self.extension_log.info(clone_outcome.stdout.decode("utf-8"))
+ self.extension_log.error(f"Could not clone 4CAT extension repository from {repository_url} - see log for "
+ f"details.")
+
+ if not cloned_correctly:
+ if extension_folder.exists():
+ shutil.rmtree(extension_folder)
+ return self.extension_log.error(f"4CAT extension {repository_url} was not installed."), None
+
+ return extension_folder, repository_url_name
+
+
+ def get_temporary_folder(self, extensions_root):
+ # clone into a temporary folder, which we will rename as needed
+ # this is because the repository name is not necessarily the extension
+ # name
+ temp_base = "new-extension"
+ temp_name = temp_base
+ temp_index = 0
+ while extensions_root.joinpath(temp_name).exists():
+ temp_index += 1
+ temp_name = f"{temp_base}-{temp_index}"
+
+ return extensions_root.joinpath(temp_name)
diff --git a/common/lib/config_definition.py b/common/lib/config_definition.py
index ee38ce70..45357965 100644
--- a/common/lib/config_definition.py
+++ b/common/lib/config_definition.py
@@ -37,6 +37,18 @@
"setting.",
"indirect": True
},
+ # Extensions
+ "extensions._intro": {
+ "type": UserInput.OPTION_INFO,
+ "help": "4CAT extensions can be disabled and disabled via the control below. When enabled, extensions may "
+ "define further settings that can typically be configured via the extension's tab on the left side of "
+ "this page. **Note that 4CAT needs to be restarted for this to take effect!**"
+ },
+ "extensions.enabled": {
+ "type": UserInput.OPTION_EXTENSIONS,
+ "default": {},
+ "help": "Extensions"
+ },
# Configure how the tool is to be named in its web interface. The backend will
# always refer to "4CAT" - the name of the software, and a "powered by 4CAT"
# notice may also show up in the web interface regardless of the value entered here.
@@ -149,6 +161,12 @@
"help": "Can restart/upgrade",
"tooltip": "Controls whether users can restart, upgrade, and manage extensions 4CAT via the Control Panel"
},
+ "privileges.admin.can_manage_extensions": {
+ "type": UserInput.OPTION_TOGGLE,
+ "default": False,
+ "help": "Can manage extensions",
+ "tooltip": "Controls whether users can install and uninstall 4CAT extensions via the Control Panel"
+ },
"privileges.can_upgrade_to_dev": {
# this is NOT an admin privilege, because all admins automatically
# get all admin privileges! users still need the above privilege
@@ -554,4 +572,5 @@
"dmi-service-manager": "DMI Service Manager",
"ui": "User interface",
"image-visuals": "Image visualization",
+ "extensions": "Extensions"
}
diff --git a/common/lib/module_loader.py b/common/lib/module_loader.py
index a036bd24..cc70f842 100644
--- a/common/lib/module_loader.py
+++ b/common/lib/module_loader.py
@@ -99,6 +99,7 @@ def load_modules(self):
# look for workers and processors in pre-defined folders and datasources
extension_path = Path(config.get('PATH_ROOT'), "extensions")
+ enabled_extensions = [e for e, s in config.get("extensions.enabled").items() if s["enabled"]]
paths = [Path(config.get('PATH_ROOT'), "processors"),
Path(config.get('PATH_ROOT'), "backend", "workers"),
@@ -124,6 +125,9 @@ def load_modules(self):
# This skips processors/datasources that were loaded by others and may not yet be captured
pass
+ if is_extension and len(module_name.split(".")) > 1 and module_name.split(".")[1] not in enabled_extensions:
+ continue
+
# try importing
try:
module = importlib.import_module(module_name)
@@ -221,7 +225,13 @@ def _load_datasource(subdirectory):
# Load extension datasources
# os.walk is used to allow for the possibility of multiple extensions, with nested "datasources" folders
- for root, dirs, files in os.walk(Path(config.get('PATH_ROOT'), "extensions"), followlinks=True):
+ enabled_extensions = [e for e, s in config.get("extensions.enabled").items() if s["enabled"]]
+ extensions_root = Path(config.get('PATH_ROOT'), "extensions")
+ for root, dirs, files in os.walk(extensions_root, followlinks=True):
+ relative_root = Path(root).relative_to(extensions_root)
+ if relative_root.parts and relative_root.parts[0] not in enabled_extensions:
+ continue
+
if "datasources" in dirs:
for subdirectory in Path(root, "datasources").iterdir():
if subdirectory.is_dir():
diff --git a/common/lib/user_input.py b/common/lib/user_input.py
index 63999083..6e10a19b 100644
--- a/common/lib/user_input.py
+++ b/common/lib/user_input.py
@@ -35,6 +35,7 @@ class UserInput:
OPTION_FILE = "file" # file upload
OPTION_HUE = "hue" # colour hue
OPTION_DATASOURCES = "datasources" # data source toggling
+ OPTION_EXTENSIONS = "extensions" # extension toggling
OPTIONS_COSMETIC = (OPTION_INFO, OPTION_DIVIDER)
@@ -143,6 +144,12 @@ def parse_all(options, input, silently_correct=True):
parsed_input[option] = [datasource for datasource, v in datasources.items() if v["enabled"]]
parsed_input[option.split(".")[0] + ".expiration"] = datasources
+ elif settings.get("type") == UserInput.OPTION_EXTENSIONS:
+ # also a special case
+ parsed_input[option] = {extension: {
+ "enabled": f"{option}-enable-{extension}" in input
+ } for extension in input[option].split(",")}
+
elif option not in input:
# not provided? use default
parsed_input[option] = settings.get("default", None)
diff --git a/webtool/static/css/control-panel.css b/webtool/static/css/control-panel.css
index dce8cd1a..15ddfa80 100644
--- a/webtool/static/css/control-panel.css
+++ b/webtool/static/css/control-panel.css
@@ -46,6 +46,10 @@ table.cp-table .actions {
vertical-align: middle;
}
+.extensions-table tr.disabled {
+ color: var(--gray-dark);
+}
+
table td[colspan] {
text-align: center;
}
@@ -89,14 +93,14 @@ table td[colspan] {
max-width: 25em;
}
-.settings .datasource-toggle-form table {
+.settings .broad-toggle-form table {
position: relative;
left: -15em;
background: var(--always-white);
width: calc(100% + 12em);
}
-.settings .datasource-toggle-form input[type=text] {
+.settings .broad-toggle-form input[type=text] {
width: 5em;
}
@@ -260,6 +264,10 @@ article .stats-container h3:not(.blocktitle) {
margin-right: 0.5em;
}
+.log-display.wrapped-log {
+ white-space: pre-line;
+}
+
/**
** Bulk dataset management
*/
diff --git a/webtool/static/css/stylesheet.css b/webtool/static/css/stylesheet.css
index acc4409b..175edbab 100644
--- a/webtool/static/css/stylesheet.css
+++ b/webtool/static/css/stylesheet.css
@@ -192,6 +192,10 @@ fieldset legend {
width: 100%;
}
+.form-notices-wrapper {
+ padding-bottom: 1em;
+}
+
p.form-notice {
text-align: center;
color: var(--accent-alternate);
diff --git a/webtool/templates/components/datasource-option.html b/webtool/templates/components/datasource-option.html
index 4ee4ba16..d46f4b61 100644
--- a/webtool/templates/components/datasource-option.html
+++ b/webtool/templates/components/datasource-option.html
@@ -108,8 +108,31 @@
data-value="{% if settings.value %}{{ settings.value }}{% else %}46{% endif %}" data-update-background="#colour-example-{{ hue_id }}">
+ {% elif settings.type == "extensions" %}
+
{% elif settings.type == "datasources" %}
-