Extension management

backend/workers/manage_extension.py

@@ -0,0 +1,266 @@
+"""
+Manage a 4CAT extension
+"""
+import subprocess
+import requests
+import logging
+import zipfile
+import shutil
+import shlex
+import json
+import ural
+import os
+import re
+
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+
+from backend.lib.worker import BasicWorker
+from common.config_manager import config
+
+
+class ExtensionManipulator(BasicWorker):
+    """
+    Manage 4CAT extensions
+
+    4CAT extensions are essentially git repositories. This worker can clone the
+    relevant git repository or delete it and clean up after it.
+
+    This is done in a worker instead of in the front-end code because cloning
+    a large git repository can take some time so it is best to do it
+    asynchronously. This is also future-proof in that it is easy to add support
+    for installation code etc here later.
+
+    Results are logged to a separate log file that can then be inspected in the
+    web interface.
+    """
+    type = "manage-extension"
+    max_workers = 1
+
+    def work(self):
+        """
+        Do something with extensions
+        """
+        extension_reference = self.job.data["remote_id"]
+        task = self.job.details.get("task")
+
+        # note that this is a databaseless config reader
+        # since we only need it for file paths
+        self.config = config
+
+        # this worker uses its own log file instead of the main 4CAT log
+        # this is so that it is easier to monitor error messages about failed
+        # installations etc and display those separately in e.g. the web
+        # interface
+
+        log_file = Path(self.config.get("PATH_ROOT")).joinpath(self.config.get("PATH_LOGS")).joinpath("extensions.log")
+        logger = logging.getLogger(self.type)
+        if not logger.handlers:
+            handler = RotatingFileHandler(log_file, backupCount=1, maxBytes=50000)
+            handler.level = logging.INFO
+            handler.setFormatter(logging.Formatter("%(asctime)-15s | %(levelname)s: %(message)s",
+                                                   "%d-%m-%Y %H:%M:%S"))
+            logger.addHandler(handler)
+        logger.level = logging.INFO
+        self.extension_log = logger
+
+        if task == "install":
+            self.install_extension(extension_reference)
+        elif task == "uninstall":
+            self.uninstall_extension(extension_reference)
+
+        self.job.finish()
+
+    def uninstall_extension(self, extension_name):
+        """
+        Remove extension
+
+        Currently as simple as deleting the folder, but could add further
+        cleaning up code later.
+
+        While an extension can define configuration settings, we do not
+        explicitly remove these here. 4CAT has general cleanup code for
+        unreferenced settings and it may be beneficial to keep them in case
+        the extension is re-installed later.
+
+        :param str extension_name:  ID of the extension (i.e. name of the
+        folder it is in)
+        """
+        extensions_root = self.config.get("PATH_ROOT").joinpath("extensions")
+        target_folder = extensions_root.joinpath(extension_name)
+
+        if not target_folder.exists():
+            return self.extension_log.error(f"Extension {extension_name} does not exist - cannot remove it.")
+
+        try:
+            shutil.rmtree(target_folder)
+            self.extension_log.info(f"Finished uninstalling extension {extension_name}.")
+        except OSError as e:
+            self.extension_log.error(f"Could not uninstall extension {extension_name}. There may be an issue with "
+                                     f"file privileges, or the extension is installed via a symbolic link which 4CAT "
+                                     f"cannot manipulate. The system error message was: '{e}'")
+
+    def install_extension(self, repository_reference, overwrite=False):
+        """
+        Install a 4CAT extension
+
+        4CAT extensions can be installed from a git URL or a zip archive. In
+        either case, the files are first put into a temporary folder, after
+        which the manifest in that folder is read to complete installation.
+
+        :param str repository_reference:  Git repository URL, or zip archive
+        path.
+        :param bool overwrite:  Overwrite extension if one exists? Set to
+        `true` to upgrade existing extensions (for example)
+        """
+        if self.job.details.get("source") == "remote":
+            extension_folder, extension_name = self.clone_from_url(repository_reference)
+        else:
+            extension_folder, extension_name = self.unpack_from_zip(repository_reference)
+
+        if not extension_name:
+            return self.extension_log.error("The 4CAT extension could not be installed.")
+
+        # read manifest file
+        manifest_file = extension_folder.joinpath("metadata.json")
+        if not manifest_file.exists():
+            shutil.rmtree(extension_folder)
+            return self.extension_log.error(f"Manifest file of newly cloned 4CAT extension {repository_reference} does "
+                                            f"not exist. Cannot install as a 4CAT extension.")
+        else:
+            try:
+                with manifest_file.open() as infile:
+                    manifest_data = json.load(infile)
+            except json.JSONDecodeError:
+                shutil.rmtree(extension_folder)
+                return self.extension_log.error(f"Manifest file of newly cloned 4CAT extension {repository_reference} "
+                                                f"could not be parsed. Cannot install as a 4CAT extension.")
+
+        canonical_name = manifest_data.get("name", extension_name)
+        canonical_id = manifest_data.get("id", extension_name)
+
+        canonical_folder = extension_folder.with_name(canonical_id)
+        existing_name = canonical_id
+        existing_version = "unknown"
+
+        if canonical_folder.exists():
+            if canonical_folder.joinpath("metadata.json").exists():
+                with canonical_folder.joinpath("metadata.json").open() as infile:
+                    try:
+                        existing_manifest = json.load(infile)
+                        existing_name = existing_manifest.get("name", canonical_id)
+                        existing_version = existing_manifest.get("version", "unknown")
+                    except json.JSONDecodeError:
+                        pass
+
+            shutil.rmtree(canonical_folder)
+            if overwrite:
+                self.extension_log.warning(f"Uninstalling existing 4CAT extension {existing_name} (version "
+                                           f"{existing_version}.")
+            else:
+                return self.extension_log.error(f"An extension with ID {canonical_id} is already installed "
+                                                f"({extension_name}, version {existing_version}). Cannot install "
+                                                f"another one with the same ID - uninstall it first.")
+
+        extension_folder.rename(canonical_folder)
+        version = f"version {manifest_data.get('version', 'unknown')}"
+        self.extension_log.info(f"Finished installing extension {canonical_name} (version {version}) with ID "
+                                f"{canonical_id}.")
+
+
+    def unpack_from_zip(self, archive_path):
+        """
+        Unpack extension files from a zip archive
+
+        Pretty straightforward - Make a temporary folder and extract the zip
+        archive's contents into it.
+
+        :param str archive_path: Path to the zip file to extract
+        :return tuple:  Tuple of folder and extension name, or `None, None` on
+        failure.
+        """
+        archive_path = Path(archive_path)
+        if not archive_path.exists():
+            return self.extension_log.error(f"Extension file does not exist at {archive_path} - cannot install."), None
+
+        extension_name = archive_path.stem
+        extensions_root = self.config.get("PATH_ROOT").joinpath("extensions")
+        temp_name = self.get_temporary_folder(extensions_root)
+        try:
+            with zipfile.ZipFile(archive_path, "r") as archive_file:
+                archive_file.extractall(temp_name)
+        except Exception as e:
+            return self.extension_log.error(f"Could not extract extension zip archive {archive_path.name}: {e}. Cannot "
+                                            f"install."), None
+        finally:
+            archive_path.unlink()
+
+        return temp_name, extension_name
+
+
+    def clone_from_url(self, repository_url):
+        """
+        Clone the extension files from a git repository URL
+
+        :param str repository_url:  Git repository URL to clone extension from
+        :return tuple:  Tuple of folder and extension name, or `None, None` on
+        failure.
+        """
+        # we only know how to install extensions from URLs for now
+        if not ural.is_url(repository_url):
+            return self.extension_log.error(f"Cannot install 4CAT extension - invalid repository url: "
+                                            f"{repository_url}"), None
+
+        # normalize URL and extract name
+        repository_url = repository_url.strip().split("#")[-1]
+        if repository_url.endswith("/"):
+            repository_url = repository_url[:-1]
+        repository_url_name = re.sub(r"\.git$", "", repository_url.split("/")[-1].split("?")[0].lower())
+
+        try:
+            test_url = requests.head(repository_url)
+            if test_url.status_code >= 400:
+                return self.extension_log.error(
+                    f"Cannot install 4CAT extension - the repository URL is unreachable (status code "
+                    f"{test_url.status_code})"), None
+        except requests.RequestException as e:
+            return self.extension_log.error(
+                f"Cannot install 4CAT extension - the repository URL seems invalid or unreachable ({e})"), None
+
+        # ok, we have a valid URL that is reachable - try cloning from it
+        extensions_root = self.config.get("PATH_ROOT").joinpath("extensions")
+        os.chdir(extensions_root)
+
+        temp_name = self.get_temporary_folder(extensions_root)
+
+        extension_folder = extensions_root.joinpath(temp_name)
+        clone_command = f"git clone {shlex.quote(repository_url)} {temp_name}"
+        clone_outcome = subprocess.run(shlex.split(clone_command), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        cloned_correctly = True
+        if clone_outcome.returncode != 0:
+            cloned_correctly = False
+            self.extension_log.info(clone_outcome.stdout.decode("utf-8"))
+            self.extension_log.error(f"Could not clone 4CAT extension repository from {repository_url} - see log for "
+                                     f"details.")
+
+        if not cloned_correctly:
+            if extension_folder.exists():
+                shutil.rmtree(extension_folder)
+            return self.extension_log.error(f"4CAT extension {repository_url} was not installed."), None
+
+        return extension_folder, repository_url_name
+
+
+    def get_temporary_folder(self, extensions_root):
+        # clone into a temporary folder, which we will rename as needed
+        # this is because the repository name is not necessarily the extension
+        # name
+        temp_base = "new-extension"
+        temp_name = temp_base
+        temp_index = 0
+        while extensions_root.joinpath(temp_name).exists():
+            temp_index += 1
+            temp_name = f"{temp_base}-{temp_index}"
+
+        return extensions_root.joinpath(temp_name)

common/lib/config_definition.py

@@ -37,6 +37,18 @@
                    "setting.",
         "indirect": True
     },
+    # Extensions
+    "extensions._intro": {
+        "type": UserInput.OPTION_INFO,
+        "help": "4CAT extensions can be disabled and disabled via the control below. When enabled, extensions may "
+                "define further settings that can typically be configured via the extension's tab on the left side of "
+                "this page. **Note that 4CAT needs to be restarted for this to take effect!**"
+    },
+    "extensions.enabled": {
+        "type": UserInput.OPTION_EXTENSIONS,
+        "default": {},
+        "help": "Extensions"
+    },
     # Configure how the tool is to be named in its web interface. The backend will
     # always refer to "4CAT" - the name of the software, and a "powered by 4CAT"
     # notice may also show up in the web interface regardless of the value entered here.
@@ -149,6 +161,12 @@
         "help": "Can restart/upgrade",
         "tooltip": "Controls whether users can restart, upgrade, and manage extensions 4CAT via the Control Panel"
     },
+    "privileges.admin.can_manage_extensions": {
+        "type": UserInput.OPTION_TOGGLE,
+        "default": False,
+        "help": "Can manage extensions",
+        "tooltip": "Controls whether users can install and uninstall 4CAT extensions via the Control Panel"
+    },
     "privileges.can_upgrade_to_dev": {
         # this is NOT an admin privilege, because all admins automatically
         # get all admin privileges! users still need the above privilege
@@ -554,4 +572,5 @@
     "dmi-service-manager": "DMI Service Manager",
     "ui": "User interface",
     "image-visuals": "Image visualization",
+    "extensions": "Extensions"
 }

common/lib/module_loader.py

@@ -99,6 +99,7 @@ def load_modules(self):
         # look for workers and processors in pre-defined folders and datasources
 
         extension_path = Path(config.get('PATH_ROOT'), "extensions")
+        enabled_extensions = [e for e, s in config.get("extensions.enabled").items() if s["enabled"]]
 
         paths = [Path(config.get('PATH_ROOT'), "processors"),
                  Path(config.get('PATH_ROOT'), "backend", "workers"),
@@ -124,6 +125,9 @@ def load_modules(self):
                     # This skips processors/datasources that were loaded by others and may not yet be captured
                     pass
 
+                if is_extension and len(module_name.split(".")) > 1 and module_name.split(".")[1] not in enabled_extensions:
+                    continue
+
                 # try importing
                 try:
                     module = importlib.import_module(module_name)
@@ -221,7 +225,13 @@ def _load_datasource(subdirectory):
 
         # Load extension datasources
         # os.walk is used to allow for the possibility of multiple extensions, with nested "datasources" folders
-        for root, dirs, files in os.walk(Path(config.get('PATH_ROOT'), "extensions"), followlinks=True):
+        enabled_extensions = [e for e, s in config.get("extensions.enabled").items() if s["enabled"]]
+        extensions_root = Path(config.get('PATH_ROOT'), "extensions")
+        for root, dirs, files in os.walk(extensions_root, followlinks=True):
+            relative_root = Path(root).relative_to(extensions_root)
+            if relative_root.parts and relative_root.parts[0] not in enabled_extensions:
+                continue
+
             if "datasources" in dirs:
                 for subdirectory in Path(root, "datasources").iterdir():
                     if subdirectory.is_dir():

common/lib/user_input.py

@@ -35,6 +35,7 @@ class UserInput:
     OPTION_FILE = "file"  # file upload
     OPTION_HUE = "hue"  # colour hue
     OPTION_DATASOURCES = "datasources"  # data source toggling
+    OPTION_EXTENSIONS = "extensions"  # extension toggling
 
     OPTIONS_COSMETIC = (OPTION_INFO, OPTION_DIVIDER)
 
@@ -143,6 +144,12 @@ def parse_all(options, input, silently_correct=True):
                 parsed_input[option] = [datasource for datasource, v in datasources.items() if v["enabled"]]
                 parsed_input[option.split(".")[0] + ".expiration"] = datasources
 
+            elif settings.get("type") == UserInput.OPTION_EXTENSIONS:
+                # also a special case
+                parsed_input[option] = {extension: {
+                    "enabled": f"{option}-enable-{extension}" in input
+                } for extension in input[option].split(",")}
+
             elif option not in input:
                 # not provided? use default
                 parsed_input[option] = settings.get("default", None)

webtool/static/css/control-panel.css

@@ -46,6 +46,10 @@ table.cp-table .actions {
     vertical-align: middle;
 }
 
+.extensions-table tr.disabled {
+    color: var(--gray-dark);
+}
+
 table td[colspan] {
     text-align: center;
 }
@@ -89,14 +93,14 @@ table td[colspan] {
     max-width: 25em;
 }
 
-.settings .datasource-toggle-form table {
+.settings .broad-toggle-form table {
     position: relative;
     left: -15em;
     background: var(--always-white);
     width: calc(100% + 12em);
 }
 
-.settings .datasource-toggle-form input[type=text] {
+.settings .broad-toggle-form input[type=text] {
     width: 5em;
 }
 
@@ -260,6 +264,10 @@ article .stats-container h3:not(.blocktitle) {
     margin-right: 0.5em;
 }
 
+.log-display.wrapped-log {
+    white-space: pre-line;
+}
+
 /**
  ** Bulk dataset management
  */