Support for importing secrets from local dir (#89)

* Support for importing secrets from local dir

bonfire/bonfire.py

@@ -32,6 +32,7 @@
     add_base_resources,
     reconcile,
 )
+from bonfire.secrets import import_secrets_from_dir
 
 log = logging.getLogger(__name__)
 
@@ -696,6 +697,18 @@ def _cmd_process(
     help="Namespace to deploy to (if none given, bonfire will try to reserve one)",
     default=None,
 )
+@click.option(
+    "--import-secrets",
+    is_flag=True,
+    help="Import secrets from local directory at deploy time",
+    default=False,
+)
+@click.option(
+    "--secrets-dir",
+    type=str,
+    help="Directory to use for secrets import (default: " "$XDG_CONFIG_HOME/bonfire/secrets/)",
+    default=conf.DEFAULT_SECRETS_DIR,
+)
 @click.option(
     "--no-release-on-fail",
     "-f",
@@ -724,11 +737,16 @@ def _cmd_config_deploy(
     timeout,
     no_release_on_fail,
     component_filter,
+    import_secrets,
+    secrets_dir,
 ):
     """Process app templates and deploy them to a cluster"""
     requested_ns = namespace
     used_ns_reservation_system, ns = _get_target_namespace(duration, retries, requested_ns)
 
+    if import_secrets:
+        import_secrets_from_dir(secrets_dir)
+
     if not clowd_env:
         # if no ClowdEnvironment name provided, see if a ClowdEnvironment is associated with this ns
         match = find_clowd_env_for_ns(ns)
@@ -812,12 +830,29 @@ def _cmd_process_clowdenv(namespace, quay_user, clowd_env, template_file):
 
 @main.command("deploy-env")
 @options(_clowdenv_process_options)
+@click.option(
+    "--import-secrets",
+    is_flag=True,
+    help="Import secrets from local directory at deploy time",
+    default=False,
+)
+@click.option(
+    "--secrets-dir",
+    type=str,
+    help=("Import secrets from this directory (default: " "$XDG_CONFIG_HOME/bonfire/secrets/)"),
+    default=conf.DEFAULT_SECRETS_DIR,
+)
 @options(_timeout_option)
-def _cmd_deploy_clowdenv(namespace, quay_user, clowd_env, template_file, timeout):
+def _cmd_deploy_clowdenv(
+    namespace, quay_user, clowd_env, template_file, timeout, import_secrets, secrets_dir
+):
     """Process ClowdEnv template and deploy to a cluster"""
     _warn_if_unsafe(namespace)
 
     try:
+        if import_secrets:
+            import_secrets_from_dir(secrets_dir)
+
         clowd_env_config = _process_clowdenv(namespace, quay_user, clowd_env, template_file)
 
         log.debug("ClowdEnvironment config:\n%s", clowd_env_config)

bonfire/config.py

@@ -4,11 +4,11 @@
 from pkg_resources import resource_filename
 import re
 import shutil
-import yaml
 import subprocess
 
 from dotenv import load_dotenv
 
+from bonfire.utils import load_file
 
 log = logging.getLogger(__name__)
 
@@ -25,6 +25,7 @@ def _get_config_path():
 
 DEFAULT_CONFIG_PATH = _get_config_path().joinpath("config.yaml")
 DEFAULT_ENV_PATH = _get_config_path().joinpath("env")
+DEFAULT_SECRETS_DIR = _get_config_path().joinpath("secrets")
 DEFAULT_CLOWDENV_TEMPLATE = resource_filename(
     "bonfire", "resources/local-cluster-clowdenvironment.yaml"
 )
@@ -59,11 +60,6 @@ def _get_config_path():
 ENV_NAME_FORMAT = os.getenv("ENV_NAME_FORMAT", "env-{namespace}")
 
 
-def _load_file(path):
-    with path.open() as fp:
-        return yaml.safe_load(fp)
-
-
 def write_default_config(outpath=None):
     outpath = Path(outpath) if outpath else DEFAULT_CONFIG_PATH
     outpath.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
@@ -99,6 +95,6 @@ def load_config(config_path=None):
                 log.info("default config not found, creating")
 
     log.info("using local config file: %s", str(config_path.absolute()))
-    local_config_data = _load_file(config_path)
+    local_config_data = load_file(config_path)
 
     return local_config_data

bonfire/secrets.py

@@ -0,0 +1,79 @@
+"""
+Handles importing of secrets from a local directory
+"""
+import glob
+import json
+import logging
+import os
+
+from bonfire.openshift import oc, get_json
+from bonfire.utils import load_file
+
+
+log = logging.getLogger(__name__)
+
+
+def _parse_secret_file(path):
+    """
+    Return a dict of all secrets in a file with key: secret name, val: parsed secret json/yaml
+    The file can contain 1 secret, or a list of secrets
+    """
+    content = load_file(path)
+    secrets = {}
+    if content.get("kind").lower() == "list":
+        items = content.get("items", [])
+    else:
+        items = [content]
+
+    for item in items:
+        if item.get("kind").lower() == "secret":
+            try:
+                secrets[item["metadata"]["name"]] = item
+            except KeyError:
+                raise ValueError("Secret at path '{}' has no metadata/name".format(path))
+
+    return secrets
+
+
+def _get_files_in_dir(path):
+    """
+    Get a list of all .yml/.yaml/.json files in a dir
+    """
+    files = list(glob.glob(os.path.join(path, "*.yaml")))
+    files.extend(list(glob.glob(os.path.join(path, "*.yml"))))
+    files.extend(list(glob.glob(os.path.join(path, "*.json"))))
+    return files
+
+
+def _import_secret(secret_name, secret_data):
+    # get existing secret in the ns (if it exists)
+    current_secret = get_json("secret", secret_name) or {}
+
+    # avoid race conditions when running multiple processes by comparing the data
+    if current_secret.get("data") != secret_data.get("data"):
+        log.info("replacing secret '%s' using local copy", secret_name)
+        # delete from dst ns so that applying 'null' values will work
+        oc("delete", "--ignore-not-found", "secret", secret_name, _silent=True)
+        oc("apply", "-f", "-", _silent=True, _in=json.dumps(secret_data))
+
+
+def import_secrets_from_dir(path):
+    if not os.path.exists(path):
+        raise ValueError(f"secrets directory not found: {path}")
+
+    if not os.path.isdir(path):
+        raise ValueError(f"invalid secrets directory: {path}")
+
+    files = _get_files_in_dir(path)
+    secrets = {}
+    log.info("importing secrets from local path: %s", path)
+    for secret_file in files:
+        secrets_in_file = _parse_secret_file(secret_file)
+        log.info("loaded %d secret(s) from file '%s'", len(secrets_in_file), secret_file)
+        for secret_name in secrets_in_file:
+            if secret_name in secrets:
+                raise ValueError(f"secret with name '{secret_name}' defined twice in secrets dir")
+        secrets.update(secrets_in_file)
+
+    for secret_name, secret_data in secrets.items():
+        _import_secret(secret_name, secret_data)

bonfire/utils.py

@@ -1,11 +1,13 @@
 import atexit
+import json
 import logging
 import os
 import re
 import requests
 import shlex
 import subprocess
 import tempfile
+import yaml
 
 from cached_property import cached_property
 
@@ -286,3 +288,24 @@ def _fetch_local(self, repo_dir=None):
         p = os.path.join(repo_dir, self.path.lstrip("/"))
         with open(p) as fp:
             return commit, fp.read()
+
+
+def load_file(path):
+    """Load a .json/.yml/.yaml file."""
+    if not os.path.isfile(path):
+        raise ValueError("Path '{}' is not a file or does not exist".format(path))
+
+    _, file_ext = os.path.splitext(path)
+
+    with open(path, "rb") as f:
+        if file_ext == ".yaml" or file_ext == ".yml":
+            content = yaml.safe_load(f)
+        elif file_ext == ".json":
+            content = json.load(f)
+        else:
+            raise ValueError("File '{}' must be a YAML or JSON file".format(path))
+
+    if not content:
+        raise ValueError("File '{}' is empty!".format(path))
+
+    return content