From 4f0cdaf4dced83c3e3836455844606bd779d66b8 Mon Sep 17 00:00:00 2001 From: Brandon Squizzato <35474886+bsquizz@users.noreply.github.com> Date: Tue, 27 Jul 2021 13:01:42 -0400 Subject: [PATCH] Support for importing secrets from local dir (#89) * Support for importing secrets from local dir --- bonfire/bonfire.py | 37 +++++++++++++++++++++- bonfire/config.py | 10 ++---- bonfire/secrets.py | 79 ++++++++++++++++++++++++++++++++++++++++++++++ bonfire/utils.py | 23 ++++++++++++++ 4 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 bonfire/secrets.py diff --git a/bonfire/bonfire.py b/bonfire/bonfire.py index a8cc2f33..b617782b 100755 --- a/bonfire/bonfire.py +++ b/bonfire/bonfire.py @@ -32,6 +32,7 @@ add_base_resources, reconcile, ) +from bonfire.secrets import import_secrets_from_dir log = logging.getLogger(__name__) @@ -696,6 +697,18 @@ def _cmd_process( help="Namespace to deploy to (if none given, bonfire will try to reserve one)", default=None, ) +@click.option( + "--import-secrets", + is_flag=True, + help="Import secrets from local directory at deploy time", + default=False, +) +@click.option( + "--secrets-dir", + type=str, + help="Directory to use for secrets import (default: " "$XDG_CONFIG_HOME/bonfire/secrets/)", + default=conf.DEFAULT_SECRETS_DIR, +) @click.option( "--no-release-on-fail", "-f", @@ -724,11 +737,16 @@ def _cmd_config_deploy( timeout, no_release_on_fail, component_filter, + import_secrets, + secrets_dir, ): """Process app templates and deploy them to a cluster""" requested_ns = namespace used_ns_reservation_system, ns = _get_target_namespace(duration, retries, requested_ns) + if import_secrets: + import_secrets_from_dir(secrets_dir) + if not clowd_env: # if no ClowdEnvironment name provided, see if a ClowdEnvironment is associated with this ns match = find_clowd_env_for_ns(ns) @@ -812,12 +830,29 @@ def _cmd_process_clowdenv(namespace, quay_user, clowd_env, template_file): @main.command("deploy-env") @options(_clowdenv_process_options) +@click.option( + "--import-secrets", + is_flag=True, + help="Import secrets from local directory at deploy time", + default=False, +) +@click.option( + "--secrets-dir", + type=str, + help=("Import secrets from this directory (default: " "$XDG_CONFIG_HOME/bonfire/secrets/)"), + default=conf.DEFAULT_SECRETS_DIR, +) @options(_timeout_option) -def _cmd_deploy_clowdenv(namespace, quay_user, clowd_env, template_file, timeout): +def _cmd_deploy_clowdenv( + namespace, quay_user, clowd_env, template_file, timeout, import_secrets, secrets_dir +): """Process ClowdEnv template and deploy to a cluster""" _warn_if_unsafe(namespace) try: + if import_secrets: + import_secrets_from_dir(secrets_dir) + clowd_env_config = _process_clowdenv(namespace, quay_user, clowd_env, template_file) log.debug("ClowdEnvironment config:\n%s", clowd_env_config) diff --git a/bonfire/config.py b/bonfire/config.py index ff30a902..bceb4438 100644 --- a/bonfire/config.py +++ b/bonfire/config.py @@ -4,11 +4,11 @@ from pkg_resources import resource_filename import re import shutil -import yaml import subprocess from dotenv import load_dotenv +from bonfire.utils import load_file log = logging.getLogger(__name__) @@ -25,6 +25,7 @@ def _get_config_path(): DEFAULT_CONFIG_PATH = _get_config_path().joinpath("config.yaml") DEFAULT_ENV_PATH = _get_config_path().joinpath("env") +DEFAULT_SECRETS_DIR = _get_config_path().joinpath("secrets") DEFAULT_CLOWDENV_TEMPLATE = resource_filename( "bonfire", "resources/local-cluster-clowdenvironment.yaml" ) @@ -59,11 +60,6 @@ def _get_config_path(): ENV_NAME_FORMAT = os.getenv("ENV_NAME_FORMAT", "env-{namespace}") -def _load_file(path): - with path.open() as fp: - return yaml.safe_load(fp) - - def write_default_config(outpath=None): outpath = Path(outpath) if outpath else DEFAULT_CONFIG_PATH outpath.parent.mkdir(mode=0o700, parents=True, exist_ok=True) @@ -99,6 +95,6 @@ def load_config(config_path=None): log.info("default config not found, creating") log.info("using local config file: %s", str(config_path.absolute())) - local_config_data = _load_file(config_path) + local_config_data = load_file(config_path) return local_config_data diff --git a/bonfire/secrets.py b/bonfire/secrets.py new file mode 100644 index 00000000..3301f214 --- /dev/null +++ b/bonfire/secrets.py @@ -0,0 +1,79 @@ +""" +Handles importing of secrets from a local directory +""" +import glob +import json +import logging +import os + +from bonfire.openshift import oc, get_json +from bonfire.utils import load_file + + +log = logging.getLogger(__name__) + + +def _parse_secret_file(path): + """ + Return a dict of all secrets in a file with key: secret name, val: parsed secret json/yaml + The file can contain 1 secret, or a list of secrets + """ + content = load_file(path) + secrets = {} + if content.get("kind").lower() == "list": + items = content.get("items", []) + else: + items = [content] + + for item in items: + if item.get("kind").lower() == "secret": + try: + secrets[item["metadata"]["name"]] = item + except KeyError: + raise ValueError("Secret at path '{}' has no metadata/name".format(path)) + + return secrets + + +def _get_files_in_dir(path): + """ + Get a list of all .yml/.yaml/.json files in a dir + """ + files = list(glob.glob(os.path.join(path, "*.yaml"))) + files.extend(list(glob.glob(os.path.join(path, "*.yml")))) + files.extend(list(glob.glob(os.path.join(path, "*.json")))) + return files + + +def _import_secret(secret_name, secret_data): + # get existing secret in the ns (if it exists) + current_secret = get_json("secret", secret_name) or {} + + # avoid race conditions when running multiple processes by comparing the data + if current_secret.get("data") != secret_data.get("data"): + log.info("replacing secret '%s' using local copy", secret_name) + # delete from dst ns so that applying 'null' values will work + oc("delete", "--ignore-not-found", "secret", secret_name, _silent=True) + oc("apply", "-f", "-", _silent=True, _in=json.dumps(secret_data)) + + +def import_secrets_from_dir(path): + if not os.path.exists(path): + raise ValueError(f"secrets directory not found: {path}") + + if not os.path.isdir(path): + raise ValueError(f"invalid secrets directory: {path}") + + files = _get_files_in_dir(path) + secrets = {} + log.info("importing secrets from local path: %s", path) + for secret_file in files: + secrets_in_file = _parse_secret_file(secret_file) + log.info("loaded %d secret(s) from file '%s'", len(secrets_in_file), secret_file) + for secret_name in secrets_in_file: + if secret_name in secrets: + raise ValueError(f"secret with name '{secret_name}' defined twice in secrets dir") + secrets.update(secrets_in_file) + + for secret_name, secret_data in secrets.items(): + _import_secret(secret_name, secret_data) diff --git a/bonfire/utils.py b/bonfire/utils.py index f07d8f0f..d7447fe9 100644 --- a/bonfire/utils.py +++ b/bonfire/utils.py @@ -1,4 +1,5 @@ import atexit +import json import logging import os import re @@ -6,6 +7,7 @@ import shlex import subprocess import tempfile +import yaml from cached_property import cached_property @@ -286,3 +288,24 @@ def _fetch_local(self, repo_dir=None): p = os.path.join(repo_dir, self.path.lstrip("/")) with open(p) as fp: return commit, fp.read() + + +def load_file(path): + """Load a .json/.yml/.yaml file.""" + if not os.path.isfile(path): + raise ValueError("Path '{}' is not a file or does not exist".format(path)) + + _, file_ext = os.path.splitext(path) + + with open(path, "rb") as f: + if file_ext == ".yaml" or file_ext == ".yml": + content = yaml.safe_load(f) + elif file_ext == ".json": + content = json.load(f) + else: + raise ValueError("File '{}' must be a YAML or JSON file".format(path)) + + if not content: + raise ValueError("File '{}' is empty!".format(path)) + + return content