Option for local/remote template processing (#168)

* Allow remote processing of oc templates

* Fix passing 'local' option to ns commands

bonfire/bonfire.py

@@ -54,6 +54,13 @@
 LOCAL_SRC = "local"
 NO_RESERVATION_SYS = "this cluster does not use a namespace reservation system"
 
+_local_option = click.option(
+    "--local",
+    help="Whether 'oc process' uses --local=true or --local=false (default: true)",
+    type=bool,
+    default=True,
+)
+
 
 def _error(msg):
     click.echo(f"ERROR: {msg}", err=True)
@@ -201,6 +208,7 @@ def _validate_reservation_duration(ctx, param, value):
         help="Duration of the reservation",
         callback=_validate_reservation_duration,
     ),
+    _local_option,
 ]
 
 
@@ -340,7 +348,6 @@ def _validate_resource_arguments(ctx, param, value):
     ),
 ]
 
-
 _process_options = [
     click.argument(
         "app_names",
@@ -426,6 +433,7 @@ def _validate_resource_arguments(ctx, param, value):
         type=str,
         multiple=True,
     ),
+    _local_option,
 ]
 
 
@@ -459,6 +467,7 @@ def _validate_resource_arguments(ctx, param, value):
         type=str,
         default=None,
     ),
+    _local_option,
 ]
 
 
@@ -538,6 +547,7 @@ def _validate_resource_arguments(ctx, param, value):
         type=str,
         default="",
     ),
+    _local_option,
 ]
 
 
@@ -594,7 +604,7 @@ def _list_namespaces(available, mine, output):
 @options(_ns_reserve_options)
 @options(_timeout_option)
 @click_exception_wrapper("namespace reserve")
-def _cmd_namespace_reserve(name, requester, duration, timeout):
+def _cmd_namespace_reserve(name, requester, duration, timeout, local):
     """Reserve an ephemeral namespace"""
     log.info("Attempting to reserve a namespace...")
     if not has_ns_operator():
@@ -606,7 +616,7 @@ def _cmd_namespace_reserve(name, requester, duration, timeout):
     if check_for_existing_reservation(requester):
         _warn_of_existing(requester)
 
-    ns = reserve_namespace(name, requester, duration, timeout)
+    ns = reserve_namespace(name, requester, duration, timeout, local)
 
     click.echo(ns.name)
 
@@ -620,16 +630,17 @@ def _cmd_namespace_reserve(name, requester, duration, timeout):
     default=False,
     help="Do not check if you own this namespace",
 )
+@options([_local_option])
 @click_exception_wrapper("namespace release")
-def _cmd_namespace_release(namespace, force):
+def _cmd_namespace_release(namespace, force, local):
     """Remove reservation from an ephemeral namespace"""
     if not has_ns_operator():
         _error(NO_RESERVATION_SYS)
 
     if not force:
         _warn_before_delete()
 
-    release_namespace(namespace)
+    release_namespace(namespace, local)
 
 
 @namespace.command("extend")
@@ -642,13 +653,14 @@ def _cmd_namespace_release(namespace, force):
     help="Amount of time to extend the reservation",
     callback=_validate_reservation_duration,
 )
+@options([_local_option])
 @click_exception_wrapper("namespace extend")
-def _cmd_namespace_extend(namespace, duration):
+def _cmd_namespace_extend(namespace, duration, local):
     """Extend a reservation of an ephemeral namespace"""
     if not has_ns_operator():
         _error(NO_RESERVATION_SYS)
 
-    extend_namespace(namespace, duration)
+    extend_namespace(namespace, duration, local)
 
 
 @namespace.command("wait-on-resources")
@@ -724,6 +736,7 @@ def _process(
     no_remove_resources,
     single_replicas,
     component_filter,
+    local,
 ):
     apps_config = _get_apps_config(source, target_env, ref_env, local_config_path)
 
@@ -739,6 +752,7 @@ def _process(
         no_remove_resources,
         single_replicas,
         component_filter,
+        local,
     )
     return processor.process()
 
@@ -767,6 +781,7 @@ def _cmd_process(
     no_remove_resources,
     single_replicas,
     component_filter,
+    local,
 ):
     """Fetch and process application templates"""
     clowd_env = _get_env_name(namespace, clowd_env)
@@ -786,11 +801,12 @@ def _cmd_process(
         no_remove_resources,
         single_replicas,
         component_filter,
+        local,
     )
     print(json.dumps(processed_templates, indent=2))
 
 
-def _get_namespace(requested_ns_name, name, requester, duration, timeout):
+def _get_namespace(requested_ns_name, name, requester, duration, timeout, local):
     reserved_new_ns = False
 
     if not has_ns_operator():
@@ -824,7 +840,7 @@ def _get_namespace(requested_ns_name, name, requester, duration, timeout):
             requester = requester if requester else _get_requester()
             if check_for_existing_reservation(requester):
                 _warn_of_existing(requester)
-            ns = reserve_namespace(name, requester, duration, timeout)
+            ns = reserve_namespace(name, requester, duration, timeout, local)
             reserved_new_ns = True
 
     return ns.name, reserved_new_ns
@@ -881,12 +897,13 @@ def _cmd_config_deploy(
     component_filter,
     import_secrets,
     secrets_dir,
+    local,
 ):
     """Process app templates and deploy them to a cluster"""
     if not has_clowder():
         _error("cluster does not have clowder operator installed")
 
-    ns, reserved_new_ns = _get_namespace(namespace, name, requester, duration, timeout)
+    ns, reserved_new_ns = _get_namespace(namespace, name, requester, duration, timeout, local)
 
     if import_secrets:
         import_secrets_from_dir(secrets_dir)
@@ -933,6 +950,7 @@ def _err_handler(err):
             no_remove_resources,
             single_replicas,
             component_filter,
+            local,
         )
         log.debug("app configs:\n%s", json.dumps(apps_config, indent=2))
         if not apps_config["items"]:
@@ -959,16 +977,16 @@ def _err_handler(err):
         click.echo(ns)
 
 
-def _process_clowdenv(target_namespace, quay_user, env_name, template_file):
+def _process_clowdenv(target_namespace, quay_user, env_name, template_file, local):
     env_name = _get_env_name(target_namespace, env_name)
-    return process_clowd_env(target_namespace, quay_user, env_name, template_file)
+    return process_clowd_env(target_namespace, quay_user, env_name, template_file, local)
 
 
 @main.command("process-env")
 @options(_clowdenv_process_options)
-def _cmd_process_clowdenv(namespace, quay_user, clowd_env, template_file):
+def _cmd_process_clowdenv(namespace, quay_user, clowd_env, template_file, local):
     """Process ClowdEnv template and print output"""
-    clowd_env_config = _process_clowdenv(namespace, quay_user, clowd_env, template_file)
+    clowd_env_config = _process_clowdenv(namespace, quay_user, clowd_env, template_file, local)
     print(json.dumps(clowd_env_config, indent=2))
 
 
@@ -1000,17 +1018,18 @@ def _cmd_deploy_clowdenv(
     name,
     requester,
     duration,
+    local,
 ):
     """Process ClowdEnv template and deploy to a cluster"""
     if not has_clowder():
         _error("cluster does not have clowder operator installed")
 
-    namespace, _ = _get_namespace(namespace, name, requester, duration, timeout)
+    namespace, _ = _get_namespace(namespace, name, requester, duration, timeout, local)
 
     if import_secrets:
         import_secrets_from_dir(secrets_dir)
 
-    clowd_env_config = _process_clowdenv(namespace, quay_user, clowd_env, template_file)
+    clowd_env_config = _process_clowdenv(namespace, quay_user, clowd_env, template_file, local)
 
     log.debug("ClowdEnvironment config:\n%s", clowd_env_config)
 
@@ -1043,6 +1062,7 @@ def _cmd_process_iqe_cji(
     requirements,
     requirements_priority,
     test_importance,
+    local,
 ):
     """Process IQE ClowdJobInvocation template and print output"""
     cji_config = process_iqe_cji(
@@ -1057,6 +1077,7 @@ def _cmd_process_iqe_cji(
         requirements,
         requirements_priority,
         test_importance,
+        local,
     )
     print(json.dumps(cji_config, indent=2))
 
@@ -1084,12 +1105,13 @@ def _cmd_deploy_iqe_cji(
     name,
     requester,
     duration,
+    local,
 ):
     """Process IQE CJI template, apply it, and wait for it to start running."""
     if not has_clowder():
         _error("cluster does not have clowder operator installed")
 
-    namespace, _ = _get_namespace(namespace, name, requester, duration, timeout)
+    namespace, _ = _get_namespace(namespace, name, requester, duration, timeout, local)
 
     cji_config = process_iqe_cji(
         clowd_app_name,
@@ -1103,6 +1125,7 @@ def _cmd_deploy_iqe_cji(
         requirements,
         requirements_priority,
         test_importance,
+        local,
     )
 
     log.debug("processed CJI config:\n%s", cji_config)

bonfire/namespaces.py

@@ -237,13 +237,13 @@ def get_namespaces(available=False, mine=False):
     return ephemeral_namespaces
 
 
-def reserve_namespace(name, requester, duration, timeout):
+def reserve_namespace(name, requester, duration, timeout, local=True):
     res = get_reservation(name)
     # Name should be unique on reservation creation.
     if res:
         raise FatalError(f"Reservation with name {name} already exists")
 
-    res_config = process_reservation(name, requester, duration)
+    res_config = process_reservation(name, requester, duration, local=local)
 
     log.debug("processed reservation:\n%s", res_config)
 
@@ -268,13 +268,14 @@ def reserve_namespace(name, requester, duration, timeout):
     return Namespace(name=ns_name)
 
 
-def release_namespace(namespace):
+def release_namespace(namespace, local=True):
     res = get_reservation(namespace=namespace)
     if res:
         res_config = process_reservation(
             res["metadata"]["name"],
             res["spec"]["requester"],
             "0s",  # on release set duration to 0s
+            local=local,
         )
 
         apply_config(None, list_resource=res_config)
@@ -283,7 +284,7 @@ def release_namespace(namespace):
         raise FatalError("Reservation lookup failed")
 
 
-def extend_namespace(namespace, duration):
+def extend_namespace(namespace, duration, local=True):
     res = get_reservation(namespace=namespace)
     if res:
         if res.get("status", {}).get("state") == "expired":
@@ -300,6 +301,7 @@ def extend_namespace(namespace, duration):
             res["metadata"]["name"],
             res["spec"]["requester"],
             _duration_fmt(new_duration),
+            local=local,
         )
 
         log.debug("processed reservation:\n%s", res_config)

bonfire/openshift.py

@@ -664,12 +664,16 @@ def copy_namespace_secrets(src_namespace, dst_namespace, secret_names):
         )
 
 
-def process_template(template_data, params):
+def process_template(template_data, params, local=True):
     valid_pnames = set(p["name"] for p in template_data.get("parameters", []))
     param_str = " ".join(f"-p {k}='{v}'" for k, v in params.items() if k in valid_pnames)
+    local_str = str(local).lower()
+
+    cmd = f"oc process --local={local_str} --ignore-unknown-parameters -o json -f - {param_str}"
+    log.debug("running: %s", cmd)
 
     proc = Popen(
-        f"oc process --local --ignore-unknown-parameters -o json -f - {param_str}",
+        cmd,
         shell=True,
         stdin=PIPE,
         stdout=PIPE,