From efa8749e9b1c0bb513d8a6073014cef441a83273 Mon Sep 17 00:00:00 2001 From: jordy Date: Thu, 24 Feb 2022 14:32:54 +0100 Subject: [PATCH 1/3] succesfully added deployment-run-script functionality, time for testing --- contents/common.py | 44 ++++++++- contents/deployment-run-script.py | 152 ++++++++++++++++++++++++++++++ contents/pods-run-script.py | 13 +-- plugin.yaml | 85 +++++++++++++++++ 4 files changed, 281 insertions(+), 13 deletions(-) create mode 100644 contents/deployment-run-script.py diff --git a/contents/common.py b/contents/common.py index 34b9f6f..f795c3a 100644 --- a/contents/common.py +++ b/contents/common.py @@ -43,7 +43,7 @@ def connect(): verify_ssl = os.environ.get('RD_CONFIG_VERIFY_SSL') ssl_ca_cert = os.environ.get('RD_CONFIG_SSL_CA_CERT') url = os.environ.get('RD_CONFIG_URL') - + token = os.environ.get('RD_CONFIG_TOKEN') if not token: token = os.environ.get('RD_CONFIG_TOKEN_STORAGE_PATH') @@ -173,6 +173,48 @@ def log_pod_parameters(logger, data): logger.debug("--------------------------") +def resolve_container_for_pod(name, namespace): + core_v1 = client.CoreV1Api() + response = core_v1.read_namespaced_pod_status( + name=name, + namespace=namespace, + pretty="True" + ) + + if response.spec.containers: + container = response.spec.containers[0].name + return container + log.error("Container not found for pod %s", name) + exit(1) + + +def get_active_pods_for_deployment(name, namespace): + """ + Retrieve all pods belonging to a deployment + """ + api = core_v1_api.CoreV1Api() + resp = None + try: + resp = api.list_namespaced_pod(namespace=namespace) + except ApiException as e: + if e.status != 404: + log.exception("Unknown error:") + exit(1) + + if not resp: + log.error("Namespace %s does not exits.", namespace) + exit(1) + pods_for_deployment = [] + for pod_spec in resp.items: + pod_labels = pod_spec.metadata.labels + if pod_labels['app'] == name and pod_spec.status.phase == 'Running': + pods_for_deployment.append(pod_spec.metadata.name) + if len(pods_for_deployment) < 1: + log.error("Did not find valid pods for deployment %s in namespace %s", name, namespace) + exit(1) + return pods_for_deployment + + def verify_pod_exists(name, namespace): """Verify pod exists.""" api = core_v1_api.CoreV1Api() diff --git a/contents/deployment-run-script.py b/contents/deployment-run-script.py new file mode 100644 index 0000000..22400c2 --- /dev/null +++ b/contents/deployment-run-script.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python -u +import logging +import sys +import os +import tempfile +import random +import common + + +logging.basicConfig(stream=sys.stderr, level=logging.INFO, + format='%(levelname)s: %(name)s: %(message)s') +log = logging.getLogger('kubernetes-model-source') + +if os.environ.get('RD_JOB_LOGLEVEL') == 'DEBUG': + log.setLevel(logging.DEBUG) + +PY = sys.version_info[0] + + +def prepare_script(script): + """ + Simple helper function to reduce lines of code in the huge main loop + """ + # Python 3 expects bytes string to transfer the data. + if PY == 3: + script = script.encode('utf-8') + + invocation = "/bin/bash" + if 'RD_CONFIG_INVOCATION' in os.environ: + invocation = os.environ.get('RD_CONFIG_INVOCATION') + + destination_path = "/tmp" + + if 'RD_NODE_FILE_COPY_DESTINATION_DIR' in os.environ: + destination_path = os.environ.get('RD_NODE_FILE_COPY_DESTINATION_DIR') + + temp = tempfile.NamedTemporaryFile() + destination_file_name = os.path.basename(temp.name) + full_path = destination_path + "/" + destination_file_name + + return script, full_path, temp, destination_path, destination_file_name, invocation + + +def main(): + """ + Runs a script on a randomly selected pod from the deployment. Optionally retries n times until success. + """ + + # start with setting up all paths/variables + common.connect() + [deployment_name, namespace, container] = common.get_core_node_parameter_list() + retries_before_failure = int(os.environ.get('RD_CONFIG_RETRYBEFOREFAILING', 0)) + pods_for_deployment = common.get_active_pods_for_deployment(name=deployment_name, namespace=namespace) + script = os.environ.get('RD_CONFIG_SCRIPT') + script, full_path, temp, destination_path, destination_file_name, invocation = prepare_script(script) + temp.write(script) + temp.seek(0) + if not container: + container = common.resolve_container_for_pod(pods_for_deployment[0], namespace) + + # loop until we've reached the max number of retries (default none) + for i in range(retries_before_failure + 1): + # randomly select one of the pods + pod_name = random.choice(pods_for_deployment) + log.debug("iteration %d", i) + common.log_pod_parameters(log, {'name': pod_name, 'namespace': namespace, 'container_name': container}) + + try: + log.debug("coping script from %s to %s", temp.name, full_path) + + common.copy_file(name=pod_name, + namespace=namespace, + container=container, + source_file=temp.name, + destination_path=destination_path, + destination_file_name=destination_file_name + ) + except Exception as e: + log.error(e) + continue + + permissions_command = ["chmod", "+x", full_path] + log.debug("setting permissions %s", permissions_command) + try: + resp = common.run_command(name=pod_name, + namespace=namespace, + container=container, + command=permissions_command + ) + + if resp.peek_stdout(): + print(resp.read_stdout()) + + if resp.peek_stderr(): + print(resp.read_stderr()) + continue + except Exception as e: + log.error(e) + continue + + # calling exec and wait for response. + exec_command = invocation.split(" ") + exec_command.append(full_path) + + try: + if 'RD_CONFIG_ARGUMENTS' in os.environ: + arguments = os.environ.get('RD_CONFIG_ARGUMENTS') + exec_command.append(arguments) + + log.debug("running script %s", exec_command) + + resp, error = common.run_interactive_command(name=pod_name, + namespace=namespace, + container=container, + command=exec_command + ) + if error: + log.error("error running script on iteration %d", i) + continue + except Exception as e: + log.error(e) + continue + + rm_command = ["rm", full_path] + log.debug("removing file %s", rm_command) + try: + resp = common.run_command(name=pod_name, + namespace=namespace, + container=container, + command=rm_command + ) + + if resp.peek_stdout(): + log.debug(resp.read_stdout()) + + if resp.peek_stderr(): + log.debug(resp.read_stderr()) + continue + except Exception as e: + log.error(e) + continue + temp.close() + log.info("Job successful on iteration %d", i) + sys.exit(0) + # if we have not reached exit 0 (at the end of the loop iteration), it means no single execution succeeded + log.error("unable to run script on any of the nodes") + temp.close() + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/contents/pods-run-script.py b/contents/pods-run-script.py index 00d379c..3ecccad 100644 --- a/contents/pods-run-script.py +++ b/contents/pods-run-script.py @@ -45,18 +45,7 @@ def main(): exit(1) if not container: - core_v1 = client.CoreV1Api() - response = core_v1.read_namespaced_pod_status( - name=name, - namespace=namespace, - pretty="True" - ) - - if response.spec.containers: - container = response.spec.containers[0].name - else: - log.error("Container not found") - exit(1) + container = common.resolve_container_for_pod(name=name, namespace=namespace) common.log_pod_parameters(log, {'name': name, 'namespace': namespace, 'container_name': container}) diff --git a/plugin.yaml b/plugin.yaml index 6a3f9ff..7eb2c11 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -815,6 +815,91 @@ providers: required: false renderingOptions: groupName: Config + - name: Kubernetes-DeploymentScript-Step + service: WorkflowNodeStep + title: Kubernetes / Deployment / Execute Script + description: 'Run a script randomly on one of the pods of a deployment' + plugin-type: script + script-interpreter: python -u + script-file: deployment-run-script.py + script-args: ${config.name} + config: + - name: name + type: String + title: "Deployment Name" + description: "Deployment Name" + required: true + - name: namespace + type: String + title: "Namespace" + description: "Namespace where the deployment is residing" + required: true + - name: script + type: String + title: "Script" + description: "Script to run" + required: true + renderingOptions: + displayType: CODE + - name: invocation + type: String + title: "Invocation String" + description: "Invocation String" + required: false + - name: arguments + type: String + title: "Arguments" + description: "Arguments of the script" + required: false + - type: Integer + name: retryBeforeFailing + title: Number of retries before failure + description: "Retries the job in case of failure n times before reporting failure (default 0)" + required: false + default: 0 + - name: config_file + type: String + title: "Kubernetes Config File Path" + description: "Leave empty if you want to pass the connection parameters" + required: false + scope: Instance + renderingOptions: + groupName: Authentication + - name: url + type: String + title: "Cluster URL" + description: "Kubernetes Cluster URL" + required: false + scope: Instance + renderingOptions: + groupName: Authentication + - name: token + type: String + title: "Token" + required: false + scope: Instance + description: "Kubernetes API Token" + renderingOptions: + groupName: Authentication + selectionAccessor: "STORAGE_PATH" + valueConversion: "STORAGE_PATH_AUTOMATIC_READ" + storage-path-root: "keys" + - name: verify_ssl + type: Boolean + title: "Verify ssl" + description: "Verify ssl for SSL connections" + required: false + scope: Instance + renderingOptions: + groupName: Authentication + - name: ssl_ca_cert + type: String + title: "SSL Certificate Path" + description: "SSL Certificate Path for SSL connections" + required: false + scope: Instance + renderingOptions: + groupName: Authentication - name: Kubernetes-Wait-StatefulSet service: WorkflowNodeStep title: Kubernetes / StatefulSet / Waitfor From 5ca0e6b3d756277dd7a044ffb155c917b3b51fb4 Mon Sep 17 00:00:00 2001 From: jordy Date: Thu, 24 Feb 2022 14:36:20 +0100 Subject: [PATCH 2/3] succesfully added deployment-run-script functionality, time for testing --- contents/deployment-run-script.py | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/contents/deployment-run-script.py b/contents/deployment-run-script.py index 22400c2..92bc42d 100644 --- a/contents/deployment-run-script.py +++ b/contents/deployment-run-script.py @@ -75,13 +75,9 @@ def main(): destination_path=destination_path, destination_file_name=destination_file_name ) - except Exception as e: - log.error(e) - continue - permissions_command = ["chmod", "+x", full_path] - log.debug("setting permissions %s", permissions_command) - try: + permissions_command = ["chmod", "+x", full_path] + log.debug("setting permissions %s", permissions_command) resp = common.run_command(name=pod_name, namespace=namespace, container=container, @@ -94,15 +90,12 @@ def main(): if resp.peek_stderr(): print(resp.read_stderr()) continue - except Exception as e: - log.error(e) - continue - # calling exec and wait for response. - exec_command = invocation.split(" ") - exec_command.append(full_path) - try: + # calling exec and wait for response. + exec_command = invocation.split(" ") + exec_command.append(full_path) + if 'RD_CONFIG_ARGUMENTS' in os.environ: arguments = os.environ.get('RD_CONFIG_ARGUMENTS') exec_command.append(arguments) @@ -117,19 +110,14 @@ def main(): if error: log.error("error running script on iteration %d", i) continue - except Exception as e: - log.error(e) - continue - rm_command = ["rm", full_path] - log.debug("removing file %s", rm_command) - try: + rm_command = ["rm", full_path] + log.debug("removing file %s", rm_command) resp = common.run_command(name=pod_name, namespace=namespace, container=container, command=rm_command ) - if resp.peek_stdout(): log.debug(resp.read_stdout()) From 87295ef38d324ce41642a4de590be8d3f7ee4f3f Mon Sep 17 00:00:00 2001 From: jordy Date: Mon, 23 May 2022 13:12:13 +0200 Subject: [PATCH 3/3] fix spelling mistake --- contents/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contents/common.py b/contents/common.py index f795c3a..19a5f20 100644 --- a/contents/common.py +++ b/contents/common.py @@ -202,7 +202,7 @@ def get_active_pods_for_deployment(name, namespace): exit(1) if not resp: - log.error("Namespace %s does not exits.", namespace) + log.error("Namespace %s does not exist.", namespace) exit(1) pods_for_deployment = [] for pod_spec in resp.items: