diff --git a/build.py b/build.py index 67e1ad2..87c25c8 100644 --- a/build.py +++ b/build.py @@ -27,7 +27,7 @@ use_plugin("filter_resources") name = "kubernator" -version = "1.0.1" +version = "1.0.2.dev" summary = "Kubernator is the a pluggable framework for K8S provisioning" authors = [Author("Express Systems USA, Inc.", "")] @@ -132,7 +132,14 @@ def publish(project): image = f"ghcr.io/karellen/kubernator" versioned_image = f"{image}:{project.dist_version}" project.set_property("docker_image", image) - check_call(["docker", "build", "-t", versioned_image, "-t", f"{image}:latest", "."]) + labels = ["-t", versioned_image] + + # Do not tag with latest if it's a development build + if project.version == project.dist_version: + labels += ["-t", f"{image}:latest"] + + check_call(["docker", "build"] + labels + ["."]) + @task def upload(project): diff --git a/src/main/python/kubernator/api.py b/src/main/python/kubernator/api.py index 957e00d..eb8bba0 100644 --- a/src/main/python/kubernator/api.py +++ b/src/main/python/kubernator/api.py @@ -28,6 +28,7 @@ from collections.abc import Callable from collections.abc import Iterable, MutableSet, Reversible from enum import Enum +from functools import cache from hashlib import sha256 from io import StringIO as io_StringIO from pathlib import Path @@ -44,6 +45,7 @@ make_logging_undefined, Template as JinjaTemplate, pass_context) +from jsonpath_ng.ext import parse as jp_parse from jsonschema import validators _CACHE_HEADER_TRANSLATION = {"etag": "if-none-match", @@ -66,6 +68,45 @@ def to_patterns(*patterns): return [re.compile(fnmatch.translate(p)) for p in patterns] +class JPath: + def __init__(self, pattern): + self.pattern = jp_parse(pattern) + + def find(self, val): + return self.pattern.find(val) + + def all(self, val): + return list(map(lambda x: x.value, self.find(val))) + + def first(self, val): + """Returns the first element or None if it doesn't exist""" + try: + return next(map(lambda x: x.value, self.find(val))) + except StopIteration: + return None + + def only(self, val): + """Returns the first and only element. + Raises ValueError if more than one value found + Raises KeyError if no value found + """ + m = map(lambda x: x.value, self.find(val)) + try: + v = next(m) + except StopIteration: + raise KeyError("no value found") + try: + next(m) + raise ValueError("more than one value returned") + except StopIteration: + return v + + +@cache +def jp(pattern) -> JPath: + return JPath(pattern) + + def scan_dir(logger, path: Path, path_filter: Callable[[os.DirEntry], bool], excludes, includes): logger.debug("Scanning %s, excluding %s, including %s", path, excludes, includes) with os.scandir(path) as it: # type: Iterable[os.DirEntry] diff --git a/src/main/python/kubernator/app.py b/src/main/python/kubernator/app.py index 3754b83..50d165e 100644 --- a/src/main/python/kubernator/app.py +++ b/src/main/python/kubernator/app.py @@ -30,7 +30,7 @@ import kubernator from kubernator.api import (KubernatorPlugin, Globs, scan_dir, PropertyDict, config_as_dict, config_parent, - download_remote_file, load_remote_file, Repository, StripNL) + download_remote_file, load_remote_file, Repository, StripNL, jp) from kubernator.proc import run, run_capturing_out TRACE = 5 @@ -123,15 +123,6 @@ def json_record(self, message, extra, record: logging.LogRecord): logger.setLevel(logging._nameToLevel[verbose]) -# class RepositoryPath: -# def __init__(self, path: Path, repository: Repository = None): -# self.path = path.absolute() -# self.repository = repository -# -# def __str__(self): -# return self.repository.url_str if self.repository else "" - - class App(KubernatorPlugin): _name = "app" @@ -325,6 +316,7 @@ def handle_init(self): download_remote_file=download_remote_file, load_remote_file=load_remote_file, register_cleanup=self.register_cleanup, + jp=jp, run=self._run, run_capturing_out=self._run_capturing_out, repository=self.repository, diff --git a/src/main/python/kubernator/plugins/istio.py b/src/main/python/kubernator/plugins/istio.py index 5c4fc51..b70c9b2 100644 --- a/src/main/python/kubernator/plugins/istio.py +++ b/src/main/python/kubernator/plugins/istio.py @@ -25,10 +25,6 @@ from shutil import which import yaml -from jsonpath_ng.ext import parse as jp_parse -from kubernetes import client -from kubernetes.client.rest import ApiException - from kubernator.api import (KubernatorPlugin, scan_dir, TemplateEngine, load_remote_file, @@ -37,7 +33,7 @@ Globs, get_golang_os, get_golang_machine, - prepend_os_path) + prepend_os_path, jp) from kubernator.plugins.k8s_api import K8SResourcePluginMixin logger = logging.getLogger("kubernator.istio") @@ -45,9 +41,7 @@ stdout_logger = StripNL(proc_logger.info) stderr_logger = StripNL(proc_logger.warning) -MESH_PILOT_JP = jp_parse('$.meshVersion[?Component="pilot"].Info.version') - -OBJECT_SCHEMA_VERSION = "1.20.6" +MESH_PILOT_JP = jp('$.meshVersion[?Component="pilot"].Info.version') class IstioPlugin(KubernatorPlugin, K8SResourcePluginMixin): @@ -108,7 +102,7 @@ def test_istioctl(self): version_out_js = json.loads(version_out) version = version_out_js["clientVersion"]["version"] logger.info("Using istioctl %r version %r with stanza %r", - self.context.istio.istioctl_file, version, self.istioctl_stanza()) + self.context.istio.istioctl_file, version, context.istio.istioctl_stanza()) logger.info("Found Istio client version %s", version) @@ -151,7 +145,7 @@ def handle_start(self): # This plugin only deals with Istio Operator, so only load that stuff self.resource_definitions_schema = load_remote_file(logger, f"https://raw.githubusercontent.com/kubernetes/kubernetes/" - f"v{OBJECT_SCHEMA_VERSION}/api/openapi-spec/swagger.json", + f"{self.context.k8s.server_version}/api/openapi-spec/swagger.json", FileType.JSON) self._populate_resource_definitions() self.add_remote_crds(f"{url_prefix}/crd-operator.yaml", FileType.YAML) @@ -175,7 +169,7 @@ def handle_after_dir(self, cwd: Path): for f in scan_dir(logger, cwd, lambda d: d.is_file(), istio.excludes, istio.includes): p = cwd / f.name display_p = context.app.display_path(p) - logger.debug("Adding Istio Operator from %s", display_p) + logger.info("Adding Istio Operator from %s", display_p) with open(p, "rt") as file: template = self.template_engine.from_string(file.read()) @@ -189,13 +183,14 @@ def handle_apply(self): logger.info("Skipping Istio as no Operator was processed") else: with tempfile.NamedTemporaryFile(mode="wt", delete=False) as operators_file: + logger.info("Saving Istio Operators to %s", operators_file.name) yaml.safe_dump_all((r.manifest for r in self.resources.values()), operators_file) if context.app.args.command == "apply": logger.info("Running Istio precheck") - context.app.run(self.istio_stanza + ["x", "precheck"], + context.app.run(context.istio.istioctl_stanza() + ["x", "precheck"], stdout_logger, stderr_logger).wait() - context.app.run(self.istio_stanza + ["validate", "-f", operators_file.name], + context.app.run(context.istio.istioctl_stanza() + ["validate", "-f", operators_file.name], stdout_logger, stderr_logger).wait() self._operator_init(operators_file, True) @@ -204,6 +199,9 @@ def handle_apply(self): self._operator_init(operators_file, False) def _operator_init(self, operators_file, dry_run): + from kubernetes import client + from kubernetes.client.rest import ApiException + context = self.context status_details = " (dry run)" if dry_run else "" @@ -231,7 +229,7 @@ def _operator_init(self, operators_file, dry_run): raise logger.info("Running Istio operator init%s", status_details) - istio_operator_init = self.istio_stanza + ["operator", "init", "-f", operators_file.name] + istio_operator_init = context.istio.istioctl_stanza() + ["operator", "init", "-f", operators_file.name] context.app.run(istio_operator_init + (["--dry-run"] if dry_run else []), stdout_logger, stderr_logger).wait() diff --git a/src/main/python/kubernator/plugins/k8s.py b/src/main/python/kubernator/plugins/k8s.py index d3add26..c306c2c 100644 --- a/src/main/python/kubernator/plugins/k8s.py +++ b/src/main/python/kubernator/plugins/k8s.py @@ -29,9 +29,6 @@ import jsonpatch import yaml -from kubernetes import client -from kubernetes.client.rest import ApiException -from kubernetes.config import load_incluster_config, load_kube_config, ConfigException from kubernator.api import (KubernatorPlugin, Globs, scan_dir, load_file, FileType, load_remote_file) from kubernator.plugins.k8s_api import (K8SResourcePluginMixin, @@ -113,6 +110,8 @@ def _kubeconfig_changed(self): self.setup_client() def setup_client(self): + from kubernetes import client + context = self.context context.k8s.client = self._setup_k8s_client() @@ -122,6 +121,8 @@ def setup_client(self): else: git_version = version.git_version + context.k8s.server_version = git_version + logger.info("Found Kubernetes %s on %s", version.git_version, context.k8s.client.configuration.host) logger.debug("Reading Kubernetes OpenAPI spec for version %s", git_version) @@ -285,6 +286,9 @@ def _apply_resource(self, create_func: Callable[[], None], delete_func: Callable[[K8SPropagationPolicy], None], status_msg): + from kubernetes import client + from kubernetes.client.rest import ApiException + rdef = resource.rdef rdef.populate_api(client, self.context.k8s.client) @@ -373,7 +377,10 @@ def _filter_resource_patch(self, patch: Iterable[Mapping], excludes: Iterable[re result.append(op) return result - def _setup_k8s_client(self) -> client.ApiClient: + def _setup_k8s_client(self): + from kubernetes import client + from kubernetes.config import load_incluster_config, load_kube_config, ConfigException + try: logger.debug("Trying K8S in-cluster configuration") load_incluster_config() diff --git a/src/main/python/kubernator/plugins/terragrunt.py b/src/main/python/kubernator/plugins/terragrunt.py index 34b8e34..4f36118 100644 --- a/src/main/python/kubernator/plugins/terragrunt.py +++ b/src/main/python/kubernator/plugins/terragrunt.py @@ -18,9 +18,10 @@ import json import logging +import tempfile import os from pathlib import Path -from shutil import which +from shutil import which, copy from kubernator.api import (KubernatorPlugin, Globs, StripNL, scan_dir, @@ -59,7 +60,11 @@ def register(self, version=None): # Download and use specific version tg_url = (f"https://github.com/gruntwork-io/terragrunt/releases/download/v{version}/" f"terragrunt_{get_golang_os()}_{get_golang_machine()}") - tg_file, _ = context.app.download_remote_file(logger, tg_url, "bin") + tg_file_cache, _ = context.app.download_remote_file(logger, tg_url, "bin") + + self.tg_dir = tempfile.TemporaryDirectory() + tg_file = Path(self.tg_dir.name) / "terragrunt" + copy(tg_file_cache, tg_file) os.chmod(tg_file, 0o500) prepend_os_path(str(self.tg_dir)) else: diff --git a/src/unittest/python/deployment.json b/src/unittest/python/deployment.json new file mode 100644 index 0000000..cd3f157 --- /dev/null +++ b/src/unittest/python/deployment.json @@ -0,0 +1,190 @@ +{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "annotations": { + "deployment.kubernetes.io/revision": "319" + }, + "creationTimestamp": "2021-07-05T17:03:01Z", + "generation": 323, + "labels": { + "remove_replicas": "true" + }, + "name": "user-api", + "namespace": "general", + "resourceVersion": "644631573", + "uid": "2bd395e5-17eb-467a-a87b-d1a4841a00fd" + }, + "spec": { + "progressDeadlineSeconds": 600, + "replicas": 1, + "revisionHistoryLimit": 10, + "selector": { + "matchLabels": { + "application": "user-api" + } + }, + "strategy": { + "rollingUpdate": { + "maxSurge": 1, + "maxUnavailable": 0 + }, + "type": "RollingUpdate" + }, + "template": { + "metadata": { + "annotations": { + "backend_app": "backend_app", + "ci_commit_sha": "c77cebe38ae82888cb362adf8c6b0b4b531dd71a", + "configmap_sha": "06e1dbf2ffd407fe77855796b6e8810b23253da0fca2bdd4ddf086607cbb3383", + "iam.amazonaws.com/role": "k8s-s3", + "java_app": "promtail", + "kubectl.kubernetes.io/restartedAt": "2022-04-11T20:16:57+04:00", + "prometheus.io/path": "/actuator/prometheus", + "prometheus.io/port": "8082", + "prometheus.io/scrape": "true", + "prometheus.istio.io/merge-metrics": "false", + "vault.hashicorp.com/agent-init-first": "true", + "vault.hashicorp.com/agent-inject": "true", + "vault.hashicorp.com/agent-inject-secret-user-api": "kv/user-api", + "vault.hashicorp.com/agent-inject-status": "update", + "vault.hashicorp.com/agent-inject-template-user-api": "{{- with secret \"kv/user-api\" -}}\n export USER_DB_URL=\"{{ .Data.data.USER_DB_URL }}\"\n export USER_DB_USER=\"{{ .Data.data.USER_DB_USER }}\"\n export USER_DB_PASSWORD=\"{{ .Data.data.USER_DB_PASSWORD }}\"\n export ACTIVEMQ_BROKER_URL=\"{{ .Data.data.ACTIVEMQ_BROKER_URL }}\"\n export ACTIVEMQ_BROKER_USER=\"{{ .Data.data.ACTIVEMQ_BROKER_USER }}\"\n export ACTIVEMQ_BROKER_PASSWORD=\"{{ .Data.data.ACTIVEMQ_BROKER_PASSWORD }}\"\n export FRONT_OAUTH_CLIENT_SECRET=\"{{ .Data.data.FRONT_OAUTH_CLIENT_SECRET }}\"\n export KEYCLOAK_MOBILE_CLIENT_SECRET=\"{{ .Data.data.KEYCLOAK_MOBILE_CLIENT_SECRET }}\"\n export KEYCLOAK_WEB_CLIENT_SECRET=\"{{ .Data.data.KEYCLOAK_WEB_CLIENT_SECRET }}\"\n export LOG_SLACK_WEBHOOK=\"{{ .Data.data.LOG_SLACK_WEBHOOK }}\"\n export PHONE_SIGNATURE_SECRET_RSA_1=\"{{ .Data.data.PHONE_SIGNATURE_SECRET_RSA_1 }}\"\n export PHONE_SIGNATURE_SECRET_RSA_2=\"{{ .Data.data.PHONE_SIGNATURE_SECRET_RSA_2 }}\"\n export PHONE_SIGNATURE_SECRET_RSA_3=\"{{ .Data.data.PHONE_SIGNATURE_SECRET_RSA_3 }}\"\n{{- end -}}\n", + "vault.hashicorp.com/role": "payperless-dev" + }, + "creationTimestamp": null, + "labels": { + "application": "user-api", + "env": "dev", + "namespace": "general" + }, + "name": "user-api" + }, + "spec": { + "containers": [ + { + "command": [ + "/bin/bash", + "-c", + "source /vault/secrets/user-api \u0026\u0026 java $JAVA_OPTS -jar user-api.jar" + ], + "env": [ + { + "name": "JAVA_OPTS", + "value": "-XX:MaxRAMPercentage=80 -Djdk.tls.client.protocols=TLSv1,TLSv1.1,TLSv1.2" + } + ], + "envFrom": [ + { + "configMapRef": { + "name": "user-api" + } + } + ], + "image": "registry.europayhub.com/covault-wallet/cb_user/dev:user-api-1.0-SNAPSHOT", + "imagePullPolicy": "Always", + "livenessProbe": { + "failureThreshold": 2, + "httpGet": { + "path": "/actuator/health/liveness", + "port": 8082, + "scheme": "HTTP" + }, + "periodSeconds": 5, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "name": "user-api", + "ports": [ + { + "containerPort": 8082, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "cpu": "3", + "memory": "2Gi" + }, + "requests": { + "cpu": "500m", + "memory": "500Mi" + } + }, + "startupProbe": { + "failureThreshold": 30, + "httpGet": { + "path": "/actuator/health/liveness", + "port": 8082, + "scheme": "HTTP" + }, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 1 + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File" + } + ], + "dnsPolicy": "ClusterFirst", + "imagePullSecrets": [ + { + "name": "gitlab-docker" + } + ], + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "payperless-dev", + "serviceAccountName": "payperless-dev", + "terminationGracePeriodSeconds": 15, + "topologySpreadConstraints": [ + { + "labelSelector": { + "matchLabels": { + "application": "user-api" + } + }, + "maxSkew": 1, + "topologyKey": "topology.kubernetes.io/zone", + "whenUnsatisfiable": "ScheduleAnyway" + }, + { + "labelSelector": { + "matchLabels": { + "application": "user-api" + } + }, + "maxSkew": 1, + "topologyKey": "kubernetes.io/hostname", + "whenUnsatisfiable": "ScheduleAnyway" + } + ] + } + } + }, + "status": { + "availableReplicas": 1, + "conditions": [ + { + "lastTransitionTime": "2022-11-08T10:09:40Z", + "lastUpdateTime": "2023-04-25T10:10:19Z", + "message": "ReplicaSet \"user-api-6cb5d95689\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + }, + { + "lastTransitionTime": "2023-07-25T21:59:06Z", + "lastUpdateTime": "2023-07-25T21:59:06Z", + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + } + ], + "observedGeneration": 323, + "readyReplicas": 1, + "replicas": 1, + "updatedReplicas": 1 + } +} diff --git a/src/unittest/python/jsonpath_tests.py b/src/unittest/python/jsonpath_tests.py new file mode 100644 index 0000000..1504928 --- /dev/null +++ b/src/unittest/python/jsonpath_tests.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Express Systems USA, Inc +# Copyright 2021 Karellen, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from gevent.monkey import patch_all, is_anything_patched + +if not is_anything_patched(): + patch_all() + +import logging +import unittest +import json +from kubernator.api import jp +from pathlib import Path + +TRACE = 5 + + +def trace(self, msg, *args, **kwargs): + """ + Log 'msg % args' with severity 'TRACE'. + + To pass exception information, use the keyword argument exc_info with + a true value, e.g. + + logger.trace("Houston, we have a %s", "interesting problem", exc_info=1) + """ + if self.isEnabledFor(TRACE): + self._log(TRACE, msg, args, **kwargs) + + +logging.addLevelName(5, "TRACE") +logging.Logger.trace = trace + + +class JsonPathTestsTestcase(unittest.TestCase): + def test_jp(self): + with open(Path(__file__).parent / "deployment.json", "rb") as f: + deployment = json.load(f) + + # if (r.group == "apps" and r.kind in ("StatefulSet", "Deployment", "DaemonSet") + # and "envFrom" in r.manifest["spec"]["template"]["spec"]["containers"][0] + # and "annotations" in r.manifest["spec"]["template"]["metadata"] + # and "backend_app" in r.manifest["spec"]["template"]["metadata"]["annotations"]): + # configmap_name = r.manifest["spec"]["template"]["spec"]["containers"][0]["envFrom"][0]["configMapRef"][ + # "name"] + # configmap_namespace = r.manifest["metadata"]["namespace"] + + self.assertEqual(jp("$.spec.template.spec.containers[*].envFrom[*].configMapRef.name").only(deployment), + "user-api") + self.assertEqual(jp("$.spec.template.metadata.annotations.backend_app").first(deployment), + "backend_app") + self.assertIsNone(jp("$.spec.template.metadata.annotations.will_not_find_this_annotation").first(deployment)) + self.assertEqual(len(jp("$.spec.template.spec.containers[*]").all(deployment)), 1)