Fix Istio plugin and add informative logging

Add JSON Path and tests
Fix Terragrunt binary name
Move K8S imports from global to local in prep for library versioning

build.py

@@ -27,7 +27,7 @@
 use_plugin("filter_resources")
 
 name = "kubernator"
-version = "1.0.1"
+version = "1.0.2.dev"
 
 summary = "Kubernator is the a pluggable framework for K8S provisioning"
 authors = [Author("Express Systems USA, Inc.", "")]
@@ -132,7 +132,14 @@ def publish(project):
     image = f"ghcr.io/karellen/kubernator"
     versioned_image = f"{image}:{project.dist_version}"
     project.set_property("docker_image", image)
-    check_call(["docker", "build", "-t", versioned_image, "-t", f"{image}:latest", "."])
+    labels = ["-t", versioned_image]
+
+    # Do not tag with latest if it's a development build
+    if project.version == project.dist_version:
+        labels += ["-t", f"{image}:latest"]
+
+    check_call(["docker", "build"] + labels + ["."])
+
 
 @task
 def upload(project):

src/main/python/kubernator/api.py

@@ -28,6 +28,7 @@
 from collections.abc import Callable
 from collections.abc import Iterable, MutableSet, Reversible
 from enum import Enum
+from functools import cache
 from hashlib import sha256
 from io import StringIO as io_StringIO
 from pathlib import Path
@@ -44,6 +45,7 @@
                     make_logging_undefined,
                     Template as JinjaTemplate,
                     pass_context)
+from jsonpath_ng.ext import parse as jp_parse
 from jsonschema import validators
 
 _CACHE_HEADER_TRANSLATION = {"etag": "if-none-match",
@@ -66,6 +68,45 @@ def to_patterns(*patterns):
     return [re.compile(fnmatch.translate(p)) for p in patterns]
 
 
+class JPath:
+    def __init__(self, pattern):
+        self.pattern = jp_parse(pattern)
+
+    def find(self, val):
+        return self.pattern.find(val)
+
+    def all(self, val):
+        return list(map(lambda x: x.value, self.find(val)))
+
+    def first(self, val):
+        """Returns the first element or None if it doesn't exist"""
+        try:
+            return next(map(lambda x: x.value, self.find(val)))
+        except StopIteration:
+            return None
+
+    def only(self, val):
+        """Returns the first and only element.
+        Raises ValueError if more than one value found
+        Raises KeyError if no value found
+        """
+        m = map(lambda x: x.value, self.find(val))
+        try:
+            v = next(m)
+        except StopIteration:
+            raise KeyError("no value found")
+        try:
+            next(m)
+            raise ValueError("more than one value returned")
+        except StopIteration:
+            return v
+
+
+@cache
+def jp(pattern) -> JPath:
+    return JPath(pattern)
+
+
 def scan_dir(logger, path: Path, path_filter: Callable[[os.DirEntry], bool], excludes, includes):
     logger.debug("Scanning %s, excluding %s, including %s", path, excludes, includes)
     with os.scandir(path) as it:  # type: Iterable[os.DirEntry]

src/main/python/kubernator/app.py

@@ -30,7 +30,7 @@
 
 import kubernator
 from kubernator.api import (KubernatorPlugin, Globs, scan_dir, PropertyDict, config_as_dict, config_parent,
-                            download_remote_file, load_remote_file, Repository, StripNL)
+                            download_remote_file, load_remote_file, Repository, StripNL, jp)
 from kubernator.proc import run, run_capturing_out
 
 TRACE = 5
@@ -123,15 +123,6 @@ def json_record(self, message, extra, record: logging.LogRecord):
     logger.setLevel(logging._nameToLevel[verbose])
 
 
-# class RepositoryPath:
-#    def __init__(self, path: Path, repository: Repository = None):
-#        self.path = path.absolute()
-#        self.repository = repository
-#
-#    def __str__(self):
-#        return self.repository.url_str if self.repository else ""
-
-
 class App(KubernatorPlugin):
     _name = "app"
 
@@ -325,6 +316,7 @@ def handle_init(self):
                                    download_remote_file=download_remote_file,
                                    load_remote_file=load_remote_file,
                                    register_cleanup=self.register_cleanup,
+                                   jp=jp,
                                    run=self._run,
                                    run_capturing_out=self._run_capturing_out,
                                    repository=self.repository,

src/main/python/kubernator/plugins/istio.py

@@ -25,10 +25,6 @@
 from shutil import which
 
 import yaml
-from jsonpath_ng.ext import parse as jp_parse
-from kubernetes import client
-from kubernetes.client.rest import ApiException
-
 from kubernator.api import (KubernatorPlugin, scan_dir,
                             TemplateEngine,
                             load_remote_file,
@@ -37,17 +33,15 @@
                             Globs,
                             get_golang_os,
                             get_golang_machine,
-                            prepend_os_path)
+                            prepend_os_path, jp)
 from kubernator.plugins.k8s_api import K8SResourcePluginMixin
 
 logger = logging.getLogger("kubernator.istio")
 proc_logger = logger.getChild("proc")
 stdout_logger = StripNL(proc_logger.info)
 stderr_logger = StripNL(proc_logger.warning)
 
-MESH_PILOT_JP = jp_parse('$.meshVersion[?Component="pilot"].Info.version')
-
-OBJECT_SCHEMA_VERSION = "1.20.6"
+MESH_PILOT_JP = jp('$.meshVersion[?Component="pilot"].Info.version')
 
 
 class IstioPlugin(KubernatorPlugin, K8SResourcePluginMixin):
@@ -108,7 +102,7 @@ def test_istioctl(self):
         version_out_js = json.loads(version_out)
         version = version_out_js["clientVersion"]["version"]
         logger.info("Using istioctl %r version %r with stanza %r",
-                    self.context.istio.istioctl_file, version, self.istioctl_stanza())
+                    self.context.istio.istioctl_file, version, context.istio.istioctl_stanza())
 
         logger.info("Found Istio client version %s", version)
 
@@ -151,7 +145,7 @@ def handle_start(self):
         # This plugin only deals with Istio Operator, so only load that stuff
         self.resource_definitions_schema = load_remote_file(logger,
                                                             f"https://raw.githubusercontent.com/kubernetes/kubernetes/"
-                                                            f"v{OBJECT_SCHEMA_VERSION}/api/openapi-spec/swagger.json",
+                                                            f"{self.context.k8s.server_version}/api/openapi-spec/swagger.json",
                                                             FileType.JSON)
         self._populate_resource_definitions()
         self.add_remote_crds(f"{url_prefix}/crd-operator.yaml", FileType.YAML)
@@ -175,7 +169,7 @@ def handle_after_dir(self, cwd: Path):
         for f in scan_dir(logger, cwd, lambda d: d.is_file(), istio.excludes, istio.includes):
             p = cwd / f.name
             display_p = context.app.display_path(p)
-            logger.debug("Adding Istio Operator from %s", display_p)
+            logger.info("Adding Istio Operator from %s", display_p)
 
             with open(p, "rt") as file:
                 template = self.template_engine.from_string(file.read())
@@ -189,13 +183,14 @@ def handle_apply(self):
             logger.info("Skipping Istio as no Operator was processed")
         else:
             with tempfile.NamedTemporaryFile(mode="wt", delete=False) as operators_file:
+                logger.info("Saving Istio Operators to %s", operators_file.name)
                 yaml.safe_dump_all((r.manifest for r in self.resources.values()), operators_file)
 
             if context.app.args.command == "apply":
                 logger.info("Running Istio precheck")
-                context.app.run(self.istio_stanza + ["x", "precheck"],
+                context.app.run(context.istio.istioctl_stanza() + ["x", "precheck"],
                                 stdout_logger, stderr_logger).wait()
-                context.app.run(self.istio_stanza + ["validate", "-f", operators_file.name],
+                context.app.run(context.istio.istioctl_stanza() + ["validate", "-f", operators_file.name],
                                 stdout_logger, stderr_logger).wait()
 
                 self._operator_init(operators_file, True)
@@ -204,6 +199,9 @@ def handle_apply(self):
                     self._operator_init(operators_file, False)
 
     def _operator_init(self, operators_file, dry_run):
+        from kubernetes import client
+        from kubernetes.client.rest import ApiException
+
         context = self.context
 
         status_details = " (dry run)" if dry_run else ""
@@ -231,7 +229,7 @@ def _operator_init(self, operators_file, dry_run):
                 raise
 
         logger.info("Running Istio operator init%s", status_details)
-        istio_operator_init = self.istio_stanza + ["operator", "init", "-f", operators_file.name]
+        istio_operator_init = context.istio.istioctl_stanza() + ["operator", "init", "-f", operators_file.name]
         context.app.run(istio_operator_init + (["--dry-run"] if dry_run else []),
                         stdout_logger,
                         stderr_logger).wait()

src/main/python/kubernator/plugins/k8s.py

@@ -29,9 +29,6 @@
 
 import jsonpatch
 import yaml
-from kubernetes import client
-from kubernetes.client.rest import ApiException
-from kubernetes.config import load_incluster_config, load_kube_config, ConfigException
 
 from kubernator.api import (KubernatorPlugin, Globs, scan_dir, load_file, FileType, load_remote_file)
 from kubernator.plugins.k8s_api import (K8SResourcePluginMixin,
@@ -113,6 +110,8 @@ def _kubeconfig_changed(self):
         self.setup_client()
 
     def setup_client(self):
+        from kubernetes import client
+
         context = self.context
 
         context.k8s.client = self._setup_k8s_client()
@@ -122,6 +121,8 @@ def setup_client(self):
         else:
             git_version = version.git_version
 
+        context.k8s.server_version = git_version
+
         logger.info("Found Kubernetes %s on %s", version.git_version, context.k8s.client.configuration.host)
 
         logger.debug("Reading Kubernetes OpenAPI spec for version %s", git_version)
@@ -285,6 +286,9 @@ def _apply_resource(self,
                         create_func: Callable[[], None],
                         delete_func: Callable[[K8SPropagationPolicy], None],
                         status_msg):
+        from kubernetes import client
+        from kubernetes.client.rest import ApiException
+
         rdef = resource.rdef
         rdef.populate_api(client, self.context.k8s.client)
 
@@ -373,7 +377,10 @@ def _filter_resource_patch(self, patch: Iterable[Mapping], excludes: Iterable[re
             result.append(op)
         return result
 
-    def _setup_k8s_client(self) -> client.ApiClient:
+    def _setup_k8s_client(self) -> "kubernetes.client.ApiClient":
+        from kubernetes import client
+        from kubernetes.config import load_incluster_config, load_kube_config, ConfigException
+
         try:
             logger.debug("Trying K8S in-cluster configuration")
             load_incluster_config()

src/main/python/kubernator/plugins/terragrunt.py

@@ -18,9 +18,10 @@
 
 import json
 import logging
+import tempfile
 import os
 from pathlib import Path
-from shutil import which
+from shutil import which, copy
 
 from kubernator.api import (KubernatorPlugin, Globs, StripNL,
                             scan_dir,
@@ -59,7 +60,11 @@ def register(self, version=None):
             # Download and use specific version
             tg_url = (f"https://github.com/gruntwork-io/terragrunt/releases/download/v{version}/"
                       f"terragrunt_{get_golang_os()}_{get_golang_machine()}")
-            tg_file, _ = context.app.download_remote_file(logger, tg_url, "bin")
+            tg_file_cache, _ = context.app.download_remote_file(logger, tg_url, "bin")
+
+            self.tg_dir = tempfile.TemporaryDirectory()
+            tg_file = Path(self.tg_dir.name) / "terragrunt"
+            copy(tg_file_cache, tg_file)
             os.chmod(tg_file, 0o500)
             prepend_os_path(str(self.tg_dir))
         else: