Merge pull request #46 from karellen/issue_45

Patch Kubernetes Client libraries

Dockerfile

@@ -3,11 +3,13 @@ FROM python:3.11-slim-bookworm
 COPY target/dist/kubernator*/dist/*.whl /tmp
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN pip install --no-input --no-cache-dir /tmp/*.whl && \
+WORKDIR /root
+
+RUN pip install --no-input /tmp/*.whl && \
     apt update && apt install git -y && \
+    kubernator --pre-cache-k8s-client $(seq 19 28) && \
+    pip cache purge && \
     rm -rf /var/lib/{apt,dpkg,cache,log}/ && \
     rm -rf /tmp/*
 
-WORKDIR /root
-RUN kubernator --pre-cache-k8s-client $(seq 19 28)
 ENTRYPOINT ["/usr/local/bin/kubernator"]

build.py

@@ -27,7 +27,7 @@
 use_plugin("filter_resources")
 
 name = "kubernator"
-version = "1.0.12"
+version = "1.0.13.dev"
 
 summary = "Kubernator is the a pluggable framework for K8S provisioning"
 authors = [Author("Express Systems USA, Inc.", "")]
@@ -61,6 +61,7 @@ def set_properties(project):
     project.depends_on("jinja2", "~=3.1")
     project.depends_on("coloredlogs", "~=15.0")
     project.depends_on("jsonschema", "<4.0")
+    project.depends_on("diff-match-patch", ">2023.0")
 
     project.set_property("coverage_break_build", False)
     project.set_property("cram_fail_if_no_tests", False)

src/integrationtest/python/issue_35/.kubernator.py → src/integrationtest/python/issue_35/crd/.kubernator.py

@@ -3,7 +3,7 @@
 
 ktor.app.register_plugin("minikube", k8s_version=os.environ["K8S_VERSION"],
                          start_fresh=os.environ["START_FRESH"],
-                         keep_running=os.environ["KEEP_RUNNING"], profile="issue-35")
+                         keep_running=True, profile="issue-35")
 ktor.app.register_plugin("k8s",
                          field_validation=os.environ["FIELD_VALIDATION"],
                          field_validation_warn_fatal=os.environ["WARN_FATAL"])

src/integrationtest/python/issue_35/crd/manifests.yaml

@@ -0,0 +1,40 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  # name must match the spec fields below, and be in the form: <plural>.<group>
+  name: crontabs.stable.example.com
+spec:
+  # group name to use for REST API: /apis/<group>/<version>
+  group: stable.example.com
+  # list of versions supported by this CustomResourceDefinition
+  versions:
+    - name: v1
+      # Each version can be enabled/disabled by Served flag.
+      served: true
+      # One and only one version must be marked as the storage version.
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                cronSpec:
+                  type: string
+                image:
+                  type: string
+                replicas:
+                  type: integer
+  # either Namespaced or Cluster
+  scope: Namespaced
+  names:
+    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
+    plural: crontabs
+    # singular name to be used as an alias on the CLI and for display
+    singular: crontab
+    # kind is normally the CamelCased singular type. Your resource manifests use this.
+    kind: CronTab
+    # shortNames allow shorter string to match your resource on the CLI
+    shortNames:
+    - ct

src/integrationtest/python/issue_35/test/.kubernator.py

@@ -0,0 +1,10 @@
+# flake8: noqa
+import os
+
+ktor.app.register_plugin("minikube", k8s_version=os.environ["K8S_VERSION"],
+                         start_fresh=False,
+                         keep_running=os.environ["KEEP_RUNNING"], profile="issue-35")
+ktor.app.register_plugin("k8s",
+                         field_validation=os.environ["FIELD_VALIDATION"],
+                         field_validation_warn_fatal=os.environ["WARN_FATAL"])
+ktor.k8s.load_crds(ktor.app.cwd / ".." / "crd" / "manifests.yaml", "yaml")

src/integrationtest/python/issue_35/manifests.yaml → src/integrationtest/python/issue_35/test/manifests.yaml

@@ -16,3 +16,13 @@ spec:
     - 'topology.kubernetes.io/zone'
     - 'topology.kubernetes.io/region'
     - '*'
+---
+apiVersion: "stable.example.com/v1"
+kind: CronTab
+metadata:
+  name: my-new-cron-object
+  namespace: default
+spec:
+  cronSpec: "* * * * */5"
+  image: my-awesome-cron-image
+  thispropertydoesntexist: so sad

src/integrationtest/python/issue_35_tests.py

@@ -22,12 +22,15 @@
 
 from pathlib import Path  # noqa: E402
 import os  # noqa: E402
+import sys  # noqa: E402
 import tempfile  # noqa: E402
 
 
 class Issue35Test(IntegrationTestSupport):
     def test_issue_35(self):
-        test_dir = Path(__file__).parent / "issue_35"
+        issue_dir = Path(__file__).parent / "issue_35"
+        crd_dir = issue_dir / "crd"
+        test_dir = issue_dir / "test"
 
         for k8s_version in ["1.20.15", "1.23.17", "1.25.16", "1.28.4"]:
             k8s_minor = int(k8s_version[2:4])
@@ -43,29 +46,47 @@ def test_issue_35(self):
                         os.environ["FIELD_VALIDATION"] = validation
                         os.environ["WARN_FATAL"] = "1" if warn_fatal else ""
 
-                        with tempfile.TemporaryDirectory() as temp_dir:
-                            log_file = str(Path(temp_dir) / "log")
-                            try:
-                                self.run_module_test("kubernator", "-p", str(test_dir),
-                                                     "--log-format", "json",
-                                                     "--log-file", log_file,
-                                                     "apply")
-                            except AssertionError:
-                                if ((not warn_fatal and validation == "Warn") or
-                                        validation == "Ignore" or
-                                        k8s_minor < 25):
-                                    raise
-                            logs = self.load_json_logs(log_file)
-                        validation_msg_found = False
-                        for log in logs:
-                            if "FAILED FIELD VALIDATION" in log["message"]:
-                                validation_msg_found = True
-                                break
+                        if phase == 0:
+                            self.run_module_test("kubernator", "-p", str(crd_dir),
+                                                 "-v", "DEBUG",
+                                                 "apply", "--yes")
+                            self.run_module_test("kubernator", "-p", str(test_dir),
+                                                 "-v", "DEBUG",
+                                                 "dump")
+                        logs = None
+                        try:
+                            with tempfile.TemporaryDirectory() as temp_dir:
+                                log_file = str(Path(temp_dir) / "log")
+                                try:
+                                    self.run_module_test("kubernator", "-p", str(test_dir),
+                                                         "--log-format", "json",
+                                                         "--log-file", log_file,
+                                                         "-v", "DEBUG",
+                                                         "apply")
+                                    logs = self.load_json_logs(log_file)
+                                except AssertionError:
+                                    logs = self.load_json_logs(log_file)
+                                    if ((not warn_fatal and validation == "Warn") or
+                                            validation == "Ignore" or
+                                            k8s_minor < 25):
+                                        raise
 
-                        if k8s_minor < 24 or validation == "Ignore":
-                            self.assertFalse(validation_msg_found)
-                        elif validation in ("Warn", "Strict"):
-                            self.assertTrue(validation_msg_found)
+                            validation_msg_found = False
+                            for log in logs:
+                                if "FAILED FIELD VALIDATION" in log["message"]:
+                                    validation_msg_found = True
+                                    break
+
+                            if k8s_minor < 24 or validation == "Ignore":
+                                self.assertFalse(validation_msg_found)
+                            elif validation in ("Warn", "Strict"):
+                                self.assertTrue(validation_msg_found)
+                        finally:
+                            if logs:
+                                for log in logs:
+                                    print(f"{log['ts']} {log['name']} {log['level']} {log['fn']}:{log['ln']} "
+                                          f"{log['message']}",
+                                          file=sys.stderr)
 
 
 if __name__ == "__main__":

src/integrationtest/python/smoke_pre_cache_tests.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+#
+#   Copyright 2020 Express Systems USA, Inc
+#   Copyright 2023 Karellen, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+from test_support import IntegrationTestSupport, unittest
+
+
+class PreCacheSmokeTest(IntegrationTestSupport):
+
+    def setUp(self):
+        self.run_module_test("kubernator", "--clear-k8s-cache")
+
+    def test_precache(self):
+        for k8s_version in range(19, 29):
+            for disable_patches in (True, False, True):
+                with self.subTest(k8s_version=k8s_version, disable_patches=disable_patches):
+                    args = ["kubernator", "--pre-cache-k8s-client", str(k8s_version)]
+                    if disable_patches:
+                        args.append("--pre-cache-k8s-client-no-patch")
+                    self.run_module_test(*args)
+
+    @classmethod
+    def tearDownClass(cls):
+        cls().run_module_test("kubernator", "--clear-k8s-cache")
+
+
+if __name__ == "__main__":
+    unittest.main()