Enable templating in extraContainers and extraInitContainers (apache#…

…38507)

* Add test cases for templating extraContainers and extraInitContainers

* Use templating for extra airflow containers

chart/files/pod-template-file.kubernetes-helm-yaml

@@ -50,7 +50,7 @@ spec:
       {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 4 }}
     {{- end }}
     {{- if .Values.workers.extraInitContainers }}
-      {{- toYaml .Values.workers.extraInitContainers | nindent 4 }}
+      {{- tpl (toYaml .Values.workers.extraInitContainers) . | nindent 4 }}
     {{- end }}
     {{- if and (semverCompare ">=2.8.0" .Values.airflowVersion) .Values.workers.kerberosInitContainer.enabled }}
     - name: kerberos-init
@@ -180,7 +180,7 @@ spec:
         {{- include "standard_airflow_environment" . | indent 6 }}
     {{- end }}
     {{- if .Values.workers.extraContainers }}
-      {{- toYaml .Values.workers.extraContainers | nindent 4 }}
+      {{- tpl (toYaml .Values.workers.extraContainers) . | nindent 4 }}
     {{- end }}
   {{- if .Values.workers.priorityClassName }}
   priorityClassName: {{ .Values.workers.priorityClassName }}

chart/templates/dag-processor/dag-processor-deployment.yaml

@@ -144,7 +144,7 @@ spec:
           {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
         {{- end }}
         {{- if .Values.dagProcessor.extraInitContainers }}
-          {{- toYaml .Values.dagProcessor.extraInitContainers | nindent 8 }}
+          {{- tpl (toYaml .Values.dagProcessor.extraInitContainers) . | nindent 8 }}
         {{- end }}
       containers:
         - name: dag-processor

chart/templates/scheduler/scheduler-deployment.yaml

@@ -169,7 +169,7 @@ spec:
           {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
         {{- end }}
         {{- if .Values.scheduler.extraInitContainers }}
-          {{- toYaml .Values.scheduler.extraInitContainers | nindent 8 }}
+          {{- tpl (toYaml .Values.scheduler.extraInitContainers) . | nindent 8 }}
         {{- end }}
       containers:
         # Always run the main scheduler container.

chart/templates/triggerer/triggerer-deployment.yaml

@@ -160,7 +160,7 @@ spec:
           {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
         {{- end }}
         {{- if .Values.triggerer.extraInitContainers }}
-          {{- toYaml .Values.triggerer.extraInitContainers | nindent 8 }}
+          {{- tpl (toYaml .Values.triggerer.extraInitContainers) . | nindent 8 }}
         {{- end }}
       containers:
         - name: triggerer

chart/templates/webserver/webserver-deployment.yaml

@@ -163,7 +163,7 @@ spec:
           {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
         {{- end }}
         {{- if .Values.webserver.extraInitContainers }}
-          {{- toYaml .Values.webserver.extraInitContainers | nindent 8 }}
+          {{- tpl (toYaml .Values.webserver.extraInitContainers) . | nindent 8 }}
         {{- end }}
       containers:
         - name: webserver

chart/templates/workers/worker-deployment.yaml

@@ -220,7 +220,7 @@ spec:
           {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
         {{- end }}
         {{- if .Values.workers.extraInitContainers }}
-          {{- toYaml .Values.workers.extraInitContainers | nindent 8 }}
+          {{- tpl (toYaml .Values.workers.extraInitContainers) . | nindent 8 }}
         {{- end }}
       containers:
         - name: worker

helm_tests/airflow_aux/test_create_user_job.py

@@ -166,6 +166,20 @@ def test_should_add_extra_containers(self):
             "image": "test-registry/test-repo:test-tag",
         } == jmespath.search("spec.template.spec.containers[-1]", docs[0])
 
+    def test_should_template_extra_containers(self):
+        docs = render_chart(
+            values={
+                "createUserJob": {
+                    "extraContainers": [{"name": "{{ .Release.Name }}-test-container"}],
+                },
+            },
+            show_only=["templates/jobs/create-user-job.yaml"],
+        )
+
+        assert {"name": "release-name-test-container"} == jmespath.search(
+            "spec.template.spec.containers[-1]", docs[0]
+        )
+
     def test_should_add_extra_volumes(self):
         docs = render_chart(
             values={

helm_tests/airflow_aux/test_migrate_database_job.py

@@ -190,6 +190,20 @@ def test_should_add_extra_containers(self):
             "image": "test-registry/test-repo:test-tag",
         } == jmespath.search("spec.template.spec.containers[-1]", docs[0])
 
+    def test_should_template_extra_containers(self):
+        docs = render_chart(
+            values={
+                "migrateDatabaseJob": {
+                    "extraContainers": [{"name": "{{ .Release.Name }}-test-container"}],
+                },
+            },
+            show_only=["templates/jobs/migrate-database-job.yaml"],
+        )
+
+        assert {"name": "release-name-test-container"} == jmespath.search(
+            "spec.template.spec.containers[-1]", docs[0]
+        )
+
     def test_set_resources(self):
         docs = render_chart(
             values={

helm_tests/airflow_aux/test_pod_template_file.py

@@ -702,6 +702,21 @@ def test_should_add_extra_init_containers(self):
             "image": "test-registry/test-repo:test-tag",
         } == jmespath.search("spec.initContainers[-1]", docs[0])
 
+    def test_should_template_extra_init_containers(self):
+        docs = render_chart(
+            values={
+                "workers": {
+                    "extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}],
+                },
+            },
+            show_only=["templates/pod-template-file.yaml"],
+            chart_dir=self.temp_chart_dir,
+        )
+
+        assert {
+            "name": "release-name-test-init-container",
+        } == jmespath.search("spec.initContainers[-1]", docs[0])
+
     def test_should_add_extra_containers(self):
         docs = render_chart(
             values={
@@ -720,6 +735,21 @@ def test_should_add_extra_containers(self):
             "image": "test-registry/test-repo:test-tag",
         } == jmespath.search("spec.containers[-1]", docs[0])
 
+    def test_should_template_extra_containers(self):
+        docs = render_chart(
+            values={
+                "workers": {
+                    "extraContainers": [{"name": "{{ .Release.Name }}-test-container"}],
+                },
+            },
+            show_only=["templates/pod-template-file.yaml"],
+            chart_dir=self.temp_chart_dir,
+        )
+
+        assert {
+            "name": "release-name-test-container",
+        } == jmespath.search("spec.containers[-1]", docs[0])
+
     def test_should_add_pod_labels(self):
         docs = render_chart(
             values={"labels": {"label1": "value1", "label2": "value2"}},

helm_tests/airflow_core/test_dag_processor.py

@@ -110,6 +110,21 @@ def test_should_add_extra_containers(self):
             "image": "test-registry/test-repo:test-tag",
         } == jmespath.search("spec.template.spec.containers[-1]", docs[0])
 
+    def test_should_template_extra_containers(self):
+        docs = render_chart(
+            values={
+                "dagProcessor": {
+                    "enabled": True,
+                    "extraContainers": [{"name": "{{ .Release.Name }}-test-container"}],
+                },
+            },
+            show_only=["templates/dag-processor/dag-processor-deployment.yaml"],
+        )
+
+        assert {"name": "release-name-test-container"} == jmespath.search(
+            "spec.template.spec.containers[-1]", docs[0]
+        )
+
     def test_should_add_extra_init_containers(self):
         docs = render_chart(
             values={
@@ -128,6 +143,21 @@ def test_should_add_extra_init_containers(self):
             "image": "test-registry/test-repo:test-tag",
         } == jmespath.search("spec.template.spec.initContainers[-1]", docs[0])
 
+    def test_should_template_extra_init_containers(self):
+        docs = render_chart(
+            values={
+                "dagProcessor": {
+                    "enabled": True,
+                    "extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}],
+                },
+            },
+            show_only=["templates/dag-processor/dag-processor-deployment.yaml"],
+        )
+
+        assert {"name": "release-name-test-init-container"} == jmespath.search(
+            "spec.template.spec.initContainers[-1]", docs[0]
+        )
+
     def test_should_add_extra_volume_and_extra_volume_mount(self):
         docs = render_chart(
             values={

helm_tests/airflow_core/test_scheduler.py

@@ -69,6 +69,21 @@ def test_should_add_extra_containers(self):
             "image": "test-registry/test-repo:test-tag",
         } == jmespath.search("spec.template.spec.containers[-1]", docs[0])
 
+    def test_should_template_extra_containers(self):
+        docs = render_chart(
+            values={
+                "executor": "CeleryExecutor",
+                "scheduler": {
+                    "extraContainers": [{"name": "{{ .Release.Name }}-test-container"}],
+                },
+            },
+            show_only=["templates/scheduler/scheduler-deployment.yaml"],
+        )
+
+        assert {"name": "release-name-test-container"} == jmespath.search(
+            "spec.template.spec.containers[-1]", docs[0]
+        )
+
     def test_disable_wait_for_migration(self):
         docs = render_chart(
             values={
@@ -100,6 +115,20 @@ def test_should_add_extra_init_containers(self):
             "image": "test-registry/test-repo:test-tag",
         } == jmespath.search("spec.template.spec.initContainers[-1]", docs[0])
 
+    def test_should_template_extra_init_containers(self):
+        docs = render_chart(
+            values={
+                "scheduler": {
+                    "extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}],
+                },
+            },
+            show_only=["templates/scheduler/scheduler-deployment.yaml"],
+        )
+
+        assert {"name": "release-name-test-init-container"} == jmespath.search(
+            "spec.template.spec.initContainers[-1]", docs[0]
+        )
+
     def test_should_add_extra_volume_and_extra_volume_mount(self):
         docs = render_chart(
             values={

helm_tests/airflow_core/test_triggerer.py

@@ -107,6 +107,20 @@ def test_should_add_extra_containers(self):
             "image": "test-registry/test-repo:test-tag",
         } == jmespath.search("spec.template.spec.containers[-1]", docs[0])
 
+    def test_should_template_extra_containers(self):
+        docs = render_chart(
+            values={
+                "triggerer": {
+                    "extraContainers": [{"name": "{{ .Release.Name }}-test-container"}],
+                },
+            },
+            show_only=["templates/triggerer/triggerer-deployment.yaml"],
+        )
+
+        assert {"name": "release-name-test-container"} == jmespath.search(
+            "spec.template.spec.containers[-1]", docs[0]
+        )
+
     def test_should_add_extra_init_containers(self):
         docs = render_chart(
             values={
@@ -124,6 +138,20 @@ def test_should_add_extra_init_containers(self):
             "image": "test-registry/test-repo:test-tag",
         } == jmespath.search("spec.template.spec.initContainers[-1]", docs[0])
 
+    def test_should_template_extra_init_containers(self):
+        docs = render_chart(
+            values={
+                "triggerer": {
+                    "extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}],
+                },
+            },
+            show_only=["templates/triggerer/triggerer-deployment.yaml"],
+        )
+
+        assert {"name": "release-name-test-init-container"} == jmespath.search(
+            "spec.template.spec.initContainers[-1]", docs[0]
+        )
+
     def test_should_add_extra_volume_and_extra_volume_mount(self):
         docs = render_chart(
             values={

helm_tests/airflow_core/test_worker.py

@@ -82,6 +82,21 @@ def test_should_add_extra_containers(self):
             "image": "test-registry/test-repo:test-tag",
         } == jmespath.search("spec.template.spec.containers[-1]", docs[0])
 
+    def test_should_template_extra_containers(self):
+        docs = render_chart(
+            values={
+                "executor": "CeleryExecutor",
+                "workers": {
+                    "extraContainers": [{"name": "{{ .Release.Name }}-test-container"}],
+                },
+            },
+            show_only=["templates/workers/worker-deployment.yaml"],
+        )
+
+        assert {"name": "release-name-test-container"} == jmespath.search(
+            "spec.template.spec.containers[-1]", docs[0]
+        )
+
     def test_disable_wait_for_migration(self):
         docs = render_chart(
             values={
@@ -113,6 +128,20 @@ def test_should_add_extra_init_containers(self):
             "image": "test-registry/test-repo:test-tag",
         } == jmespath.search("spec.template.spec.initContainers[-1]", docs[0])
 
+    def test_should_template_extra_init_containers(self):
+        docs = render_chart(
+            values={
+                "workers": {
+                    "extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}],
+                },
+            },
+            show_only=["templates/workers/worker-deployment.yaml"],
+        )
+
+        assert {"name": "release-name-test-init-container"} == jmespath.search(
+            "spec.template.spec.initContainers[-1]", docs[0]
+        )
+
     def test_should_add_extra_volume_and_extra_volume_mount(self):
         docs = render_chart(
             values={

helm_tests/webserver/test_webserver.py

@@ -200,6 +200,21 @@ def test_should_add_extra_containers(self):
             "image": "test-registry/test-repo:test-tag",
         } == jmespath.search("spec.template.spec.containers[-1]", docs[0])
 
+    def test_should_template_extra_containers(self):
+        docs = render_chart(
+            values={
+                "executor": "CeleryExecutor",
+                "webserver": {
+                    "extraContainers": [{"name": "{{ .Release.Name }}-test-container"}],
+                },
+            },
+            show_only=["templates/webserver/webserver-deployment.yaml"],
+        )
+
+        assert {
+            "name": "release-name-test-container",
+        } == jmespath.search("spec.template.spec.containers[-1]", docs[0])
+
     def test_should_add_extraEnvs(self):
         docs = render_chart(
             values={
@@ -315,6 +330,20 @@ def test_should_add_extra_init_containers(self):
             "image": "test-registry/test-repo:test-tag",
         } == jmespath.search("spec.template.spec.initContainers[-1]", docs[0])
 
+    def test_should_template_extra_init_containers(self):
+        docs = render_chart(
+            values={
+                "webserver": {
+                    "extraInitContainers": [{"name": "{{ .Release.Name }}-init-container"}],
+                },
+            },
+            show_only=["templates/webserver/webserver-deployment.yaml"],
+        )
+
+        assert {
+            "name": "release-name-init-container",
+        } == jmespath.search("spec.template.spec.initContainers[-1]", docs[0])
+
     def test_should_add_component_specific_labels(self):
         docs = render_chart(
             values={