Cleanup to refactor the common helm helpers and to try to support ins…

…talling the capture proxy by itself. Signed-off-by: Greg Schohn <[email protected]>
opensearch-project · Oct 31, 2024 · 0b89305 · 0b89305
1 parent a7ce390
commit 0b89305
Show file tree

Hide file tree

Showing 20 changed files with 105 additions and 47 deletions.
diff --git a/deployment/k8s/.gitignore b/deployment/k8s/.gitignore
@@ -0,0 +1 @@
+**/charts/*.tgz
diff --git a/deployment/k8s/charts/aggregates/mockCustomerClusters/Chart.lock b/deployment/k8s/charts/aggregates/mockCustomerClusters/Chart.lock
@@ -0,0 +1,12 @@
+dependencies:
+- name: elasticsearch
+  repository: https://helm.elastic.co
+  version: 8.5.1
+- name: capture-proxy
+  repository: file://../../components/captureProxy
+  version: 0.1.0
+- name: opensearch
+  repository: https://opensearch-project.github.io/helm-charts/
+  version: 2.23.1
+digest: sha256:4a32b1cb130067ad9ac20d5e12caaab2dcf4463bce2b214bc87107f407a2f592
+generated: "2024-10-28T23:14:40.432564-04:00"
diff --git a/...8s/charts/mockCustomerClusters/Chart.yaml → ...ggregates/mockCustomerClusters/Chart.yaml b/...8s/charts/mockCustomerClusters/Chart.yaml → ...ggregates/mockCustomerClusters/Chart.yaml
@@ -8,7 +8,7 @@ dependencies:
     alias: source
   - name: capture-proxy
     version: "0.1.0"
-    repository: file://../captureProxy
+    repository: file://../../components/captureProxy
     alias: proxy
   - name: opensearch
     version: "2.23.1"

diff --git a/deployment/k8s/charts/aggregates/mockCustomerClusters/charts/captureProxy b/deployment/k8s/charts/aggregates/mockCustomerClusters/charts/captureProxy
@@ -0,0 +1 @@
+../../../components/captureProxy
diff --git a/deployment/k8s/charts/components/captureProxy/Chart.yaml b/deployment/k8s/charts/components/captureProxy/Chart.yaml
@@ -1,3 +1,7 @@
 apiVersion: v2
 name: capture-proxy
 version: 0.1.0
+dependencies:
+  - name: helm-common
+    repository: file://../helm-common
+    version: 0.1.0
diff --git a/deployment/k8s/charts/components/captureProxy/charts/helm-common b/deployment/k8s/charts/components/captureProxy/charts/helm-common
@@ -0,0 +1 @@
+../../helm-common
diff --git a/.../captureProxy/templates/proxyService.yaml → ...ts/captureProxy/templates/deployment.yaml b/.../captureProxy/templates/proxyService.yaml → ...ts/captureProxy/templates/deployment.yaml
@@ -19,25 +19,11 @@ spec:
           image: migrations/capture_proxy:latest
           imagePullPolicy: IfNotPresent
           env:
-          {{ include "generic.pullEnvVarsFromConfigMaps" (dict "PackageName" (include "generic.fullname" .) "Values" .Values "include" .Template.Include "Template" .Template) | indent 10 }}
+            {{- include "generic.pullEnvVarsFromConfigMaps" (dict "NameSpace" (include "generic.fullname" .) "Parameters" .Values.parameters "include" .Template.Include "Template" .Template) | indent 10 }}
           command:
             - "/bin/sh"
             - "-c"
             - |
-              {{- include "generic.buildCommandBuilderScript" (dict "Command" "/runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy" "PackageName" (include "generic.fullname" .) "Values" .Values "include" .Template.Include "Template" .Template) | nindent 14 }}
+              {{- include "generic.buildCommandBuilderScript" (dict "Command" "/runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy" "PackageName" (include "generic.fullname" .) "Parameters" .Values.parameters "include" .Template.Include "Template" .Template) | nindent 14 }}
           ports:
             - containerPort: 9200
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ include "generic.fullname" . }}
-spec:
-  selector:
-    app: {{ include "generic.fullname" . }}
-    env: v1
-  ports:
-    - protocol: TCP
-      port: 9200
-      targetPort: 9200
-  type: ClusterIP
diff --git a/deployment/k8s/charts/components/captureProxy/templates/service.yaml b/deployment/k8s/charts/components/captureProxy/templates/service.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "generic.fullname" . }}
+spec:
+  selector:
+    app: {{ include "generic.fullname" . }}
+    env: v1
+  ports:
+    - protocol: TCP
+      port: 9200
+      targetPort: 9200
+  type: ClusterIP
diff --git a/deployment/k8s/charts/components/captureProxy/values.yaml b/deployment/k8s/charts/components/captureProxy/values.yaml
@@ -1,6 +1,5 @@
 nameOverride: ""
 fullnameOverride: ""
-protectedParameterNamespace: "baseline"
 # Set to true if you want the chart to create the namespace
 createNamespace: true
 # Optional: specify namespace name (defaults to .Release.Namespace)
@@ -19,9 +18,18 @@ parameters:
   noCapture:
     present: false
     allowRuntimeOverride: true
+  kafkaBrokersList:
+    list:
+      - kafka1
+      - kafka2
   insecureDestination:
     present: true
     allowRuntimeOverride: false
+
+    # Proxy... --destinationUri  "http://destination:9200" --listenPort 9200 --noCapture --kafkaBrokersList kafka1 kafka2 --insecureDestination
+
+
+
 # TODO - setup cert management
 #  sslConfigFile:
 #    value: "/usr/share/elasticsearch/config/proxy_tls.yml"

diff --git a/deployment/k8s/charts/components/helm-common/.helmignore b/deployment/k8s/charts/components/helm-common/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/deployment/k8s/charts/components/helm-common/Chart.yaml b/deployment/k8s/charts/components/helm-common/Chart.yaml
@@ -0,0 +1,5 @@
+apiVersion: v2
+name: helm-common
+version: 0.1.0
+description: A common library for the Migration Assistant
+type: library
diff --git a/...nents/captureProxy/templates/_command.tpl → ...onents/helm-common/templates/_command.tpl b/...nents/captureProxy/templates/_command.tpl → ...onents/helm-common/templates/_command.tpl
@@ -3,15 +3,15 @@ set -e
 CMD="{{ .Command }}"
 
 {{- /* Default environment variables if not set */}}
-{{- range $key, $param := .Values.parameters }}
+{{- range $key, $param := .Parameters }}
 {{- $envVarName := include "toSnakeCase" $key }}
 if [ -z "${{ $envVarName }}" ]; then
   export {{ $envVarName }}="${{ $envVarName }}_DEFAULT"
 fi
 {{- end }}
 
 {{- /* Construct the command based on parameter types */}}
-{{- range $key, $param := .Values.parameters }}
+{{- range $key, $param := .Parameters }}
 {{- $envVarName := include "toSnakeCase" $key }}
 {{- if hasKey $param "value" }}
 if [ -n "${{ $envVarName }}" ]; then

diff --git a/...plates/_createConfigMapsForParameters.tpl → ...plates/_createConfigMapsForParameters.tpl b/...plates/_createConfigMapsForParameters.tpl → ...plates/_createConfigMapsForParameters.tpl
@@ -6,7 +6,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ $packageName }}-{{ $key | lower }}-default
+  name: {{ $key | lower }}-default
   namespace: {{ $namespace }}
   labels:
     type: default
@@ -26,7 +26,7 @@ data:
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ $packageName }}-{{ $key | lower }}
+  name: {{ $key | lower }}
   namespace: {{ $namespace }}
   labels:
     type: override

diff --git a/...ptureProxy/templates/_createNamespace.tpl → ...elm-common/templates/_createNamespace.tpl b/...ptureProxy/templates/_createNamespace.tpl → ...elm-common/templates/_createNamespace.tpl
diff --git a/...nents/captureProxy/templates/_helpers.tpl → ...onents/helm-common/templates/_helpers.tpl b/...nents/captureProxy/templates/_helpers.tpl → ...onents/helm-common/templates/_helpers.tpl
diff --git a/...plates/_pullEnvVariablesFromConfigMap.tpl → ...plates/_pullEnvVariablesFromConfigMap.tpl b/...plates/_pullEnvVariablesFromConfigMap.tpl → ...plates/_pullEnvVariablesFromConfigMap.tpl
@@ -1,16 +1,18 @@
 {{- define "generic.pullEnvVarsFromConfigMaps" }}
-{{- $packageName := .PackageName }}
-{{- range $key, $param := .Values.parameters }}
+{{- $namespace := .NameSpace }}
+{{- range $key, $param := .Parameters }}
   - name: {{ include "toSnakeCase" $key }}_DEFAULT
     valueFrom:
       configMapKeyRef:
-        name: {{ $packageName }}-{{ $key | lower }}-default
-        key: {{ if hasKey $param "value" }}value{{ else if hasKey $param "list" }}list{{ else }}present{{ end }}
+        name: {{ $key | lower }}-default
+        namespace: {{ $namespace }}
+        key: {{ if hasKey $param "value" }}value{{ else if hasKey $param "list" }}list{{ else }}present{{ end }} {{/*TODO be explicit*/}}
 {{- if hasKey $param "allowRuntimeOverride" | ternary $param.allowRuntimeOverride true }}
   - name: {{ include "toSnakeCase" $key }}
     valueFrom:
       configMapKeyRef:
-        name: {{ $packageName }}-{{ $key | lower }}
+        name: {{ $key | lower }}
+        namespace: {{ $namespace }}
         key: {{ if hasKey $param "value" }}value{{ else if hasKey $param "list" }}list{{ else }}present{{ end }}
         optional: true
 {{- end }}

diff --git a/...s/captureProxy/templates/_toSnakeCase.tpl → ...ts/helm-common/templates/_toSnakeCase.tpl b/...s/captureProxy/templates/_toSnakeCase.tpl → ...ts/helm-common/templates/_toSnakeCase.tpl
diff --git a/deployment/k8s/helmValues/localTesting/captureProxy.yaml b/deployment/k8s/helmValues/localTesting/captureProxy.yaml
@@ -1,2 +1,3 @@
-proxy:
-  destinationUrl: "http://elasticsearch:9200"
+parameters:
+  destinationUri:
+    value: "http://elasticsearch:9200"
diff --git a/deployment/k8s/helmValues/localTesting/sourceElasticsearchCluster.yaml b/deployment/k8s/helmValues/localTesting/sourceElasticsearchCluster.yaml
@@ -1,17 +1,19 @@
-source:
-  imageTag: 7.10.2
-  image: docker.elastic.co/elasticsearch/elasticsearch-oss
-  replicas: 1
-  minimumMasterNodes: 1
-  roles: [master, data, ingest]
-  antiAffinity: "soft"
-  esJavaOpts: "-Xmx128m -Xms128m"
-  protocol: http
+imageTag: "7.10.2"
+image: "docker.elastic.co/elasticsearch/elasticsearch-oss"
+fullnameOverride: "elasticsearch"
+roles: [master, data, ingest]
+antiAffinity: "soft"
+esJavaOpts: "-Xmx128m -Xms128m"
+protocol: http
+# for simplest dev testing
+replicas: 2 # k8s replica count, not ES shard replicas
+createCert: false
 #  esConfig:
 #    elasticsearch.yml: |
 #      discovery.type: single-node
-  createCert: false
-  fullnameOverride: "elasticsearch"
+
+# for load-testing deployments
+minimumMasterNodes: 1
 
 #  extraEnvs:
 #    - name: "xpack.security.enabled"

diff --git a/deployment/k8s/helmValues/localTesting/targetOpenSearchCluster.yaml b/deployment/k8s/helmValues/localTesting/targetOpenSearchCluster.yaml
@@ -1,7 +1,6 @@
-target:
-  extraEnvs:
-    - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD
-      value: myStrongPassword123!
-  singleNode: true
-  persistence:
-    enabled: false
+extraEnvs:
+  - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD
+    value: myStrongPassword123!
+singleNode: true
+persistence:
+  enabled: false