fixes 114,150: create a single Helm package for dracon-dev and releva…

…nt make targets

This commit unifies the installation of dracon-dev and its dependencies
via one single helm package. It installs all necessary dependencies for
development such as Elasticsearch, Kibana, Postgres, Mongo and Tekton.
Dependencies that need to be installed on a namespace different than the
dracon components such as Tekton, Nginx, ElasticOperator and ArangoDB are
installed via Make targets.

This commit also refactors Make targets introducing `deploy-dracon-dev`
which builds and loads dracon container images and installs the helm
chart.

Last this commit runs the necessary dracon migrations job as a helm
post-install hook

.gitignore

@@ -6,3 +6,4 @@ bin
 .custom_image
 tests/output
 .vscode/
+deploy/dracon/charts

Makefile

@@ -153,18 +153,8 @@ print-%:
 ########################################
 ########## DEPLOYMENT TARGETS ##########
 ########################################
-.PHONY: deploy-arangodb-crds deploy-arangodb dev-deploy deploy-elasticsearch deploy-mongodb deploy-pg deploy-tektoncd-pipeline tektoncd-pipeline-helm tektoncd-dashboard-helm
-
-deploy-arangodb-crds:
-	@helm upgrade arangodb-crds https://github.com/arangodb/kube-arangodb/releases/download/$(ARANGODB_VERSION)/kube-arangodb-crd-$(ARANGODB_VERSION).tgz \
-		--install
-
-deploy-arangodb: deploy-arangodb-crds
-	@helm upgrade arangodb-instance deploy/arangodb/ \
-		--install \
-		--namespace $(ARANGODB_NS) \
-		--create-namespace \
-		--values=deploy/arangodb/values.yaml
+.PHONY: deploy-nginx deploy-arangodb-crds deploy-arangodb-operator add-es-helm-repo deploy-elasticoperator \
+		tektoncd-dashboard-helm deploy-tektoncd-dashboard add-bitnami-repo deploy-dracon-dev dev-deploy dev-teardown
 
 deploy-nginx:
 	@helm upgrade nginx-ingress https://github.com/kubernetes/ingress-nginx/releases/download/helm-chart-$(NGINX_INGRESS_VERSION)/ingress-nginx-$(NGINX_INGRESS_VERSION).tgz \
@@ -173,6 +163,13 @@ deploy-nginx:
 		--create-namespace \
 		--set "controller.admissionWebhooks.enabled=false"
 
+deploy-arangodb-crds:
+	@helm upgrade arangodb-crds https://github.com/arangodb/kube-arangodb/releases/download/$(ARANGODB_VERSION)/kube-arangodb-crd-$(ARANGODB_VERSION).tgz \
+		--install
+
+deploy-arangodb-operator:
+	@helm install --generate-name https://github.com/arangodb/kube-arangodb/releases/download/1.2.40/kube-arangodb-1.2.40.tgz
+
 add-es-helm-repo:
 	@helm repo add elastic https://helm.elastic.co
 	@helm repo update
@@ -184,37 +181,6 @@ deploy-elasticoperator: add-es-helm-repo
 		--create-namespace \
 		--version=$(ES_OPERATOR_VERSION)
 
-deploy-elasticsearch: deploy-elasticoperator
-	@helm upgrade dracon-es deploy/elasticsearch/ \
-		--install \
-		--set version=$(ES_VERSION) \
-		--namespace $(DRACON_NS) \
-		--create-namespace
-
-deploy-kibana: deploy-elasticsearch
-	@helm upgrade dracon-kb deploy/kibana/ \
-		--install \
-		--set version=$(ES_VERSION) \
-		--set es_name=dracon-es-elasticsearch \
-		--namespace $(DRACON_NS) \
-		--version $(ES_VERSION)
-
-deploy-mongodb:
-	@helm upgrade consumer-mongodb https://charts.bitnami.com/bitnami/mongodb-$(MONGODB_VERSION).tgz \
-		--install \
-		--namespace $(DRACON_NS) \
-		--create-namespace \
-		--set "auth.usernames[0]=consumer-mongodb" \
-		--set "auth.passwords[0]=consumer-mongodb" \
-		--set "auth.databases[0]=consumer-mongodb"
-
-deploy-pg:
-	@helm upgrade pg https://charts.bitnami.com/bitnami/postgresql-$(PG_VERSION).tgz \
-		--install \
-		--namespace $(DRACON_NS) \
-		--create-namespace \
-		--values=deploy/enrichment-db/values.yaml
-
 deploy/tektoncd/pipeline/release-v$(TEKTON_VERSION).yaml:
 	@wget "https://storage.googleapis.com/tekton-releases/pipeline/previous/v$(TEKTON_VERSION)/release.yaml" -O $@
 
@@ -239,4 +205,22 @@ deploy-tektoncd-dashboard: tektoncd-dashboard-helm
 		--values ./deploy/tektoncd/dashboard/values.yaml \
 		--namespace $(TEKTON_NS)
 
-dev-deploy: deploy-nginx deploy-arangodb deploy-kibana deploy-mongodb deploy-pg deploy-tektoncd-pipeline deploy-tektoncd-dashboard
+add-bitnami-repo:
+	@helm repo add bitnami https://charts.bitnami.com/bitnami
+
+deploy-dracon-dev: deploy-elasticoperator deploy-arangodb-crds add-bitnami-repo
+	@echo "fetching dependencies if needed"
+	@helm dependency build ./deploy/dracon/
+	@echo "deploying dracon in dev mode"
+	@helm upgrade dracon ./deploy/dracon/ \
+	 	  --install \
+		  --values ./deploy/dracon/values.dev.yaml \
+		  --create-namespace \
+		  --namespace $(DRACON_NS) \
+		  --set "enrichmentDB.migrations.image=kind-registry:5000/ocurity/dracon/draconctl:$(DRACON_VERSION)"
+		  --wait
+
+dev-deploy: deploy-nginx deploy-tektoncd-pipeline deploy-tektoncd-dashboard deploy-dracon-dev
+
+dev-teardown:
+	@kind delete clusters dracon-demo

deploy/dracon/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deploy/dracon/Chart.lock

@@ -0,0 +1,9 @@
+dependencies:
+- name: postgresql
+  repository: https://charts.bitnami.com/bitnami
+  version: 15.2.5
+- name: mongodb
+  repository: https://charts.bitnami.com/bitnami
+  version: 15.1.5
+digest: sha256:f5b463f7862318ed8de9439769a72f14320f271c72c80ec7a2a1f1b209959d7a
+generated: "2024-05-02T22:03:05.901032098+03:00"

deploy/dracon/Chart.yaml

@@ -0,0 +1,15 @@
+apiVersion: v2
+name: dracon
+description: A Helm chart for Kubernetes
+type: application
+version: 0.1.0
+appVersion: 0.1.0
+dependencies:
+  - name: postgresql
+    version: 15.2.5
+    repository: https://charts.bitnami.com/bitnami
+    condition: postgres.enabled
+  - name: mongodb
+    version: 15.1.5
+    repository: https://charts.bitnami.com/bitnami
+    condition: mongodb.enabled

deploy/kibana/templates/_helpers.tpl → deploy/dracon/templates/_helpers.tpl

@@ -1,7 +1,7 @@
 {{/*
 Expand the name of the chart.
 */}}
-{{- define "kibana.name" -}}
+{{- define "dracon.name" -}}
 {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
@@ -10,7 +10,7 @@ Create a default fully qualified app name.
 We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
 If release name contains chart name it will be used as a full name.
 */}}
-{{- define "kibana.fullname" -}}
+{{- define "dracon.fullname" -}}
 {{- if .Values.fullnameOverride }}
 {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
 {{- else }}
@@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name.
 {{/*
 Create chart name and version as used by the chart label.
 */}}
-{{- define "kibana.chart" -}}
+{{- define "dracon.chart" -}}
 {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
 {{/*
 Common labels
 */}}
-{{- define "kibana.labels" -}}
-helm.sh/chart: {{ include "kibana.chart" . }}
-{{ include "kibana.selectorLabels" . }}
+{{- define "dracon.labels" -}}
+helm.sh/chart: {{ include "dracon.chart" . }}
+{{ include "dracon.selectorLabels" . }}
 {{- if .Chart.AppVersion }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
@@ -45,17 +45,17 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{/*
 Selector labels
 */}}
-{{- define "kibana.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "kibana.name" . }}
+{{- define "dracon.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "dracon.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
 {{/*
 Create the name of the service account to use
 */}}
-{{- define "kibana.serviceAccountName" -}}
+{{- define "dracon.serviceAccountName" -}}
 {{- if .Values.serviceAccount.create }}
-{{- default (include "kibana.fullname" .) .Values.serviceAccount.name }}
+{{- default (include "dracon.fullname" .) .Values.serviceAccount.name }}
 {{- else }}
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}

deploy/dracon/templates/arango-deployment.yaml

@@ -0,0 +1,15 @@
+{{ if .Values.arangodb.enabled }}
+apiVersion: database.arangodb.com/v1
+kind: ArangoDeployment
+metadata:
+  name: {{ include "dracon.fullname" . }}
+  labels:
+    {{- include "dracon.labels" . | nindent 4 }}
+spec:
+  mode: Single
+  externalAccess:
+    type: None
+  auth:
+    jwtSecretName: None # Disable auth
+  image: docker.io/arangodb/arangodb:{{ .Values.arangodb.version }}
+{{ end }}

deploy/dracon/templates/arango-ingress.yaml

@@ -0,0 +1,26 @@
+{{ if .Values.arangodb.enabled }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ include "dracon.fullname" . }}-arangodb
+  annotations:
+    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
+    nginx.ingress.kubernetes.io/ssl-passthrough: "false"
+  labels:
+    {{- include "dracon.labels" . | nindent 4 }}
+spec:
+  ingressClassName: {{ .Values.arangodb.ingress.className }}
+  defaultBackend:
+    service:
+      name: {{ include "dracon.fullname" . }}
+      port:
+        name: server
+  rules:
+  - host: {{ .Values.arangodb.ingress.host }}
+  {{- if .Values.arangodb.ingress.tlsEnabled }}
+  tls:
+  - hosts:
+    - {{ .Values.arangodb.ingress.host }}
+    secretName: cert-kibana
+  {{- end }}
+{{ end }}

deploy/elasticsearch/templates/deployment.yaml → deploy/dracon/templates/es-deployment.yaml

@@ -1,14 +1,15 @@
+{{ if .Values.elasticsearch.enabled }}
 apiVersion: elasticsearch.k8s.elastic.co/v1
 kind: Elasticsearch
 metadata:
-  name: {{ include "elasticsearch.fullname" . }}
+  name: {{ include "dracon.fullname" . }}
   annotations:
     "helm.sh/hook": post-install,post-upgrade
   labels:
-    {{- include "elasticsearch.labels" . | nindent 4 }}
+    {{- include "dracon.labels" . | nindent 4 }}
 spec:
-  version: {{ .Values.version }}
-  image: docker.elastic.co/elasticsearch/elasticsearch:{{ .Values.version }}
+  version: {{ .Values.elasticsearch.version }}
+  image: docker.elastic.co/elasticsearch/elasticsearch:{{ .Values.elasticsearch.version }}
   http:
     tls:
       selfSignedCertificate:
@@ -23,3 +24,4 @@ spec:
           username: anonymous
           roles: superuser
           authz_exception: false
+{{ end }}

deploy/kibana/templates/deployment.yaml → deploy/dracon/templates/kb-deployment.yaml

@@ -1,19 +1,18 @@
+{{ if .Values.kibana.enabled }}
 apiVersion: kibana.k8s.elastic.co/v1
 kind: Kibana
 metadata:
-  name: {{ include "kibana.fullname" . }}
+  name: {{ include "dracon.fullname" . }}
   annotations:
     "helm.sh/hook": post-install,post-upgrade
   labels:
-    {{- include "kibana.labels" . | nindent 4 }}
+    {{- include "dracon.labels" . | nindent 4 }}
 spec:
-  version: {{ .Values.version }}
-  image: docker.elastic.co/kibana/kibana:{{ .Values.version }}
+  version: {{ .Values.kibana.version }}
+  image: docker.elastic.co/kibana/kibana:{{ .Values.kibana.version }}
   count: 1
   elasticsearchRef:
-    name: {{ .Values.es_name }}
-  # config:
-  #   xpack.security.enabled: false
+    name: {{ include "dracon.fullname" . }}
   http:
     tls:
       selfSignedCertificate:
@@ -41,3 +40,4 @@ spec:
             periodSeconds: 10
             successThreshold: 1
             timeoutSeconds: 5
+{{ end }}

deploy/dracon/templates/kb-ingress.yaml

@@ -0,0 +1,25 @@
+{{ if .Values.kibana.enabled }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ include "dracon.fullname" . }}-kb
+  labels:
+    {{- include "dracon.labels" . | nindent 4 }}
+  annotations:
+    nginx.ingress.kubernetes.io/backend-protocol: "http"
+spec:
+  ingressClassName: {{ .Values.kibana.ingress.className }}
+  defaultBackend:
+    service:
+      name: {{ include "dracon.fullname" . }}-kb-http
+      port:
+        name: http
+  rules:
+  - host: {{ .Values.kibana.ingress.host }}
+  {{- if .Values.kibana.ingress.tlsEnabled }}
+  tls:
+  - hosts:
+    - {{ .Values.kibana.ingress.host }}
+    secretName: cert-kibana
+  {{- end }}
+{{ end }}