diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8196cfbd3..4ec6ae936 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -35,5 +35,14 @@ jobs: set -e DRACON_VERSION_SEMVER=$(sed 's/v//' <<< ${{ github.ref_name }}) make cmd/draconctl/bin + # Package, Publish Components bin/cmd/draconctl components package --version ${{ github.ref_name }} --chart-version ${DRACON_VERSION_SEMVER} --name dracon-oss-components ./components helm push dracon-oss-components-${DRACON_VERSION_SEMVER}.tgz oci://ghcr.io/ocurity/dracon/charts + + # Package, Publish Migrations + helm package -u\ + --version ${{ github.ref_name }}\ + --chart-version ${DRACON_VERSION_SEMVER}\ + --name dracon-enrichment-db-migrations\ + ./deploy/enrichment-db-migrations/chart + helm push dracon-enrichment-db-migrations oci://ghcr.io/ocurity/dracon/charts diff --git a/Makefile b/Makefile index e24ef8d61..88c02ac0e 100644 --- a/Makefile +++ b/Makefile @@ -210,12 +210,14 @@ dev-dracon: deploy-elasticoperator deploy-arangodb-crds add-bitnami-repo @echo "fetching dependencies if needed" @helm dependency build ./deploy/dracon/chart @echo "deploying dracon in dev mode" + @helm dependency update ./deploy/dracon/chart + @helm dependency build ./deploy/dracon/chart @helm upgrade dracon ./deploy/dracon/chart \ --install \ --values ./deploy/dracon/values/dev.yaml \ --create-namespace \ --namespace $(DRACON_NS) \ - --set "enrichmentDB.migrations.image=$(CONTAINER_REPO)/draconctl:$(DRACON_VERSION)" + --set "enrichment-db-migrations.image.tag=$(DRACON_VERSION)" --wait @helm upgrade dracon-oss-components oci://ghcr.io/ocurity/dracon/charts/dracon-oss-components \ --install \ diff --git a/deploy/dracon/chart/Chart.lock b/deploy/dracon/chart/Chart.lock index e342a25b2..eea9eec28 100644 --- a/deploy/dracon/chart/Chart.lock +++ b/deploy/dracon/chart/Chart.lock @@ -5,5 +5,8 @@ dependencies: - name: mongodb repository: https://charts.bitnami.com/bitnami version: 15.1.5 -digest: sha256:f5b463f7862318ed8de9439769a72f14320f271c72c80ec7a2a1f1b209959d7a -generated: "2024-05-02T22:03:05.901032098+03:00" +- name: enrichment-db-migrations + repository: file://../../enrichment-db-migrations/chart + version: 0.7.0 +digest: sha256:c38df8a4fdc32cc93962c87c8a7e8ef6da7fee48f03c0972b467691c124adfef +generated: "2024-07-14T18:24:02.622213661+01:00" diff --git a/deploy/dracon/chart/Chart.yaml b/deploy/dracon/chart/Chart.yaml index b3b72a48c..883ee6ca4 100644 --- a/deploy/dracon/chart/Chart.yaml +++ b/deploy/dracon/chart/Chart.yaml @@ -14,3 +14,6 @@ dependencies: version: 15.1.5 repository: https://charts.bitnami.com/bitnami condition: mongodb.enabled + - name: enrichment-db-migrations + version: 0.7.0 + repository: oci://ghcr.io/ocurity/dracon/charts \ No newline at end of file diff --git a/deploy/dracon/chart/templates/migrations-job.yaml b/deploy/dracon/chart/templates/migrations-job.yaml deleted file mode 100644 index 0a3dced3c..000000000 --- a/deploy/dracon/chart/templates/migrations-job.yaml +++ /dev/null @@ -1,35 +0,0 @@ -{{ if .Values.enrichmentDB.migrations.enabled }} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "dracon.fullname" . }}-migrations - labels: - {{- include "dracon.labels" . | nindent 4 }} - annotations: - # This is what defines this resource as a hook. Without this line, the - # job is considered part of the release. - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-weight": "-5" - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded, hook-failed -spec: - template: - metadata: - name: dracon-migrations - labels: - {{- include "dracon.labels" . | nindent 8 }} - spec: - restartPolicy: Never - containers: - - name: dracon-migrations - image: {{ .Values.enrichmentDB.migrations.image | default (print .Values.global.image.registry "/ocurity/dracon/draconctl:" .Chart.AppVersion) }} - env: - - name: DRACONCTL_MIGRATIONS_PATH - value: "/etc/dracon/migrations/enrichment" - command: - - draconctl - - migrations - - apply - - --url - - {{.Values.enrichmentDB.connectionStr | quote}} - serviceAccountName: {{ include "dracon.fullname" . }}-migrations -{{ end }} \ No newline at end of file diff --git a/deploy/dracon/chart/templates/migrations-rolebinding.yaml b/deploy/dracon/chart/templates/migrations-rolebinding.yaml deleted file mode 100644 index 18b8d3130..000000000 --- a/deploy/dracon/chart/templates/migrations-rolebinding.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "dracon.fullname" . }}-migrations -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ include "dracon.fullname" . }}-migrations -subjects: -- kind: ServiceAccount - name: {{ include "dracon.fullname" . }}-migrations diff --git a/deploy/dracon/chart/templates/serviceaccount.yaml b/deploy/dracon/chart/templates/serviceaccount.yaml deleted file mode 100644 index d725dbfb7..000000000 --- a/deploy/dracon/chart/templates/serviceaccount.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "dracon.fullname" . }}-migrations diff --git a/deploy/dracon/chart/values.yaml b/deploy/dracon/chart/values.yaml index 4db9933dc..b3a793450 100644 --- a/deploy/dracon/chart/values.yaml +++ b/deploy/dracon/chart/values.yaml @@ -46,22 +46,20 @@ postgresql: password: "" database: "" postgresPassword: "" + querystringargs: "" fullnameOverride: "" # this section controls aspects of managing a database used to store enrichments # the database should use the Postgres dialect. -enrichmentDB: - migrations: - # if set, a Job will be deployed that applies migrations to the enrichment database - # the Job will run as part of the post-install/post-upgrade hook - enabled: true - # image to use for applying the migrations - image: "" +enrichment_db_migrations: + # if set, a Job will be deployed that applies migrations to the enrichment database + # the Job will run as part of the post-install/post-upgrade hook + enabled: true + # image to use for applying the migrations + migrationsImage: "" # connection string for the migrations and potentially other processes to connect to the # enrichment database - connectionStr: "" -global: - image: - # registry to use for all - registry: "" +image: + # registry to use for all + registry: "" diff --git a/deploy/dracon/values/dev.yaml b/deploy/dracon/values/dev.yaml index fad78b773..be7628142 100644 --- a/deploy/dracon/values/dev.yaml +++ b/deploy/dracon/values/dev.yaml @@ -27,6 +27,21 @@ arangodb: className: nginx host: arangodb.dracon.localhost +global: + image: + registry: kind-registry:5000/ocurity/dracon + + postgresql: + enabled: true + auth: + username: dracon + password: dracon + database: dracon + postgresPassword: dracon + querystringargs: "sslmode=disable" + host: dracon-postgresql:5432 + +# necessary duplication because postgres does not support global variables postgresql: enabled: true auth: @@ -34,16 +49,13 @@ postgresql: password: dracon database: dracon postgresPassword: dracon - fullnameOverride: dracon-enrichment-db + querystringargs: "sslmode=disable" + host: dracon-postgresql:5432 tekton: enabled: true -enrichmentDB: - migrations: - enabled: true - connectionStr: postgresql://dracon:dracon@dracon-enrichment-db?sslmode=disable +enrichment-db-migrations: + migrationsImage: "" + enabled: true -global: - image: - registry: kind-registry:5000 diff --git a/deploy/enrichment-db-migrations/chart/.helmignore b/deploy/enrichment-db-migrations/chart/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/deploy/enrichment-db-migrations/chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/enrichment-db-migrations/chart/Chart.lock b/deploy/enrichment-db-migrations/chart/Chart.lock new file mode 100644 index 000000000..35aa804d7 --- /dev/null +++ b/deploy/enrichment-db-migrations/chart/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 15.2.5 +digest: sha256:8145d7210428553425c22a5cc40850a0d8da6d02a7b1dd3b05766e0e2e48b6c0 +generated: "2024-07-12T14:51:36.398614928+01:00" diff --git a/deploy/enrichment-db-migrations/chart/Chart.yaml b/deploy/enrichment-db-migrations/chart/Chart.yaml new file mode 100644 index 000000000..5afcb78f5 --- /dev/null +++ b/deploy/enrichment-db-migrations/chart/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: enrichment-db-migrations +description: | + A Helm chart for Kubernetes containing the dracon migrations needed for Dracon pipelines to run. Please check the documentation for more information +type: application +version: 0.7.0 +appVersion: 0.7.0 +dependencies: + - name: postgresql + version: 15.5.4 + repository: https://charts.bitnami.com/bitnami + condition: postgres.enabled \ No newline at end of file diff --git a/deploy/enrichment-db-migrations/chart/charts/postgresql-15.2.5.tgz b/deploy/enrichment-db-migrations/chart/charts/postgresql-15.2.5.tgz new file mode 100644 index 000000000..f63e2e970 Binary files /dev/null and b/deploy/enrichment-db-migrations/chart/charts/postgresql-15.2.5.tgz differ diff --git a/deploy/enrichment-db-migrations/chart/templates/_helpers.tpl b/deploy/enrichment-db-migrations/chart/templates/_helpers.tpl new file mode 100644 index 000000000..85742e1ce --- /dev/null +++ b/deploy/enrichment-db-migrations/chart/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "enrichment_db_migrations.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "enrichment_db_migrations.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "enrichment_db_migrations.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "enrichment_db_migrations.labels" -}} +helm.sh/chart: {{ include "enrichment_db_migrations.chart" . }} +{{ include "enrichment_db_migrations.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "enrichment_db_migrations.selectorLabels" -}} +app.kubernetes.io/name: {{ include "enrichment_db_migrations.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "enrichment_db_migrations.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "enrichment_db_migrations.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/deploy/enrichment-db-migrations/chart/templates/migrations-job.yaml b/deploy/enrichment-db-migrations/chart/templates/migrations-job.yaml new file mode 100644 index 000000000..87cbed5b2 --- /dev/null +++ b/deploy/enrichment-db-migrations/chart/templates/migrations-job.yaml @@ -0,0 +1,35 @@ +{{ if .Values.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "enrichment_db_migrations.fullname" . }} + labels: + {{- include "enrichment_db_migrations.labels" . | nindent 4 }} + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded, hook-failed +spec: + template: + metadata: + name: enrichment-db-migrations + labels: + {{- include "enrichment_db_migrations.labels" . | nindent 8 }} + spec: + restartPolicy: Never + containers: + - name: enrichment-db-migrations + image: "{{ .Values.global.image.registry }}/draconctl:{{ .Values.image.tag | default .Chart.AppVersion }}" + env: + - name: DRACONCTL_MIGRATIONS_PATH + value: "/etc/dracon/migrations/enrichment" + command: + - draconctl + - migrations + - apply + - --url + - "postgresql://{{ .Values.global.postgresql.auth.username }}:{{ .Values.global.postgresql.auth.password }}@{{ .Values.global.postgresql.host }}/{{ .Values.global.postgresql.auth.database }}?{{ .Values.global.postgresql.auth.querystringargs}}" + serviceAccountName: {{ include "enrichment_db_migrations.fullname" . }} +{{ end }} \ No newline at end of file diff --git a/deploy/dracon/chart/templates/migrations-role.yaml b/deploy/enrichment-db-migrations/chart/templates/migrations-role.yaml similarity index 76% rename from deploy/dracon/chart/templates/migrations-role.yaml rename to deploy/enrichment-db-migrations/chart/templates/migrations-role.yaml index 65abc4b41..617d9c06e 100644 --- a/deploy/dracon/chart/templates/migrations-role.yaml +++ b/deploy/enrichment-db-migrations/chart/templates/migrations-role.yaml @@ -1,7 +1,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: {{ include "dracon.fullname" . }}-migrations + name: {{ include "enrichment_db_migrations.fullname" . }} rules: - apiGroups: - coordination.k8s.io diff --git a/deploy/enrichment-db-migrations/chart/templates/migrations-rolebinding.yaml b/deploy/enrichment-db-migrations/chart/templates/migrations-rolebinding.yaml new file mode 100644 index 000000000..2fbf6a790 --- /dev/null +++ b/deploy/enrichment-db-migrations/chart/templates/migrations-rolebinding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "enrichment_db_migrations.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "enrichment_db_migrations.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "enrichment_db_migrations.fullname" . }} diff --git a/deploy/enrichment-db-migrations/chart/templates/serviceaccount.yaml b/deploy/enrichment-db-migrations/chart/templates/serviceaccount.yaml new file mode 100644 index 000000000..0f9a9fb25 --- /dev/null +++ b/deploy/enrichment-db-migrations/chart/templates/serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "enrichment_db_migrations.fullname" . }} diff --git a/deploy/enrichment-db-migrations/chart/values.yaml b/deploy/enrichment-db-migrations/chart/values.yaml new file mode 100644 index 000000000..1bc2731e2 --- /dev/null +++ b/deploy/enrichment-db-migrations/chart/values.yaml @@ -0,0 +1,22 @@ +# this section controls aspects of managing a database used to store enrichments +# the database should use the Postgres dialect. + +# image to use for applying the migrations +migrationsImage: "" +global: + image: + # registry to use for all + registry: "" + +enabled: true + +serviceAccount: + create: false + +postgresql: + enabled: true + auth: + username: "" + password: "" + database: "" + postgresPassword: "" \ No newline at end of file diff --git a/deploy/enrichment-db-migrations/values/dev.yaml b/deploy/enrichment-db-migrations/values/dev.yaml new file mode 100644 index 000000000..402f18a27 --- /dev/null +++ b/deploy/enrichment-db-migrations/values/dev.yaml @@ -0,0 +1,25 @@ + +global: + postgresql: + enabled: true + auth: + username: dracon + password: dracon + database: dracon + postgresPassword: dracon + querystringargs: "sslmode=disable" + fullnameOverride: dracon-enrichment-db + + image: + registry: kind-registry:5000 + +# necessary duplication because postgrs does not support global variables +postgresql: + enabled: true + auth: + username: dracon + password: dracon + database: dracon + postgresPassword: dracon + querystringargs: "sslmode=disable" + fullnameOverride: dracon-enrichment-db