feat: add support for extraEnvFrom

README.md

@@ -28,3 +28,26 @@ The way storage config is defined changed and you have to update your helm value
 The changes are:
 - the `config.s3` and `config.azure_blob` values are no more supported. You now have to use the storage config as defined in the [docs](https://quickwit.io/docs/configuration/storage-config).
 - the keys of secrets have changed: `s3.secret_key` is replaced by `storage.s3.secret_access_key` and `azure_blob.access_key` is replace by `storage.azure.access_key`.
+
+## Upgrade helm chart from 0.5.0 to 0.6.0
+
+The way the `config` value works has changed in 0.6.0. It is now copied "as is"
+to the Quickwit nodes' configurations. In particular:
+
+- the `config.postgres` section does not support the following attributes
+  anymore
+```
+host: ""
+port: 5432
+database: metastore
+username: quickwit
+assword: ""
+```
+Configure `QW_METASTORE_URI` in `extraEnvFrom` instead (see
+[documentation](https://quickwit.io/docs/configuration/metastore-config) for
+more details).
+
+- the seed configuration has moved from `config` to a dedicated attribute. The
+  changes are:
+  - the `config.indexes` field is moved to `seed.indexes`
+  - the `config.sources` field is moved to `seed.sources`

charts/quickwit/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: quickwit
 description: Sub-second search & analytics engine on cloud storage.
 type: application
-version: 0.5.16
+version: 0.6.0
 appVersion: "v0.8.1"
 keywords:
   - quickwit

charts/quickwit/templates/_helpers.tpl

@@ -49,7 +49,7 @@ helm.sh/chart: {{ include "quickwit.chart" . }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{ include "quickwit.additionalLabels" . }}
+{{- include "quickwit.additionalLabels" . }}
 {{- end }}
 
 {{/*
@@ -147,63 +147,16 @@ Quickwit environment
   value: node.yaml
 - name: QW_CLUSTER_ID
   value: {{ .Release.Namespace }}-{{ include "quickwit.fullname" . }}
-{{- if ((.Values.config.storage).s3).access_key_id }}
-- name: AWS_ACCESS_KEY_ID
-  value: {{ .Values.config.storage.s3.access_key_id }}
-{{- end }}
-{{- if ((.Values.config.storage).s3).secret_access_key }}
-- name: AWS_SECRET_ACCESS_KEY
-  valueFrom:
-    secretKeyRef:
-      name: {{ include "quickwit.fullname" $ }}
-      key: storage.s3.secret_access_key
-{{- end }}
-{{- if ((.Values.config.storage).azure).account }}
-- name: QW_AZURE_STORAGE_ACCOUNT
-  value: {{ .Values.config.storage.azure.account }}
-{{- end }}
-{{- if ((.Values.config.storage).azure).access_key }}
-- name: QW_AZURE_STORAGE_ACCESS_KEY
-  valueFrom:
-    secretKeyRef:
-      name: {{ include "quickwit.fullname" $ }}
-      key: storage.azure.access_key
-{{- end }}
 - name: QW_NODE_ID
   value: "$(POD_NAME)"
 - name: QW_PEER_SEEDS
   value: {{ include "quickwit.fullname" . }}-headless
 - name: QW_ADVERTISE_ADDRESS
   value: "$(POD_IP)"
+- name: QW_CLUSTER_ENDPOINT
+  value: http://{{ include "quickwit.fullname" $ }}-metastore.{{ $.Release.Namespace }}.svc.cluster.local:7280
 {{- range $key, $value := .Values.environment }}
 - name: "{{ $key }}"
   value: "{{ $value }}"
 {{- end }}
 {{- end }}
-
-{{/*
-Quickwit metastore environment
-*/}}
-{{- define "quickwit.metastore.environment" -}}
-{{ include "quickwit.environment" . }}
-{{- if .Values.config.metastore_uri }}
-- name: QW_METASTORE_URI
-  value: {{ .Values.config.metastore_uri }}
-{{- else if .Values.config.postgres }}
-- name: POSTGRES_HOST
-  value: {{ required "A valid config.postgres.host is required!" .Values.config.postgres.host }}
-- name: POSTGRES_PORT
-  value: {{ .Values.config.postgres.port | default 5432 | quote }}
-- name: POSTGRES_DATABASE
-  value: {{ .Values.config.postgres.database | default "metastore" }}
-- name: POSTGRES_USERNAME
-  value: {{ .Values.config.postgres.username | default "quickwit" }}
-- name: POSTGRES_PASSWORD
-  valueFrom:
-    secretKeyRef:
-      name: {{ include "quickwit.fullname" . }}
-      key: postgres.password
-- name: QW_METASTORE_URI
-  value: "postgres://$(POSTGRES_USERNAME):$(POSTGRES_PASSWORD)@$(POSTGRES_HOST):$(POSTGRES_PORT)/$(POSTGRES_DATABASE)"      
-{{- end }}
-{{- end }}

charts/quickwit/templates/configmap-bootstrap.yaml

@@ -1,16 +1,16 @@
-{{- if and (or .Values.config.sources .Values.config.indexes) .Values.bootstrap.enabled -}}
+{{- if and (or .Values.seed.sources .Values.seed.indexes) .Values.bootstrap.enabled -}}
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ include "quickwit.fullname" . }}-bootstrap
   labels:
     {{- include "quickwit.labels" . | nindent 4 }}
 data:
-  {{- range .Values.config.indexes }}
+  {{- range .Values.seed.indexes }}
   {{ .index_id }}.yaml: |-
     {{- toYaml . | nindent 4 }}
   {{- end }}
-  {{- range .Values.config.sources }}
+  {{- range .Values.seed.sources }}
   {{ .source.source_id }}.yaml: |-
     {{- toYaml .source | nindent 4 }}
   {{- end }}

charts/quickwit/templates/configmap.yaml

@@ -6,33 +6,4 @@ metadata:
     {{- include "quickwit.labels" . | nindent 4 }}
 data:
   node.yaml: |-
-    version: 0.8
-    listen_address: 0.0.0.0
-    gossip_listen_port: 7282
-    data_dir: /quickwit/qwdata
-    default_index_root_uri: {{ .Values.config.default_index_root_uri }}
-    {{- with .Values.config.indexer }}
-    indexer:
-      {{- toYaml . | nindent 6 }}
-    {{- end }}
-    {{- with .Values.config.storage }}
-    storage:
-      {{- toYaml . | nindent 6 }}
-    {{- end }}
-    {{- with .Values.config.ingest_api }}
-    ingest_api:
-      {{- toYaml . | nindent 6 }}
-    {{- end }}
-    {{- with .Values.config.searcher }}
-    searcher:
-      {{- toYaml . | nindent 6 }}
-    {{- end }}
-    {{- with .Values.config.jaeger }}
-    jaeger:
-      {{- toYaml . | nindent 6 }}
-    {{- end }}
-    {{- if (.Values.config.postgres).max_num_connections }}
-    metastore:
-      postgres:
-        max_num_connections: {{ .Values.config.postgres.max_num_connections }}
-    {{- end }}
+    {{- toYaml .Values.config | nindent 4 }}

charts/quickwit/templates/control-plane-deployment.yaml

@@ -52,6 +52,15 @@ spec:
             - name: "{{ $key }}"
               value: "{{ $value }}"
             {{- end }}
+          {{- if or (.Values.environmentFrom) (.Values.control_plane.extraEnvFrom) }}
+          envFrom:
+          {{- with .Values.environmentFrom }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.control_plane.extraEnvFrom }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- end }}
           ports:
             {{- include "quickwit.ports" . | nindent 12 }}
           startupProbe:

charts/quickwit/templates/indexer-statefulset.yaml

@@ -55,6 +55,15 @@ spec:
             - name: "{{ $key }}"
               value: "{{ $value }}"
             {{- end }}
+          {{- if or (.Values.environmentFrom) (.Values.indexer.extraEnvFrom) }}
+          envFrom:
+          {{- with .Values.environmentFrom }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.indexer.extraEnvFrom }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- end }}
           args: ["run", "--service", "indexer"]
           ports:
             {{- include "quickwit.ports" . | nindent 12 }}

charts/quickwit/templates/janitor-deployment.yaml

@@ -53,6 +53,15 @@ spec:
             - name: "{{ $key }}"
               value: "{{ $value }}"
             {{- end }}
+          {{- if or (.Values.environmentFrom) (.Values.janitor.extraEnvFrom) }}
+          envFrom:
+          {{- with .Values.environmentFrom }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.janitor.extraEnvFrom }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- end }}
           ports:
             {{- include "quickwit.ports" . | nindent 12 }}
           startupProbe:

charts/quickwit/templates/job-create-indices.yaml

@@ -1,5 +1,5 @@
 {{- if .Values.bootstrap.enabled -}}
-{{- range .Values.config.indexes }}
+{{- range .Values.seed.indexes }}
 ---
 apiVersion: batch/v1
 kind: Job
@@ -35,13 +35,20 @@ spec:
         imagePullPolicy: {{ $.Values.image.pullPolicy }}
         command: ["/bin/bash","-c","quickwit index describe --index {{ .index_id }} --endpoint ${QW_CLUSTER_ENDPOINT} || quickwit index create --index-config {{ .index_id }}.yaml --endpoint ${QW_CLUSTER_ENDPOINT}"]
         env:
-          - name: QW_CLUSTER_ENDPOINT
-            value: http://{{ include "quickwit.fullname" $ }}-metastore.{{ $.Release.Namespace }}.svc.cluster.local:7280
-          {{- include "quickwit.metastore.environment" $ | nindent 10 }}
+          {{- include "quickwit.environment" $ | nindent 10 }}
           {{- range $key, $value := $.Values.bootstrap.extraEnv }}
           - name: "{{ $key }}"
             value: "{{ $value }}"
           {{- end }}
+        {{- if or ($.Values.environmentFrom) ($.Values.bootstrap.extraEnvFrom) }}
+        envFrom:
+        {{- with $.Values.environmentFrom }}
+          {{- toYaml . | nindent 12 }}
+        {{- end }}
+        {{- with $.Values.bootstrap.extraEnvFrom }}
+          {{- toYaml . | nindent 12 }}
+        {{- end }}
+        {{- end }}
         volumeMounts:
           - name: config
             mountPath: /quickwit/node.yaml

charts/quickwit/templates/job-create-sources.yaml

@@ -1,5 +1,5 @@
 {{- if .Values.bootstrap.enabled -}}
-{{- range .Values.config.sources }}
+{{- range .Values.seed.sources }}
 ---
 apiVersion: batch/v1
 kind: Job
@@ -35,18 +35,25 @@ spec:
         imagePullPolicy: {{ $.Values.image.pullPolicy }}
         command: ["/bin/bash","-c","quickwit source describe --index {{ .index }} --source {{ .source.source_id }} --endpoint ${QW_CLUSTER_ENDPOINT}|| quickwit source create --index {{ .index }} --source-config {{ .source.source_id }}.yaml --endpoint ${QW_CLUSTER_ENDPOINT}"]
         env:
-          - name: QW_CLUSTER_ENDPOINT
-            value: http://{{ include "quickwit.fullname" $ }}-metastore.{{ $.Release.Namespace }}.svc.cluster.local:7280
-          {{- include "quickwit.metastore.environment" $ | nindent 10 }}
+          {{- include "quickwit.environment" $ | nindent 10 }}
           {{- range $key, $value := $.Values.bootstrap.extraEnv }}
           - name: "{{ $key }}"
             value: "{{ $value }}"
           {{- end }}
+        {{- if or ($.Values.environmentFrom) ($.Values.bootstrap.extraEnvFrom) }}
+        envFrom:
+        {{- with $.Values.environmentFrom }}
+          {{- toYaml . | nindent 12 }}
+        {{- end }}
+        {{- with $.Values.bootstrap.extraEnvFrom }}
+          {{- toYaml . | nindent 12 }}
+        {{- end }}
+        {{- end }}
         volumeMounts:
           - name: config
             mountPath: /quickwit/node.yaml
             subPath: node.yaml
-          {{- if $.Values.config.sources }}
+          {{- if $.Values.seed.sources }}
           - name: source
             mountPath: /quickwit/{{ .source.source_id }}.yaml
             subPath: {{ .source.source_id }}.yaml

charts/quickwit/templates/metastore-deployment.yaml

@@ -46,11 +46,20 @@ spec:
           imagePullPolicy: {{ .Values.image.pullPolicy }}        
           args: ["run", "--service", "metastore"]
           env:
-            {{- include "quickwit.metastore.environment" . | nindent 12 }}
+            {{- include "quickwit.environment" . | nindent 12 }}
             {{- range $key, $value := .Values.metastore.extraEnv }}
             - name: "{{ $key }}"
               value: "{{ $value }}"
             {{- end }}
+          {{- if or (.Values.environmentFrom) (.Values.metastore.extraEnvFrom) }}
+          envFrom:
+          {{- with .Values.environmentFrom }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.metastore.extraEnvFrom }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- end }}
           ports:
             {{- include "quickwit.ports" . | nindent 12 }}
           startupProbe:

charts/quickwit/templates/searcher-statefulset.yaml

@@ -55,6 +55,15 @@ spec:
             - name: "{{ $key }}"
               value: "{{ $value }}"
             {{- end }}
+          {{- if or (.Values.environmentFrom) (.Values.searcher.extraEnvFrom) }}
+          envFrom:
+          {{- with .Values.environmentFrom }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.searcher.extraEnvFrom }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- end }}
           args: ["run", "--service", "searcher"]
           ports:
             {{- include "quickwit.ports" . | nindent 12 }}