feat: add support for gcs (parseablehq#1033)

parmesant · Dec 15, 2024 · c0fb214 · c0fb214
1 parent 80a3ff4
commit c0fb214
Show file tree

Hide file tree

Showing 4 changed files with 63 additions and 8 deletions.
diff --git a/helm/templates/ingestor-statefulset.yaml b/helm/templates/ingestor-statefulset.yaml
@@ -42,7 +42,7 @@ spec:
         imagePullPolicy: {{ .Values.parseable.image.pullPolicy }}
         args:
           - /usr/bin/parseable
-          - {{ .Values.parseable.store }}
+          - {{ if eq .Values.parseable.store "gcs-store" }}"s3-store"{{ else }}{{ .Values.parseable.store | quote }}{{ end }}
           - --ingestor-endpoint=$(HOSTNAME).{{ include "parseable.fullname" . }}-ingestor-headless.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.parseable.highAvailability.ingestor.port }}
         env:
           {{- range $key, $value :=  .Values.parseable.highAvailability.ingestor.env }}
@@ -69,6 +69,20 @@ spec:
           {{- end }}
           {{- end }}
 
+          {{- if and .Values.parseable.gcsModeSecret .Values.parseable.gcsModeSecret.enabled }}
+          {{- range $secret := .Values.parseable.gcsModeSecret.secrets }}
+          {{- range $key := $secret.keys }}
+          {{- $envPrefix := $secret.prefix | default "" | upper }}
+          {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
+          - name: {{ $envPrefix }}{{ $envKey | replace "GCS" "S3"}}
+            valueFrom:
+              secretKeyRef:
+                name: {{ $secret.name }}
+                key: {{ $key }}
+          {{- end }}
+          {{- end }}
+          {{- end }}
+
           {{- if and .Values.parseable.blobModeSecret .Values.parseable.blobModeSecret.enabled }}
           {{- range $secret := .Values.parseable.blobModeSecret.secrets }}
           {{- range $key := $secret.keys }}

diff --git a/helm/templates/querier-statefulset.yaml b/helm/templates/querier-statefulset.yaml
@@ -43,9 +43,7 @@ spec:
           {{- toYaml .Values.parseable.securityContext | nindent 8 }}
         image: {{ .Values.parseable.image.repository }}:{{ .Values.parseable.image.tag | default .Chart.AppVersion }}
         imagePullPolicy: {{ .Values.parseable.image.pullPolicy }}
-        args: 
-        - /usr/bin/parseable
-        - {{ .Values.parseable.store }}
+        args: ["/usr/bin/parseable", {{ if eq .Values.parseable.store "gcs-store" }}"s3-store"{{ else }}{{ .Values.parseable.store | quote }}{{ end }}]
         env:
           - name: HOSTNAME
             valueFrom:
@@ -73,6 +71,20 @@ spec:
           {{- end }}
           {{- end }}
 
+          {{- if and .Values.parseable.gcsModeSecret .Values.parseable.gcsModeSecret.enabled }}
+          {{- range $secret := .Values.parseable.gcsModeSecret.secrets }}
+          {{- range $key := $secret.keys }}
+          {{- $envPrefix := $secret.prefix | default "" | upper }}
+          {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
+          - name: {{ $envPrefix }}{{ $envKey | replace "GCS" "S3"}}
+            valueFrom:
+              secretKeyRef:
+                name: {{ $secret.name }}
+                key: {{ $key }}
+          {{- end }}
+          {{- end }}
+          {{- end }}
+
           {{- if and .Values.parseable.blobModeSecret .Values.parseable.blobModeSecret.enabled }}
           {{- range $secret := .Values.parseable.blobModeSecret.secrets }}
           {{- range $key := $secret.keys }}

diff --git a/helm/templates/standalone-deployment.yaml b/helm/templates/standalone-deployment.yaml
@@ -36,7 +36,7 @@ spec:
           imagePullPolicy: {{ .Values.parseable.image.pullPolicy }}
           # Uncomment to debug
           # command: [ "/bin/sh", "-c", "sleep 1000000" ]
-          args: ["/usr/bin/parseable", {{ .Values.parseable.store | quote }}]
+          args: ["/usr/bin/parseable", {{ if eq .Values.parseable.store "gcs-store" }}"s3-store"{{ else }}{{ .Values.parseable.store | quote }}{{ end }}]
           env:
             - name: HOSTNAME
               valueFrom:
@@ -75,6 +75,20 @@ spec:
             {{- end }}
             {{- end }}
 
+            {{- if and .Values.parseable.gcsModeSecret .Values.parseable.gcsModeSecret.enabled }}
+            {{- range $secret := .Values.parseable.gcsModeSecret.secrets }}
+            {{- range $key := $secret.keys }}
+            {{- $envPrefix := $secret.prefix | default "" | upper }}
+            {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
+            - name: {{ $envPrefix }}{{ $envKey | replace "GCS" "S3"}}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $secret.name }}
+                  key: {{ $key }}
+            {{- end }}
+            {{- end }}
+            {{- end }}
+
             {{- if and .Values.parseable.blobModeSecret .Values.parseable.blobModeSecret.enabled }}
             {{- range $secret := .Values.parseable.blobModeSecret.secrets }}
             {{- range $key := $secret.keys }}

diff --git a/helm/values.yaml b/helm/values.yaml
@@ -3,8 +3,7 @@ parseable:
     repository: containers.parseable.com/parseable/parseable
     tag: v1.6.3
     pullPolicy: Always
-  ## object store can be local-store, s3-store or blob-store.
-  ## local needs to be false if set to object store.
+  ## object store can be local-store, s3-store, blob-store or gcs-store.
   store: local-store
   ## Set to true if you want to deploy Parseable in a HA mode (multiple ingestors)
   ## Please note that highAvailability is not supported in local mode
@@ -116,6 +115,22 @@ parseable:
           - s3.secret.key
           - s3.bucket
           - s3.region
+  gcsModeSecret:
+    enabled: false
+    secrets:
+      - name: parseable-env-secret
+        prefix: P_
+        keys:
+          - addr
+          - username
+          - password
+          - staging.dir
+          - fs.dir
+          - gcs.url
+          - gcs.access.key
+          - gcs.secret.key
+          - gcs.bucket
+          - gcs.region
   serviceAccount:
     create: true
     name: "parseable"
@@ -295,7 +310,7 @@ vector:
 # See fluent-bit helm documentation to learn more:
 # https://github.com/fluent/helm-charts/tree/main/charts/fluent-bit
 fluent-bit:
-  enabled: true
+  enabled: false
   kind: DaemonSet
   serverHost: parseable.parseable.svc.cluster.local
   replicaCount: 1