Add clickhouse deployment and setup clickhouse migrations

langchain-ai · Dec 19, 2023 · 3f14a94 · 3f14a94
1 parent c3fe2ca
commit 3f14a94
Show file tree

Hide file tree

Showing 8 changed files with 109 additions and 11 deletions.
diff --git a/charts/langsmith/templates/_helpers.tpl b/charts/langsmith/templates/_helpers.tpl
@@ -156,6 +156,14 @@ Template containing common environment variables that are used by several servic
 {{- end -}}
 {{- end -}}
 
+{{- define "clickhouse.serviceAccountName" -}}
+{{- if .Values.clickhouse.serviceAccount.create -}}
+    {{ default (printf "%s-%s" (include "langsmith.fullname" .) .Values.clickhouse.name) .Values.clickhouse.serviceAccount.name | trunc 63 | trimSuffix "-" }}
+{{- else -}}
+    {{ default "default" .Values.clickhouse.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
 {{- define "frontend.serviceAccountName" -}}
 {{- if .Values.frontend.serviceAccount.create -}}
     {{ default (printf "%s-%s" (include "langsmith.fullname" .) .Values.frontend.name) .Values.frontend.serviceAccount.name | trunc 63 | trimSuffix "-" }}

diff --git a/charts/langsmith/templates/backend/run-migrations.yaml b/charts/langsmith/templates/backend/run-migrations.yaml
@@ -54,6 +54,28 @@ spec:
             {{- toYaml .Values.backend.migrations.resources | nindent 12 }}
           securityContext:
             {{- toYaml .Values.backend.migrations.securityContext | nindent 12 }}
+        - name: {{ .Values.backend.name }}-clickhouse
+          command:
+          - migrate
+          - -source
+          - file://clickhouse/migrations
+          - -database
+          - "clickhouse://{{ include "langsmith.fullname" . }}-{{ .Values.clickhouse.name }}:{{ .Values.clickhouse.containerNativePort }}?username=clickhouse_admin&password=password&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+          - up
+          env:
+            {{- include "langsmith.commonEnv" . | nindent 12 }}
+            {{- with .Values.backend.migrations.extraEnv }}
+              {{- toYaml . | nindent 12 }}
+            {{- end }}
+          envFrom:
+            - configMapRef:
+                name: {{ include "langsmith.fullname" . }}-config
+          image: "{{ .Values.images.backendImage.repository }}:{{ .Values.images.backendImage.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.images.backendImage.pullPolicy }}
+          resources:
+            {{- toYaml .Values.backend.migrations.resources | nindent 12 }}
+          securityContext:
+            {{- toYaml .Values.backend.migrations.securityContext | nindent 12 }}
         {{- with .Values.backend.migrations.sidecars }}
           {{- toYaml . | nindent 8 }}
         {{- end }}

diff --git a/charts/langsmith/templates/clickhouse/config-map.yaml b/charts/langsmith/templates/clickhouse/config-map.yaml
@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "langsmith.fullname" . }}-{{ .Values.clickhouse.name }}
+  labels:
+    {{- include "langsmith.labels" . | nindent 4 }}
+  annotations:
+    {{- include "langsmith.annotations" . | nindent 4 }}
+data:
+  users.xml: |
+    <clickhouse>
+        <profiles>
+            <default>
+                <readonly>0</readonly>
+            </default>
+        </profiles>
+        <users>
+            <default>
+                <password></password>
+                <access_management>1</access_management>
+                <named_collection_control>1</named_collection_control>
+                <show_named_collections>1</show_named_collections>
+                <show_named_collections_secrets>1</show_named_collections_secrets>
+            </default>
+        </users>
+    </clickhouse>
diff --git a/charts/langsmith/templates/clickhouse/service-account.yaml b/charts/langsmith/templates/clickhouse/service-account.yaml
@@ -0,0 +1,16 @@
+{{- if .Values.clickhouse.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "clickhouse.serviceAccountName" . }}
+  labels:
+    {{- include "langsmith.labels" . | nindent 4 }}
+    {{- with.Values.clickhouse.serviceAccount.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  annotations:
+    {{- include "langsmith.annotations" . | nindent 4 }}
+    {{- with.Values.clickhouse.serviceAccount.annotations }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+{{- end }}
diff --git a/charts/langsmith/templates/clickhouse/service.yaml b/charts/langsmith/templates/clickhouse/service.yaml
@@ -16,11 +16,15 @@ metadata:
 spec:
   type: {{ .Values.clickhouse.service.type }}
   ports:
-    - name: clickhouse
-      port: {{ .Values.clickhouse.service.port }}
-      targetPort: clickhouse
+    - name: ch
+      port: {{ .Values.clickhouse.service.httpPort }}
+      targetPort: ch
+      protocol: TCP
+    - name: ch-native
+      port: {{ .Values.clickhouse.service.nativePort }}
+      targetPort: ch-native
       protocol: TCP
   selector:
     {{- include "langsmith.selectorLabels" . | nindent 4 }}
-    app.kubernetes.io/name: {{ include "langsmith.fullname" . }}-{{ .Values.clickhouse.name }}
+    app.kubernetes.io/component: {{ include "langsmith.fullname" . }}-{{ .Values.clickhouse.name }}
 {{- end }}
diff --git a/charts/langsmith/templates/clickhouse/stateful-set.yaml b/charts/langsmith/templates/clickhouse/stateful-set.yaml
@@ -19,7 +19,7 @@ spec:
   selector:
     matchLabels:
       {{- include "langsmith.selectorLabels" . | nindent 6 }}
-      app.kubernetes.io/name: {{ include "langsmith.fullname" . }}-{{ .Values.clickhouse.name }}
+      app.kubernetes.io/component: {{ include "langsmith.fullname" . }}-{{ .Values.clickhouse.name }}
   template:
     metadata:
       {{- with .Values.clickhouse.statefulSet.annotations }}
@@ -31,7 +31,7 @@ spec:
             {{- toYaml . | nindent 8 }}
         {{- end }}
         {{- include "langsmith.labels" . | nindent 8 }}
-        app.kubernetes.io/name: {{ include "langsmith.fullname" . }}-{{ .Values.clickhouse.name }}
+        app.kubernetes.io/component: {{ include "langsmith.fullname" . }}-{{ .Values.clickhouse.name }}
     spec:
       {{- with .Values.images.imagePullSecrets }}
       imagePullSecrets:
@@ -43,11 +43,18 @@ spec:
         - name: {{ .Values.clickhouse.name }}
           image: "{{ .Values.images.clickhouseImage.repository }}:{{ .Values.images.clickhouseImage.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.images.clickhouseImage.pullPolicy }}
+          env:
+            - name: CLICKHOUSE_DB
+              value: default
+            - name: CLICKHOUSE_USER
+              value: clickhouse_admin
+            - name: CLICKHOUSE_PASSWORD
+              value: password
           ports:
-            - name: clickhouse
+            - name: ch
               containerPort: {{ .Values.clickhouse.containerHttpPort }}
               protocol: TCP
-            - name: clickhouse-native
+            - name: ch-native
               containerPort: {{ .Values.clickhouse.containerNativePort }}
               protocol: TCP
           startupProbe:
@@ -78,8 +85,9 @@ spec:
           volumeMounts:
             - mountPath: /var/lib/clickhouse
               name: data
-            - mountPath: /var/lib/clickhouse
-              name: config
+            - mountPath: /etc/clickhouse-server/users.d/users.xml
+              name: clickhouse-conf
+              subPath: users.xml
       {{- with .Values.clickhouse.statefulSet.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
@@ -92,14 +100,21 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      volumes:
+        - name: clickhouse-conf
+          configMap:
+            name: {{ include "langsmith.fullname" . }}-{{ .Values.clickhouse.name }}
+            items:
+              - key: users.xml
+                path: users.xml
   volumeClaimTemplates:
     - apiVersion: v1
       kind: PersistentVolumeClaim
       metadata:
         name: data
         labels:
           {{- include "langsmith.labels" . | nindent 10 }}
-          app.kubernetes.io/name: {{ include "langsmith.fullname" . }}-{{ .Values.clickhouse.name }}
+          app.kubernetes.io/component: {{ include "langsmith.fullname" . }}-{{ .Values.clickhouse.name }}
       spec:
         accessModes:
           - ReadWriteOnce

diff --git a/charts/langsmith/templates/frontend/config-map.yaml b/charts/langsmith/templates/frontend/config-map.yaml
@@ -4,6 +4,8 @@ metadata:
   name: {{ include "langsmith.fullname" . }}-{{ .Values.frontend.name }}
   labels:
     {{- include "langsmith.labels" . | nindent 4 }}
+  annotations:
+    {{- include "langsmith.annotations" . | nindent 4 }}
 data:
   nginx.conf: |
     server {

diff --git a/charts/langsmith/values.yaml b/charts/langsmith/values.yaml
@@ -139,6 +139,11 @@ clickhouse:
     nativePort: 9000
     labels: {}
     annotations: {}
+  serviceAccount:
+    create: true
+    name: ""
+    labels: { }
+    annotations: { }
 
 frontend:
   name: "frontend"