Refresh Metaflow charts and create a github pages to host them

.github/workflows/publish_helm_charts.yml

@@ -0,0 +1,34 @@
+name: Release Charts
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  release:
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Configure Git
+      run: |
+        git config user.name "$GITHUB_ACTOR"
+        git config user.email "[email protected]"
+
+    - name: Install Helm
+      uses: azure/setup-helm@v4
+      env:
+        GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+
+    - name: Run chart-releaser
+      uses: helm/[email protected]
+      with:
+        charts_dir: charts/metaflow
+      env:
+        CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

charts/metaflow/Chart.lock

@@ -0,0 +1,12 @@
+dependencies:
+- name: metaflow-service
+  repository: ""
+  version: 0.2.0
+- name: metaflow-ui
+  repository: ""
+  version: 0.3.0
+- name: postgresql
+  repository: https://charts.bitnami.com/bitnami
+  version: 14.3.3
+digest: sha256:c9397633f678654b4d610387e66930c1a08461e639d58c4cba2e460362a22dd7
+generated: "2024-12-17T14:23:51.728248-08:00"

charts/metaflow/Chart.yaml

@@ -0,0 +1,28 @@
+apiVersion: v2
+appVersion: v2.4.13
+dependencies:
+  - name: metaflow-service
+    repository: ""
+    version: 0.2.0
+    condition: metaflow-service.enabled
+  - name: metaflow-ui
+    repository: ""
+    version: 0.3.0
+    condition: metaflow-ui.enabled
+  - name: postgresql
+    repository: https://charts.bitnami.com/bitnami
+    version: "^14"
+    condition: postgresql.enabled
+description: Machine Learning platform
+home: https://metaflow.org
+keywords:
+  - mlops
+  - metaflow
+  - metadata service
+  - metaflow ui
+name: metaflow
+sources:
+  - https://github.com/Netflix/metaflow
+  - https://github.com/outerbounds/metaflow-tools
+type: application
+version: 0.0.2

charts/metaflow/README.md

@@ -0,0 +1,10 @@
+# Helm Chart for Metaflow Services
+
+This Helm chart provides a comprehensive deployment solution for Metaflow Services in a Kubernetes cluster. It includes three main components:
+
+* Metaflow Metadata Service - Manages workflow metadata and execution state
+* Metaflow UI (Frontend and Backend) - Provides web-based visualization and monitoring
+* PostgreSQL Database - Stores metadata for Metaflow services
+
+While the included PostgreSQL sub-chart enables quick evaluation and development setups, for production environments we strongly recommend using a managed PostgreSQL service (like AWS RDS). To use an external database, disable the PostgreSQL sub-chart by setting `postgresql.enabled: false`.
+

k8s/helm/metaflow/charts/metaflow-ui/.helmignore → charts/metaflow/charts/metaflow-service/.helmignore

@@ -2,6 +2,9 @@
 # This supports shell glob matching, relative path matching, and
 # negation (prefixed with !). Only one pattern per line.
 .DS_Store
+Thumbs.db
+ehthumbs.db
+
 # Common VCS dirs
 .git/
 .gitignore
@@ -10,14 +13,40 @@
 .hg/
 .hgignore
 .svn/
+
 # Common backup files
 *.swp
 *.bak
 *.tmp
 *.orig
 *~
+
 # Various IDEs
 .project
 .idea/
 *.tmproj
 .vscode/
+*.suo
+*.user
+*.userossc
+*.sln.docstates
+
+# Logs
+*.log
+
+# Node.js
+node_modules/
+npm-debug.log
+yarn-error.log
+
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+.Python
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/

charts/metaflow/charts/metaflow-service/Chart.yaml

@@ -0,0 +1,19 @@
+apiVersion: v2
+appVersion: v2.4.13
+description: A Helm chart to deploy Metadata Service for Metaflow
+name: metaflow-service
+type: application
+version: 0.2.0
+maintainers:
+  - name: Savin Goyal
+    email: [email protected]
+  - name: Oleg Avdeev
+    email: [email protected]
+keywords:
+  - metaflow
+  - metadata
+  - service
+sources:
+  - https://github.com/Netflix/metaflow-service
+  - https://github.com/outerbounds/metaflow-tools
+icon: https://metaflow.org/images/metaflow.svg

k8s/helm/metaflow/charts/metaflow-service/templates/NOTES.txt → charts/metaflow/charts/metaflow-service/templates/NOTES.txt

@@ -1,22 +1,34 @@
 1. Get the application URL by running these commands:
 {{- if .Values.ingress.enabled }}
-{{- range $host := .Values.ingress.hosts }}
-  {{- range .paths }}
-  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
+  {{- range $host := .Values.ingress.hosts }}
+    {{- range .paths }}
+      http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
+    {{- end }}
   {{- end }}
-{{- end }}
 {{- else if contains "NodePort" .Values.service.type }}
   export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "metaflow-service.fullname" . }})
   export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  echo http://$NODE_IP:$NODE_PORT
+  if [ -z "$NODE_PORT" ] || [ -z "$NODE_IP" ]; then
+    echo "Error: Unable to retrieve NodePort or Node IP."
+  else
+    echo http://$NODE_IP:$NODE_PORT
+  fi
 {{- else if contains "LoadBalancer" .Values.service.type }}
-     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "metaflow-service.fullname" . }}'
+  NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+        You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "metaflow-service.fullname" . }}'
   export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "metaflow-service.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
-  echo http://$SERVICE_IP:{{ .Values.service.port }}
+  if [ -z "$SERVICE_IP" ]; then
+    echo "Error: Unable to retrieve LoadBalancer IP."
+  else
+    echo http://$SERVICE_IP:{{ .Values.service.port }}
+  fi
 {{- else if contains "ClusterIP" .Values.service.type }}
   export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "metaflow-service.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
   export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
-  echo "Visit http://127.0.0.1:8080 to use your application"
-  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+  if [ -z "$POD_NAME" ] || [ -z "$CONTAINER_PORT" ]; then
+    echo "Error: Unable to retrieve Pod name or Container port."
+  else
+    echo "Visit http://127.0.0.1:8080 to use your application"
+    kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+  fi
 {{- end }}

k8s/helm/metaflow/charts/metaflow-service/templates/deployment.yaml → charts/metaflow/charts/metaflow-service/templates/deployment.yaml

@@ -6,7 +6,7 @@ metadata:
     {{- include "metaflow-service.labels" . | nindent 4 }}
 spec:
   {{- if not .Values.autoscaling.enabled }}
-  replicas: {{ .Values.replicaCount }}
+  replicas: {{ .Values.replicaCount | default 1 }}
   {{- end }}
   selector:
     matchLabels:
@@ -46,7 +46,7 @@ spec:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           command: ["/opt/latest/bin/python3", "-m", "services.metadata_service.server" ]
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          imagePullPolicy: {{ .Values.image.pullPolicy | default "IfNotPresent" }}
           ports:
             - name: http
               containerPort: 8080

charts/metaflow/charts/metaflow-service/templates/hpa.yaml

@@ -0,0 +1,27 @@
+{{ if .Values.autoscaling.enabled }}
+apiVersion: autoscaling/v2beta2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "metaflow-service.fullname"  . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "metaflow-service.fullname"  . }}
+  minReplicas: {{ .Values.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.autoscaling.maxReplicas }}
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
+{{ end }}

k8s/helm/metaflow/charts/metaflow-service/templates/ingress.yaml → charts/metaflow/charts/metaflow-service/templates/ingress.yaml

@@ -1,14 +1,15 @@
 {{- if .Values.ingress.enabled -}}
 {{- $fullName := include "metaflow-service.fullname" . -}}
 {{- $svcPort := .Values.service.port -}}
-{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
+{{- $kubeVersion := default "1.14.0" (lookup "version" "") -}}
+{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" $kubeVersion)) }}
   {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
   {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
   {{- end }}
 {{- end }}
-{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
+{{- if semverCompare ">=1.19-0" $kubeVersion -}}
 apiVersion: networking.k8s.io/v1
-{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+{{- else if semverCompare ">=1.14-0" $kubeVersion -}}
 apiVersion: networking.k8s.io/v1beta1
 {{- else -}}
 apiVersion: extensions/v1beta1
@@ -23,7 +24,7 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
 spec:
-  {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
+  {{- if and .Values.ingress.className (semverCompare ">=1.18-0" $kubeVersion) }}
   ingressClassName: {{ .Values.ingress.className }}
   {{- end }}
   {{- if .Values.ingress.tls }}
@@ -43,11 +44,11 @@ spec:
         paths:
           {{- range .paths }}
           - path: {{ .path }}
-            {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
+            {{- if and .pathType (semverCompare ">=1.18-0" $kubeVersion) }}
             pathType: {{ .pathType }}
             {{- end }}
             backend:
-              {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+              {{- if semverCompare ">=1.19-0" $kubeVersion }}
               service:
                 name: {{ $fullName }}
                 port:

k8s/helm/metaflow/charts/metaflow-service/templates/service.yaml → charts/metaflow/charts/metaflow-service/templates/service.yaml

@@ -9,7 +9,7 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
 spec:
-  type: {{ .Values.service.type }}
+  type: {{ .Values.service.type | default "ClusterIP" }} # Use default value
   ports:
     - name: metadata
       port: {{ .Values.service.port }}

charts/metaflow/charts/metaflow-service/templates/serviceaccount.yaml

@@ -0,0 +1,12 @@
+{{ if .Values.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "metaflow-service.serviceAccountName" . }}
+  labels:
+    {{ include "metaflow-service.labels" . | nindent 4 }}
+  {{ with .Values.serviceAccount.annotations }}
+  annotations:
+    {{ toYaml . | nindent 4 }}
+  {{ end }}
+{{ end }}

k8s/helm/metaflow/charts/metaflow-service/templates/tests/test-connection.yaml → charts/metaflow/charts/metaflow-service/templates/tests/test-connection.yaml

@@ -6,10 +6,20 @@ metadata:
     {{- include "metaflow-service.labels" . | nindent 4 }}
   annotations:
     "helm.sh/hook": test
+    "helm.sh/hook-weight": "0"
 spec:
   containers:
     - name: wget
       image: busybox
       command: ['wget']
       args: ['{{ include "metaflow-service.fullname" . }}:{{ .Values.service.port }}']
+      resources:
+        requests:
+          memory: "64Mi"
+          cpu: "50m"
+        limits:
+          memory: "128Mi"
+          cpu: "100m"
   restartPolicy: Never
+  serviceAccountName: default
+  activeDeadlineSeconds: 60