Anaselmhamdi/add workload launcher and api

.vscode/settings.json

@@ -39,8 +39,13 @@
     "editor.formatOnSave": true,
     "editor.defaultFormatter": "esbenp.prettier-vscode",
     "editor.codeActionsOnSave": {
-      "source.fixAll.stylelint": true
+      "source.fixAll.stylelint": "explicit"
     }
   },
-  "typescript.tsdk": "./airbyte-webapp/node_modules/typescript/lib"
+  "typescript.tsdk": "./airbyte-webapp/node_modules/typescript/lib",
+  "[javascript][typescript][typescriptreact]": {
+    "editor.codeActionsOnSave": {
+      "source.organizeImports": "never"
+    }
+  }
 }

charts/airbyte-worker/airbyte-worker/index.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+entries:
+  worker:
+  - apiVersion: v2
+    appVersion: dev
+    created: "2023-11-28T00:46:00.644774+01:00"
+    dependencies:
+    - name: common
+      repository: https://charts.bitnami.com/bitnami
+      tags:
+      - bitnami-common
+      version: 1.x.x
+    description: Helm chart to deploy airbyte-worker
+    digest: e229f66b86933fad9cc0217edd757743fa85d566a458d7bcf24584635f039ff4
+    name: worker
+    type: application
+    urls:
+    - https://core-ops-charts.storage.googleapis.com/airbyte-worker/worker-0.50.2.tgz
+    version: 0.50.2
+generated: "2023-11-28T00:46:00.642993+01:00"

charts/airbyte-worker/templates/autoscaler.yaml

@@ -0,0 +1,26 @@
+{{- if .Values.autoscaler  -}}
+  {{- if .Values.autoscaler.enabled -}}
+{{- if semverCompare ">=1.23-0" $.Capabilities.KubeVersion.GitVersion -}}
+apiVersion: autoscaling/v2
+{{- else -}}
+apiVersion: autoscaling/v2beta2
+{{- end }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "common.names.fullname" . }}-autoscaler
+  namespace: {{ .Release.Namespace }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "common.names.fullname" . }}
+  minReplicas: {{ .Values.autoscaler.minReplicas | default 1 }}
+  maxReplicas: {{ .Values.autoscaler.maxReplicas | default 1 }}
+  {{- if .Values.autoscaler.extra }}{{ toYaml .Values.autoscaler.extra | trim | nindent 2 }}{{ end }}
+  {{- if .Values.autoscaler.metrics }}
+  metrics:
+  {{- toYaml .Values.autoscaler.metrics | nindent 4 }}
+  {{- end }}
+---
+  {{- end }}
+{{- end }}

charts/airbyte-worker/values.yaml

@@ -129,7 +129,18 @@ global:
 enabled: true
 ##  worker.replicaCount Number of worker replicas
 replicaCount: 1
-
+autoscaler:
+  enabled: true
+  minReplicas: 2
+  maxReplicas: 4
+  extra: {}
+  metrics:
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: 70
 ##  worker.image.repository The repository to use for the airbyte worker image.
 ##  worker.image.pullPolicy the pull policy to use for the airbyte worker image
 ##  worker.image.tag The airbyte worker image tag. Defaults to the chart's AppVersion

charts/airbyte-workload-api-server/index.yaml

@@ -0,0 +1,48 @@
+apiVersion: v1
+entries:
+  common:
+  - annotations:
+      category: Infrastructure
+    apiVersion: v2
+    appVersion: 1.17.1
+    created: "2023-11-28T01:13:54.839209+01:00"
+    description: A Library Helm Chart for grouping common logic between bitnami charts.
+      This chart is not deployable by itself.
+    digest: 2a623d42475796af0ff0d18bbfa3b76ead1ccd7efab8280ea079b6a53f5ac627
+    home: https://github.com/bitnami/charts/tree/master/bitnami/common
+    icon: https://bitnami.com/downloads/logos/bitnami-mark.png
+    keywords:
+    - common
+    - helper
+    - template
+    - function
+    - bitnami
+    maintainers:
+    - name: Bitnami
+      url: https://github.com/bitnami/charts
+    name: common
+    sources:
+    - https://github.com/bitnami/charts
+    - https://www.bitnami.com/
+    type: library
+    urls:
+    - https://core-ops-charts.storage.googleapis.com/airbyte-workload-api/charts/common-1.17.1.tgz
+    version: 1.17.1
+  workload-api:
+  - apiVersion: v2
+    appVersion: dev
+    created: "2023-11-28T01:13:54.838446+01:00"
+    dependencies:
+    - name: common
+      repository: https://charts.bitnami.com/bitnami
+      tags:
+      - bitnami-common
+      version: 1.x.x
+    description: Helm chart to deploy the workload-api service
+    digest: 461432c40528e865ac85b16475fece58e79de7b68f2fe5f0e43be80519f4bdd5
+    name: workload-api
+    type: application
+    urls:
+    - https://core-ops-charts.storage.googleapis.com/airbyte-workload-api/workload-api-0.50.2.tgz
+    version: 0.50.2
+generated: "2023-11-28T01:13:54.836317+01:00"

charts/airbyte-workload-launcher/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: common
+  repository: https://charts.bitnami.com/bitnami
+  version: 1.17.1
+digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690
+generated: "2023-11-28T00:55:50.536518+01:00"

charts/airbyte-workload-launcher/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.49.6
+version: 0.50.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/airbyte-workload-launcher/index.yaml

@@ -0,0 +1,48 @@
+apiVersion: v1
+entries:
+  common:
+  - annotations:
+      category: Infrastructure
+    apiVersion: v2
+    appVersion: 1.17.1
+    created: "2023-11-28T01:04:30.131034+01:00"
+    description: A Library Helm Chart for grouping common logic between bitnami charts.
+      This chart is not deployable by itself.
+    digest: 2a623d42475796af0ff0d18bbfa3b76ead1ccd7efab8280ea079b6a53f5ac627
+    home: https://github.com/bitnami/charts/tree/master/bitnami/common
+    icon: https://bitnami.com/downloads/logos/bitnami-mark.png
+    keywords:
+    - common
+    - helper
+    - template
+    - function
+    - bitnami
+    maintainers:
+    - name: Bitnami
+      url: https://github.com/bitnami/charts
+    name: common
+    sources:
+    - https://github.com/bitnami/charts
+    - https://www.bitnami.com/
+    type: library
+    urls:
+    - https://core-ops-charts.storage.googleapis.com/airbyte-workload-launcher/charts/common-1.17.1.tgz
+    version: 1.17.1
+  workload-launcher:
+  - apiVersion: v2
+    appVersion: dev
+    created: "2023-11-28T01:04:30.130597+01:00"
+    dependencies:
+    - name: common
+      repository: https://charts.bitnami.com/bitnami
+      tags:
+      - bitnami-common
+      version: 1.x.x
+    description: Helm chart to deploy airbyte-workload-launcher
+    digest: dfec8392c8fdb77ea8fcaa9706bd99cdf86ad97c8146e63c38fd16a83495d5bc
+    name: workload-launcher
+    type: application
+    urls:
+    - https://core-ops-charts.storage.googleapis.com/airbyte-workload-launcher/workload-launcher-0.50.2.tgz
+    version: 0.50.2
+generated: "2023-11-28T01:04:30.128486+01:00"

charts/airbyte/Chart.lock

@@ -18,8 +18,8 @@ dependencies:
   repository: https://airbytehq.github.io/helm-charts/
   version: 0.50.2
 - name: worker
-  repository: https://airbytehq.github.io/helm-charts/
-  version: 0.50.2
+  repository: https://core-ops-charts.storage.googleapis.com/airbyte-worker
+  version: 0.49.6
 - name: pod-sweeper
   repository: https://airbytehq.github.io/helm-charts/
   version: 0.50.2
@@ -38,5 +38,11 @@ dependencies:
 - name: keycloak-setup
   repository: https://airbytehq.github.io/helm-charts/
   version: 0.50.2
-digest: sha256:2bde5f6938e63e5e5d3e219beb21e0cfb745d5d026f3c0ecf799e9941d604cdd
-generated: "2023-11-17T19:50:35.272998636Z"
+- name: workload-launcher
+  repository: https://core-ops-charts.storage.googleapis.com/airbyte-workload-launcher
+  version: 0.50.2
+- name: workload-api
+  repository: https://core-ops-charts.storage.googleapis.com/airbyte-workload-api
+  version: 0.50.2
+digest: sha256:db6cb5f7d55ba6fce9735c1926b30d543ad4193deb3b6722574c75ba78b9bedb
+generated: "2023-11-28T01:16:09.148304+01:00"

charts/airbyte/Chart.yaml

@@ -51,7 +51,7 @@ dependencies:
     version: 0.50.2
   - condition: worker.enabled
     name: worker
-    repository: https://airbytehq.github.io/helm-charts/
+    repository: https://core-ops-charts.storage.googleapis.com/airbyte-worker
     version: 0.50.2
   - condition: pod-sweeper.enabled
     name: pod-sweeper
@@ -77,3 +77,11 @@ dependencies:
     name: keycloak-setup
     repository: https://airbytehq.github.io/helm-charts/
     version: 0.50.2
+  - condition: workload-launcher.enabled
+    name: workload-launcher
+    repository: https://core-ops-charts.storage.googleapis.com/airbyte-workload-launcher
+    version: 0.50.2
+  - condition: workload-api.enabled
+    name: workload-api
+    repository: https://core-ops-charts.storage.googleapis.com/airbyte-workload-api
+    version: 0.50.2