From b6f35add1227c930e9208103235d89fe4b864b8e Mon Sep 17 00:00:00 2001 From: Cornelis Boon Date: Fri, 22 Mar 2024 16:06:16 +0100 Subject: [PATCH] add first version of gke-starter values file (#5026) * add first version of gke-starter values file Signed-off-by: Cornelis Boon * remove AWS metadata env var, add service account annotations, update templateUri Signed-off-by: Cornelis Boon * let user fill in GCP project ID in templateUri Signed-off-by: Cornelis Boon * add link to GCP workload identity configuration Signed-off-by: Cornelis Boon * run make helm Signed-off-by: Cornelis Boon * bump flyte-binary chart version to v0.1.11 Signed-off-by: Cornelis Boon * Revert "bump flyte-binary chart version to v0.1.11" This reverts commit ddfe8402182fe594edd6a47e096b344b712d0985. Signed-off-by: Cornelis Boon * add default configuration to allow scheduling on gpu nodes in GKE Signed-off-by: Cornelis Boon * add default gpu-partition-size label to allow scheduling on multi-instance GPUs in GKE Signed-off-by: Cornelis Boon * run make helm Signed-off-by: Cornelis Boon * fix linting errors Signed-off-by: Cornelis Boon --------- Signed-off-by: Cornelis Boon --- charts/flyte-binary/Chart.yaml | 1 + charts/flyte-binary/gke-starter.yaml | 146 ++++++++++++++++++ .../manifests/complete-agent.yaml | 4 +- .../sandbox-bundled/manifests/complete.yaml | 4 +- docker/sandbox-bundled/manifests/dev.yaml | 4 +- 5 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 charts/flyte-binary/gke-starter.yaml diff --git a/charts/flyte-binary/Chart.yaml b/charts/flyte-binary/Chart.yaml index 730b109d0d..9c8ebc3d24 100644 --- a/charts/flyte-binary/Chart.yaml +++ b/charts/flyte-binary/Chart.yaml @@ -7,6 +7,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) + version: v0.1.10 # VERSION # This is the version number of the application being deployed. This version number should be diff --git a/charts/flyte-binary/gke-starter.yaml b/charts/flyte-binary/gke-starter.yaml new file mode 100644 index 0000000000..f9052a27fd --- /dev/null +++ b/charts/flyte-binary/gke-starter.yaml @@ -0,0 +1,146 @@ +# configuration Specify configuration for Flyte +configuration: + # database Specify configuration for Flyte's database connection + database: + # username Name for user to connect to database as + username: postgres + # password Password to connect to database with + # If set, a Secret will be created with this value and mounted to Flyte pod + password: "" + # host Hostname of database instance + host: 127.0.0.1 + # dbname Name of database to use + dbname: flyteadmin + # storage Specify configuration for object store + storage: + # metadataContainer Bucket to store Flyte metadata + metadataContainer: "my-organization-flyte-container" + # userDataContainer Bucket to store Flyte user data + userDataContainer: "my-organization-flyte-container" + # provider Object store provider (Supported values: s3, gcs) + provider: gcs + # providerConfig Additional object store provider-specific configuration + providerConfig: + # gcs Provider configuration for GCS object store + gcs: + # project Google Cloud project in which bucket resides + project: "my-organization-gcp-project" + # logging Specify configuration for logs emitted by Flyte + logging: + # level Set the log level + level: 5 + # plugins Specify additional logging plugins + plugins: + # stackdriver Configure logging plugin to have logs visible in StackDriver + stackdriver: + enabled: true + templateUri: | + "https://console.cloud.google.com/logs/query;query=resource.labels.namespace_name%3D%22{{.namespace}}%22%0Aresource.labels.pod_name%3D%22{{.podName}}%22%0Aresource.labels.container_name%3D%22{{.containerName}}%22?project=&angularJsUrl=%2Flogs%2Fviewer%3Fproject%3D" + # auth Specify configuration for Flyte authentication + auth: + # enabled Enable Flyte authentication + enabled: false + # oidc OIDC configuration for Flyte authentication + oidc: + # baseUrl URL for OIDC provider + baseUrl: "" + # clientId Flyte application client ID + clientId: "" + # clientSecret Flyte application client secret + clientSecret: "" + # internal Configuration for internal authentication + # The settings for internal still need to be defined if you wish to use an external auth server + # These credentials are used during communication between the FlyteAdmin and Propeller microservices + internal: + # clientId Client ID for internal authentication - set to flytepropeller or external auth server + clientId: flytepropeller + # clientSecret Client secret for internal authentication + clientSecret: "" + # clientSecretHash Bcrypt hash of clientSecret + clientSecretHash: "" + # authorizedUris Set of URIs that clients are allowed to visit the service on + authorizedUris: [] + + # inline Specify additional configuration or overrides for Flyte, to be merged with the base configuration + inline: + #This section automates the IAM Role annotation for the default KSA on each project namespace to enable IRSA + #Learn more: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html + cluster_resources: + customData: + - production: + - defaultIamServiceAccount: + value: + - staging: + - defaultIamServiceAccount: + value: + - development: + - defaultIamServiceAccount: + value: + plugins: + k8s: + inject-finalizer: true + gpu-device-node-label: cloud.google.com/gke-accelerator + gpu-partition-size-node-label: cloud.google.com/gke-gpu-partition-size + resource-tolerations: + - nvidia.com/gpu: + - key: "nvidia.com/gpu" + operator: "Equal" + value: "present" + effect: "NoSchedule" + # Configuration for the Datacatalog engine, used when caching is enabled + # Learn more: https://docs.flyte.org/en/latest/deployment/configuration/generated/datacatalog_config.html + storage: + cache: + max_size_mbs: 10 + target_gc_percent: 100 + tasks: + task-plugins: + enabled-plugins: + - container + - sidecar + - K8S-ARRAY #used for MapTasks + default-for-task-types: + - container: container + - container_array: K8S-ARRAY + +# clusterResourceTemplates Specify templates for Kubernetes resources that should be created for new Flyte projects +clusterResourceTemplates: + # inline Specify additional cluster resource templates, to be merged with the base configuration + inline: + #This section automates the creation of the project-domain namespaces + 001_namespace.yaml: | + apiVersion: v1 + kind: Namespace + metadata: + name: '{{ namespace }}' + # This block performs the automated annotation of KSAs across all project-domain namespaces. Make sure to bind the KSA to the GSA after KSAs are created: https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#authenticating_to + 002_serviceaccount.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: default + namespace: '{{ namespace }}' + annotations: + iam.gke.io/gcp-service-account: '{{ defaultIamServiceAccount }}' + +# serviceAccount Configure Flyte ServiceAccount +serviceAccount: + # create Create ServiceAccount for Flyte + create: true + #Automates annotation of default flyte-binary KSA. Make sure to bind the KSA to the GSA: https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#authenticating_to + annotations: + iam.gke.io/gcp-service-account: +# rbac Configure Kubernetes RBAC for Flyte +rbac: + # create Create ClusterRole and ClusterRoleBinding resources + create: true + # extraRules Add additional rules to the ClusterRole + extraRules: + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - patch diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml index 1779389183..3d40c5a8f8 100644 --- a/docker/sandbox-bundled/manifests/complete-agent.yaml +++ b/docker/sandbox-bundled/manifests/complete-agent.yaml @@ -816,7 +816,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: allvNmJ4bUxTcVo2Z0lObw== + haSharedSecret: QWVsREJpZnlIR2N1UXJSMg== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1412,7 +1412,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 7e2eea3d2c604971389c67f39e7d553b6329ea37af5254119febf0a125e55e64 + checksum/secret: 6eadd3a29b61a78cf3a7712f3370a10fc0ec1a61c40753a48c7fa8bea69a6ec6 labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml index 05e557ad96..69739d52d7 100644 --- a/docker/sandbox-bundled/manifests/complete.yaml +++ b/docker/sandbox-bundled/manifests/complete.yaml @@ -796,7 +796,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: d2Fqb3NpcVh5a1JUaGR4Vg== + haSharedSecret: NmtkWjAwUWhadWlzb0xNcA== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1360,7 +1360,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 087a8de1fb03ba21b367df5aa3a47d77ec2acbdeb9b6d2fa66c0caa04c304246 + checksum/secret: 98727a2bd78c4e88ef413663ebff406f78c8fdbda001f7ba7b6b784934cd4d4a labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml index 71a34f8bed..fd77ad44e0 100644 --- a/docker/sandbox-bundled/manifests/dev.yaml +++ b/docker/sandbox-bundled/manifests/dev.yaml @@ -499,7 +499,7 @@ metadata: --- apiVersion: v1 data: - haSharedSecret: WGtoeXNQV2FrV0lGeWJMeg== + haSharedSecret: WG01UkdoN2dNTzBMRjJDVA== proxyPassword: "" proxyUsername: "" kind: Secret @@ -934,7 +934,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 63c7525d8f4e16616715d985d9581611a83fe095e65b51cc25c61f9009f595da + checksum/secret: 5400c48803b4ae9d08115e0f52f00245498c0b06d11d318a36590b01f91e2753 labels: app: docker-registry release: flyte-sandbox