From 51ceacc5aa3d2664f130ef962b2eb96048d2b1c6 Mon Sep 17 00:00:00 2001 From: davidmirror-ops Date: Fri, 15 Dec 2023 10:54:14 -0500 Subject: [PATCH 1/8] Comment and address missing info in values file Signed-off-by: davidmirror-ops --- charts/flyte-binary/eks-production.yaml | 97 ++++++++----------------- 1 file changed, 31 insertions(+), 66 deletions(-) diff --git a/charts/flyte-binary/eks-production.yaml b/charts/flyte-binary/eks-production.yaml index 2db827b804..267ffd0b4e 100644 --- a/charts/flyte-binary/eks-production.yaml +++ b/charts/flyte-binary/eks-production.yaml @@ -1,35 +1,42 @@ configuration: database: + username: postgres password: host: - dbname: app + dbname: flyteadmin storage: + #Learn more about how Flyte handles data: https://docs.flyte.org/en/latest/concepts/data_management.html metadataContainer: userDataContainer: provider: s3 providerConfig: s3: - region: "us-east-2" + region: "" authType: "iam" + #For logging to work, you need to setup an agent. + # Learn more: https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Container-Insights-EKS-logs.html logging: level: 5 plugins: cloudwatch: enabled: true templateUri: |- - https://console.aws.amazon.com/cloudwatch/home?region=#logEventViewer:group=/eks/opta-development/cluster;stream=var.log.containers.{{ .podName }}_{{ .namespace }}_{{ .containerName }}-{{ .containerId }}.log + https://console.aws.amazon.com/cloudwatch/home?region=#logEventViewer:group=/aws/eks//cluster;stream=var.log.containers.{{ .podName }}_{{ .namespace }}_{{ .containerName }}-{{ .containerId }}.log + # To configure auth, refer to https://docs.flyte.org/en/latest/deployment/configuration/auth_setup.html auth: - enabled: true + enabled: false oidc: - baseUrl: https://signin.hosted.unionai.cloud/oauth2/default + baseUrl: clientId: clientSecret: internal: clientSecret: clientSecretHash: authorizedUris: - - https://flyte.company.com + - https://flyte.company.com #change to your authorized URI inline: + #This section automates the IAM Role annotation for the default KSA on each project namespace to enable IRSA + #Learn more: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html cluster_resources: customData: - production: @@ -49,22 +56,8 @@ configuration: default-env-vars: - AWS_METADATA_SERVICE_TIMEOUT: 5 - AWS_METADATA_SERVICE_NUM_ATTEMPTS: 20 - spark: - spark-config-default: - - spark.hadoop.fs.s3a.aws.credentials.provider: com.amazonaws.auth.DefaultAWSCredentialsProviderChain - - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" - - spark.kubernetes.allocation.batch.size: "50" - - spark.hadoop.fs.s3a.acl.default: BucketOwnerFullControl - - spark.hadoop.fs.s3n.impl: org.apache.hadoop.fs.s3a.S3AFileSystem - - spark.hadoop.fs.AbstractFileSystem.s3n.impl: org.apache.hadoop.fs.s3a.S3A - - spark.hadoop.fs.s3.impl: org.apache.hadoop.fs.s3a.S3AFileSystem - - spark.hadoop.fs.AbstractFileSystem.s3.impl: org.apache.hadoop.fs.s3a.S3A - - spark.hadoop.fs.s3a.impl: org.apache.hadoop.fs.s3a.S3AFileSystem - - spark.hadoop.fs.AbstractFileSystem.s3a.impl: org.apache.hadoop.fs.s3a.S3A - - spark.hadoop.fs.s3a.multipart.threshold: "536870912" - - spark.blacklist.enabled: "true" - - spark.blacklist.timeout: 5m - - spark.task.maxfailures: "8" + # Configuration for the Datacatalog engine, used when caching is enabed + # Learn more: https://docs.flyte.org/en/latest/deployment/configuration/generated/datacatalog_config.html storage: cache: max_size_mbs: 10 @@ -74,65 +67,43 @@ configuration: enabled-plugins: - container - sidecar - - K8S-ARRAY - - spark + - K8S-ARRAY #used for MapTasks default-for-task-types: - container: container - container_array: K8S-ARRAY - - spark: spark clusterResourceTemplates: inline: + #This section automates the creation of the project-domain namespaces 001_namespace.yaml: | apiVersion: v1 kind: Namespace metadata: name: '{{ namespace }}' - 010_spark_role.yaml: | - apiVersion: rbac.authorization.k8s.io/v1 - kind: Role - metadata: - name: spark-role - namespace: '{{ namespace }}' - rules: - - apiGroups: - - "" - resources: - - pods - - services - - configmaps - verbs: - - '*' - 011_spark_service_account.yaml: | + # This block performs the automated annotation of KSAs across all project-domain namespaces + 002_serviceaccount.yaml: | apiVersion: v1 kind: ServiceAccount metadata: - name: spark + name: default namespace: '{{ namespace }}' annotations: eks.amazonaws.com/role-arn: '{{ defaultIamRole }}' - 012_spark_role_binding.yaml: | - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: spark-role-binding - namespace: '{{ namespace }}' - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: spark-role - subjects: - - kind: ServiceAccount - name: spark - namespace: '{{ namespace }}' ingress: create: true commonAnnotations: - kubernetes.io/ingress.class: nginx + #This section assumes you are using the ALB Ingress controller + alb.ingress.kubernetes.io/certificate-arn: 'arn:aws:acm:::certificate/' + alb.ingress.kubernetes.io/group.name: flyte + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/ssl-redirect: '443' + alb.ingress.kubernetes.io/target-type: ip + kubernetes.io/ingress.class: alb httpAnnotations: - nginx.ingress.kubernetes.io/app-root: /console + alb.ingress.kubernetes.io/actions.app-root: '{"Type": "redirect", "RedirectConfig": {"Path": "/console", "StatusCode": "HTTP_302"}}' grpcAnnotations: - nginx.ingress.kubernetes.io/backend-protocol: GRPC - host: # change for the URL you'll use to connect to Flyte + alb.ingress.kubernetes.io/backend-protocol-version: GRPC + host: flyte.mydomain.com #replace with your fully-qualified domain name rbac: extraRules: - apiGroups: @@ -164,12 +135,6 @@ rbac: - list - patch - update - - apiGroups: - - sparkoperator.k8s.io - resources: - - sparkapplications - verbs: - - "*" serviceAccount: create: true annotations: From bf627968b5e7b7fd79325d5e7de2c607f259b40a Mon Sep 17 00:00:00 2001 From: davidmirror-ops Date: Fri, 15 Dec 2023 11:22:43 -0500 Subject: [PATCH 2/8] Remove Spark-specific extra RBAC rules Signed-off-by: davidmirror-ops --- charts/flyte-binary/eks-production.yaml | 31 ------------------------- 1 file changed, 31 deletions(-) diff --git a/charts/flyte-binary/eks-production.yaml b/charts/flyte-binary/eks-production.yaml index 267ffd0b4e..7cd3555617 100644 --- a/charts/flyte-binary/eks-production.yaml +++ b/charts/flyte-binary/eks-production.yaml @@ -104,37 +104,6 @@ ingress: grpcAnnotations: alb.ingress.kubernetes.io/backend-protocol-version: GRPC host: flyte.mydomain.com #replace with your fully-qualified domain name -rbac: - extraRules: - - apiGroups: - - "" - resources: - - pods - - services - - configmaps - verbs: - - "*" - - apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - get - - list - - patch - - update - - apiGroups: - - rbac.authorization.k8s.io - resources: - - rolebindings - - roles - verbs: - - create - - get - - list - - patch - - update serviceAccount: create: true annotations: From 5044e4f4edf332d8fada4a35165d0d84a4bfd53a Mon Sep 17 00:00:00 2001 From: davidmirror-ops Date: Fri, 15 Dec 2023 12:26:33 -0500 Subject: [PATCH 3/8] Update references to Ingress annotations Signed-off-by: davidmirror-ops --- charts/flyte-binary/eks-production.yaml | 16 +++++++++++-- .../deployment/cloud_production.rst | 23 ++++--------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/charts/flyte-binary/eks-production.yaml b/charts/flyte-binary/eks-production.yaml index 7cd3555617..ec385041d2 100644 --- a/charts/flyte-binary/eks-production.yaml +++ b/charts/flyte-binary/eks-production.yaml @@ -90,15 +90,27 @@ clusterResourceTemplates: eks.amazonaws.com/role-arn: '{{ defaultIamRole }}' ingress: create: true + ##-- Uncomment the following section if you plan to use NGINX Ingress Controller + #ingressClassName: nginx + #commonAnnotations: + # ingress.kubernetes.io/rewrite-target: / + # nginx.ingress.kubernetes.io/ssl-redirect: "true" + #httpAnnotations: + # nginx.ingress.kubernetes.io/app-root: /console + #grpcAnnotations: + # nginx.ingress.kubernetes.io/backend-protocol: GRPC + #host: # change for the URL you'll use to connect to Flyte + ## --- + + #This section assumes you are using the ALB Ingress controller. + ingressClassName: alb commonAnnotations: - #This section assumes you are using the ALB Ingress controller alb.ingress.kubernetes.io/certificate-arn: 'arn:aws:acm:::certificate/' alb.ingress.kubernetes.io/group.name: flyte alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' alb.ingress.kubernetes.io/scheme: internet-facing alb.ingress.kubernetes.io/ssl-redirect: '443' alb.ingress.kubernetes.io/target-type: ip - kubernetes.io/ingress.class: alb httpAnnotations: alb.ingress.kubernetes.io/actions.app-root: '{"Type": "redirect", "RedirectConfig": {"Path": "/console", "StatusCode": "HTTP_302"}}' grpcAnnotations: diff --git a/rsts/deployment/deployment/cloud_production.rst b/rsts/deployment/deployment/cloud_production.rst index 1736f1eb4c..c5993dbe53 100644 --- a/rsts/deployment/deployment/cloud_production.rst +++ b/rsts/deployment/deployment/cloud_production.rst @@ -30,27 +30,14 @@ To turn on ingress, update your ``values.yaml`` file to include the following bl .. literalinclude:: ../../../charts/flyte-binary/eks-production.yaml :caption: charts/flyte-binary/eks-production.yaml :language: yaml - :lines: 127-135 + :lines: 93-102 .. group-tab:: ``flyte-binary``/ on EKS using ALB - .. code-block:: yaml - - ingress: - create: true - commonAnnotations: - alb.ingress.kubernetes.io/certificate-arn: '' - alb.ingress.kubernetes.io/group.name: flyte - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' - alb.ingress.kubernetes.io/scheme: internet-facing - alb.ingress.kubernetes.io/ssl-redirect: '443' - alb.ingress.kubernetes.io/target-type: ip - kubernetes.io/ingress.class: alb - httpAnnotations: - alb.ingress.kubernetes.io/actions.app-root: '{"Type": "redirect", "RedirectConfig": {"Path": "/console", "StatusCode": "HTTP_302"}}' - grpcAnnotations: - alb.ingress.kubernetes.io/backend-protocol-version: GRPC - host: #use a DNS CNAME pointing to your ALB + .. literalinclude:: ../../../charts/flyte-binary/eks-production.yaml + :caption: charts/flyte-binary/eks-production.yaml + :language: yaml + :lines: 106-118 .. group-tab:: ``flyte-core`` on GCP using NGINX From 75a8628cde55e31723ff6b469c94905009769393 Mon Sep 17 00:00:00 2001 From: davidmirror-ops Date: Fri, 15 Dec 2023 12:30:03 -0500 Subject: [PATCH 4/8] Fix typo Signed-off-by: davidmirror-ops --- charts/flyte-binary/eks-production.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/flyte-binary/eks-production.yaml b/charts/flyte-binary/eks-production.yaml index ec385041d2..51fd15f112 100644 --- a/charts/flyte-binary/eks-production.yaml +++ b/charts/flyte-binary/eks-production.yaml @@ -56,7 +56,7 @@ configuration: default-env-vars: - AWS_METADATA_SERVICE_TIMEOUT: 5 - AWS_METADATA_SERVICE_NUM_ATTEMPTS: 20 - # Configuration for the Datacatalog engine, used when caching is enabed + # Configuration for the Datacatalog engine, used when caching is enabled # Learn more: https://docs.flyte.org/en/latest/deployment/configuration/generated/datacatalog_config.html storage: cache: From 8d3b4e645c475a3fad2e8ca7b4c3098715eac20e Mon Sep 17 00:00:00 2001 From: davidmirror-ops Date: Fri, 15 Dec 2023 12:38:34 -0500 Subject: [PATCH 5/8] Fix reference to YAML config Signed-off-by: davidmirror-ops --- .../deployment/cloud_production.rst | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/docs/deployment/deployment/cloud_production.rst b/docs/deployment/deployment/cloud_production.rst index 1736f1eb4c..744d1fb666 100644 --- a/docs/deployment/deployment/cloud_production.rst +++ b/docs/deployment/deployment/cloud_production.rst @@ -30,27 +30,15 @@ To turn on ingress, update your ``values.yaml`` file to include the following bl .. literalinclude:: ../../../charts/flyte-binary/eks-production.yaml :caption: charts/flyte-binary/eks-production.yaml :language: yaml - :lines: 127-135 + :lines: 93-102 .. group-tab:: ``flyte-binary``/ on EKS using ALB - .. code-block:: yaml - - ingress: - create: true - commonAnnotations: - alb.ingress.kubernetes.io/certificate-arn: '' - alb.ingress.kubernetes.io/group.name: flyte - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' - alb.ingress.kubernetes.io/scheme: internet-facing - alb.ingress.kubernetes.io/ssl-redirect: '443' - alb.ingress.kubernetes.io/target-type: ip - kubernetes.io/ingress.class: alb - httpAnnotations: - alb.ingress.kubernetes.io/actions.app-root: '{"Type": "redirect", "RedirectConfig": {"Path": "/console", "StatusCode": "HTTP_302"}}' - grpcAnnotations: - alb.ingress.kubernetes.io/backend-protocol-version: GRPC - host: #use a DNS CNAME pointing to your ALB + .. literalinclude:: ../../../charts/flyte-binary/eks-production.yaml + :caption: charts/flyte-binary/eks-production.yaml + :language: yaml + :lines: 106-118 + .. group-tab:: ``flyte-core`` on GCP using NGINX From 56e6f5dd33c33cae3710610e742b78d5261e903e Mon Sep 17 00:00:00 2001 From: davidmirror-ops Date: Mon, 18 Dec 2023 12:25:41 -0500 Subject: [PATCH 6/8] Unify default db name Signed-off-by: davidmirror-ops --- charts/flyte-binary/eks-production.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/flyte-binary/eks-production.yaml b/charts/flyte-binary/eks-production.yaml index 51fd15f112..b73da0278a 100644 --- a/charts/flyte-binary/eks-production.yaml +++ b/charts/flyte-binary/eks-production.yaml @@ -3,7 +3,7 @@ configuration: username: postgres password: host: - dbname: flyteadmin + dbname: flyte storage: #Learn more about how Flyte handles data: https://docs.flyte.org/en/latest/concepts/data_management.html metadataContainer: From 4b4aa779b53a1677c43a9b11b4502dd53b84bd26 Mon Sep 17 00:00:00 2001 From: davidmirror-ops Date: Tue, 19 Dec 2023 15:57:49 -0500 Subject: [PATCH 7/8] Unify values files Signed-off-by: davidmirror-ops --- charts/flyte-binary/eks-starter.yaml | 103 +++++++++++++++++++++++++-- 1 file changed, 98 insertions(+), 5 deletions(-) diff --git a/charts/flyte-binary/eks-starter.yaml b/charts/flyte-binary/eks-starter.yaml index c7ca135340..51fd15f112 100644 --- a/charts/flyte-binary/eks-starter.yaml +++ b/charts/flyte-binary/eks-starter.yaml @@ -1,29 +1,122 @@ configuration: database: - username: + username: postgres password: host: - dbname: flyteadmin () + dbname: flyteadmin storage: + #Learn more about how Flyte handles data: https://docs.flyte.org/en/latest/concepts/data_management.html metadataContainer: userDataContainer: provider: s3 providerConfig: s3: - region: "" + region: "" authType: "iam" + #For logging to work, you need to setup an agent. + # Learn more: https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Container-Insights-EKS-logs.html + logging: + level: 5 + plugins: + cloudwatch: + enabled: true + templateUri: |- + https://console.aws.amazon.com/cloudwatch/home?region=#logEventViewer:group=/aws/eks//cluster;stream=var.log.containers.{{ .podName }}_{{ .namespace }}_{{ .containerName }}-{{ .containerId }}.log + # To configure auth, refer to https://docs.flyte.org/en/latest/deployment/configuration/auth_setup.html + auth: + enabled: false + oidc: + baseUrl: + clientId: + clientSecret: + internal: + clientSecret: + clientSecretHash: + authorizedUris: + - https://flyte.company.com #change to your authorized URI inline: + #This section automates the IAM Role annotation for the default KSA on each project namespace to enable IRSA + #Learn more: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html + cluster_resources: + customData: + - production: + - defaultIamRole: + value: + - staging: + - defaultIamRole: + value: + - development: + - defaultIamRole: + value: + flyteadmin: + roleNameKey: "iam.amazonaws.com/role" plugins: k8s: inject-finalizer: true default-env-vars: - AWS_METADATA_SERVICE_TIMEOUT: 5 - AWS_METADATA_SERVICE_NUM_ATTEMPTS: 20 + # Configuration for the Datacatalog engine, used when caching is enabled + # Learn more: https://docs.flyte.org/en/latest/deployment/configuration/generated/datacatalog_config.html storage: cache: - max_size_mbs: 100 + max_size_mbs: 10 target_gc_percent: 100 + tasks: + task-plugins: + enabled-plugins: + - container + - sidecar + - K8S-ARRAY #used for MapTasks + default-for-task-types: + - container: container + - container_array: K8S-ARRAY +clusterResourceTemplates: + inline: + #This section automates the creation of the project-domain namespaces + 001_namespace.yaml: | + apiVersion: v1 + kind: Namespace + metadata: + name: '{{ namespace }}' + # This block performs the automated annotation of KSAs across all project-domain namespaces + 002_serviceaccount.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: default + namespace: '{{ namespace }}' + annotations: + eks.amazonaws.com/role-arn: '{{ defaultIamRole }}' +ingress: + create: true + ##-- Uncomment the following section if you plan to use NGINX Ingress Controller + #ingressClassName: nginx + #commonAnnotations: + # ingress.kubernetes.io/rewrite-target: / + # nginx.ingress.kubernetes.io/ssl-redirect: "true" + #httpAnnotations: + # nginx.ingress.kubernetes.io/app-root: /console + #grpcAnnotations: + # nginx.ingress.kubernetes.io/backend-protocol: GRPC + #host: # change for the URL you'll use to connect to Flyte + ## --- + + #This section assumes you are using the ALB Ingress controller. + ingressClassName: alb + commonAnnotations: + alb.ingress.kubernetes.io/certificate-arn: 'arn:aws:acm:::certificate/' + alb.ingress.kubernetes.io/group.name: flyte + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/ssl-redirect: '443' + alb.ingress.kubernetes.io/target-type: ip + httpAnnotations: + alb.ingress.kubernetes.io/actions.app-root: '{"Type": "redirect", "RedirectConfig": {"Path": "/console", "StatusCode": "HTTP_302"}}' + grpcAnnotations: + alb.ingress.kubernetes.io/backend-protocol-version: GRPC + host: flyte.mydomain.com #replace with your fully-qualified domain name serviceAccount: create: true annotations: - eks.amazonaws.com/role-arn: "" + eks.amazonaws.com/role-arn: "" \ No newline at end of file From 31712d455bd5b9ad00559d1f028a1008a977ca74 Mon Sep 17 00:00:00 2001 From: davidmirror-ops Date: Wed, 20 Dec 2023 16:10:17 -0500 Subject: [PATCH 8/8] Update reference to values file Signed-off-by: davidmirror-ops --- docs/deployment/deployment/cloud_production.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/deployment/deployment/cloud_production.rst b/docs/deployment/deployment/cloud_production.rst index 744d1fb666..dce2d2d085 100644 --- a/docs/deployment/deployment/cloud_production.rst +++ b/docs/deployment/deployment/cloud_production.rst @@ -27,15 +27,15 @@ To turn on ingress, update your ``values.yaml`` file to include the following bl .. group-tab:: ``flyte-binary`` on EKS using NGINX - .. literalinclude:: ../../../charts/flyte-binary/eks-production.yaml - :caption: charts/flyte-binary/eks-production.yaml + .. literalinclude:: ../../../charts/flyte-binary/eks-starter.yaml + :caption: charts/flyte-binary/eks-starter.yaml :language: yaml - :lines: 93-102 + :lines: 94-102 .. group-tab:: ``flyte-binary``/ on EKS using ALB - .. literalinclude:: ../../../charts/flyte-binary/eks-production.yaml - :caption: charts/flyte-binary/eks-production.yaml + .. literalinclude:: ../../../charts/flyte-binary/eks-starter.yaml + :caption: charts/flyte-binary/eks-starter.yaml :language: yaml :lines: 106-118