From db159e121d7843121ada8af851e04593188e2160 Mon Sep 17 00:00:00 2001 From: davidmirror-ops Date: Mon, 28 Aug 2023 15:21:54 -0500 Subject: [PATCH 1/2] Updates to deployment guides Signed-off-by: davidmirror-ops --- charts/flyte-binary/eks-production.yaml | 2 +- .../deployment/cloud_production.rst | 15 ++--- rsts/deployment/deployment/cloud_simple.rst | 8 +++ rsts/deployment/deployment/index.rst | 32 ++-------- rsts/deployment/deployment/multicluster.rst | 63 +++++++++---------- rsts/deployment/deployment/sandbox.rst | 16 ++--- 6 files changed, 54 insertions(+), 82 deletions(-) diff --git a/charts/flyte-binary/eks-production.yaml b/charts/flyte-binary/eks-production.yaml index 727b9b10fa..bda8356c98 100644 --- a/charts/flyte-binary/eks-production.yaml +++ b/charts/flyte-binary/eks-production.yaml @@ -132,7 +132,7 @@ ingress: nginx.ingress.kubernetes.io/app-root: /console grpcAnnotations: nginx.ingress.kubernetes.io/backend-protocol: GRPC - host: development.uniondemo.run + host: development.uniondemo.run # change for the URL you'll use to connect to Flyte rbac: extraRules: - apiGroups: diff --git a/rsts/deployment/deployment/cloud_production.rst b/rsts/deployment/deployment/cloud_production.rst index 90997556c9..ff8e182d98 100644 --- a/rsts/deployment/deployment/cloud_production.rst +++ b/rsts/deployment/deployment/cloud_production.rst @@ -28,18 +28,18 @@ To turn on ingress, update your ``values.yaml`` file to include the following bl .. literalinclude:: ../../../charts/flyte-binary/eks-production.yaml :caption: charts/flyte-binary/eks-production.yaml :language: yaml - :lines: 123-131 + :lines: 127-135 .. note:: - This currently assumes that you have nginx ingress. We'll be updating these - in the near future to use the ALB ingress controller instead. + This section assumes that you're using the NGINX Ingress controller. Instructions and annotations for the ALB controller + are covered in the `Flyte The Hard Way `__ tutorial. *************** Authentication *************** -Authentication comes with Flyte in the form of OAuth 2. Please see the +Authentication comes with Flyte in the form of OAuth 2.0. Please see the `authentication guide `__ for instructions. .. note:: @@ -60,10 +60,3 @@ compatibility being maintained, for the most part. If you're using the :ref:`multi-cluster ` deployment model for Flyte, components should be upgraded together. - -.. note:: - - Expect to see minor version releases roughly 4-6 times a year - we aim to - release monthly, or whenever there is a large enough set of features to - warrant a release. Expect to see patch releases at more regular intervals, - especially for flytekit, the Python SDK. diff --git a/rsts/deployment/deployment/cloud_simple.rst b/rsts/deployment/deployment/cloud_simple.rst index b675df00b9..b280546708 100644 --- a/rsts/deployment/deployment/cloud_simple.rst +++ b/rsts/deployment/deployment/cloud_simple.rst @@ -115,6 +115,14 @@ hello world example: cd flytesnacks/cookbook pyflyte run --remote core/flyte_basics/hello_world.py my_wf +*********************************** +Flyte in on-premises infrastructure +*********************************** + +Sometimes, it's also helpful to be able to set up a Flyte environment in an on-premises Kubernetes environment or even on a laptop for testing and development purposes. +Check out `this community-maintained tutorial `__ to learn how to setup the required dependencies and deploy the `flyte-binary` chart to a local Kubernetes cluster. + + ************* What's Next? ************* diff --git a/rsts/deployment/deployment/index.rst b/rsts/deployment/deployment/index.rst index ac0765412a..e253ae480c 100644 --- a/rsts/deployment/deployment/index.rst +++ b/rsts/deployment/deployment/index.rst @@ -49,29 +49,6 @@ deployment comes with a containerized `Minio `__, which offers - **GCP**: `GCS `__ - **Azure**: `Azure Blob Storage `__ - -Cluster Configuration -===================== - -Flyte configures K8s clusters to work with it. For example, as your Flyte userbase evolves, adding new projects is as -simple as registering them through the command line: - -.. prompt:: bash $ - - flytectl create project \ - --id my-flyte-project \ - --name "My Flyte Project" \ - --description "My first project onboarding onto Flyte" - -Once you invoke this command, this project should immediately show up in the Flyte console after refreshing. - -Flyte runs at a configurable cadence that ensures that all Kubernetes resources necessary for the new project are -created and new workflows can successfully be registered and executed within it. - -.. note:: - - For more information, see :std:ref:`flytectl `. - ************************ Flyte Deployment Paths ************************ @@ -108,7 +85,7 @@ There are three different paths for deploying a Flyte cluster: This option is appropriate if all your compute can `fit on one EKS cluster `__ . As of this writing, a single Flyte cluster can handle more than 13,000 nodes. - Whatever path you choose, note that ``FlytePropeller`` itself can be sharded as well, though typically it's not required. + Regardless of using single or multiple Kubernetes clusters for Flyte, note that ``FlytePropeller`` -tha main data plane component- can be sharded as well, if scale demands require it. Helm ==== @@ -156,10 +133,13 @@ Deployment Tips and Tricks Due to the many choices and constraints that you may face in your organization, the specific steps for deploying Flyte can vary significantly. For example, which cloud platform to use is typically a big fork in the road for many, and there -are many choices to make in terms of ingresses, auth providers, and versions of different dependent libraries that +are many choices to make in terms of Ingress controllers, auth providers, and versions of different dependent libraries that may interact with other parts of your stack. -In addition to searching and posting on the `Flyte Slack community `__, +Considering the above, we recommend checking out the `"Flyte The Hard Way" `__ set of community-maintained tutorials that can guide you through the process of preparing the infrastructure and +deploying Flyte. + +In addition to searching and posting on the `#flyte-deployment Slack channel `__, we have a `Github Discussion `__ section dedicated to deploying Flyte. Feel free to submit any hints you've found helpful as a discussion, ask questions, or simply document what worked or what didn't work for you. diff --git a/rsts/deployment/deployment/multicluster.rst b/rsts/deployment/deployment/multicluster.rst index 69c34989ae..2b8e15084c 100644 --- a/rsts/deployment/deployment/multicluster.rst +++ b/rsts/deployment/deployment/multicluster.rst @@ -8,8 +8,8 @@ Multiple K8s Cluster Deployment .. note:: - The multicluster deployment described in this doc assumes you have deployed - the ``flyte`` Helm chart, which runs the individual Flyte services separately. + The multicluster deployment described in this section, assumes you have deployed + the ``flyte-core`` Helm chart, which runs the individual Flyte services separately. This is needed because in a multicluster setup, the execution engine is deployed to multiple K8s clusters. This will not work with the ``flyte-binary`` Helm chart, since that chart deploys all Flyte service as one single binary. @@ -24,23 +24,22 @@ Scaling Beyond Kubernetes execution. The data plane fulfills these workflows by launching pods in Kubernetes. -At very large companies, total compute needs could exceed the limits of a single +At large organizations, total compute needs could exceed the limits of a single Kubernetes cluster. To address this, you can deploy the data plane to multiple Kubernetes clusters. The control plane (FlyteAdmin) can be configured to load-balance workflows across these individual data planes, protecting you from failure in a single Kubernetes -cluster increasing scalability. +cluster, thus increasing scalability. -To achieve this, first, you have to create additional Kubernetes clusters. -For now, let's assume you have three Kubernetes clusters and that you can access +To achieve this, first you have to create additional Kubernetes clusters. + +This gude assumes that you have three Kubernetes clusters and that you can access them all with ``kubectl``. Let's call these clusters ``cluster1``, ``cluster2``, and ``cluster3``. -Next, deploy *only* the data planes to these clusters. To do this, remove the -data plane components from the ``flyte`` overlay, and create a new overlay -containing *only* the data plane resources. +Next, deploy *only* the data planes to these clusters. To do this, use the `values-dataplane.yaml `__ provided with the Helm chart. Data Plane Deployment ********************* @@ -61,16 +60,16 @@ Install Flyte data plane Helm chart .. code-block:: - helm upgrade flyte -n flyte flyteorg/flyte-core values.yaml \ + helm upgrade -n flyte -f values.yaml \ -f values-eks.yaml \ -f values-dataplane.yaml \ - --create-namespace flyte --install + --create-namespace flyte flyteorg/flyte-core --install .. tabbed:: GCP .. code-block:: - helm upgrade flyte -n flyte flyteorg/flyte-core values.yaml \ + helm upgrade flyte -n flyte flyteorg/flyte-core -f values.yaml \ -f values-gcp.yaml \ -f values-dataplane.yaml \ --create-namespace flyte --install @@ -83,24 +82,24 @@ Some Flyte deployments may choose to run the control plane separate from the dat plane. FlyteAdmin is designed to create Kubernetes resources in one or more Flyte data plane clusters. For the admin to access remote clusters, it needs credentials to each cluster. +Flyte makes use of Kubernetess Service Accounts to enable every data plane cluster to perform +authenticated requests to the K8s API Server. +The default behaviour is that ``FlyteAdmin`` creates a `ServiceAccount `_ +in each data plane cluster. +In order to verify requests, the API Server expects a `signed bearer token `__ +attached to the Service Account. -In Kubernetes, scoped service credentials are created by configuring a "Role" -resource in a Kubernetes cluster. When you attach the role to a "ServiceAccount", -Kubernetes generates a bearer token that permits access. Hence, create a -FlyteAdmin `ServiceAccount `_ -in each data plane cluster to generate these tokens. -.. warning:: - - **Never delete a ServiceAccount 🛑** - - When you first create the FlyteAdmin ``ServiceAccount`` in a new cluster, a - bearer token is generated and will continue to allow access unless the - "ServiceAccount" is deleted. +.. note:: + As of Kubernetes 1.24 an above, the bearer token has to be generated manually for a Service Account, using the following command: -To feed the credentials to FlyteAdmin, you must retrieve them from your new data plane cluster and upload them to admin (for example, within Lyft, `Confidant `__ is used). + .. prompt:: bash $ + + kubectl create token -n + +To feed the credentials to FlyteAdmin, you must retrieve them from your new data plane cluster and upload them to ``FlyteAmin``. -The credentials have two parts ("ca cert" and "bearer token"). Find the generated secret via: +The credentials have two parts (``ca cert`` and ``bearer token``). Find the generated secret via: .. prompt:: bash $ @@ -133,12 +132,12 @@ file named ``secrets.yaml`` that looks like: namespace: flyte type: Opaque data: - cluster_1_token: {{ cluster 1 token here }} - cluster_1_cacert: {{ cluster 1 cacert here }} - cluster_2_token: {{ cluster 2 token here }} - cluster_2_cacert: {{ cluster 2 cacert here }} - cluster_3_token: {{ cluster 3 token here }} - cluster_3_cacert: {{ cluster 3 cacert here }} + cluster_1_token: "cluster-1-token-here" + cluster_1_cacert: "cluster-1-cacert-here" + cluster_2_token: "cluster-2-token-here" + cluster_2_cacert: "cluster-2-cacert-here" + cluster_3_token: "cluster-3-token-here" + cluster_3_cacert: "cluster-3-cacert-here" Create cluster credentials secret in the control plane cluster. diff --git a/rsts/deployment/deployment/sandbox.rst b/rsts/deployment/deployment/sandbox.rst index 073125e5cc..98d1f48582 100644 --- a/rsts/deployment/deployment/sandbox.rst +++ b/rsts/deployment/deployment/sandbox.rst @@ -6,11 +6,11 @@ Sandbox Deployment .. tags:: Kubernetes, Infrastructure, Basic -A sandbox deployment of Flyte is bundles together portable versions of Flyte's +A sandbox deployment of Flyte bundles together portable versions of Flyte's dependencies such as a relational database and durable object store. For the blob store requirements, Flyte Sandbox uses `Minio `__, -which offers an S3 compatible interface, and for Postgres, we use the stock +which offers an S3 compatible interface, and for Postgres, it uses the stock Postgres Docker image and Helm chart. .. important:: @@ -41,7 +41,7 @@ Requirements - Install `docker `__ or any other OCI-compatible tool, like Podman or LXD. - Install `flytectl `__, the official CLI for Flyte. -While Flyte can run any OCI-compatible task image, using the default Kubernetes container runtime (cri-o), the Flyte +While Flyte can run any OCI-compatible task image using the default Kubernetes container runtime (cri-o), the Flyte core maintainers typically use Docker. Note that the ``flytectl demo`` command does rely on Docker APIs, but as this demo environment is just one self-contained image, you can also run the image directly using another run time. @@ -79,12 +79,4 @@ who wish to dig deeper into the storage layer. 📂 The Minio API is hosted on localhost:30002. Use http://localhost:30080/minio/login for Minio console Now that you have the sandbox cluster running, you can now go to the :ref:`User Guide ` or -:ref:`Tutorials ` to run tasks and workflows written in ``flytekit``, the Python SDK for Flyte. - -************************** -Flyte Sandbox on the Cloud -************************** - -Sometimes it's also helpful to be able to install a sandboxed environment on a cloud provider. That is, you have access -to an EKS or GKE cluster, but provisioning a separate database or blob storage bucket is harder because of a lack of -infrastructure support. Instructions for how to do this will be forthcoming. +:ref:`Tutorials ` to run tasks and workflows written in ``flytekit``, the Python SDK for Flyte. \ No newline at end of file From 3afc4fec83ea2f44826fe531e5fd0fee33416292 Mon Sep 17 00:00:00 2001 From: davidmirror-ops Date: Fri, 15 Sep 2023 14:01:13 -0500 Subject: [PATCH 2/2] Remove unmaintained Opta code Signed-off-by: davidmirror-ops --- opta/README.md | 9 --- opta/aws/env.yaml | 15 ----- opta/aws/flyte.yaml | 133 ------------------------------------- opta/gcp/env.yaml | 16 ----- opta/gcp/flyte.yaml | 156 -------------------------------------------- 5 files changed, 329 deletions(-) delete mode 100644 opta/README.md delete mode 100644 opta/aws/env.yaml delete mode 100644 opta/aws/flyte.yaml delete mode 100644 opta/gcp/env.yaml delete mode 100644 opta/gcp/flyte.yaml diff --git a/opta/README.md b/opta/README.md deleted file mode 100644 index e256ae3ce2..0000000000 --- a/opta/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Easy Flyte Deployment with Opta -[Opta](https://github.com/run-x/opta) is an open source cli tool made for developers to quickly get production-grade -infrastructure up and running. With minimal yaml configuration, a user can set up an environment within their -account/project on their cloud provider (e.g. AWS, GCP) and deploy containerized applications therein. -The Flyte team has partnered with the Opta team to make Opta the recommended production deployment tool for Flyte. -For instructions on how to deploy, please refer to the -[Flyte Opta AWS docs](https://docs.flyte.org/en/latest/deployment/aws/opta.html) -and [Flyte Opta GCP docs](https://docs.flyte.org/en/latest/deployment/gcp/opta.html). -You may also find additional docs for Opta [here](https://docs.opta.dev/). diff --git a/opta/aws/env.yaml b/opta/aws/env.yaml deleted file mode 100644 index 244f449498..0000000000 --- a/opta/aws/env.yaml +++ /dev/null @@ -1,15 +0,0 @@ -name: -org_name: -providers: - aws: - region: - account_id: -modules: - - type: base - - type: dns - domain: - delegated: false # set to true once ready https://docs.opta.dev/miscellaneous/ingress/ - - type: k8s-cluster - max_nodes: 15 - - type: k8s-base - # - type: aws-ses # needs to be done after dns delegation if you wish to send emails via AWS diff --git a/opta/aws/flyte.yaml b/opta/aws/flyte.yaml deleted file mode 100644 index 9e79ba43fc..0000000000 --- a/opta/aws/flyte.yaml +++ /dev/null @@ -1,133 +0,0 @@ -environments: - - name: default - path: "./env.yaml" # NOTE: relative path to environment - variables: - region: - account_id: -name: service-flyte -modules: - - name: postgres - type: aws-postgres - - name: s3 - type: aws-s3 - bucket_name: "{parent_name}-{layer_name}" - - name: adminflyterole - type: aws-iam-role - extra_iam_policies: - - "arn:aws:iam::aws:policy/CloudWatchEventsFullAccess" - # - "arn:aws:iam::{vars.account_id}:policy/{env}-{env}-awsses-sender" # Uncomment out for SES - allowed_k8s_services: - - namespace: "*" - service_name: "*" - links: - - s3: ["write"] - - name: userflyterole - type: aws-iam-role - extra_iam_policies: - - "arn:aws:iam::aws:policy/CloudWatchEventsFullAccess" - # - "arn:aws:iam::{vars.account_id}:policy/{env}-{env}-awsses-sender" # Uncomment out for SESre, change the templating - allowed_k8s_services: - - namespace: "*" - service_name: "*" - links: - - s3: ["write"] - - type: helm-chart - chart: "../../charts/flyte-core" # NOTE: relative path to chart - namespace: flyte - timeout: 600 - create_namespace: true - values_file: "../../charts/flyte-core/values-eks.yaml" # NOTE: relative path to values yaml - # Additional overrides to the values provided by the chart. Opta enables piping through produced outputs from prior modules/steps. - values: - db: - datacatalog: - database: - port: 5432 - username: "${{module.postgres.db_user}}" - host: "${{module.postgres.db_host}}" - dbname: "${{module.postgres.db_name}}" - admin: - database: - port: 5432 - username: "${{module.postgres.db_user}}" - host: "${{module.postgres.db_host}}" - dbname: "${{module.postgres.db_name}}" - common: - ingress: - albSSLRedirect: false - host: "{parent.domain}" - annotations: - kubernetes.io/ingress.class: "nginx" - nginx.ingress.kubernetes.io/app-root: /console - databaseSecret: - secretManifest: - stringData: - pass.txt: "${{module.postgres.db_password}}" - storage: - bucketName: "{parent_name}-{layer_name}" - s3: - region: "{vars.region}" - flyteadmin: - serviceAccount: - create: true - annotations: - eks.amazonaws.com/role-arn: "${{module.adminflyterole.role_arn}}" - datacatalog: - serviceAccount: - create: true - annotations: - eks.amazonaws.com/role-arn: "${{module.adminflyterole.role_arn}}" - flytepropeller: - serviceAccount: - create: true - annotations: - eks.amazonaws.com/role-arn: "${{module.adminflyterole.role_arn}}" - flytescheduler: - serviceAccount: - create: true - annotations: - "eks.amazonaws.com/role-arn": "${{module.adminflyterole.role_arn}}" - workflow_scheduler: - enabled: true - workflow_notifications: - enabled: false - configmap: - remoteData: - remoteData: - region: "{vars.region}" - scheme: aws - signedUrls: - durationMinutes: 3 - task_logs: - plugins: - logs: - cloudwatch-region: "{vars.region}" - core: - propeller: - rawoutput-prefix: "s3://{parent_name}-{layer_name}" - cluster_resource_manager: - enabled: true - config: - cluster_resources: - customData: - - production: - - defaultIamRole: - value: "${{module.userflyterole.role_arn}}" - - projectQuotaCpu: - value: "6" - - projectQuotaMemory: - value: "6000Mi" - - staging: - - defaultIamRole: - value: "${{module.userflyterole.role_arn}}" - - projectQuotaCpu: - value: "6" - - projectQuotaMemory: - value: "6000Mi" - - development: - - defaultIamRole: - value: "${{module.userflyterole.role_arn}}" - - projectQuotaCpu: - value: "6" - - projectQuotaMemory: - value: "6000Mi" diff --git a/opta/gcp/env.yaml b/opta/gcp/env.yaml deleted file mode 100644 index e413e3d495..0000000000 --- a/opta/gcp/env.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: -org_name: -providers: - google: - region: - project: -modules: - - type: base - - type: dns - domain: - delegated: false # set to true once ready https://docs.opta.dev/miscellaneous/ingress/ - - type: k8s-cluster - min_nodes: 3 - max_nodes: 6 - node_instance_type: e2-medium - - type: k8s-base \ No newline at end of file diff --git a/opta/gcp/flyte.yaml b/opta/gcp/flyte.yaml deleted file mode 100644 index 9a0e2f2aed..0000000000 --- a/opta/gcp/flyte.yaml +++ /dev/null @@ -1,156 +0,0 @@ -environments: - - name: default - path: "./env.yaml" # NOTE: relative path to environment - variables: - google_project_id: -name: service-flyte -modules: - - name: postgres - type: gcp-postgres - - name: gcs - type: gcp-gcs - bucket_name: "{parent_name}-{layer_name}" - - name: adminflyteaccount - type: gcp-service-account - explicit_name: gsa-flyteadmin - allowed_k8s_services: - - namespace: flyte - service_account_name: flyteadmin - links: - - gcs: [ "write" ] - - name: datacatalogaccount - type: gcp-service-account - explicit_name: gsa-datacatalog - allowed_k8s_services: - - namespace: flyte - service_account_name: datacatalog - links: - - gcs: [ "write" ] - - name: flytescheduleraccount - type: gcp-service-account - explicit_name: gsa-flytescheduler - allowed_k8s_services: - - namespace: flyte - service_account_name: flytescheduler - links: - - gcs: [ "write" ] - - name: flytepropelleraccount - type: gcp-service-account - explicit_name: gsa-flytepropeller - allowed_k8s_services: - - namespace: flyte - service_account_name: flytepropeller - links: - - gcs: [ "write" ] - - name: flyteproductionaccount - type: gcp-service-account - explicit_name: gsa-production - allowed_k8s_services: - - namespace: production - service_account_name: default - links: - - gcs: [ "write" ] - - name: flytestagingaccount - type: gcp-service-account - explicit_name: gsa-staging - allowed_k8s_services: - - namespace: staging - service_account_name: default - links: - - gcs: [ "write" ] - - name: flytedevelopmentaccount - type: gcp-service-account - explicit_name: gsa-development - allowed_k8s_services: - - namespace: development - service_account_name: default - links: - - gcs: [ "write" ] - - type: helm-chart - chart: "../../charts/flyte-core" # NOTE: relative path to chart - namespace: flyte - timeout: 600 - create_namespace: true - values_file: "../../charts/flyte-core/values-gcp.yaml" # NOTE: relative path to values yaml - # Additional overrides to the values provided by the chart. Opta enables piping through produced outputs from prior modules/steps. - values: - postgres: - enabled: false - db: - datacatalog: - database: - port: 5432 - username: "${{module.postgres.db_user}}" - host: "${{module.postgres.db_host}}" - dbname: "${{module.postgres.db_name}}" - admin: - database: - port: 5432 - username: "${{module.postgres.db_user}}" - host: "${{module.postgres.db_host}}" - dbname: "${{module.postgres.db_name}}" - common: - ingress: - albSSLRedirect: false - host: "{parent.domain}" - annotations: - kubernetes.io/ingress.class: "nginx" - nginx.ingress.kubernetes.io/app-root: /console - databaseSecret: - secretManifest: - stringData: - pass.txt: "${{module.postgres.db_password}}" - storage: - bucketName: "${{module.gcs.bucket_name}}" - gcs: - projectId: "{vars.google_project_id}" - flyteadmin: - serviceAccount: - create: true - annotations: - "iam.gke.io/gcp-service-account": "${{module.adminflyteaccount.service_account_email}}" - datacatalog: - serviceAccount: - create: true - annotations: - "iam.gke.io/gcp-service-account": "${{module.datacatalogaccount.service_account_email}}" - flytepropeller: - serviceAccount: - create: true - annotations: - "iam.gke.io/gcp-service-account": "${{module.flytepropelleraccount.service_account_email}}" - flytescheduler: - serviceAccount: - create: true - annotations: - "iam.gke.io/gcp-service-account": "${{module.flytescheduleraccount.service_account_email}}" - configmap: - core: - propeller: - rawoutput-prefix: "gs://${{module.gcs.bucket_name}}/" - cluster_resource_manager: - enabled: true - config: - cluster_resources: - customData: - - production: - - gsa: - value: "${{module.flyteproductionaccount.service_account_email}}" - - projectQuotaCpu: - value: "5" - - projectQuotaMemory: - value: "4000Mi" - - staging: - - gsa: - value: "${{module.flytestagingaccount.service_account_email}}" - - projectQuotaCpu: - value: "2" - - projectQuotaMemory: - value: "3000Mi" - - development: - - gsa: - value: "${{module.flytedevelopmentaccount.service_account_email}}" - - projectQuotaCpu: - value: "2" - - projectQuotaMemory: - value: "3000Mi" \ No newline at end of file