Merge remote-tracking branch 'origin/master' into kuberay-v1

.github/workflows/flyteidl-release.yml

@@ -19,26 +19,21 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install setuptools wheel twine
-      - name: Set version in python package
-        run: |
-          # from refs/tags/v1.2.3 get 1.2.3
-          VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')
-          VERSION=$VERSION make update_pyversion
-        shell: bash
+          pip install build twine
       - name: Build and publish
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
-          python setup.py sdist bdist_wheel
+          python -m build
           twine upload dist/*
   deploy-to-npm:
     runs-on: ubuntu-latest
     defaults:
       run:
         working-directory: flyteidl
     steps:
+      - uses: actions/checkout@v2
       - uses: actions/setup-node@v1
         with:
           node-version: "12.x"

.github/workflows/tests.yml

@@ -40,6 +40,7 @@ jobs:
       - shell: bash -el {0}
         run: |
           conda activate monodocs-env
+          pip install -e ./flyteidl
           conda info
           conda list
           conda config --show-sources

.readthedocs.yml

@@ -11,6 +11,7 @@ build:
   jobs:
     post_install:
       - conda-lock install --name $READTHEDOCS_VERSION monodocs-environment.lock.yaml
+      - pip install ./flyteidl
       - conda info
       - conda env list
       - conda list

CHANGELOG/CHANGELOG-v1.10.7-b3.md

@@ -0,0 +1,3 @@
+# Flyte v1.10.7-b3 Release
+
+Pre-release testing.

CHANGELOG/CHANGELOG-v1.10.7-b4.md

@@ -0,0 +1,3 @@
+# Flyte v1.10.7-b4 Release
+
+Pre-release testing.

CODEOWNERS

@@ -0,0 +1,3 @@
+# Automatically request docs team for docs PR review
+/docs/          @neverett @ppiegaze
+

Makefile

@@ -53,10 +53,6 @@ deploy_sandbox:
 install-piptools: ## Install pip-tools
 	pip install -U pip-tools
 
-.PHONY: doc-requirements.txt
-doc-requirements.txt: doc-requirements.in install-piptools
-	$(call PIP_COMPILE,doc-requirements.in)
-
 .PHONY: install-conda-lock
 install-conda-lock:
 	pip install conda-lock

charts/flyte-binary/README.md

@@ -42,7 +42,7 @@ Chart for basic single Flyte executable deployment
 | configuration.auth.oidc.clientId | string | `""` |  |
 | configuration.auth.oidc.clientSecret | string | `""` |  |
 | configuration.co-pilot.image.repository | string | `"cr.flyte.org/flyteorg/flytecopilot"` |  |
-| configuration.co-pilot.image.tag | string | `"v1.10.7-b2"` |  |
+| configuration.co-pilot.image.tag | string | `"v1.10.7-b4"` |  |
 | configuration.database.dbname | string | `"flyte"` |  |
 | configuration.database.host | string | `"127.0.0.1"` |  |
 | configuration.database.options | string | `"sslmode=disable"` |  |

charts/flyte-binary/values.yaml

@@ -159,7 +159,7 @@ configuration:
       # repository CoPilot sidecar image repository
       repository: cr.flyte.org/flyteorg/flytecopilot # FLYTECOPILOT_IMAGE
       # tag CoPilot sidecar image tag
-      tag: v1.10.7-b2 # FLYTECOPILOT_TAG
+      tag: v1.10.7-b4 # FLYTECOPILOT_TAG
   # agentService Flyte Agent configuration
   agentService:
     defaultAgent:

charts/flyte-core/templates/admin/deployment.yaml

@@ -38,6 +38,12 @@ spec:
           volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }}
           - mountPath: /etc/flyte/config
             name: base-config-volume
+          {{- if .Values.flyteadmin.env }}
+          env:
+            {{- with .Values.flyteadmin.env -}}
+            {{- tpl (toYaml .) $ | nindent 12 }}
+            {{- end }}
+            {{- end }}
         {{- if .Values.flyteadmin.initialProjects }}
         - command:
           - flyteadmin
@@ -54,6 +60,12 @@ spec:
           volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }}
           - mountPath: /etc/flyte/config
             name: base-config-volume
+          {{- if .Values.flyteadmin.env }}
+          env:
+            {{- with .Values.flyteadmin.env -}}
+            {{- tpl (toYaml .) $ | nindent 12 }}
+            {{- end }}
+            {{- end }}
         {{- end }}
         {{- if and (.Values.cluster_resource_manager.enabled) (not .Values.cluster_resource_manager.standalone_deploy) }}
         - command:
@@ -77,6 +89,12 @@ spec:
           {{ tpl (toYaml .) $ | nindent 10 }}
           {{- end }}
           {{- end }}
+          {{- if .Values.flyteadmin.env }}
+          env:
+            {{- with .Values.flyteadmin.env -}}
+            {{- tpl (toYaml .) $ | nindent 12 }}
+            {{- end }}
+            {{- end }}
         {{- end }}
         - name: generate-secrets
           image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}"
@@ -96,6 +114,9 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            {{- with .Values.flyteadmin.env -}}
+            {{- tpl (toYaml .) $ | nindent 12 }}
+            {{- end }}
       containers:
       - command:
         - flyteadmin

charts/flyte-core/templates/clusterresourcesync/deployment.yaml

@@ -70,4 +70,7 @@ spec:
           secret:
             secretName: flyte-secret-auth
         {{- end }}
+      {{- with .Values.cluster_resource_manager.nodeSelector }}
+      nodeSelector: {{ tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
   {{- end }}

charts/flyte-core/templates/common/ingress.yaml

@@ -163,6 +163,7 @@ metadata:
   annotations: {{ tpl (toYaml .) $ | nindent 4}}
   {{- end }}
 spec:
+  ingressClassName: {{ .Values.common.ingress.ingressClassName | quote }}
   rules:
     - http:
         paths:
@@ -477,14 +478,13 @@ kind: Ingress
 metadata:
   name: {{ template "flyte.name" . }}-grpc
   namespace: {{ template "flyte.namespace" . }}
-  annotations:
-  {{- with .Values.common.ingress.annotations }}
-  {{- tpl (toYaml .) $ | nindent 4}}
-  {{- end }}
-  {{- with .Values.common.ingress.separateGrpcIngressAnnotations }}
-  {{- toYaml . | nindent 4}}
+  {{- $annotations := .Values.common.ingress.annotations | deepCopy -}}
+  {{- $_ := merge $annotations .Values.common.ingress.separateGrpcIngressAnnotations -}}
+  {{- with $annotations }}
+  annotations: {{ tpl (toYaml .) $ | nindent 4}}
   {{- end }}
 spec:
+  ingressClassName: {{ .Values.common.ingress.ingressClassName | quote }}
   rules:
     - host: {{ tpl (toYaml .Values.common.ingress.host) $ }}
       http:

charts/flyte-core/templates/console/deployment.yaml

@@ -18,6 +18,10 @@ spec:
         {{- end }}
       labels: {{ include "flyteconsole.podLabels" . | nindent 8 }}
     spec:
+    {{- with .Values.flyteconsole.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
       securityContext:
         runAsUser: 1000
         fsGroupChangePolicy: "OnRootMismatch"
@@ -33,7 +37,9 @@ spec:
             name: flyte-console-config
         ports:
         - containerPort: 8080
+        {{- if or .Values.flyteconsole.ga.enabled .Values.flyteconsole.podEnv }}
         env:
+        {{- end }}
         {{- if .Values.flyteconsole.ga.enabled }}
         - name: ENABLE_GA
           value: "{{ .Values.flyteconsole.ga.enabled }}"

charts/flyte-core/templates/datacatalog/deployment.yaml

@@ -38,6 +38,12 @@ spec:
         volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }}
         - mountPath: /etc/datacatalog/config
           name: config-volume
+        {{- if .Values.datacatalog.podEnv }}
+        env:
+          {{- with .Values.datacatalog.podEnv }}
+          {{- toYaml . | nindent 8 }}
+          {{- end }}
+        {{- end }}
       containers:
       - command:
         - datacatalog

charts/flyte-core/templates/flytescheduler/deployment.yaml

@@ -33,6 +33,12 @@ spec:
         - precheck
         - --config
         - {{ .Values.flytescheduler.configPath }}
+        {{- if .Values.flytescheduler.podEnv }}
+        env:
+          {{- with .Values.flytescheduler.podEnv -}}
+          {{- toYaml . | nindent 8 }}
+          {{- end }}
+        {{- end }}
         image: "{{ .Values.flytescheduler.image.repository }}:{{ .Values.flytescheduler.image.tag }}"
         imagePullPolicy: "{{ .Values.flytescheduler.image.pullPolicy }}"
         name: flytescheduler-check

charts/flyte-core/templates/propeller/configmap.yaml

@@ -55,3 +55,8 @@ data:
   task_logs.yaml: | {{ tpl (toYaml .) $ | nindent 4 }}
 {{- end }}
 {{- end }}
+{{- if .Values.flyteagent.enabled }}
+{{- with .Values.flyteagent.plugin_config }}
+  agent_service.yaml: | {{ tpl (toYaml .) $ | nindent 4 }}
+{{- end }}
+{{- end }}

charts/flyte-core/templates/propeller/webhook.yaml

@@ -61,6 +61,11 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: metadata.namespace
+        {{- if .Values.flytepropeller.podEnv -}}
+        {{- with .Values.flytepropeller.podEnv -}}
+        {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
         volumeMounts:
           - name: config-volume
             mountPath: /etc/flyte/config
@@ -89,6 +94,8 @@ spec:
         {{- toYaml . | nindent 12 }}
         {{- end }}
         {{- end }}
+          ports:
+          - containerPort: 9443
           volumeMounts:
             - name: config-volume
               mountPath: /etc/flyte/config
@@ -103,6 +110,9 @@ spec:
         - name: webhook-certs
           secret:
             secretName: flyte-pod-webhook
+      {{- with .Values.flytepropeller.nodeSelector }}
+      nodeSelector: {{ tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
 ---
 # Service
 apiVersion: v1
@@ -154,6 +164,9 @@ metadata:
   annotations: {{ tpl (toYaml .) $ | nindent 4 }}
   {{- end}}
 {{- end }}
+  {{- with .Values.webhook.serviceAccount.imagePullSecrets }}
+imagePullSecrets: {{ tpl (toYaml .) $ | nindent 2 }}
+  {{- end }}
 ---
 # Create a binding from Role -> ServiceAccount
 kind: ClusterRoleBinding

charts/flyte-core/values.yaml

@@ -16,7 +16,7 @@ flyteadmin:
   image:
     # -- Docker image for Flyteadmin deployment
     repository: cr.flyte.org/flyteorg/flyteadmin # FLYTEADMIN_IMAGE
-    tag: v1.10.7-b2 # FLYTEADMIN_TAG
+    tag: v1.10.7-b4 # FLYTEADMIN_TAG
     pullPolicy: IfNotPresent
   # -- Additional flyteadmin container environment variables
   #
@@ -134,7 +134,7 @@ flytescheduler:
     # -- Docker image for Flytescheduler deployment
     repository: cr.flyte.org/flyteorg/flytescheduler # FLYTESCHEDULER_IMAGE
     # -- Docker image tag
-    tag: v1.10.7-b2 # FLYTESCHEDULER_TAG
+    tag: v1.10.7-b4 # FLYTESCHEDULER_TAG
     # -- Docker image pull policy
     pullPolicy: IfNotPresent
   # -- Default resources requests and limits for Flytescheduler deployment
@@ -192,7 +192,7 @@ datacatalog:
     # -- Docker image for Datacatalog deployment
     repository: cr.flyte.org/flyteorg/datacatalog # DATACATALOG_IMAGE
     # -- Docker image tag
-    tag: v1.10.7-b2 # DATACATALOG_TAG
+    tag: v1.10.7-b4 # DATACATALOG_TAG
     # -- Docker image pull policy
     pullPolicy: IfNotPresent
   # -- Default resources requests and limits for Datacatalog deployment
@@ -249,6 +249,12 @@ datacatalog:
 
 flyteagent:
   enabled: false
+  plugin_config:
+    plugins:
+      agentService:
+        defaultAgent:
+          endpoint: "dns:///flyteagent.flyte.svc.cluster.local:8000"
+          insecure: true
 
 #
 # FLYTEPROPELLER SETTINGS
@@ -264,7 +270,7 @@ flytepropeller:
   image:
     # -- Docker image for Flytepropeller deployment
     repository: cr.flyte.org/flyteorg/flytepropeller # FLYTEPROPELLER_IMAGE
-    tag: v1.10.7-b2 # FLYTEPROPELLER_TAG
+    tag: v1.10.7-b4 # FLYTEPROPELLER_TAG
     pullPolicy: IfNotPresent
   # -- Default resources requests and limits for Flytepropeller deployment
   resources:
@@ -374,6 +380,8 @@ flyteconsole:
     tracking_id: "G-0QW4DJWJ20"
   # -- Sets priorityClassName for flyte console pod(s).
   priorityClassName: ""
+   # -- ImagePullSecrets to assign to the Flyteconsole deployment
+  imagePullSecrets: []
 
 # It will enable the redoc route in ingress
 deployRedoc: false
@@ -424,6 +432,8 @@ common:
     # -- Specify your Secret (with sensitive data) or pseudo-manifest (without sensitive data). See https://github.com/godaddy/kubernetes-external-secrets
     secretManifest: {}
   ingress:
+    # --- Sets the ingressClassName
+    ingressClassName:
     # --- Enable or disable creating Ingress for Flyte. Relevant to disable when using e.g. Istio as ingress controller.
     enabled: true
     # --- Enable or disable HMR route to flyteconsole. This is useful only for frontend development.
@@ -671,7 +681,7 @@ configmap:
         # -- Structure documented [here](https://pkg.go.dev/github.com/lyft/[email protected]/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig)
         co-pilot:
           name: flyte-copilot-
-          image: cr.flyte.org/flyteorg/flytecopilot:v1.10.7-b2 # FLYTECOPILOT_IMAGE
+          image: cr.flyte.org/flyteorg/flytecopilot:v1.10.7-b4 # FLYTECOPILOT_IMAGE
           start-timeout: 30s
 
   # -- Core propeller configuration
@@ -750,11 +760,6 @@ configmap:
         #  DEFAULT_ENV_VAR: VALUE
         default-cpus: 100m
         default-memory: 100Mi
-      # Uncomment and modify to include configuration for Flyte Agent
-      # agent-service:
-      #   defaultGrpcEndpoint: flyteagent.flyte.svc.cluster.local:8000
-      #   supportedTaskTypes:
-      #     - bigquery_query_job_task
   remoteData:
     remoteData:
       region: "us-east-1"
@@ -845,6 +850,8 @@ cluster_resource_manager:
   podEnv: {}
   # -- Labels for ClusterResource pods
   podLabels: {}
+  # -- nodeSelector for ClusterResource deployment
+  nodeSelector: {}
   # -- Configmap for ClusterResource parameters
   config:
     # -- ClusterResource parameters