Merge branch 'master' into go-1.21

Signed-off-by: Eduardo Apolinario <[email protected]>
flyteorg · Dec 8, 2023 · 220c85f · 220c85f
2 parents 429ba4c + f440278
commit 220c85f
Show file tree

Hide file tree

Showing 14 changed files with 16,766 additions and 128 deletions.
diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml
@@ -25,4 +25,4 @@ jobs:
       - name: Codespell
         uses: codespell-project/actions-codespell@v2
         with:
-          skip: "*.pb"
+          skip: "*.pb,monodocs-environment.lock.yaml"
diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml
@@ -79,25 +79,16 @@ jobs:
       - name: Register P0 tests
         if: ${{ inputs.priorities == 'P0' }}
         run: |
-          for f in \
-                  basics/basics/hello_world.py \
-                  basics/basics/workflow.py \
-                  basics/basics/named_outputs.py \
-                  advanced_composition/advanced_composition/chain_entities.py \
-                  advanced_composition/advanced_composition/dynamics.py \
-                  advanced_composition/advanced_composition/map_task.py \
-                  advanced_composition/advanced_composition/subworkflows.py \
-                  data_types_and_io/data_types_and_io/dataclass.py \
-                  data_types_and_io/data_types_and_io/structured_dataset.py ;
+          while read -r line;
           do
               pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                   register \
                   --project flytesnacks \
                   --domain development \
                   --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
                   --version ${{ env.FLYTESNACKS_VERSION }} \
-                  flytesnacks/examples/$f;
-          done
+                  flytesnacks/$line;
+          done < flytesnacks/flyte_tests.txt
       - name: Register all flytesnacks examples
         if: ${{ inputs.priorities != 'P0' }}
         uses: unionai/[email protected]

diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml
@@ -27,7 +27,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.21'
+          go-version: "1.21"
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
@@ -129,8 +129,8 @@ jobs:
   sandbox-bundled-functional-tests:
     runs-on: ubuntu-latest
     env:
-      FLYTESNACKS_PRIORITIES: 'P0'
-      FLYTESNACKS_VERSION: ''
+      FLYTESNACKS_PRIORITIES: "P0"
+      FLYTESNACKS_VERSION: ""
     timeout-minutes: 20
     needs: [build-and-push-single-binary-image]
     steps:
@@ -164,7 +164,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
+          python-version: "3.11"
       - uses: unionai/[email protected]
       - name: Setup sandbox
         run: |
@@ -193,25 +193,16 @@ jobs:
           # ref: ${{ env.FLYTESNACKS_VERSION }}
       - name: Register specific tests
         run: |
-          for f in \
-                  basics/basics/hello_world.py \
-                  basics/basics/workflow.py \
-                  basics/basics/named_outputs.py \
-                  advanced_composition/advanced_composition/chain_entities.py \
-                  advanced_composition/advanced_composition/dynamics.py \
-                  advanced_composition/advanced_composition/map_task.py \
-                  advanced_composition/advanced_composition/subworkflows.py \
-                  data_types_and_io/data_types_and_io/dataclass.py \
-                  data_types_and_io/data_types_and_io/structured_dataset.py ;
+          while read -r line;
           do
               pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                   register \
                   --project flytesnacks \
                   --domain development \
                   --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
                   --version ${{ env.FLYTESNACKS_VERSION }} \
-                  flytesnacks/examples/$f;
-          done
+                  flytesnacks/$line;
+          done < flytesnacks/flyte_tests.txt
       - name: End2End
         run: |
           make end2end_execute

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -31,14 +31,15 @@ jobs:
         uses: actions/checkout@v4
       - uses: conda-incubator/setup-miniconda@v3
         with:
+          auto-update-conda: true
           python-version: 3.9
-          mamba-version: "*"
-          channels: conda-forge
-          channel-priority: true
-          activate-environment: monodocs-env
-          environment-file: monodocs-environment.yaml
       - shell: bash -el {0}
         run: |
+          conda install -c conda-forge conda-lock
+          conda-lock install -n monodocs-env monodocs-environment.lock.yaml
+      - shell: bash -el {0}
+        run: |
+          conda activate monodocs-env
           conda info
           conda list
           conda config --show-sources
@@ -48,7 +49,9 @@ jobs:
         uses: ts-graphviz/setup-graphviz@v1
       - name: Build the documentation
         shell: bash -el {0}
-        run: make docs
+        run: |
+          conda activate monodocs-env
+          make docs
 
   generate_kustomize:
     runs-on: ubuntu-latest

diff --git a/Makefile b/Makefile
@@ -57,6 +57,14 @@ install-piptools: ## Install pip-tools
 doc-requirements.txt: doc-requirements.in install-piptools
 	$(call PIP_COMPILE,doc-requirements.in)
 
+.PHONY: install-conda-lock
+install-conda-lock:
+	pip install conda-lock
+
+.PHONY: conda-lock
+conda-lock: install-conda-lock
+	conda-lock -f monodocs-environment.yaml --without-cuda --lockfile monodocs-environment.lock.yaml
+
 .PHONY: stats
 stats:
 	@generate-dashboard -o deployment/stats/prometheus/flytepropeller-dashboard.json stats/flytepropeller.dashboard.py

diff --git a/boilerplate/flyte/end2end/run-tests.py b/boilerplate/flyte/end2end/run-tests.py
@@ -1,11 +1,9 @@
 #!/usr/bin/env python3
-
-import datetime
 import json
 import sys
 import time
 import traceback
-from typing import Dict, List, Mapping, Tuple, Optional
+from typing import Dict, List, Optional
 
 import click
 import requests
@@ -17,79 +15,6 @@
 WAIT_TIME = 10
 MAX_ATTEMPTS = 200
 
-# This dictionary maps the names found in the flytesnacks manifest to a list of workflow names and
-# inputs. This is so we can progressively cover all priorities in the original flytesnacks manifest,
-# starting with "core".
-FLYTESNACKS_WORKFLOW_GROUPS: Mapping[str, List[Tuple[str, dict]]] = {
-    "lite": [
-        ("basics.hello_world.hello_world_wf", {}),
-    ],
-    "core": [
-        # ("development_lifecycle.decks.image_renderer_wf", {}),
-        # The chain_workflows example in flytesnacks expects to be running in a sandbox.
-        ("advanced_composition.chain_entities.chain_workflows_wf", {}),
-        ("advanced_composition.dynamics.wf", {"s1": "Pear", "s2": "Earth"}),
-        ("advanced_composition.map_task.my_map_workflow", {"a": [1, 2, 3, 4, 5]}),
-        # Workflows that use nested executions cannot be launched via flyteremote.
-        # This issue is being tracked in https://github.com/flyteorg/flyte/issues/1482.
-        # ("control_flow.run_conditions.multiplier", {"my_input": 0.5}),
-        # ("control_flow.run_conditions.multiplier_2", {"my_input": 10}),
-        # ("control_flow.run_conditions.multiplier_3", {"my_input": 5}),
-        # ("control_flow.run_conditions.basic_boolean_wf", {"seed": 5}),
-        # ("control_flow.run_conditions.bool_input_wf", {"b": True}),
-        # ("control_flow.run_conditions.nested_conditions", {"my_input": 0.4}),
-        # ("control_flow.run_conditions.consume_outputs", {"my_input": 0.4, "seed": 7}),
-        # ("control_flow.run_merge_sort.merge_sort", {"numbers": [5, 4, 3, 2, 1], "count": 5}),
-        ("advanced_composition.subworkflows.parent_workflow", {"my_input1": "hello"}),
-        ("advanced_composition.subworkflows.nested_parent_wf", {"a": 3}),
-        ("basics.workflow.simple_wf", {"x": [1, 2, 3], "y": [1, 2, 3]}),
-        # TODO: enable new files and folders workflows
-        # ("basics.files.rotate_one_workflow", {"in_image": "https://upload.wikimedia.org/wikipedia/commons/d/d2/Julia_set_%28C_%3D_0.285%2C_0.01%29.jpg"}),
-        # ("basics.folders.download_and_rotate", {}),
-        ("basics.hello_world.hello_world_wf", {}),
-        ("basics.named_outputs.simple_wf_with_named_outputs", {}),
-        # # Getting a 403 for the wikipedia image
-        # # ("basics.reference_task.wf", {}),
-        ("data_types_and_io.dataclass.dataclass_wf", {"x": 10, "y": 20}),
-        # Enums are not supported in flyteremote
-        # ("type_system.enums.enum_wf", {"c": "red"}),
-        ("data_types_and_io.structured_dataset.simple_sd_wf", {"a": 42}),
-        # ("my.imperative.workflow.example", {"in1": "hello", "in2": "foo"}),
-    ],
-    "integrations-k8s-spark": [
-        (
-            "k8s_spark_plugin.pyspark_pi.my_spark",
-            {"triggered_date": datetime.datetime.now()},
-        ),
-    ],
-    "integrations-kfpytorch": [
-        ("kfpytorch_plugin.pytorch_mnist.pytorch_training_wf", {}),
-    ],
-    "integrations-kftensorflow": [
-        ("kftensorflow_plugin.tf_mnist.mnist_tensorflow_workflow", {}),
-    ],
-    # "integrations-pod": [
-    #     ("pod.pod.pod_workflow", {}),
-    # ],
-    "integrations-pandera_examples": [
-        ("pandera_plugin.basic_schema_example.process_data", {}),
-        # TODO: investigate type mismatch float -> numpy.float64
-        # ("pandera_plugin.validating_and_testing_ml_pipelines.pipeline", {"data_random_state": 42, "model_random_state": 99}),
-    ],
-    "integrations-modin_examples": [
-        ("modin_plugin.knn_classifier.pipeline", {}),
-    ],
-    "integrations-papermilltasks": [
-        ("papermill_plugin.simple.nb_to_python_wf", {"f": 3.1415926535}),
-    ],
-    "integrations-greatexpectations": [
-        ("greatexpectations_plugin.task_example.simple_wf", {}),
-        ("greatexpectations_plugin.task_example.file_wf", {}),
-        ("greatexpectations_plugin.task_example.schema_wf", {}),
-        ("greatexpectations_plugin.task_example.runtime_wf", {}),
-    ],
-}
-
 
 def execute_workflow(
     remote: FlyteRemote,
@@ -137,6 +62,7 @@ def schedule_workflow_groups(
     workflow_groups: List[str],
     remote: FlyteRemote,
     terminate_workflow_on_failure: bool,
+    parsed_manifest: List[dict],
     cluster_pool_name: Optional[str] = None,
 ) -> Dict[str, bool]:
     """
@@ -146,7 +72,12 @@ def schedule_workflow_groups(
     executions_by_wfgroup = {}
     # Schedule executions for each workflow group,
     for wf_group in workflow_groups:
-        workflows = FLYTESNACKS_WORKFLOW_GROUPS.get(wf_group, [])
+        workflow_group_item = list(
+            filter(lambda item: item["name"] == wf_group, parsed_manifest)
+        )
+        workflows = []
+        if workflow_group_item:
+            workflows = workflow_group_item[0]["examples"]
         executions_by_wfgroup[wf_group] = [
             execute_workflow(remote, tag, workflow[0], workflow[1], cluster_pool_name)
             for workflow in workflows
@@ -188,12 +119,12 @@ def schedule_workflow_groups(
     return results
 
 
-def valid(workflow_group):
+def valid(workflow_group, parsed_manifest):
     """
-    Return True if a workflow group is contained in FLYTESNACKS_WORKFLOW_GROUPS,
+    Return True if a workflow group is contained in parsed_manifest,
     False otherwise.
     """
-    return workflow_group in FLYTESNACKS_WORKFLOW_GROUPS.keys()
+    return workflow_group in set(wf_group["name"] for wf_group in parsed_manifest)
 
 
 def run(
@@ -233,7 +164,7 @@ def run(
     results = []
     valid_workgroups = []
     for workflow_group in workflow_groups:
-        if not valid(workflow_group):
+        if not valid(workflow_group, parsed_manifest):
             results.append(
                 {
                     "label": workflow_group,
@@ -249,6 +180,7 @@ def run(
         valid_workgroups,
         remote,
         terminate_workflow_on_failure,
+        parsed_manifest,
         cluster_pool_name,
     )
 

diff --git a/boilerplate/flyte/golang_test_targets/Makefile b/boilerplate/flyte/golang_test_targets/Makefile
@@ -3,9 +3,6 @@
 #
 # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
 
-.PHONY: codespell
-codespell:
-	git ls-files | grep -vE 'go\.mod|go\.sum|flyteidl/|\.pb$$|\.git|\.pdf|\.svg|requirements\.txt|gen/' | xargs codespell -w
 
 .PHONY: download_tooling
 download_tooling: #download dependencies (including test deps) for the package
@@ -16,8 +13,8 @@ generate: download_tooling #generate go code
 	@boilerplate/flyte/golang_test_targets/go-gen.sh
 
 .PHONY: lint
-lint: codespell download_tooling #lints the package for common code smells
-	GL_DEBUG=linters_output,env golangci-lint run --fix --deadline=5m --exclude deprecated -v
+lint: download_tooling #lints the package for common code smells
+	GL_DEBUG=linters_output,env golangci-lint run --deadline=5m --exclude deprecated -v
 
 # If code is failing goimports linter, this will fix.
 # skips 'vendor'

diff --git a/boilerplate/flyte/golang_test_targets/download_tooling.sh b/boilerplate/flyte/golang_test_targets/download_tooling.sh
@@ -19,6 +19,7 @@ tools=(
   "github.com/EngHabu/mockery/cmd/mockery"
   "github.com/flyteorg/flytestdlib/cli/pflags@latest"
   "github.com/golangci/golangci-lint/cmd/golangci-lint"
+  "github.com/daixiang0/gci"
   "github.com/alvaroloes/enumer"
   "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc"
 )

diff --git a/boilerplate/flyte/golang_test_targets/goimports b/boilerplate/flyte/golang_test_targets/goimports
@@ -6,3 +6,4 @@
 # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
 
 goimports -w $(find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./pkg/client/*" -not -path "./boilerplate/*")
+gci write -s standard -s default -s "prefix(github.com/flyteorg)" --custom-order --skip-generated .
diff --git a/docs/.readthedocs.yaml b/docs/.readthedocs.yaml
@@ -8,10 +8,15 @@ build:
   os: "ubuntu-22.04"
   tools:
     python: "mambaforge-22.9"
+  commands:
+    - cat monodocs-environment.lock.yaml
+    - mamba install -c conda-forge conda-lock
+    - conda-lock install -p /home/docs/monodocs-env monodocs-environment.lock.yaml
+    - conda info
+    - conda env list
+    - cat docs/conf.py
+    - cd docs && /home/docs/monodocs-env/bin/python -m sphinx -T -E -b html -d docs/_build/doctrees -D language=en . $READTHEDOCS_OUTPUT/html
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
   configuration: docs/conf.py
-
-conda:
-  environment: monodocs-environment.yaml
diff --git a/docs/_ext/import_projects.py b/docs/_ext/import_projects.py
@@ -33,6 +33,11 @@ class Project:
 
 
 def update_sys_path_for_flytekit(import_project_config: ImportProjectsConfig):
+    # create flytekit/_version.py file manually
+    with open(f"{import_project_config.flytekit_api_dir}/flytekit/_version.py", "w") as f:
+        f.write(f'__version__ = "dev"')
+
+
     # add flytekit to python path
     flytekit_dir = os.path.abspath(import_project_config.flytekit_api_dir)
     flytekit_src_dir = os.path.abspath(os.path.join(flytekit_dir, "flytekit"))

diff --git a/docs/integrations.md b/docs/integrations.md
@@ -49,13 +49,12 @@ Data is automatically marshalled and unmarshalled in and out of the plugin. User
 {py:class}`~flytekit.core.base_task.PythonTask` API defined in Flytekit.
 
 Flytekit Plugins are lazily loaded and can be released independently like libraries. We follow a convention to name the
-plugin like `flytekitplugins-*`, where * indicates the package to be integrated into Flytekit. For example
+plugin like `flytekitplugins-*`, where \* indicates the package to be integrated into Flytekit. For example
 `flytekitplugins-papermill` enables users to author Flytekit tasks using [Papermill](https://papermill.readthedocs.io/en/latest/).
 
 You can find the plugins maintained by the core Flyte team [here](https://github.com/flyteorg/flytekit/tree/master/plugins).
 :::
 
-
 ## Native Backend Plugins
 
 Native Backend Plugins are the plugins that can be executed without any external service dependencies because the compute is
@@ -83,7 +82,6 @@ orchestrated by Flyte itself, within its provisioned Kubernetes clusters.
 
 (external_service_backend_plugins)=
 
-
 ## External Service Backend Plugins
 
 As the term suggests, external service backend plugins relies on external services like
@@ -138,7 +136,6 @@ Flyte uses Kustomize to generate the the deployment configuration which can be l
 
 ::::
 
-
 ## Custom Container Tasks
 
 Because Flyte uses executable docker containers as the smallest unit of compute, you can write custom tasks with the
@@ -208,6 +205,7 @@ flytesnacks/examples/athena_plugin/index
 flytesnacks/examples/aws_batch_plugin/index
 flytesnacks/examples/hive_plugin/index
 flytesnacks/examples/mmcloud_plugin/index
+flytesnacks/examples/sensor/index
 flytesnacks/examples/snowflake_plugin/index
 flytesnacks/examples/databricks_plugin/index
 flytesnacks/examples/bigquery_plugin/index
Original file line number	Diff line number	Diff line change
Expand Up		@@ -6,3 +6,4 @@
		# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst

		goimports -w $(find . -type f -name '.go' -not -path "./vendor/" -not -path "./pkg/client/" -not -path "./boilerplate/")
		gci write -s standard -s default -s "prefix(github.com/flyteorg)" --custom-order --skip-generated .