Use uv in CI and to build images (flyteorg#2403)

* Use uv in CI and to build images Signed-off-by: Eduardo Apolinario <[email protected]> * Source venv and remove extraneous -y Signed-off-by: Eduardo Apolinario <[email protected]> * Use format that uv understands Signed-off-by: Eduardo Apolinario <[email protected]> * Fix lint errors Signed-off-by: Eduardo Apolinario <[email protected]> * Test sourcing the venv in hypothesis job Signed-off-by: Eduardo Apolinario <[email protected]> * Source everywhere Signed-off-by: Eduardo Apolinario <[email protected]> * Remove redundant shellcheck Signed-off-by: Eduardo Apolinario <[email protected]> * Add activate-uv-venv target Signed-off-by: Eduardo Apolinario <[email protected]> * Install uv in the system python Signed-off-by: Eduardo Apolinario <[email protected]> * Add --system Signed-off-by: Eduardo Apolinario <[email protected]> * Install plugin dev-requirements in the system python Signed-off-by: Eduardo Apolinario <[email protected]> * Rename setup-uv to setup-global-uv Signed-off-by: Eduardo Apolinario <[email protected]> * Fix call to install flytekit in dev image Signed-off-by: Eduardo Apolinario <[email protected]> * Comment out the dolt unit tests Signed-off-by: Eduardo Apolinario <[email protected]> * Fix papermill and great expectations enviroment setup Signed-off-by: Eduardo Apolinario <[email protected]> --------- Signed-off-by: Eduardo Apolinario <[email protected]> Co-authored-by: Eduardo Apolinario <[email protected]>
jasonlai1218 · May 10, 2024 · 0d4981b · 0d4981b
1 parent c0eca6a
commit 0d4981b
Show file tree

Hide file tree

Showing 7 changed files with 58 additions and 36 deletions.
diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml
@@ -57,9 +57,10 @@ jobs:
           key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
       - name: Install dependencies
         run: |
-          make setup
-          pip uninstall -y pandas
-          pip freeze
+          pip install uv
+          make setup-global-uv
+          uv pip uninstall --system pandas
+          uv pip freeze
       - name: Test with coverage
         run: |
           make unit_test_codecov
@@ -95,9 +96,10 @@ jobs:
           key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
       - name: Install dependencies
         run: |
-          make setup
-          pip uninstall -y pandas
-          pip freeze
+          pip install uv
+          make setup-global-uv
+          uv pip uninstall --system pandas
+          uv pip freeze
       - name: Run extras unit tests with coverage
         # Skip this step if running on python 3.12 due to https://github.com/tensorflow/tensorflow/issues/62003
         # and https://github.com/pytorch/pytorch/issues/110436
@@ -137,9 +139,10 @@ jobs:
           key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
       - name: Install dependencies
         run: |
-          make setup
-          pip install --force-reinstall "${{ matrix.pandas }}"
-          pip freeze
+          pip install uv
+          make setup-global-uv
+          uv pip install --system --force-reinstall "${{ matrix.pandas }}"
+          uv pip freeze
       - name: Test with coverage
         run: |
           make unit_test_codecov
@@ -172,7 +175,10 @@ jobs:
           # Look to see if there is a cache hit for the corresponding requirements files
           key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
       - name: Install dependencies
-        run: make setup && pip freeze
+        run: |
+          pip install uv
+          make setup-global-uv
+          uv pip freeze
       - name: Test with coverage
         env:
           FLYTEKIT_HYPOTHESIS_PROFILE: ci
@@ -207,7 +213,10 @@ jobs:
           # Look to see if there is a cache hit for the corresponding requirements files
           key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
       - name: Install dependencies
-        run: make setup && pip freeze
+        run: |
+          pip install uv
+          make setup-global-uv
+          uv pip freeze
       - name: Test with coverage
         run: |
           make test_serialization_codecov
@@ -244,7 +253,10 @@ jobs:
           # Look to see if there is a cache hit for the corresponding requirements files
           key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
       - name: Install dependencies
-        run: make setup && pip freeze
+        run: |
+          pip install uv
+          make setup-global-uv
+          uv pip freeze
       - name: Install FlyteCTL
         uses: unionai-oss/flytectl-setup-action@master
       - name: Setup Flyte Sandbox
@@ -272,7 +284,8 @@ jobs:
           FLYTEKIT_IMAGE: localhost:30000/flytekit:dev
           FLYTEKIT_CI: 1
           PYTEST_OPTS: -n2
-        run: make integration_test_codecov
+        run: |
+          make integration_test_codecov
       - name: Codecov
         uses: codecov/[email protected]
         with:
@@ -299,7 +312,8 @@ jobs:
           - flytekit-data-fsspec
           - flytekit-dbt
           - flytekit-deck-standard
-          - flytekit-dolt
+          # TODO: remove dolt plugin - https://github.com/flyteorg/flyte/issues/5350
+          # flytekit-dolt
           - flytekit-duckdb
           - flytekit-envd
           - flytekit-flyteinteractive
@@ -395,14 +409,15 @@ jobs:
           key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('plugins/{0}/requirements.txt', matrix.plugin-names ))) }}
       - name: Install dependencies
         run: |
+          pip install uv
           # TODO: double-check if checking out all tags solves the issue
           export SETUPTOOLS_SCM_PRETEND_VERSION="3.0.0"
-          make setup
+          make setup-global-uv
           cd plugins/${{ matrix.plugin-names }}
-          pip install .
-          if [ -f dev-requirements.in ]; then pip install -r dev-requirements.in; fi
-          pip install -U $GITHUB_WORKSPACE
-          pip freeze
+          uv pip install --system .
+          if [ -f dev-requirements.in ]; then uv pip install --system -r dev-requirements.in; fi
+          uv pip install --system -U $GITHUB_WORKSPACE
+          uv pip freeze
       - name: Test with coverage
         run: |
           cd plugins/${{ matrix.plugin-names }}
@@ -435,12 +450,9 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          make setup
-          pip freeze
+          pip install uv
+          make setup-global-uv
+          uv pip freeze
       - name: Lint
         run: |
           make lint
-      - name: ShellCheck
-        uses: ludeeus/action-shellcheck@master
-        with:
-          ignore_paths: boilerplate
diff --git a/Dockerfile b/Dockerfile
@@ -21,7 +21,8 @@ ARG DOCKER_IMAGE
 # 3. Clean up the apt cache to reduce image size. Reference: https://gist.github.com/marvell/7c812736565928e602c4
 # 4. Create a non-root user 'flytekit' and set appropriate permissions for directories.
 RUN apt-get update && apt-get install build-essential -y \
-    && pip install --no-cache-dir -U flytekit==$VERSION \
+    && pip install uv \
+    && uv pip install --system --no-cache-dir -U flytekit==$VERSION \
         flytekitplugins-pod==$VERSION \
         flytekitplugins-deck-standard==$VERSION \
         scikit-learn \

diff --git a/Dockerfile.dev b/Dockerfile.dev
@@ -17,6 +17,7 @@ ENV FLYTE_SDK_RICH_TRACEBACKS 0
 # Flytekit version of flytekit to be installed in the image
 ARG PSEUDO_VERSION
 
+
 # Note: Pod tasks should be exposed in the default image
 # Note: Some packages will create config files under /home by default, so we need to make sure it's writable
 # Note: There are use cases that require reading and writing files under /tmp, so we need to change its permissions.
@@ -26,15 +27,18 @@ ARG PSEUDO_VERSION
 # 2. Install Flytekit and its plugins.
 # 3. Clean up the apt cache to reduce image size. Reference: https://gist.github.com/marvell/7c812736565928e602c4
 # 4. Create a non-root user 'flytekit' and set appropriate permissions for directories.
-RUN apt-get update && apt-get install build-essential vim libmagic1 git -y
-RUN pip install scikit-learn
+RUN apt-get update && apt-get install build-essential vim libmagic1 git -y \
+    && pip install uv
+
 COPY . /flytekit
-RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEKIT=$PSEUDO_VERSION pip install --no-cache-dir -U \
+
+RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEKIT=$PSEUDO_VERSION uv pip install --system --no-cache-dir -U \
         "git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl" \
         -e /flytekit \
         -e /flytekit/plugins/flytekit-k8s-pod \
         -e /flytekit/plugins/flytekit-deck-standard \
         -e /flytekit/plugins/flytekit-flyteinteractive \
+        scikit-learn \
     && apt-get clean autoclean \
     && apt-get autoremove --yes \
     && rm -rf /var/lib/{apt,dpkg,cache,log}/ \

diff --git a/Makefile b/Makefile
@@ -27,6 +27,11 @@ update_boilerplate:
 setup: install-piptools ## Install requirements
 	pip install -r dev-requirements.in
 
+# Warning: this will install the requirements in your system python
+.PHONY: setup-global-uv
+setup-global-uv:
+	uv pip install --system -r dev-requirements.in
+
 .PHONY: fmt
 fmt:
 	pre-commit run ruff --all-files || true
@@ -105,7 +110,7 @@ requirements: doc-requirements.txt ${MOCK_FLYTE_REPO}/requirements.txt ## Compil
 # TODO: Change this in the future to be all of flytekit
 .PHONY: coverage
 coverage:
-	coverage run -m pytest tests/flytekit/unit/core flytekit/types -m "not sandbox_test"
+	coverage run -m $(PYTEST) tests/flytekit/unit/core flytekit/types -m "not sandbox_test"
 	coverage report -m --include="flytekit/core/*,flytekit/types/*"
 
 .PHONY: build-dev

diff --git a/dev-requirements.in b/dev-requirements.in
@@ -1,5 +1,5 @@
--e file:.#egg=flytekit
-git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl
+-e file:.
+flyteidl @ git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl
 
 coverage[toml]
 hypothesis

diff --git a/plugins/flytekit-greatexpectations/dev-requirements.in b/plugins/flytekit-greatexpectations/dev-requirements.in
@@ -1 +1 @@
--e file:../../.#egg=flytekitplugins-spark&subdirectory=plugins/flytekit-spark
+-e file:../flytekit-spark/.#egg=flytekitplugins-spark
diff --git a/plugins/flytekit-papermill/dev-requirements.in b/plugins/flytekit-papermill/dev-requirements.in
@@ -1,4 +1,4 @@
 -e file:../../.#egg=flytekit
--e file:../../.#egg=flytekitplugins-pod&subdirectory=plugins/flytekit-k8s-pod
--e file:../../.#egg=flytekitplugins-spark&subdirectory=plugins/flytekit-spark
--e file:../../.#egg=flytekitplugins-awsbatch&subdirectory=plugins/flytekit-aws-batch
+-e file:../flytekit-k8s-pod/.#egg=flytekitplugins-pod
+-e file:../flytekit-spark/.#egg=flytekitplugins-spark
+-e file:../flytekit-aws-batch/.#egg=flytekitplugins-awsbatch
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		-e file:../../.#egg=flytekitplugins-spark&subdirectory=plugins/flytekit-spark
		-e file:../flytekit-spark/.#egg=flytekitplugins-spark