diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index c2e5d96544..c3aa79c93f 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -57,9 +57,10 @@ jobs: key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }} - name: Install dependencies run: | - make setup - pip uninstall -y pandas - pip freeze + pip install uv + make setup-global-uv + uv pip uninstall --system pandas + uv pip freeze - name: Test with coverage run: | make unit_test_codecov @@ -95,9 +96,10 @@ jobs: key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }} - name: Install dependencies run: | - make setup - pip uninstall -y pandas - pip freeze + pip install uv + make setup-global-uv + uv pip uninstall --system pandas + uv pip freeze - name: Run extras unit tests with coverage # Skip this step if running on python 3.12 due to https://github.com/tensorflow/tensorflow/issues/62003 # and https://github.com/pytorch/pytorch/issues/110436 @@ -137,9 +139,10 @@ jobs: key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }} - name: Install dependencies run: | - make setup - pip install --force-reinstall "${{ matrix.pandas }}" - pip freeze + pip install uv + make setup-global-uv + uv pip install --system --force-reinstall "${{ matrix.pandas }}" + uv pip freeze - name: Test with coverage run: | make unit_test_codecov @@ -172,7 +175,10 @@ jobs: # Look to see if there is a cache hit for the corresponding requirements files key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }} - name: Install dependencies - run: make setup && pip freeze + run: | + pip install uv + make setup-global-uv + uv pip freeze - name: Test with coverage env: FLYTEKIT_HYPOTHESIS_PROFILE: ci @@ -207,7 +213,10 @@ jobs: # Look to see if there is a cache hit for the corresponding requirements files key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }} - name: Install dependencies - run: make setup && pip freeze + run: | + pip install uv + make setup-global-uv + uv pip freeze - name: Test with coverage run: | make test_serialization_codecov @@ -244,7 +253,10 @@ jobs: # Look to see if there is a cache hit for the corresponding requirements files key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }} - name: Install dependencies - run: make setup && pip freeze + run: | + pip install uv + make setup-global-uv + uv pip freeze - name: Install FlyteCTL uses: unionai-oss/flytectl-setup-action@master - name: Setup Flyte Sandbox @@ -272,7 +284,8 @@ jobs: FLYTEKIT_IMAGE: localhost:30000/flytekit:dev FLYTEKIT_CI: 1 PYTEST_OPTS: -n2 - run: make integration_test_codecov + run: | + make integration_test_codecov - name: Codecov uses: codecov/codecov-action@v3.1.0 with: @@ -299,7 +312,8 @@ jobs: - flytekit-data-fsspec - flytekit-dbt - flytekit-deck-standard - - flytekit-dolt + # TODO: remove dolt plugin - https://github.com/flyteorg/flyte/issues/5350 + # flytekit-dolt - flytekit-duckdb - flytekit-envd - flytekit-flyteinteractive @@ -395,14 +409,15 @@ jobs: key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('plugins/{0}/requirements.txt', matrix.plugin-names ))) }} - name: Install dependencies run: | + pip install uv # TODO: double-check if checking out all tags solves the issue export SETUPTOOLS_SCM_PRETEND_VERSION="3.0.0" - make setup + make setup-global-uv cd plugins/${{ matrix.plugin-names }} - pip install . - if [ -f dev-requirements.in ]; then pip install -r dev-requirements.in; fi - pip install -U $GITHUB_WORKSPACE - pip freeze + uv pip install --system . + if [ -f dev-requirements.in ]; then uv pip install --system -r dev-requirements.in; fi + uv pip install --system -U $GITHUB_WORKSPACE + uv pip freeze - name: Test with coverage run: | cd plugins/${{ matrix.plugin-names }} @@ -435,12 +450,9 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - make setup - pip freeze + pip install uv + make setup-global-uv + uv pip freeze - name: Lint run: | make lint - - name: ShellCheck - uses: ludeeus/action-shellcheck@master - with: - ignore_paths: boilerplate diff --git a/Dockerfile b/Dockerfile index 63e4d301bc..0a20d8aefa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,8 @@ ARG DOCKER_IMAGE # 3. Clean up the apt cache to reduce image size. Reference: https://gist.github.com/marvell/7c812736565928e602c4 # 4. Create a non-root user 'flytekit' and set appropriate permissions for directories. RUN apt-get update && apt-get install build-essential -y \ - && pip install --no-cache-dir -U flytekit==$VERSION \ + && pip install uv \ + && uv pip install --system --no-cache-dir -U flytekit==$VERSION \ flytekitplugins-pod==$VERSION \ flytekitplugins-deck-standard==$VERSION \ scikit-learn \ diff --git a/Dockerfile.dev b/Dockerfile.dev index f4f56d0d4a..59b4e595f2 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -17,6 +17,7 @@ ENV FLYTE_SDK_RICH_TRACEBACKS 0 # Flytekit version of flytekit to be installed in the image ARG PSEUDO_VERSION + # Note: Pod tasks should be exposed in the default image # Note: Some packages will create config files under /home by default, so we need to make sure it's writable # Note: There are use cases that require reading and writing files under /tmp, so we need to change its permissions. @@ -26,15 +27,18 @@ ARG PSEUDO_VERSION # 2. Install Flytekit and its plugins. # 3. Clean up the apt cache to reduce image size. Reference: https://gist.github.com/marvell/7c812736565928e602c4 # 4. Create a non-root user 'flytekit' and set appropriate permissions for directories. -RUN apt-get update && apt-get install build-essential vim libmagic1 git -y -RUN pip install scikit-learn +RUN apt-get update && apt-get install build-essential vim libmagic1 git -y \ + && pip install uv + COPY . /flytekit -RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEKIT=$PSEUDO_VERSION pip install --no-cache-dir -U \ + +RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEKIT=$PSEUDO_VERSION uv pip install --system --no-cache-dir -U \ "git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl" \ -e /flytekit \ -e /flytekit/plugins/flytekit-k8s-pod \ -e /flytekit/plugins/flytekit-deck-standard \ -e /flytekit/plugins/flytekit-flyteinteractive \ + scikit-learn \ && apt-get clean autoclean \ && apt-get autoremove --yes \ && rm -rf /var/lib/{apt,dpkg,cache,log}/ \ diff --git a/Makefile b/Makefile index a8317faa17..4605e57719 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,11 @@ update_boilerplate: setup: install-piptools ## Install requirements pip install -r dev-requirements.in +# Warning: this will install the requirements in your system python +.PHONY: setup-global-uv +setup-global-uv: + uv pip install --system -r dev-requirements.in + .PHONY: fmt fmt: pre-commit run ruff --all-files || true @@ -105,7 +110,7 @@ requirements: doc-requirements.txt ${MOCK_FLYTE_REPO}/requirements.txt ## Compil # TODO: Change this in the future to be all of flytekit .PHONY: coverage coverage: - coverage run -m pytest tests/flytekit/unit/core flytekit/types -m "not sandbox_test" + coverage run -m $(PYTEST) tests/flytekit/unit/core flytekit/types -m "not sandbox_test" coverage report -m --include="flytekit/core/*,flytekit/types/*" .PHONY: build-dev diff --git a/dev-requirements.in b/dev-requirements.in index 3baa218102..7f42851061 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -1,5 +1,5 @@ --e file:.#egg=flytekit -git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl +-e file:. +flyteidl @ git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl coverage[toml] hypothesis diff --git a/plugins/flytekit-greatexpectations/dev-requirements.in b/plugins/flytekit-greatexpectations/dev-requirements.in index 35fcaf1b07..c61448aa45 100644 --- a/plugins/flytekit-greatexpectations/dev-requirements.in +++ b/plugins/flytekit-greatexpectations/dev-requirements.in @@ -1 +1 @@ --e file:../../.#egg=flytekitplugins-spark&subdirectory=plugins/flytekit-spark +-e file:../flytekit-spark/.#egg=flytekitplugins-spark diff --git a/plugins/flytekit-papermill/dev-requirements.in b/plugins/flytekit-papermill/dev-requirements.in index 3dc10d1afc..ef684746d2 100644 --- a/plugins/flytekit-papermill/dev-requirements.in +++ b/plugins/flytekit-papermill/dev-requirements.in @@ -1,4 +1,4 @@ -e file:../../.#egg=flytekit --e file:../../.#egg=flytekitplugins-pod&subdirectory=plugins/flytekit-k8s-pod --e file:../../.#egg=flytekitplugins-spark&subdirectory=plugins/flytekit-spark --e file:../../.#egg=flytekitplugins-awsbatch&subdirectory=plugins/flytekit-aws-batch +-e file:../flytekit-k8s-pod/.#egg=flytekitplugins-pod +-e file:../flytekit-spark/.#egg=flytekitplugins-spark +-e file:../flytekit-aws-batch/.#egg=flytekitplugins-awsbatch