From 4dd15bc8a62f4a816e3d7328fda8c6b3001b87d8 Mon Sep 17 00:00:00 2001 From: Alex Langenfeld Date: Wed, 23 Oct 2024 20:38:16 -0500 Subject: [PATCH] drop dagster-airflow docker pin (#25475) the urrlib2 pin dropped in https://github.com/dagster-io/dagster/pull/25451 was related to this ## How I Tested These Changes bk --- .../images/test-project-base/Dockerfile | 59 ++++++++++--------- .../test-project-base/last_updated.yaml | 10 ++-- python_modules/dagster-test/Dockerfile | 8 +-- .../libraries/dagster-airflow/setup.py | 1 - .../libraries/dagster-docker/setup.py | 6 +- 5 files changed, 43 insertions(+), 41 deletions(-) diff --git a/python_modules/automation/automation/docker/images/test-project-base/Dockerfile b/python_modules/automation/automation/docker/images/test-project-base/Dockerfile index 8471502833101..f069a5fe797a4 100644 --- a/python_modules/automation/automation/docker/images/test-project-base/Dockerfile +++ b/python_modules/automation/automation/docker/images/test-project-base/Dockerfile @@ -12,30 +12,31 @@ FROM "${BASE_IMAGE}" AS snapshot_builder RUN apt-get update -yqq \ && apt-get install -yqq \ - build-essential \ - cron \ - g++ \ - gcc \ - git \ - make - + build-essential \ + cron \ + g++ \ + gcc \ + git \ + make + +RUN pip install -U uv RUN git clone --depth 1 https://github.com/dagster-io/dagster.git \ && cd dagster \ - && pip install \ - -e python_modules/dagster \ - -e python_modules/dagster-pipes \ - -e python_modules/dagster-graphql \ - -e python_modules/dagster-webserver \ - -e python_modules/libraries/dagster-celery[flower,redis,kubernetes] \ - -e python_modules/libraries/dagster-postgres \ - -e python_modules/libraries/dagster-pandas \ - -e python_modules/libraries/dagster-aws \ - -e python_modules/libraries/dagster-gcp \ - -e python_modules/libraries/dagster-k8s \ - -e python_modules/libraries/dagster-celery-k8s \ + && python -m uv pip install \ + -e python_modules/dagster \ + -e python_modules/dagster-pipes \ + -e python_modules/dagster-graphql \ + -e python_modules/dagster-webserver \ + -e python_modules/libraries/dagster-celery[flower,redis,kubernetes] \ + -e python_modules/libraries/dagster-postgres \ + -e python_modules/libraries/dagster-pandas \ + -e python_modules/libraries/dagster-aws \ + -e python_modules/libraries/dagster-gcp \ + -e python_modules/libraries/dagster-k8s \ + -e python_modules/libraries/dagster-docker \ + -e python_modules/libraries/dagster-celery-k8s \ && pip freeze --exclude-editable > /snapshot-reqs.txt - FROM "${BASE_IMAGE}" COPY --from=snapshot_builder /snapshot-reqs.txt . @@ -43,15 +44,15 @@ COPY --from=snapshot_builder /snapshot-reqs.txt . # gcc etc needed for building gevent RUN apt-get update -yqq \ && apt-get install -yqq \ - build-essential \ - cron \ - g++ \ - gcc \ - git \ - make - -RUN pip install -r /snapshot-reqs.txt \ - && rm /snapshot-reqs.txt + build-essential \ + cron \ + g++ \ + gcc \ + git \ + make + +RUN pip install -U uv +RUN python -m uv pip install -r /snapshot-reqs.txt # New versions of debian require allowlisting folders where `git` can run RUN git config --system --add safe.directory \* diff --git a/python_modules/automation/automation/docker/images/test-project-base/last_updated.yaml b/python_modules/automation/automation/docker/images/test-project-base/last_updated.yaml index b7ae3529385e5..7f8691be303ef 100644 --- a/python_modules/automation/automation/docker/images/test-project-base/last_updated.yaml +++ b/python_modules/automation/automation/docker/images/test-project-base/last_updated.yaml @@ -1,5 +1,5 @@ -"3.10": 2024-02-14T041706 -"3.11": 2024-02-14T041706 -"3.12": 2024-02-14T041706 -"3.8": 2024-02-14T041706 -"3.9": 2024-02-14T041706 +"3.10": 2024-10-23T230432 +"3.11": 2024-10-23T230432 +"3.12": 2024-10-23T230432 +"3.8": 2024-10-23T230432 +"3.9": 2024-10-23T230432 diff --git a/python_modules/dagster-test/Dockerfile b/python_modules/dagster-test/Dockerfile index 4cbf7918c3c74..6b8b2b2a68446 100644 --- a/python_modules/dagster-test/Dockerfile +++ b/python_modules/dagster-test/Dockerfile @@ -13,12 +13,10 @@ ENV DAGSTER_DISABLE_TELEMETRY=true ENV PYTHONUNBUFFERED=1 RUN pip install -U uv -# dagster-celery specified twice to deal with pip resolution in pip 20.3.3 when only -# extras are specified + RUN python -m uv pip install \ -e modules/dagster \ -e modules/dagster-graphql \ - -e modules/dagster-celery \ -e modules/dagster-celery[flower,redis,kubernetes] \ -e modules/dagster-webserver \ -e modules/dagster-postgres \ @@ -32,8 +30,8 @@ RUN python -m uv pip install \ -e modules/dagster-airflow \ -e modules/dagstermill \ -e modules/dagster-pipes \ - -e . \ - pyparsing\<3.0.0 + -e . + RUN ! (pip list --exclude-editable | grep -e dagster) diff --git a/python_modules/libraries/dagster-airflow/setup.py b/python_modules/libraries/dagster-airflow/setup.py index 0b11d8b110b11..9d3fd19d24650 100644 --- a/python_modules/libraries/dagster-airflow/setup.py +++ b/python_modules/libraries/dagster-airflow/setup.py @@ -34,7 +34,6 @@ def get_version() -> str: python_requires=">=3.8,<3.13", install_requires=[ f"dagster{pin}", - "docker>=5.0.3,<6.0.0", "lazy_object_proxy", "setuptools<71.0.0", "pendulum", diff --git a/python_modules/libraries/dagster-docker/setup.py b/python_modules/libraries/dagster-docker/setup.py index 050960d7c7236..203c0c996bb93 100644 --- a/python_modules/libraries/dagster-docker/setup.py +++ b/python_modules/libraries/dagster-docker/setup.py @@ -35,6 +35,10 @@ def get_version() -> str: packages=find_packages(exclude=["dagster_docker_tests*"]), include_package_data=True, python_requires=">=3.8,<3.13", - install_requires=[f"dagster{pin}", "docker", "docker-image-py"], + install_requires=[ + f"dagster{pin}", + "docker", + "docker-image-py", + ], zip_safe=False, )