From f430df276cf7927044c7bfc3a21d9407f64a79f6 Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Fri, 16 Jun 2023 22:59:31 +0100 Subject: [PATCH] Ensure that the pip cache directory is writable by the astro user (#19) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise it's owned by root and astro can't actually cache anything as we are installing the virtualenvs Build output show it's being used: ``` #25 [stage-0 14/17] RUN --mount=type=cache,uid=50000,gid=0,target=/home/astro/.cache/pip /home/astro/.venv/snowpark/bin/pip --cache-dir=/home/astro/.cache/pip install -r /home/astro/.venv/ snowpark/requirements.txt #25 0.902 + /home/astro/.venv/snowpark/bin/pip --cache-dir=/home/astro/.cache/pip install -r /home/astro/.venv/snowpark/requirements.txt #25 2.110 Looking in indexes: https://pip.astronomer.io/v2/ #25 2.110 Looking in links: https://pip.astronomer.io/simple/astronomer-fab-security-manager/, https://pip.astronomer.io/simple/astronomer-airflow-version-check/ #25 2.645 Collecting googleads #25 2.649 Using cached googleads-38.0.0.tar.gz (27 kB) #25 2.664 Preparing metadata (setup.py): started #25 3.407 Preparing metadata (setup.py): finished with status 'done' ``` And in the built image ``` ❯ docker run --rm -ti 90a591231bee0e0312a995fd3ed017600f86a054ec1176bc84bef85e1c245aac bash -c 'ls -al /home/astro/.cache/pip' total 8 drwxr-xr-x 2 astro astro 4096 Jun 16 21:54 . drwxr-xr-x 3 astro astro 4096 Jun 16 21:54 .. ``` --- buildkit/internal/transform/transforms.go | 19 +++++++++++++++---- .../internal/transform/transforms_test.go | 8 ++++---- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/buildkit/internal/transform/transforms.go b/buildkit/internal/transform/transforms.go index 68ce38c..1914965 100644 --- a/buildkit/internal/transform/transforms.go +++ b/buildkit/internal/transform/transforms.go @@ -26,11 +26,11 @@ RUN /sbin/ldconfig /usr/local/lib RUN ln -s /usr/local/include/python{{.PythonMajorMinor}} /usr/local/include/python{{.PythonMajorMinor}}m USER astro ` - virtualEnvTemplate = `RUN mkdir -p /home/astro/.venv/{{.Name}} -{{if .RequirementsFile}}COPY {{.RequirementsFile}} /home/astro/.venv/{{.Name}}/requirements.txt{{end}} + virtualEnvTemplate = `RUN mkdir -p /home/astro/.cache/pip /home/astro/.venv/{{.Name}} +{{if .RequirementsFile}}COPY --chown={{.AstroUid}}:0 {{.RequirementsFile}} /home/astro/.venv/{{.Name}}/requirements.txt{{end}} RUN /usr/local/bin/python{{.PythonMajorMinor}} -m venv /home/astro/.venv/{{.Name}} ENV ASTRO_PYENV_{{.Name}} /home/astro/.venv/{{.Name}}/bin/python -{{if .RequirementsFile}}RUN --mount=type=cache,target=/home/astro/.cache/pip /home/astro/.venv/{{.Name}}/bin/pip --cache-dir=/home/astro/.cache/pip install -r /home/astro/.venv/{{.Name}}/requirements.txt{{end}} +{{if .RequirementsFile}}RUN --mount=type=cache,uid={{.AstroUid}},gid=0,target=/home/astro/.cache/pip /home/astro/.venv/{{.Name}}/bin/pip --cache-dir=/home/astro/.cache/pip install -r /home/astro/.venv/{{.Name}}/requirements.txt{{end}} ` fromCommand = "FROM" argCommand = "ARG" @@ -38,6 +38,8 @@ ENV ASTRO_PYENV_{{.Name}} /home/astro/.venv/{{.Name}}/bin/python astroRuntimeImage = "quay.io/astronomer/astro-runtime" defaultImageFlavour = "slim-bullseye" + // Sadly for the `RUN --mount,uid=$uid` we need to use a numeric ID. + defaultAstroUid = 50000 ) var ( @@ -229,9 +231,18 @@ func (r *Transformer) addVirtualEnvironment(venv *virtualEnv) (*parser.Node, err return nil, err } buf := &bytes.Buffer{} - if err := tpl.Execute(buf, venv); err != nil { + params := struct { + *virtualEnv + AstroUid int + }{ + venv, + defaultAstroUid, + } + + if err := tpl.Execute(buf, params); err != nil { return nil, err } + parsedNodes, err := parser.Parse(buf) if err != nil { return nil, err diff --git a/buildkit/internal/transform/transforms_test.go b/buildkit/internal/transform/transforms_test.go index 699942f..25cbefa 100644 --- a/buildkit/internal/transform/transforms_test.go +++ b/buildkit/internal/transform/transforms_test.go @@ -31,11 +31,11 @@ RUN /sbin/ldconfig /usr/local/lib RUN ln -s /usr/local/include/python3.8 /usr/local/include/python3.8m USER astro -RUN mkdir -p /home/astro/.venv/venv1 -COPY reqs/venv1.txt /home/astro/.venv/venv1/requirements.txt +RUN mkdir -p /home/astro/.cache/pip /home/astro/.venv/venv1 +COPY --chown=50000:0 reqs/venv1.txt /home/astro/.venv/venv1/requirements.txt RUN /usr/local/bin/python3.8 -m venv /home/astro/.venv/venv1 ENV ASTRO_PYENV_venv1 /home/astro/.venv/venv1/bin/python -RUN --mount=type=cache,target=/home/astro/.cache/pip /home/astro/.venv/venv1/bin/pip --cache-dir=/home/astro/.cache/pip install -r /home/astro/.venv/venv1/requirements.txt +RUN --mount=type=cache,uid=50000,gid=0,target=/home/astro/.cache/pip /home/astro/.venv/venv1/bin/pip --cache-dir=/home/astro/.cache/pip install -r /home/astro/.venv/venv1/requirements.txt COPY foo bar USER root COPY --link --from=python:3.10-slim-bullseye /usr/local/bin/*3.10* /usr/local/bin/ @@ -47,7 +47,7 @@ RUN /sbin/ldconfig /usr/local/lib RUN ln -s /usr/local/include/python3.10 /usr/local/include/python3.10m USER astro -RUN mkdir -p /home/astro/.venv/venv2 +RUN mkdir -p /home/astro/.cache/pip /home/astro/.venv/venv2 RUN /usr/local/bin/python3.10 -m venv /home/astro/.venv/venv2 ENV ASTRO_PYENV_venv2 /home/astro/.venv/venv2/bin/python