Skip to content

Commit

Permalink
Ensure that the pip cache directory is writable by the astro user (#19)
Browse files Browse the repository at this point in the history
Otherwise it's owned by root and astro can't actually cache anything as we are installing the virtualenvs

Build output show it's being used:

```
#25 [stage-0 14/17] RUN --mount=type=cache,uid=50000,gid=0,target=/home/astro/.cache/pip /home/astro/.venv/snowpark/bin/pip --cache-dir=/home/astro/.cache/pip install -r /home/astro/.venv/
snowpark/requirements.txt
#25 0.902 + /home/astro/.venv/snowpark/bin/pip --cache-dir=/home/astro/.cache/pip install -r /home/astro/.venv/snowpark/requirements.txt
#25 2.110 Looking in indexes: https://pip.astronomer.io/v2/
#25 2.110 Looking in links: https://pip.astronomer.io/simple/astronomer-fab-security-manager/, https://pip.astronomer.io/simple/astronomer-airflow-version-check/
#25 2.645 Collecting googleads
#25 2.649   Using cached googleads-38.0.0.tar.gz (27 kB)
#25 2.664   Preparing metadata (setup.py): started
#25 3.407   Preparing metadata (setup.py): finished with status 'done'
```


And in the built image

```
❯ docker run --rm -ti 90a591231bee0e0312a995fd3ed017600f86a054ec1176bc84bef85e1c245aac bash -c 'ls -al /home/astro/.cache/pip'
total 8
drwxr-xr-x 2 astro astro 4096 Jun 16 21:54 .
drwxr-xr-x 3 astro astro 4096 Jun 16 21:54 ..
```
  • Loading branch information
ashb authored Jun 16, 2023
1 parent ab47ac7 commit f430df2
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 8 deletions.
19 changes: 15 additions & 4 deletions buildkit/internal/transform/transforms.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,18 +26,20 @@ RUN /sbin/ldconfig /usr/local/lib
RUN ln -s /usr/local/include/python{{.PythonMajorMinor}} /usr/local/include/python{{.PythonMajorMinor}}m
USER astro
`
virtualEnvTemplate = `RUN mkdir -p /home/astro/.venv/{{.Name}}
{{if .RequirementsFile}}COPY {{.RequirementsFile}} /home/astro/.venv/{{.Name}}/requirements.txt{{end}}
virtualEnvTemplate = `RUN mkdir -p /home/astro/.cache/pip /home/astro/.venv/{{.Name}}
{{if .RequirementsFile}}COPY --chown={{.AstroUid}}:0 {{.RequirementsFile}} /home/astro/.venv/{{.Name}}/requirements.txt{{end}}
RUN /usr/local/bin/python{{.PythonMajorMinor}} -m venv /home/astro/.venv/{{.Name}}
ENV ASTRO_PYENV_{{.Name}} /home/astro/.venv/{{.Name}}/bin/python
{{if .RequirementsFile}}RUN --mount=type=cache,target=/home/astro/.cache/pip /home/astro/.venv/{{.Name}}/bin/pip --cache-dir=/home/astro/.cache/pip install -r /home/astro/.venv/{{.Name}}/requirements.txt{{end}}
{{if .RequirementsFile}}RUN --mount=type=cache,uid={{.AstroUid}},gid=0,target=/home/astro/.cache/pip /home/astro/.venv/{{.Name}}/bin/pip --cache-dir=/home/astro/.cache/pip install -r /home/astro/.venv/{{.Name}}/requirements.txt{{end}}
`
fromCommand = "FROM"
argCommand = "ARG"
pyenvCommand = "PYENV"
astroRuntimeImage = "quay.io/astronomer/astro-runtime"

defaultImageFlavour = "slim-bullseye"
// Sadly for the `RUN --mount,uid=$uid` we need to use a numeric ID.
defaultAstroUid = 50000
)

var (
Expand Down Expand Up @@ -229,9 +231,18 @@ func (r *Transformer) addVirtualEnvironment(venv *virtualEnv) (*parser.Node, err
return nil, err
}
buf := &bytes.Buffer{}
if err := tpl.Execute(buf, venv); err != nil {
params := struct {
*virtualEnv
AstroUid int
}{
venv,
defaultAstroUid,
}

if err := tpl.Execute(buf, params); err != nil {
return nil, err
}

parsedNodes, err := parser.Parse(buf)
if err != nil {
return nil, err
Expand Down
8 changes: 4 additions & 4 deletions buildkit/internal/transform/transforms_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ RUN /sbin/ldconfig /usr/local/lib
RUN ln -s /usr/local/include/python3.8 /usr/local/include/python3.8m
USER astro
RUN mkdir -p /home/astro/.venv/venv1
COPY reqs/venv1.txt /home/astro/.venv/venv1/requirements.txt
RUN mkdir -p /home/astro/.cache/pip /home/astro/.venv/venv1
COPY --chown=50000:0 reqs/venv1.txt /home/astro/.venv/venv1/requirements.txt
RUN /usr/local/bin/python3.8 -m venv /home/astro/.venv/venv1
ENV ASTRO_PYENV_venv1 /home/astro/.venv/venv1/bin/python
RUN --mount=type=cache,target=/home/astro/.cache/pip /home/astro/.venv/venv1/bin/pip --cache-dir=/home/astro/.cache/pip install -r /home/astro/.venv/venv1/requirements.txt
RUN --mount=type=cache,uid=50000,gid=0,target=/home/astro/.cache/pip /home/astro/.venv/venv1/bin/pip --cache-dir=/home/astro/.cache/pip install -r /home/astro/.venv/venv1/requirements.txt
COPY foo bar
USER root
COPY --link --from=python:3.10-slim-bullseye /usr/local/bin/*3.10* /usr/local/bin/
Expand All @@ -47,7 +47,7 @@ RUN /sbin/ldconfig /usr/local/lib
RUN ln -s /usr/local/include/python3.10 /usr/local/include/python3.10m
USER astro
RUN mkdir -p /home/astro/.venv/venv2
RUN mkdir -p /home/astro/.cache/pip /home/astro/.venv/venv2
RUN /usr/local/bin/python3.10 -m venv /home/astro/.venv/venv2
ENV ASTRO_PYENV_venv2 /home/astro/.venv/venv2/bin/python
Expand Down

0 comments on commit f430df2

Please sign in to comment.