diff --git a/.github/workflows/dockercompose.yml b/.github/workflows/dockercompose.yml index 3ba0883f18..893c299c3d 100644 --- a/.github/workflows/dockercompose.yml +++ b/.github/workflows/dockercompose.yml @@ -16,6 +16,8 @@ jobs: - name: docker-compose build run: | + echo "UID=$(id -u)" >> .env + echo "GID=$(id -g)" >> .env docker-compose build - name: Verify containers stay running at least 1 minute diff --git a/Dockerfile b/Dockerfile index cf8d0fb79f..56c0626ebc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,10 +30,12 @@ ENV DEBIAN_FRONTEND noninteractive RUN echo 'deb-src http://deb.debian.org/debian bullseye main' >> /etc/apt/sources.list.d/srcpkg.list && \ echo 'deb-src http://security.debian.org/debian-security bullseye-security main' >> /etc/apt/sources.list.d/srcpkg.list -RUN apt-get update && \ +RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ + --mount=target=/var/cache/apt,type=cache,sharing=locked \ + apt-get update && \ apt-get -y --no-install-recommends install \ locales \ - python3-dbg gdb \ + python3-dbg python3-venv gdb \ sudo python3-dev python3-pip python3-virtualenv build-essential supervisor \ debian-keyring debian-archive-keyring ca-certificates curl gpg @@ -53,7 +55,9 @@ RUN echo "${TIMEZONE}" > /etc/timezone && cp /usr/share/zoneinfo/${TIMEZONE} /et #### Install various build and runtime requirements as Debian packages #### -RUN apt-get update \ +RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ + --mount=target=/var/cache/apt,type=cache,sharing=locked \ + apt-get update \ && apt-get -y --no-install-recommends install \ git-core \ libsnmp40 \ @@ -76,10 +80,26 @@ RUN apt-get update \ iputils-ping \ snmp -RUN adduser --system --group --no-create-home --home=/source --shell=/bin/bash nav - -RUN pip3 install --upgrade 'setuptools>=61' wheel && \ - pip3 install --upgrade 'pip<=23.1.0' pip-tools build +# Make an unprivileged nav user that corresponds to the user building this image. +# Allow this user to run sudo commands and make a virtualenv for them to install NAV in +ARG UID +ARG GID +RUN groupadd --gid "$GID" nav ; adduser --home=/source --shell=/bin/bash --uid=$UID --gid=$GID nav +RUN echo "nav ALL =(ALL: ALL) NOPASSWD: ALL" > /etc/sudoers.d/nav +# Ensure the virtualenv's bin directory is on everyone's PATH variable +RUN sed -e 's,^Defaults.*secure_path.*,Defaults secure_path="/opt/venvs/nav/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",' -i /etc/sudoers +RUN sed -e 's,^ENV_SUPATH.*,ENV_SUPATH PATH=/opt/venvs/nav/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",' -i /etc/login.defs +RUN sed -e 's,^ENV_PATH.*,ENV_PATH PATH=/opt/venvs/nav/bin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games",' -i /etc/login.defs + +RUN --mount=type=cache,target=/source/.cache \ + mkdir -p /opt/venvs/nav && chown nav /opt/venvs/nav && \ + mkdir -p /etc/nav && chown nav /etc/nav && \ + chown -R nav /source/.cache +USER nav +ENV PATH=/opt/venvs/nav/bin:$PATH +RUN python3.9 -m venv /opt/venvs/nav +RUN --mount=type=cache,target=/source/.cache \ + pip install --upgrade setuptools wheel pip-tools build ################################################################################# ### COPYing the requirements file to pip-install Python requirements may bust ### @@ -89,25 +109,26 @@ RUN pip3 install --upgrade 'setuptools>=61' wheel && \ COPY tools/docker/supervisord.conf /etc/supervisor/conf.d/nav.conf +# Make an initial install of all NAV requirements into the virtualenv, to make +# builds inside the container go faster COPY requirements/ /requirements COPY requirements.txt / COPY constraints.txt / COPY tests/requirements.txt /test-requirements.txt COPY doc/requirements.txt /doc-requirements.txt -# Since we used pip3 to install pip globally, pip should now be for Python 3 -RUN pip-compile --resolver=backtracking --output-file /requirements.txt.lock -c /constraints.txt /requirements.txt /test-requirements.txt /doc-requirements.txt -RUN pip install -r /requirements.txt.lock +RUN --mount=type=cache,target=/source/.cache \ + cd /opt/venvs/nav && \ + pip-compile --resolver=backtracking --output-file ./requirements.txt.lock -c /constraints.txt /requirements.txt /test-requirements.txt /doc-requirements.txt ; \ + pip install -r ./requirements.txt.lock ARG CUSTOM_PIP=ipython -RUN pip install ${CUSTOM_PIP} +RUN --mount=type=cache,target=/source/.cache \ + pip install ${CUSTOM_PIP} COPY tools/docker/full-nav-restore.sh /usr/local/sbin/full-nav-restore.sh -# Set up for mounting live source code from git repo at /source -RUN git config --global --add safe.directory /source VOLUME ["/source"] ENV DJANGO_SETTINGS_MODULE nav.django.settings -EXPOSE 80 +EXPOSE 8080 -ENTRYPOINT ["/source/tools/docker/entrypoint.sh"] CMD ["/source/tools/docker/run.sh"] diff --git a/Makefile b/Makefile index a805f48b6e..04c6367cf7 100644 --- a/Makefile +++ b/Makefile @@ -27,4 +27,9 @@ doc: doc/reference/alerttypes.rst doc/reference/alerttypes.rst: .FORCE python3 doc/exts/alerttypes.py > $@ +.env: .FORCE + echo "# This file was generated by 'make .env'" > .env + echo "UID=$(shell id -u)" >> .env + echo "GID=$(shell id -g)" >> .env + .FORCE: diff --git a/doc/hacking/using-docker.rst b/doc/hacking/using-docker.rst index 3c5b8f749f..f80bc12ca5 100644 --- a/doc/hacking/using-docker.rst +++ b/doc/hacking/using-docker.rst @@ -11,38 +11,80 @@ For more information on Docker visit their homepage_ or read the documentation_. Installing Docker and docker-compose ------------------------------------ -Docker has updated documentation on how to install it for most Linux -distributions [*]_. +Docker provides up-to-date documentation on how to install it for most popular +operating systems [*]_. NAV's Docker definitions should work smoothly on +Linux, but may have some rough edges on Docker Desktop for Mac. .. Tip:: To avoid having to use sudo with docker commands, it is recommended to add your user to the ``docker`` group. You may need to relogin for it to take effect. -Building the Docker image -------------------------- +Getting started with Docker Compose +----------------------------------- -First you will need to obtain the NAV source code. +After installing Docker, you will need to obtain the NAV source code. -The source contains a configuration file for `Docker Compose`_ to build a -suite of containers for PostgreSQL, Graphite and NAV itself. Simply run this -command to build and run everything:: +The source code contains a :file:`docker-compose.yml` configuration file for +`Docker Compose`_. This configuration defines a fully integrated NAV runtime +environment, with all its dependencies. This environment is designed to run +NAV directly from the checked out source code, and as such it defines an +environment for developers, not for production use of NAV. The alternative is +to manage all the dependencies and integrations on your own host machine. - docker-compose up +The quickest way to build the container images and start all the services for +the first time is by running these commands:: + + make .env + docker compose up .. Tip:: The first time you run this would be the perfect time to grab some coffee (and maybe redecorate your living room), as the initial build may take a while. +Troubleshooting +~~~~~~~~~~~~~~~ + +The container images for NAV development are designed to bind-mount your source +code directory inside the running containers. In order to avoid leaving files +owned by strange user IDs in your source code directory, the images will create +a non-privileged ``nav`` user with a specific user-id and group-id. These IDs +should match that of your user account on the host system, so therefore the +Docker Compose build process needs to know your ``UID`` and ``GID``. + +.. note:: This UID/GID mapping is not really relevant if you are running Docker + Desktop on a Mac, since it uses an entirely different mechanism for + bind-mounted volumes. You still will need to set the ``UID`` and + ``GID`` arguments for the build to work, though. + +The quickest way to go about this is the :kbd:`make .env` command. This will +attempt to generate a :file:`.env` file in your top-level source code +directory, which will set the ``UID`` and ``GID`` variables from your running +environment. Docker Compose will implicitly read the environment variables in +this file when it builds or runs the services defined in +:file:`docker-compose.yml`. If, for some reason, the :kbd:`make .env` command +does not work for you, you can create the :file:`.env` file by hand (but supply +real values if you're on Linux): + +.. code-block:: shell + :caption: :file:`.env` example + + UID=1337 + GID=100 + Using the container(s) ---------------------- -The Docker Compose specificiation creates these containers (called "services" -in Docker Compose lingo): +The Docker Compose specificiation creates several containers (called "services" +in Docker Compose lingo). Several of them will mount the checked out source +code directory internally on the `/source` directory, allowing them to always +be up-to-date with the latest changes you are making in your favorite editor. + +These are the defined services: nav This container runs the NAV backend processes and cron jobs. It also runs the - "sass-watcher" job, which will watch ``*.scss`` files for modifications and + ``sass-watcher`` job, which will watch ``*.scss`` files for modifications and recompile NAV's CSS when changes do occur. web @@ -74,31 +116,54 @@ NAV daemons using the ``nav`` command, adjust the running config, or whatever) by running a bash shell inside the container, like so (for the ``nav`` container):: - docker-compose exec nav /bin/bash + docker compose exec nav /bin/bash Manually restarting the web server ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To manually restart the web server, all you need is:: - docker-compose restart web + docker compose restart web Rebuilding the NAV code from scratch ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A complete rebuild of the NAV code can be initiated by:: - docker-compose restart nav + docker compose restart nav Rebuilding the containers ~~~~~~~~~~~~~~~~~~~~~~~~~ -If you are switching between branches, though, you may need to rebuild the -images the containers are based on (as different development branches may have -different requirements, and therefore different Dockerfiles). Stop the existing -containers and run this:: - - docker-compose build +Running :kdb:`docker compose up` will normally build the container images, +before starting them, if they don't exist already. However, if the image +definitions have changed (e.g. when you are switching between development +branches or changed the :file:`Dockerfile` definitions, or any of the files +used as part of the image definitions), you may need to rebuild the images. To +initiate a full build (which will still utilize Docker's build cache), run +this:: + + docker compose build + +Another valid method is to use the ``--build`` option when starting the +containers. This will ensure the images are always rebuilt if necessary as +part of the startup process:: + + docker compose up --build + +Sometimes, you may find that a rebuild isn't enough to clear out all the cruft +after switching development branches or adding or changing NAV's default +configuration file examples. The Docker Compose environment defines two +persistent volumes that will retain their data between restarts and rebuilds: +``nav_cache`` and ``nav_config``. The former exist just to share some caching +data between the various service containers. The second ensures the set of NAV +config files remain persistent between restarts or rebuilds, and also that all +service containers can share the same set of files. When you really want to +start from scratch, you can fully nuke the Docker Compose environment and the +persistent volumes using this command (before initiating a new ``up`` or +``build`` command):: + + docker compose down --volumes Controlling processes inside the nav container diff --git a/docker-compose.yml b/docker-compose.yml index ecc619066a..8643011c8b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,11 @@ version: '3' services: nav: - build: . + build: + context: . + args: + UID: $UID + GID: $GID environment: - NONAVSTART=0 # Set to 1 to disable startup of NAV backend processes when container starts - PGHOST=postgres @@ -58,10 +62,14 @@ services: # - postgres web: - build: . + build: + context: . + args: + UID: $UID + GID: $GID command: ["/source/tools/docker/web.sh"] ports: - - "80:80" + - "80:8080" volumes_from: - nav depends_on: @@ -86,7 +94,11 @@ services: # Add a service to continuously rebuild the Sphinx documentation if the doc # directory is modified: docbuild: - build: . + build: + context: . + args: + UID: $UID + GID: $GID volumes_from: - nav command: /source/tools/docker/doc-watch.sh diff --git a/python/nav/startstop.py b/python/nav/startstop.py index 693f4845ad..16cfea6788 100644 --- a/python/nav/startstop.py +++ b/python/nav/startstop.py @@ -126,7 +126,7 @@ def start(self, silent=False): if not self.service_dict.get("privileged", False): # run command as regular nav user user = NAV_CONFIG.get("NAV_USER", "navcron") - command = 'su - {user} -c "{command}"'.format( + command = 'su {user} -c "{command}"'.format( command=self._command, user=user ) else: diff --git a/tools/docker/build.sh b/tools/docker/build.sh index 41fa1e5f2f..08c5d19aa7 100755 --- a/tools/docker/build.sh +++ b/tools/docker/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -ex if [[ ! -f "/source/setup.py" ]]; then echo NAV source code does not appear to be mounted at /source @@ -8,14 +8,12 @@ if [[ ! -f "/source/setup.py" ]]; then fi cd /source -sudo -u nav python3 -m build -pip install -e . -sudo -u nav python3 setup.py build_sass +pip install -vv -e . +python setup.py build_sass if [[ ! -f "/etc/nav/nav.conf" ]]; then echo "Copying initial NAV config files into this container" nav config install --verbose /etc/nav - chown -R nav:nav /etc/nav cd /etc/nav sed -e 's/^#\s*\(DJANGO_DEBUG.*\)$/\1/' -i nav.conf # Enable django debug. sed -e 's/^NAV_USER\s*=.*/NAV_USER=nav/' -i nav.conf # Set the nav user @@ -25,4 +23,4 @@ if [[ ! -f "/etc/nav/nav.conf" ]]; then cp /source/tools/docker/graphite.conf /etc/nav/graphite.conf fi -chown -R nav:nav /tmp/nav_cache +sudo chown -R nav /tmp/nav_cache diff --git a/tools/docker/doc-watch.sh b/tools/docker/doc-watch.sh index 371a85cab9..a47dd257ce 100755 --- a/tools/docker/doc-watch.sh +++ b/tools/docker/doc-watch.sh @@ -1,13 +1,12 @@ -#!/bin/bash -e +#!/bin/bash -ex # Rebuilds Sphinx documentation on changes # cd /source # Build once first -sudo -u nav python3 -m build # ensure build data and .eggs aren't stored as root -pip install -e . -sudo -u nav sphinx-build doc/ build/sphinx/html/ +pip install -vv -e . +sphinx-build doc/ build/sphinx/html/ # Then re-build on any changes to the doc directory while inotifywait -e modify -e move -e create -e delete -r --exclude \# /source/doc /source/NOTES.rst do - sudo -u nav sphinx-build doc/ build/sphinx/html/ + sphinx-build doc/ build/sphinx/html/ done diff --git a/tools/docker/entrypoint.sh b/tools/docker/entrypoint.sh deleted file mode 100755 index c8490dfa97..0000000000 --- a/tools/docker/entrypoint.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -if [ "$EUID" == "0" ]; then - # Sneakily modify the nav user to match the UID/GID of the real user who owns - # the mounted /source volume. - uid=$(stat -c '%u' /source) - gid=$(stat -c '%g' /source) - # stuff appears like it's owned by root on OSX host, but is still accessible - if [ "$uid" -ne 0 ]; then - usermod --uid "$uid" nav - groupmod --gid "$gid" nav - fi -fi - -exec "$@" diff --git a/tools/docker/run.sh b/tools/docker/run.sh index e58b919ab8..3546839cde 100755 --- a/tools/docker/run.sh +++ b/tools/docker/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -ex # Ensure latest NAV code is built mydir=$(dirname $0) @@ -10,5 +10,5 @@ mydir=$(dirname $0) # Start supervisor to control the rest of the runtime [[ -f /source/tools/docker/supervisord.conf ]] && \ - cp /source/tools/docker/supervisord.conf /etc/supervisor/conf.d/nav.conf -exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf + sudo cp /source/tools/docker/supervisord.conf /etc/supervisor/conf.d/nav.conf +exec sudo /usr/bin/supervisord -c /etc/supervisor/supervisord.conf diff --git a/tools/docker/sass-watch.sh b/tools/docker/sass-watch.sh index 6eb5141d26..4837eccf5e 100755 --- a/tools/docker/sass-watch.sh +++ b/tools/docker/sass-watch.sh @@ -4,5 +4,5 @@ cd /source while inotifywait -e modify -e move -e create -e delete -r --exclude \# /source/python/nav/web/sass do - python3 setup.py build_sass + /opt/venvs/nav/bin/python setup.py build_sass done diff --git a/tools/docker/syncdb.sh b/tools/docker/syncdb.sh index 781ca67353..596be27f88 100755 --- a/tools/docker/syncdb.sh +++ b/tools/docker/syncdb.sh @@ -1,5 +1,5 @@ -#!/bin/bash -e +#!/bin/bash -ex cd /source export PGHOST=postgres PGUSER=postgres psql -l -t | grep -q '^ *nav' || navsyncdb -c -sudo -u nav navsyncdb -o +navsyncdb -o diff --git a/tools/docker/web.sh b/tools/docker/web.sh index 054f35bee1..1e23935e69 100755 --- a/tools/docker/web.sh +++ b/tools/docker/web.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -ex # Ensure latest NAV code is built mydir=$(dirname $0) @@ -8,4 +8,4 @@ mydir=$(dirname $0) cd /source -django-admin check && exec django-admin runserver --insecure 0.0.0.0:80 +django-admin check && exec django-admin runserver --insecure 0.0.0.0:8080