Merge pull request #2859 from lunkwill42/feature/docker-python-improv…

…ements

Rework docker development environment

.github/workflows/dockercompose.yml

@@ -16,6 +16,8 @@ jobs:
 
       - name: docker-compose build
         run: |
+          echo "UID=$(id -u)" >> .env
+          echo "GID=$(id -g)" >> .env
           docker-compose build
 
       - name: Verify containers stay running at least 1 minute

Dockerfile

@@ -30,10 +30,12 @@ ENV DEBIAN_FRONTEND noninteractive
 
 RUN echo 'deb-src http://deb.debian.org/debian bullseye main' >> /etc/apt/sources.list.d/srcpkg.list && \
     echo 'deb-src http://security.debian.org/debian-security bullseye-security main' >> /etc/apt/sources.list.d/srcpkg.list
-RUN apt-get update && \
+RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \
+    --mount=target=/var/cache/apt,type=cache,sharing=locked \
+    apt-get update && \
     apt-get -y --no-install-recommends install \
             locales \
-            python3-dbg gdb \
+            python3-dbg python3-venv gdb \
             sudo python3-dev python3-pip python3-virtualenv build-essential supervisor \
 	    debian-keyring debian-archive-keyring ca-certificates curl gpg
 
@@ -53,7 +55,9 @@ RUN echo "${TIMEZONE}" > /etc/timezone && cp /usr/share/zoneinfo/${TIMEZONE} /et
 
 #### Install various build and runtime requirements as Debian packages ####
 
-RUN apt-get update \
+RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \
+    --mount=target=/var/cache/apt,type=cache,sharing=locked \
+    apt-get update \
     && apt-get -y --no-install-recommends install \
        git-core \
        libsnmp40 \
@@ -76,10 +80,26 @@ RUN apt-get update \
        iputils-ping \
        snmp
 
-RUN adduser --system --group --no-create-home --home=/source --shell=/bin/bash nav
-
-RUN pip3 install --upgrade 'setuptools>=61' wheel && \
-    pip3 install --upgrade 'pip<=23.1.0' pip-tools build
+# Make an unprivileged nav user that corresponds to the user building this image.
+# Allow this user to run sudo commands and make a virtualenv for them to install NAV in
+ARG UID
+ARG GID
+RUN groupadd --gid "$GID" nav ; adduser --home=/source --shell=/bin/bash --uid=$UID --gid=$GID nav
+RUN echo "nav    ALL =(ALL: ALL) NOPASSWD: ALL" > /etc/sudoers.d/nav
+# Ensure the virtualenv's bin directory is on everyone's PATH variable
+RUN sed -e 's,^Defaults.*secure_path.*,Defaults        secure_path="/opt/venvs/nav/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",' -i /etc/sudoers
+RUN sed -e 's,^ENV_SUPATH.*,ENV_SUPATH      PATH=/opt/venvs/nav/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",' -i /etc/login.defs
+RUN sed -e 's,^ENV_PATH.*,ENV_PATH        PATH=/opt/venvs/nav/bin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games",' -i /etc/login.defs
+
+RUN --mount=type=cache,target=/source/.cache \
+    mkdir -p /opt/venvs/nav && chown nav /opt/venvs/nav && \
+    mkdir -p /etc/nav && chown nav /etc/nav && \
+    chown -R nav /source/.cache
+USER nav
+ENV PATH=/opt/venvs/nav/bin:$PATH
+RUN python3.9 -m venv /opt/venvs/nav
+RUN --mount=type=cache,target=/source/.cache \
+    pip install --upgrade setuptools wheel pip-tools build
 
 #################################################################################
 ### COPYing the requirements file to pip-install Python requirements may bust ###
@@ -89,25 +109,26 @@ RUN pip3 install --upgrade 'setuptools>=61' wheel && \
 
 COPY tools/docker/supervisord.conf /etc/supervisor/conf.d/nav.conf
 
+# Make an initial install of all NAV requirements into the virtualenv, to make
+# builds inside the container go faster
 COPY requirements/ /requirements
 COPY requirements.txt /
 COPY constraints.txt /
 COPY tests/requirements.txt /test-requirements.txt
 COPY doc/requirements.txt /doc-requirements.txt
-# Since we used pip3 to install pip globally, pip should now be for Python 3
-RUN pip-compile --resolver=backtracking --output-file /requirements.txt.lock -c /constraints.txt /requirements.txt /test-requirements.txt /doc-requirements.txt
-RUN pip install -r /requirements.txt.lock
+RUN --mount=type=cache,target=/source/.cache \
+    cd /opt/venvs/nav && \
+    pip-compile --resolver=backtracking --output-file ./requirements.txt.lock -c /constraints.txt /requirements.txt /test-requirements.txt /doc-requirements.txt ; \
+    pip install -r ./requirements.txt.lock
 
 ARG CUSTOM_PIP=ipython
-RUN pip install ${CUSTOM_PIP}
+RUN --mount=type=cache,target=/source/.cache \
+    pip install ${CUSTOM_PIP}
 
 COPY tools/docker/full-nav-restore.sh /usr/local/sbin/full-nav-restore.sh
 
-# Set up for mounting live source code from git repo at /source
-RUN    git config --global --add safe.directory /source
 VOLUME ["/source"]
 ENV    DJANGO_SETTINGS_MODULE nav.django.settings
-EXPOSE 80
+EXPOSE 8080
 
-ENTRYPOINT ["/source/tools/docker/entrypoint.sh"]
 CMD        ["/source/tools/docker/run.sh"]

Makefile

@@ -27,4 +27,9 @@ doc: doc/reference/alerttypes.rst
 doc/reference/alerttypes.rst: .FORCE
 	python3 doc/exts/alerttypes.py > $@
 
+.env: .FORCE
+	echo "# This file was generated by 'make .env'" > .env
+	echo "UID=$(shell id -u)" >> .env
+	echo "GID=$(shell id -g)" >> .env
+
 .FORCE:

doc/hacking/using-docker.rst

@@ -11,38 +11,80 @@ For more information on Docker visit their homepage_ or read the documentation_.
 Installing Docker and docker-compose
 ------------------------------------
 
-Docker has updated documentation on how to install it for most Linux
-distributions [*]_.
+Docker provides up-to-date documentation on how to install it for most popular
+operating systems [*]_.  NAV's Docker definitions should work smoothly on
+Linux, but may have some rough edges on Docker Desktop for Mac.
 
 .. Tip:: To avoid having to use sudo with docker commands, it is recommended
          to add your user to the ``docker`` group. You may need to relogin for it to
          take effect.
 
-Building the Docker image
--------------------------
+Getting started with Docker Compose
+-----------------------------------
 
-First you will need to obtain the NAV source code.
+After installing Docker, you will need to obtain the NAV source code.
 
-The source contains a configuration file for `Docker Compose`_ to build a
-suite of containers for PostgreSQL, Graphite and NAV itself. Simply run this
-command to build and run everything::
+The source code contains a :file:`docker-compose.yml` configuration file for
+`Docker Compose`_.  This configuration defines a fully integrated NAV runtime
+environment, with all its dependencies.  This environment is designed to run
+NAV directly from the checked out source code, and as such it defines an
+environment for developers, not for production use of NAV.  The alternative is
+to manage all the dependencies and integrations on your own host machine.
 
-    docker-compose up
+The quickest way to build the container images and start all the services for
+the first time is by running these commands::
+
+    make .env
+    docker compose up
 
 .. Tip:: The first time you run this would be the perfect time to grab some
          coffee (and maybe redecorate your living room), as the initial build
          may take a while.
 
+Troubleshooting
+~~~~~~~~~~~~~~~
+
+The container images for NAV development are designed to bind-mount your source
+code directory inside the running containers.  In order to avoid leaving files
+owned by strange user IDs in your source code directory, the images will create
+a non-privileged ``nav`` user with a specific user-id and group-id.  These IDs
+should match that of your user account on the host system, so therefore the
+Docker Compose build process needs to know your ``UID`` and ``GID``.
+
+.. note:: This UID/GID mapping is not really relevant if you are running Docker
+          Desktop on a Mac, since it uses an entirely different mechanism for
+          bind-mounted volumes.  You still will need to set the ``UID`` and
+          ``GID`` arguments for the build to work, though.
+
+The quickest way to go about this is the :kbd:`make .env` command.  This will
+attempt to generate a :file:`.env` file in your top-level source code
+directory, which will set the ``UID`` and ``GID`` variables from your running
+environment.  Docker Compose will implicitly read the environment variables in
+this file when it builds or runs the services defined in
+:file:`docker-compose.yml`.  If, for some reason, the :kbd:`make .env` command
+does not work for you, you can create the :file:`.env` file by hand (but supply
+real values if you're on Linux):
+
+.. code-block:: shell
+   :caption: :file:`.env` example
+
+   UID=1337
+   GID=100
+
 
 Using the container(s)
 ----------------------
 
-The Docker Compose specificiation creates these containers (called "services"
-in Docker Compose lingo):
+The Docker Compose specificiation creates several containers (called "services"
+in Docker Compose lingo).  Several of them will mount the checked out source
+code directory internally on the `/source` directory, allowing them to always
+be up-to-date with the latest changes you are making in your favorite editor.
+
+These are the defined services:
 
 nav
   This container runs the NAV backend processes and cron jobs. It also runs the
-  "sass-watcher" job, which will watch ``*.scss`` files for modifications and
+  ``sass-watcher`` job, which will watch ``*.scss`` files for modifications and
   recompile NAV's CSS when changes do occur.
 
 web
@@ -74,31 +116,54 @@ NAV daemons using the ``nav`` command, adjust the running config, or whatever)
 by running a bash shell inside the container, like so (for the ``nav``
 container)::
 
-  docker-compose exec nav /bin/bash
+  docker compose exec nav /bin/bash
 
 Manually restarting the web server
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 To manually restart the web server, all you need is::
 
-  docker-compose restart web
+  docker compose restart web
 
 Rebuilding the NAV code from scratch
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 A complete rebuild of the NAV code can be initiated by::
 
-  docker-compose restart nav
+  docker compose restart nav
 
 Rebuilding the containers
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
-If you are switching between branches, though, you may need to rebuild the
-images the containers are based on (as different development branches may have
-different requirements, and therefore different Dockerfiles). Stop the existing
-containers and run this::
-
-  docker-compose build
+Running :kdb:`docker compose up` will normally build the container images,
+before starting them, if they don't exist already.  However, if the image
+definitions have changed (e.g. when you are switching between development
+branches or changed the :file:`Dockerfile` definitions, or any of the files
+used as part of the image definitions), you may need to rebuild the images.  To
+initiate a full build (which will still utilize Docker's build cache), run
+this::
+
+  docker compose build
+
+Another valid method is to use the ``--build`` option when starting the
+containers.  This will ensure the images are always rebuilt if necessary as
+part of the startup process::
+
+  docker compose up --build
+
+Sometimes, you may find that a rebuild isn't enough to clear out all the cruft
+after switching development branches or adding or changing NAV's default
+configuration file examples.  The Docker Compose environment defines two
+persistent volumes that will retain their data between restarts and rebuilds:
+``nav_cache`` and ``nav_config``.  The former exist just to share some caching
+data between the various service containers.  The second ensures the set of NAV
+config files remain persistent between restarts or rebuilds, and also that all
+service containers can share the same set of files.  When you really want to
+start from scratch, you can fully nuke the Docker Compose environment and the
+persistent volumes using this command (before initiating a new ``up`` or
+``build`` command)::
+
+  docker compose down --volumes
 
 
 Controlling processes inside the nav container

docker-compose.yml

@@ -7,7 +7,11 @@ version: '3'
 
 services:
   nav:
-    build: .
+    build:
+      context: .
+      args:
+        UID: $UID
+        GID: $GID
     environment:
       - NONAVSTART=0  # Set to 1 to disable startup of NAV backend processes when container starts
       - PGHOST=postgres
@@ -58,10 +62,14 @@ services:
 #      - postgres
 
   web:
-    build: .
+    build:
+      context: .
+      args:
+        UID: $UID
+        GID: $GID
     command: ["/source/tools/docker/web.sh"]
     ports:
-      - "80:80"
+      - "80:8080"
     volumes_from:
       - nav
     depends_on:
@@ -86,7 +94,11 @@ services:
   # Add a service to continuously rebuild the Sphinx documentation if the doc
   # directory is modified:
   docbuild:
-    build: .
+    build:
+      context: .
+      args:
+        UID: $UID
+        GID: $GID
     volumes_from:
       - nav
     command: /source/tools/docker/doc-watch.sh

python/nav/startstop.py

@@ -126,7 +126,7 @@ def start(self, silent=False):
         if not self.service_dict.get("privileged", False):
             # run command as regular nav user
             user = NAV_CONFIG.get("NAV_USER", "navcron")
-            command = 'su - {user} -c "{command}"'.format(
+            command = 'su {user} -c "{command}"'.format(
                 command=self._command, user=user
             )
         else:

tools/docker/build.sh

@@ -1,21 +1,19 @@
 #!/bin/bash
 
-set -e
+set -ex
 
 if [[ ! -f "/source/setup.py" ]]; then
   echo NAV source code does not appear to be mounted at /source
   exit 1
 fi
 
 cd /source
-sudo -u nav python3 -m build
-pip install -e .
-sudo -u nav python3 setup.py build_sass
+pip install -vv -e .
+python setup.py build_sass
 
 if [[ ! -f "/etc/nav/nav.conf" ]]; then
     echo "Copying initial NAV config files into this container"
     nav config install --verbose /etc/nav
-    chown -R nav:nav /etc/nav
     cd /etc/nav
     sed -e 's/^#\s*\(DJANGO_DEBUG.*\)$/\1/' -i nav.conf  # Enable django debug.
     sed -e 's/^NAV_USER\s*=.*/NAV_USER=nav/' -i nav.conf  # Set the nav user
@@ -25,4 +23,4 @@ if [[ ! -f "/etc/nav/nav.conf" ]]; then
     cp /source/tools/docker/graphite.conf /etc/nav/graphite.conf
 fi
 
-chown -R nav:nav /tmp/nav_cache
+sudo chown -R nav /tmp/nav_cache

tools/docker/doc-watch.sh

@@ -1,13 +1,12 @@
-#!/bin/bash -e
+#!/bin/bash -ex
 # Rebuilds Sphinx documentation on changes
 #
 cd /source
 # Build once first
-sudo -u nav python3 -m build  # ensure build data and .eggs aren't stored as root
-pip install -e .
-sudo -u nav sphinx-build doc/ build/sphinx/html/
+pip install -vv -e .
+sphinx-build doc/ build/sphinx/html/
 # Then re-build on any changes to the doc directory
 while inotifywait -e modify -e move -e create -e delete -r --exclude \# /source/doc /source/NOTES.rst
 do
-  sudo -u nav sphinx-build doc/ build/sphinx/html/
+  sphinx-build doc/ build/sphinx/html/
 done

tools/docker/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -e
+set -ex
 
 # Ensure latest NAV code is built
 mydir=$(dirname $0)
@@ -10,5 +10,5 @@ mydir=$(dirname $0)
 
 # Start supervisor to control the rest of the runtime
 [[ -f /source/tools/docker/supervisord.conf ]] && \
-  cp /source/tools/docker/supervisord.conf /etc/supervisor/conf.d/nav.conf
-exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
+  sudo cp /source/tools/docker/supervisord.conf /etc/supervisor/conf.d/nav.conf
+exec sudo /usr/bin/supervisord -c /etc/supervisor/supervisord.conf