From d40d77200df70778fc52b684767db6ee1ffdada9 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 6 Oct 2024 18:02:46 -0700 Subject: [PATCH] infra(docs): containerize and incremental builds --- .github/workflows/ci.yml | 2 +- .gitignore | 2 + CHANGELOG.md | 2 + docs/README.md | 78 +++++++++++++++++++--------------- docs/ci.sh | 8 ++++ docs/docker.sh | 16 ------- docs/docker/Dockerfile | 30 +++++++++++++ docs/docker/build-docs.sh | 10 +++++ docs/docker/docker-compose.yml | 15 +++++++ docs/source/conf.py | 9 ++-- 10 files changed, 117 insertions(+), 55 deletions(-) create mode 100755 docs/ci.sh delete mode 100755 docs/docker.sh create mode 100644 docs/docker/Dockerfile create mode 100755 docs/docker/build-docs.sh create mode 100644 docs/docker/docker-compose.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f18e2d2a..4844288f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -325,7 +325,7 @@ jobs: - name: Test building docs run: | - cd docs && ./docker.sh + cd docs && ./ci.sh test-readme: diff --git a/.gitignore b/.gitignore index 11c34dea2..104d69b49 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,8 @@ coverage.xml # Sphinx documentation docs/_build/ +docs/doctrees/ +docs/source/demos/ # PyBuilder target/ diff --git a/CHANGELOG.md b/CHANGELOG.md index cf2495175..5ffda364b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm * Python: Add Python 12 to CI and document support * Docs: Udated dependencies - Sphinx 8, Python 12, and various related * Docs: Added nbsphinx - hub url grounding, ... +* Docs: Redo as a docker compose flow with incremental builds (docker, sphinx) +* Docs: Updated instructions for new flow ### Fix diff --git a/docs/README.md b/docs/README.md index 8681145a4..8c7f4d1d2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,54 +5,62 @@ Uses Sphinx to extract .RST from parent .PY and converts into docs/build/ .HTML ## Run ```bash -docs $ sudo ./docker.sh +docs $ ./ci.sh ``` -This is a shim to `build.sh`, which CI also uses +It is volume-mounted to emit `docs/_build/html/index.html` , as well as epub and pdf versions -It is volume-mounted to emit `docs/build/html/graphistry.html` +## Architecture -CI will reject documentation warnings and errors +### Containerized sphinx and nbconvert -## Develop + `ci.sh` runs `docker compose build` and then `docker compose run --rm sphinx` to build the docs -Sphinx has to install project dev deps, so to work more quickly: + - It also volume mounts demos/ (.ipynb files) into docs/source/demos -1. Start a docker session: + - As part of the run, nbsphinx calls nbconvert and then sphinx to convert the .ipynb files into .rst files -Run from `docs/`: + - nbconvert rewrites relative graph.html urls to assume hub.graphistry.com -``` -docker run \ - --entrypoint=/bin/bash \ - --rm -it \ - -e USER_ID=$UID \ - -e PIP_CACHE_DIR=/cache/pip \ - -v $(pwd)/..:/doc \ - -v ~/.cache/pip:/cache/pip \ - -v ~/.cache/apt:/cache/apt \ - sphinxdoc/sphinx:8.0.2 -``` + - We currently use the preexisting .ipynb output, but in the future should reexecute the notebooks to ensure they are up to date -2. Install deps in a sandbox: +### Caching -```python -cp -r /doc /pygraphistry \ -&& apt update && apt -o dir::cache=/cache/apt install -y pandoc \ -&& ( cd /pygraphistry && python -m pip install -e .[docs] ) \ -&& (test -d /doc/docs/source/demos || ln -s /doc/demos /doc/docs/source/demos) \ -&& (cd /docs/docs/source/demos && rm -f /doc/docs/source/demos/demos || echo ok) -``` +The setup aggressively caches for fast edit cycles: -This prevents your host env from getting littered + - Standard docker layer caching -3. Edit and build! + - Sphinx caching to avoid recompiling unchanged files on reruns -```bash -cd /doc/docs -make clean html SPHINXOPTS="-W --keep-going -n" -``` + - docs/doctrees for tracking + - docs/_build for output + +### Error checking + +Sphinx in strict mode: + + - Ex: .rst files that aren't included in any TOC + + - Ex: broken links in .ipynb and .rst files + + - Ex: verifies .ipynb follow .rst conventions like first element is `# a title` and only one of them + +### Output artifacts + +- The output is docs/_build + + - epub, html, and latexpdf + + +## CI + +CI runs `ci.sh` and checks for warnings and errors. If there are any, it will fail the build. + +## Develop + +- Edit the `demo/` notebooks, `graphistry/` Python code, and `docs/source` .rst files + +- Rerun `cd docs && ./ci.sh` to see the changes, with the benefit of docker & sphinx incremental builds -This emits `$PWD/docs/build/html/index.html` on your host, which you can open in a browser +- Check your results in `docs/_build/html/index.html` or the equivalent epub and pdf files -If there are warnings or errors, it will run to completetion, emit them all, and then exit with a failure. CI will reject these bugs, so you need to make sure it works. diff --git a/docs/ci.sh b/docs/ci.sh new file mode 100755 index 000000000..bbfad9155 --- /dev/null +++ b/docs/ci.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -ex + +( + cd docker \ + && docker compose build \ + && docker compose run --rm sphinx +) diff --git a/docs/docker.sh b/docs/docker.sh deleted file mode 100755 index 58cae6fed..000000000 --- a/docs/docker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -ex - -mkdir -p build - -RUN_INSTALLS="(apt update && apt install -y pandoc && cp -r /data /pygraphistry && cd /pygraphistry && python -m pip install -e .[docs,build] ) && ln -s /data/demos /data/docs/source/demos" -RUN_SPHINX="cd /data/docs && ./build.sh || ( echo 'Printing /tmp/sphinx*' && ( cat /tmp/sphinx* || echo no err_file ) && exit 1 )" - -#TODO make a docker layer so we can cache RUN_INSTALLS -docker run \ - --entrypoint=/bin/bash \ - --rm \ - -e USER_ID=$UID \ - -v $(pwd)/..:/data \ - sphinxdoc/sphinx:8.0.2 \ - -c "${RUN_INSTALLS} && ${RUN_SPHINX}" diff --git a/docs/docker/Dockerfile b/docs/docker/Dockerfile new file mode 100644 index 000000000..1c75b01c7 --- /dev/null +++ b/docs/docker/Dockerfile @@ -0,0 +1,30 @@ +# Use the official Sphinx base image +FROM sphinxdoc/sphinx:8.0.2 + +# Set working directory for docs +WORKDIR /docs + +# Step 1: Install necessary system dependencies including Pandoc, with caching for apt-get +RUN --mount=type=cache,target=/var/cache/apt \ + --mount=type=cache,target=/var/lib/apt \ + apt-get update && apt-get install -y pandoc && rm -rf /var/lib/apt/lists/* + +# Step 2: Copy project files needed for pip install (setup.py, setup.cfg, versioneer.py, README.md) +COPY ../../setup.py ../../setup.cfg ../../versioneer.py ../../README.md ./ + +# Step 3: Install the project dependencies with pip, including docs extras +RUN python3 -m pip install --no-cache-dir -e .[docs] + +# Step 4: Copy the graphistry source files into the container +COPY ../../graphistry /graphistry +ENV PYTHONPATH="/graphistry:${PYTHONPATH}" + +# Step 5: Copy the build script into the container +COPY docs/docker/build-docs.sh /build-docs.sh +RUN chmod +x /build-docs.sh + +# Step 6: Set the working directory for Sphinx to the `source/` folder +WORKDIR /docs/source + +# Step 7: Run the script to build the documentation +CMD ["/build-docs.sh"] diff --git a/docs/docker/build-docs.sh b/docs/docker/build-docs.sh new file mode 100755 index 000000000..b8bdefe77 --- /dev/null +++ b/docs/docker/build-docs.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# Build the HTML documentation incrementally +sphinx-build -b html -d /docs/doctrees . /docs/_build/html + +# Build the EPUB documentation incrementally +sphinx-build -b epub -d /docs/doctrees . /docs/_build/epub + +# Build the PDF documentation incrementally +sphinx-build -b latex -d /docs/doctrees . /docs/_build/latexpdf diff --git a/docs/docker/docker-compose.yml b/docs/docker/docker-compose.yml new file mode 100644 index 000000000..8bef2fa9a --- /dev/null +++ b/docs/docker/docker-compose.yml @@ -0,0 +1,15 @@ +# docs/docker/docker-compose.yml +version: '3' +services: + sphinx: + build: + context: ../.. # Use project root as context + dockerfile: docs/docker/Dockerfile # Specify the Dockerfile + volumes: + - ../../docs/source:/docs/source # Mount the `docs/source` folder to allow live editing + - ../../demos:/docs/source/demos # Mount the `demos` folder for live editing + - ../../docs/_build:/docs/_build # Mount the Sphinx build output to a directory on the host + - ../../docs/doctrees:/docs/doctrees # Mount doctrees to persist build state + environment: + - USER_ID=${UID} # Pass user ID for permission handling + diff --git a/docs/source/conf.py b/docs/source/conf.py index 5a8ded859..07ec11869 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -285,6 +285,12 @@ # This pattern also affects html_static_path and html_extra_path. exclude_patterns = [ + '_build', + '**/_build/**', + 'doctrees', + '**/doctrees/**', + 'demos/.ipynb_checkpoints', + # nbsphinx stalls on these 'demos/ai/Introduction/Ask-HackerNews-Demo.ipynb', 'demos/ai/OSINT/jack-donations.ipynb', @@ -364,9 +370,6 @@ 'demos/ai/OSINT/jack-donations.ipynb', 'demos/ai/OSINT/Chavismo.ipynb', - # Preserve exclusion of Jupyter Notebook checkpoints - 'demos/**/.ipynb_checkpoints/**', - ] pygments_style = "sphinx"