From 831f3f1485c0c42a2178330f7c3d461cd4750cf5 Mon Sep 17 00:00:00 2001
From: Benjamin Bannier <benjamin.bannier@corelight.com>
Date: Wed, 11 Dec 2024 11:07:03 +0100
Subject: [PATCH] Make sure autogen-docs pre-commit hook can always run in CI.

We would previously exit early from this script if we could not generate
the architecture diagram for which we needed the Python `diagrams`
package (which implicitly depends on `dot` as well). This setup made it
impossible to automatically update generated documentation unless one
had both `diagrams` and `dot` installed. Additionally the generated
`architecture.svg` was not stable across different versions of `dot`.

This patch rewrites `./doc/scripts/autogen-docs` so it can update
documentation even if the dependencies for diagram generation are
present. Since the `diagrams` output depends on dot (and its version) I
currently see no stable, platform-indepdendent way to generate the
diagram in a pre-commit hook, so we do not even attempt to make
`diagrams` available to the pre-commit hook.

We also fix a few other issues in the generator script:

- make sure it works when `gsed` is not present
- make sorting order of `reserved-keywords.txt` determistic across GNU
  and BSD toolchains
---
 doc/scripts/autogen-docs | 48 +++++++++++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 15 deletions(-)

diff --git a/doc/scripts/autogen-docs b/doc/scripts/autogen-docs
index d1f0290fd..8dfe4bafe 100755
--- a/doc/scripts/autogen-docs
+++ b/doc/scripts/autogen-docs
@@ -8,6 +8,17 @@
 set -o errexit
 set -o nounset
 
+# GNU `sort` sorts case-insensitive while macOS `sed` sorts case-sensitive which
+# we want. Force GNU sed into case-sensitive mode.
+export LC_COLLATE=C
+
+# We expect a GNU sed. If we find `gsed` use that instead since we are likely
+# on macOS which by default provides a BSD sed.
+SED="sed"
+if command -v gsed >/dev/null 2>&1; then
+    SED=gsed
+fi
+
 ROOTDIR="$(cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/../.."
 BUILDDIR="${ROOTDIR}/build"
 SPICYDOC="${BUILDDIR}/bin/spicy-doc"
@@ -15,17 +26,32 @@ SPICYDTR="${ROOTDIR}/doc/scripts/spicy-doc-to-rst"
 AUTOGEN_FINAL="${ROOTDIR}/doc/autogen"
 AUTOGEN_STAGE=$(mktemp -d -t spicy-autogen-docs.XXXXXXXXXX)
 
+update_architecture_diagram() {
+    # autogen-architecture-diagram needs the diagrams package and dot.
+    if ! python3 -c "import diagrams" >/dev/null 2>&1; then
+        >&2 echo "Warning: Need Python diagrams to run autogen-docs, skipping"
+        return
+    fi
+    if ! command -v dot >/dev/null 2>&1; then
+        >&2 echo "Warning: Need 'dot' to run autogen-docs, skipping"
+        return
+    fi
+
+    "${ROOTDIR}/doc/scripts/autogen-architecture-diagram" "${AUTOGEN_STAGE}/architecture" || exit 1
+
+    # Remove comments and titles from SVG to make content stable.
+    ${SED} -i"" -e '/<!--/d' -e '/-->/d' "${AUTOGEN_STAGE}/architecture.svg"
+    ${SED} -i"" -e '/<title/d' -e '/title>/d' "${AUTOGEN_STAGE}/architecture.svg"
+
+    # Delete unstable DOT and PDF outputs.
+    rm -f "${AUTOGEN_STAGE}/architecture.dot" "${AUTOGEN_STAGE}/architecture.pdf"
+}
+
 if ! command -v rsync >/dev/null 2>&1; then
     >&2 echo "Warning: Need rsync to run autogen-docs, aborting"
     exit 0
 fi
 
-# autogen-architecture-diagram needs the diagrams package
-if ! command -v python -c 'import diagrams' >/dev/null 2>&1; then
-    >&2 echo "Warning: Need Python diagrams to run autogen-docs, aborting"
-    exit 0
-fi
-
 if [[ ! -x ${BUILDDIR}/bin/spicy-doc ]]; then
     >&2 echo "Warning: Could not find required executable ${BUILDDIR}/bin/spicy-doc, aborting"
     exit 0
@@ -49,15 +75,7 @@ cat ${ROOTDIR}/spicy/toolchain/src/compiler/parser/scanner.ll \
     | awk '{print $1}' \
     | sort >${AUTOGEN_STAGE}/reserved-keywords.txt
 
-# Render architecture diagram.
-"${ROOTDIR}/doc/scripts/autogen-architecture-diagram" "${AUTOGEN_STAGE}/architecture" || exit 1
-
-# Remove comments and titles from SVG to make content stable.
-gsed -i"" -e '/<!--/d' -e '/-->/d' "${AUTOGEN_STAGE}/architecture.svg"
-gsed -i"" -e '/<title/d' -e '/title>/d' "${AUTOGEN_STAGE}/architecture.svg"
-
-# Delete unstable DOT and PDF outputs.
-rm -f "${AUTOGEN_STAGE}/architecture.dot" "${AUTOGEN_STAGE}/architecture.pdf"
+update_architecture_diagram
 
 # All done, move staged files to final location where changed.
     # "-rlpgo" is "-a" minus "-tD".