Generic project improvements

.github/workflows/release.yml

@@ -160,9 +160,9 @@ jobs:
         uses: dtolnay/rust-toolchain@nightly
       - uses: Swatinem/rust-cache@v2
         with:
-          workspaces: "./rust -> target"
-          cache-on-failure: "true"
-          cache-all-crates: "true"
+          workspaces: ./rust -> target
+          cache-on-failure: 'true'
+          cache-all-crates: 'true'
 
       - name: Build docs
         run: |

.github/workflows/tests.yml

@@ -97,6 +97,11 @@ jobs:
 
       - name: Install rust toolchain (doc builds use nightly features)
         uses: dtolnay/rust-toolchain@nightly
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: "./rust -> target"
+          cache-on-failure: "true"
+          cache-all-crates: "true"
 
       - name: Install dependencies
         run: |
@@ -141,6 +146,7 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         run: |
           COVERAGE=${{ matrix.coverage }} ./dev_scripts/test.sh py
+
       - name: Run tests (windows)
         if: matrix.os == 'windows-latest'
         run: |
@@ -215,6 +221,7 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         run: |
           ./dev_scripts/test.sh py_rust
+
       - name: Run tests (windows)
         if: matrix.os == 'windows-latest'
         run: |
@@ -249,6 +256,7 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         run: |
           ./dev_scripts/test.sh rust --no-fail-fast
+
       - name: Run tests (windows)
         if: matrix.os == 'windows-latest'
         run: |

.pre-commit-config.yaml

@@ -66,6 +66,7 @@ repos:
       - id: bandit
         args: ["-c", "./py/bandit.yml", "./py"]
 
+
   # Cargo/clippy: linting and formatting for rust code:
   - repo: local
     hooks:

bitbazaar.code-workspace

@@ -76,6 +76,7 @@
       "editor.formatOnSave": true,
       "editor.formatOnPaste": false,
       "editor.codeActionsOnSave": {
+        // Ruff will auto fix issues and sort imports (needs extension https://marketplace.visualstudio.com/items?itemName=charliermarsh.ruff)
         "source.fixAll": "explicit"
       },
       // Ruff (replaces black) formats python code (needs extension https://marketplace.visualstudio.com/items?itemName=charliermarsh.ruff)

dev_scripts/_internal/match_substring.py

@@ -0,0 +1,23 @@
+"""Used internally by utils.sh match_substring."""
+
+import re
+import sys
+
+r = sys.argv[1]
+s = sys.argv[2]
+
+
+def err_inf():
+    """Helper for error message."""
+    return "\n\nString:\n{}\n\nRegex: '{}'\n\n".format(s, r)
+
+
+res = re.findall(r, s)
+if len(res) != 1:
+    raise ValueError("{}Expected 1 match, got: {}.".format(err_inf(), len(res)))
+
+match = res[0]
+if match == "":
+    raise ValueError("{}Matched empty string.".format(err_inf()))
+
+print(match.strip())

dev_scripts/docs.sh

@@ -15,6 +15,7 @@ js_sub_build () {
 
 # Builds the nested js site:
 rust_sub_build () {
+    # Nightly features used for docs (e.g. feature(doc_auto_cfg))
     rm -rf ./docs/rust_ref && cargo +nightly doc --no-deps --manifest-path ./rust/Cargo.toml --target-dir ./docs/rust_ref --all-features
 }
 

dev_scripts/initial_setup.sh

@@ -20,15 +20,16 @@ _install_yaml_fmt () {
     echo "yamlfmt version $1 installed!"
 }
 
+
 _install_openobserve() {
     echo "Installing openobserve version $1..."
 
     # os lowercase:
     OS=$(uname -s | tr '[:upper:]' '[:lower:]')
     ARCH=$(uname -m)
 
-    echo "Downloading openobserve version $1 for $OS-$ARCH..."
-    curl -L https://github.com/openobserve/openobserve/releases/download/v$1/openobserve-v$1-$OS-$ARCH.tar.gz -o openobserve.tar.gz -f
+    echo "Downloading openobserve version $1 for ${OS}-${ARCH}..."
+    curl -L https://github.com/openobserve/openobserve/releases/download/v$1/openobserve-v$1-${OS}-${ARCH}.tar.gz -o openobserve.tar.gz -f
     tar -xzf openobserve.tar.gz
     rm openobserve.tar.gz
     chmod +x openobserve
@@ -64,15 +65,67 @@ _ensure_openobserve() {
     fi
 }
 
+_install_otlp_collector () {
+    echo "Installing otlp_collector version $1..."
+
+    # os lowercase:
+    OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+    ARCH=$(uname -m)
+
+    # If ARCH == aarch64, replace with arm64:
+    if [ "${ARCH}" == "aarch64" ]; then
+        ARCH="arm64"
+    fi
+
+    echo "Downloading otlp_collector version $1 for ${OS}-${ARCH}..."
+    curl -L https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v$1/otelcol-contrib_$1_${OS}_${ARCH}.tar.gz \
+        -o otlp_collector.tar.gz -f
+    tar -xzf otlp_collector.tar.gz
+    rm otlp_collector.tar.gz
+    # Comes out as otelcol-contrib:
+    mv otelcol-contrib otlp_collector
+    chmod +x otlp_collector
+    sudo mv otlp_collector /usr/local/bin
+}
+
+_ensure_otlp_collector() {
+    req_ver="$1"
+
+    if [[ -z "$req_ver" ]]; then
+        echo "otlp_collector version not provided!"
+        exit 1
+    fi
+
+    if version=$(otlp_collector --version 2>/dev/null); then
+        # Will be "otelcol-contrib version $ver", make sure starts with "otelcol-contrib version " and remove that:
+        if [[ ! "$version" =~ ^otelcol-contrib\ version\  ]]; then
+            echo "otlp_collector version not found in expected format, expected 'otelcol-contrib version x.x.x', got '$version'!"
+            exit 1
+        fi
+
+        # Strip prefix:
+        version=${version#otelcol-contrib version }
+
+        if [[ "$version" == "$req_ver" ]]; then
+            echo "otlp_collector already installed with correct version $version!"
+        else
+            echo "otlp_collector incorrect version, upgrading to $version..."
+            _install_otlp_collector $req_ver
+        fi
+    else
+        _install_otlp_collector $req_ver
+    fi
+}
+
 _install_biome () {
     echo "Installing biome version $1..."
 
     # os lowercase:
     OS=$(uname -s | tr '[:upper:]' '[:lower:]')
     ARCH=$(uname -m)
 
-    echo "Downloading biome version $1 for $OS-$ARCH..."
-    curl -L https://github.com/biomejs/biome/releases/download/cli%2Fv$1/biome-$OS-$ARCH -o biome -f
+    echo "Downloading biome version $1 for ${OS}-${ARCH}..."
+    curl -L https://github.com/biomejs/biome/releases/download/cli%2Fv$1/biome-${OS}-${ARCH} -o biome -f
     chmod +x biome
     sudo mv biome /usr/local/bin
 }
@@ -121,6 +174,7 @@ initial_setup () {
 
     # Make sure openobserve is installed for dev open telemetry logging:
     _ensure_openobserve "0.8.0"
+    _ensure_otlp_collector "0.94.0"
 
     # Make sure biome is installed for linting and formatting various files:
     _ensure_biome "1.5.3"

dev_scripts/process.sh

@@ -0,0 +1,128 @@
+#!/bin/bash
+
+# Stop on error:
+set -e
+
+PROCESS_PREFIX="dev_script_process"
+
+# Start a process with this system. $1: process_name, $2: stringified command to run
+# These processes will be tracked and are listable and stopable.
+# All processes should stop when terminal is shut
+start() {
+    if [ "$#" -ne 2 ]; then
+        echo "Usage: $0 <process_name> <stringified command to run>"
+        return 1
+    fi
+
+    # If process_name is empty, error
+    if [ -z "$1" ]; then
+        echo "Process name cannot be empty!"
+        return 1
+    fi
+
+    local process_name="$1"
+    local process_command="$2"
+    local process_id_file="/tmp/${PROCESS_PREFIX}_${process_name}.pid"
+
+    # Check if the process is already running
+    if [ -e "$process_id_file" ]; then
+        local existing_pid=$(<"$process_id_file")
+        if ps -p "$existing_pid" > /dev/null; then
+            echo "Process '$process_name' is already running with PID $existing_pid"
+            return 1
+        else
+            # Remove stale PID file
+            rm "$process_id_file"
+        fi
+    fi
+
+    # Start the process and write the processes output to $(pwd)/process_data/processes/$process_name.log
+    local log_file="$(pwd)/logs/proc_$process_name.log"
+    mkdir -p "$(dirname "$log_file")"
+    # Clear the logfile to start from scratch:
+    > "$log_file"
+
+    # Start the process in its own process group, rerouting all output to the logfile:
+    eval "$process_command" > "$log_file" 2>&1 &
+
+    # Capture the PID of the process and write it to a file
+    local new_pid=$!
+
+    # Wait for 0.2 seconds, if the process has exited already with a non-zero exit code, then print the log file
+    sleep 0.2
+    if ! ps -p "$new_pid" > /dev/null; then
+        cat "$log_file"
+        echo "Process '$process_name' failed (wasn't running after 0.2 seconds), output in logfile printed above."
+        return 1
+    fi
+
+    echo "$new_pid" > "$process_id_file"
+    echo "Process '$process_name' started with PID $new_pid, output will write to '$log_file'"
+}
+
+# Stop processes with a given namespace started with this system. $1: process_name
+# E.g. scr process.sh stop my_process would stop my_process and any processes with that as a prefix.
+stop() {
+    if [ "$#" -ne 1 ]; then
+        echo "Usage: $0 <process_name>"
+        return 1
+    fi
+
+    local process_name="$1"
+    local process_id_files=(/tmp/${PROCESS_PREFIX}_${process_name}*.pid)
+
+    echo "Stopping processes matching or prefixed by '$process_name'..."
+    for process_id_file in "${process_id_files[@]}"; do
+        if [ -e "$process_id_file" ]; then
+            while IFS= read -r pid_to_kill; do
+                echo "Stopping process with PID: $pid_to_kill"
+                terminate "$pid_to_kill"
+            done < "$process_id_file"
+            rm "$process_id_file"
+        fi
+    done
+    echo "Stopped ${#process_id_files[@]} process."
+}
+
+# List all processes actively running processes that were started with start()
+list() {
+    local pid_files=$(ls /tmp/${PROCESS_PREFIX}_*.pid 2>/dev/null)
+
+    if [ -z "$pid_files" ]; then
+        echo "No processes running!"
+        return
+    fi
+
+    echo "Processes:"
+    for pid_file in $pid_files; do
+        local process_name=$(basename "$pid_file" | sed "s/${PROCESS_PREFIX}_//g" | sed "s/.pid//g")
+        local pid=$(<"$pid_file")
+        echo "  Process: $process_name, PID: $pid"
+    done
+}
+
+# Terminate a process and all of its child processes
+terminate() {
+    local parent_pid="$1"
+    local IS_CHILD=$2
+
+    # Terminate the child processes of the parent PID
+    local child_pids=$(pgrep -P "$parent_pid")
+    for pid in $child_pids; do
+        terminate "$pid" "true"
+    done
+
+    # Terminate the parent PID
+    if ps -p "$parent_pid" > /dev/null; then
+        if [ "$IS_CHILD" = "true" ]; then
+            echo "Terminating child: $parent_pid"
+        else
+            echo "Terminating root: $parent_pid"
+        fi
+        # Or true to not error if the process is already dead:
+        kill -9 "$parent_pid" > /dev/null 2>&1 || true
+    fi
+}
+
+# Has to come at the end of these files:
+source ./dev_scripts/_scr_setup/setup.sh "$@"