Open telemetry properly supported by python and rust log implementati…

…ons, otel collector for collecting and openobserve for viewing (#17)

* Open telemetry properly supported by python and rust log implementations, otel collector for collecting and openobserve for viewing

* Fix ci by disabling coverage for some blocks, is tested but not in CI

* Fix ci by disabling coverage for some blocks, is tested but not in CI

.gitignore

@@ -6,6 +6,7 @@ private/
 !*private*.*.zetch
 
 process_data/
+logs/
 
 # Tempate files
 .cop.*.yml

bitbazaar.code-workspace

@@ -164,6 +164,8 @@
       "**/package-lock.json": true,
       "**/Cargo.lock": true,
       "**/pdm.lock": true,
+      "**/logs/**": true,
+      "**/process_data/**": true,
     }
   },
   "launch": {

dev_scripts/docker.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e # Exit on error
+
+# Hook into live logs from a detached running container.
+logs () {
+    # Check if a container name is provided
+    if [ $# -eq 0 ]; then
+    echo "Usage: $0 CONTAINER_NAME"
+    exit 1
+    fi
+
+    # Retrieve the container name
+    CONTAINER_NAME=$1
+
+    # Trap Ctrl+C to disconnect from logs
+    trap 'echo "Ctrl+C pressed. Disconnecting from logs..."' INT
+
+    # Hook into the logs of the specified container
+    docker logs -f "$CONTAINER_NAME"
+
+    # Script execution continues after disconnecting from logs
+    echo "Disconnected from logs, but container is still running."
+}
+
+# Has to come at the end of these files:
+source ./dev_scripts/_scr_setup/setup.sh "$@"

dev_scripts/initial_setup.sh

@@ -20,48 +20,110 @@ _install_yaml_fmt () {
     echo "yamlfmt version $1 installed!"
 }
 
+_install_openobserve() {
+    echo "Installing openobserve version $1..."
+
+    # os lowercase:
+    OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+    ARCH=$(uname -m)
+
+    echo "Downloading openobserve version $1 for $OS-$ARCH..."
+    curl -L https://github.com/openobserve/openobserve/releases/download/v$1/openobserve-v$1-$OS-$ARCH.tar.gz -o openobserve.tar.gz -f
+    tar -xzf openobserve.tar.gz
+    rm openobserve.tar.gz
+    chmod +x openobserve
+    sudo mv openobserve /usr/local/bin
+}
+
+_ensure_openobserve() {
+    req_ver="$1"
+
+    if [[ -z "$req_ver" ]]; then
+        echo "openobserve version not provided!"
+        exit 1
+    fi
+
+    if version=$(openobserve --version 2>/dev/null); then
+        # Will be "openobserve v$ver", make sure starts with "openobserve v" and remove that:
+        if [[ ! "$version" =~ ^openobserve\ v ]]; then
+            echo "openobserve version not found in expected format, expected 'openobserve vx.x.x', got '$version'!"
+            exit 1
+        fi
+
+        # Strip prefix:
+        version=${version#openobserve v}
+
+        if [[ "$version" == "$req_ver" ]]; then
+            echo "openobserve already installed with correct version $version!"
+        else
+            echo "openobserve incorrect version, upgrading to $version..."
+            _install_openobserve $req_ver
+        fi
+    else
+        _install_openobserve $req_ver
+    fi
+}
+
 _install_biome () {
     echo "Installing biome version $1..."
 
     # os lowercase:
     OS=$(uname -s | tr '[:upper:]' '[:lower:]')
     ARCH=$(uname -m)
 
-    curl -L https://github.com/biomejs/biome/releases/download/cli%2Fv$1/biome-$OS-$ARCH -o biome
+    echo "Downloading biome version $1 for $OS-$ARCH..."
+    curl -L https://github.com/biomejs/biome/releases/download/cli%2Fv$1/biome-$OS-$ARCH -o biome -f
     chmod +x biome
     sudo mv biome /usr/local/bin
 }
 
-initial_setup () {
-    # Make sure zetch is installed and up to date:
-    if command -v zetch > /dev/null 2>&1; then
-        echo "zetch already installed"
-    else
-        echo "zetch could not be found, installing..."
-        pipx install zetch
+_ensure_biome() {
+    req_ver="$1"
+
+    if [[ -z "$req_ver" ]]; then
+        echo "biome version not provided!"
+        exit 1
     fi
 
-    # Install biome if missing:
-    biome_req_ver="1.5.3"
     if version=$(biome --version 2>/dev/null); then
         # Will be "Version: $ver", make sure starts with "Version: " and remove that:
         if [[ ! "$version" =~ ^Version:\  ]]; then
-            echo "biome version not found in expected format, expected 'Version: x.x.x', got $version!"
+            echo "Biome version not found in expected format, expected 'Version: x.x.x', got '$version'!"
             exit 1
         fi
 
         # Strip prefix:
         version=${version#Version: }
 
-        if [[ "$version" == "$biome_req_ver" ]]; then
+        if [[ "$version" == "$req_ver" ]]; then
             echo "biome already installed with correct version $version!"
         else
-            echo "biome incorrect version, upgrading..."
-            _install_biome $biome_req_ver
+            echo "biome incorrect version, upgrading to $version..."
+            _install_biome $req_ver
         fi
     else
-        _install_biome $biome_req_ver
+        _install_biome $req_ver
     fi
+}
+
+initial_setup () {
+    # Install useful local directories (might be unused):
+    mkdir -p ./process_data
+    mkdir -p ./logs
+
+    # Make sure zetch is installed and up to date:
+    if command -v zetch > /dev/null 2>&1; then
+        echo "zetch already installed"
+    else
+        echo "zetch could not be found, installing..."
+        pipx install zetch
+    fi
+
+    # Make sure openobserve is installed for dev open telemetry logging:
+    _ensure_openobserve "0.8.0"
+
+    # Make sure biome is installed for linting and formatting various files:
+    _ensure_biome "1.5.3"
 
     # Make sure bun installed:
     if command -v bun > /dev/null 2>&1; then

dev_scripts/run.sh

@@ -3,6 +3,38 @@
 # Stop on error:
 set -e
 
+# Starts the open telemetry collector in a docker container if not already started
+ensure_collector () {
+    # Define the name of the Docker container
+    CONTAINER_NAME="collector_bitbazaar"
+
+    if [ "$(./dev_scripts/utils.sh in_ci)" = "true" ]; then
+        echo "In CI, not starting open telemetry collector."
+    else
+        # Check if the container is already running
+        if [ "$(docker inspect -f '{{.State.Running}}' $CONTAINER_NAME 2>/dev/null)" = "true" ]; then
+            echo "Open telemetry collector as container '$CONTAINER_NAME' already running!"
+        else
+            # Start the container
+            echo "Starting open telemetry collector as container '$CONTAINER_NAME'..."
+            # - Link the config file
+            # - Link the ./logs/ directory to /logs/ in the container
+            # - Collector listens for inputs from programs on 4317
+            # - Runs in detached mode
+            docker run --rm --name $CONTAINER_NAME \
+                -v $(pwd)/opencollector.yaml:/etc/otelcol-contrib/config.yaml \
+                -v $(pwd)/logs:/logs \
+                -p 127.0.0.1:4317:4317 \
+                -d \
+                otel/opentelemetry-collector-contrib:0.94.0
+        fi
+    fi
+}
+
+# Starts the openobserve server to look at dev logs/traces/metrics
+oo () {
+    ZO_ROOT_USER_EMAIL="[email protected]" ZO_ROOT_USER_PASSWORD="pass" openobserve
+}
 
 # Has to come at the end of these files:
 source ./dev_scripts/_scr_setup/setup.sh "$@"

dev_scripts/test.sh

@@ -60,6 +60,8 @@ qa () {
 }
 
 py () {
+    ./dev_scripts/run.sh ensure_collector # Needed for open telemetry tests in bitbazaar
+
     cd ./py/
     # Check for COVERAGE=False/false, which is set in some workflow runs to make faster:
     if [[ "$COVERAGE" == "False" ]] || [[ "$COVERAGE" == "false" ]]; then
@@ -109,6 +111,7 @@ py_rust () {
 }
 
 rust () {
+    ./dev_scripts/run.sh ensure_collector # Needed for open telemetry tests in bitbazaar
 
     cargo nextest run --cargo-profile dev --manifest-path ./rust/Cargo.toml --all-features $@
 }

dev_scripts/utils.sh

@@ -24,7 +24,15 @@ replace_text () {
     awk "{sub(\"$1\",\"$2\")} {print}" $3 > temp.txt && mv temp.txt $3
 }
 
-
+# Returns "true" if looks like in_ci, "false" otherwise:
+in_ci () {
+    # Check if any of the CI/CD environment variables are set
+    if [ -n "$GITHUB_ACTIONS" ] || [ -n "$TRAVIS" ] || [ -n "$CIRCLECI" ] || [ -n "$GITLAB_CI" ]; then
+        echo "true"
+    else
+        echo "false"
+    fi
+}
 
 # Has to come at the end of these files:
 source ./dev_scripts/_scr_setup/setup.sh "$@"