fixed runnables

added telemetry

.env.production

@@ -4,4 +4,6 @@ CORS=*
 CLOCK_OFFSET=0.0
 PLANNING_WINDOW=31
 PLANNING_INTERVAL=1
-REDIS_URL=redis://redis:6379/0
+REDIS_URL=redis://redis:6379/0
+OLTP_INSECURE=True
+OLTP_COLLECTOR_URL=http://collector:4317

README.md

@@ -36,7 +36,7 @@ The docker way, either use the `devcontainer` or run with `docker-compose`:
 ```bash
 1. Clone the repository
 2. Run the project
-  a. `docker-compose up`
+  a. `docker-compose -f docker-compose.yml build`
   b. `docker-compose -f docker-compose.yml up`
 ```
 

docker-compose.yml

@@ -1,6 +1,11 @@
-version: '3'
-
 services:
+  collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    volumes:
+      - ./otel-config.yaml:/etc/otelcol-contrib/config.yaml
+    command: [ "--config=/etc/otelcol-contrib/config.yaml" ]
+    ports:
+      - "4317:4317"
   redis:
     image: redis:7.4.1
     container_name: redis

docs/Telemetry.md

@@ -0,0 +1,64 @@
+# Telemetry
+
+`HORAO` is telemetrised using [OpenTelemetry](https://opentelemetry.io). 
+Telemetry is switched on by default, but can be configured turned off by explicitly setting the environment variable `TELEMETRY` to `OFF`.
+
+```dotenv 
+TELEMETRY=OFF
+```
+
+The following sections describe telemetry configuration for `HORAO`.
+
+## Logging level
+
+The logging level can be changed using the `DEBUG` environment variable. The default is `INFO`. 
+The logging level can be set in the `.env` file as follows:
+```dotenv 
+DEBUG=True
+```
+Be aware that setting the logging level to `DEBUG` will generate a lot of output.
+
+## Sending telemetry to a collector
+
+The telemetry is sent using [OLTP exporter](https://github.com/open-telemetry/opentelemetry-python/tree/main/exporter/opentelemetry-exporter-otlp).
+The default protocol is `grpc` this can be switched to `http` by setting the environment variable `OLTP_HTTP` to True.
+```dotenv
+OLTP_HTTP=True
+```
+The default mechanism is to send telemetry `securely`, this can be switched to `insecure` by setting the environment variable `OLTP_INSECURE` to True.
+```dotenv
+OLTP_INSECURE=True
+```
+
+A specific collector URL needs to be specified, otherwise the Telemetry will not be sent.
+```dotenv
+OLTP_COLLECTOR_URL=http://localhost:4317
+```
+
+## Various environment variables that can be set to configure telemetry
+
+To exclude certain URLs from tracking
+```dotenv
+OTEL_PYTHON_STARLETTE_EXCLUDED_URLS="client/.*/info,health"
+```
+will exclude requests such as https://site/client/123/info and https://site/health from being traced.
+
+
+To capture HTTP request headers as span attributes
+```dotenv
+OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST="content-type,custom_request_header"
+```
+will extract content-type and custom_request_header from the request headers and add them as span attributes.
+
+
+To capture HTTP response headers as span attributes
+```dotenv
+OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE="content-type,custom_response_header"
+```
+will extract content-type and custom_response_header from the response headers and add them as span attributes.
+
+In order to prevent storing sensitive data such as personally identifiable information (PII), session keys, passwords, etc.
+```dotenv
+OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS=".*session.*,set-cookie"
+```
+will replace the value of headers such as session-id and set-cookie with [REDACTED] in the span.

docs/index.md

@@ -1,8 +1,9 @@
-# HORAO: management engine for hybrid multi-cloud environments
+# Management engine for hybrid multi-cloud environments
 
 There are various cloud based management engines available. These tend to be either very specific to a certain cloud provider, or very generic and complex. The goal of HORAO is to provide a simple, yet powerful, management engine that can be used to manage hybrid multi-cloud environments. One of the key design features is to provide a model-based approach to managing resources, which allows for a high level of abstraction and automation.
 
 These pages are currently available:
 - [Architecture](./Architecture.md)
 - [Development notes](./Development.md)
 - [Creating a virtual environment](./CreateVirtualEnv.md)
+- [Telemetry](./Telemetry.md)

horao/__init__.py

@@ -10,12 +10,36 @@
 import logging
 import os
 
+from opentelemetry import metrics, trace  # type: ignore
+from opentelemetry.instrumentation.logging import LoggingInstrumentor  # type: ignore
+from opentelemetry.instrumentation.starlette import (
+    StarletteInstrumentor,  # type: ignore
+)
+from opentelemetry.sdk.metrics import MeterProvider  # type: ignore
+from opentelemetry.sdk.metrics.export import (
+    PeriodicExportingMetricReader,  # type: ignore
+)
+
+if bool(os.getenv("OLTP_HTTP", False)):
+    from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
+        OTLPMetricExporter,  # type: ignore
+    )
+    from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
+        OTLPSpanExporter,  # type: ignore
+    )
+else:
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter  # type: ignore
+    from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter  # type: ignore
+
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource  # type: ignore
+from opentelemetry.sdk.trace import TracerProvider  # type: ignore
+from opentelemetry.sdk.trace.export import BatchSpanProcessor  # type: ignore
 from starlette.applications import Starlette  # type: ignore
 from starlette.middleware import Middleware  # type: ignore
 from starlette.middleware.authentication import AuthenticationMiddleware
 from starlette.middleware.cors import CORSMiddleware  # type: ignore
-from starlette.responses import HTMLResponse
-from starlette.routing import Route
+from starlette.responses import HTMLResponse  # type: ignore
+from starlette.routing import Route  # type: ignore
 from starlette.schemas import SchemaGenerator  # type: ignore
 
 import horao.api
@@ -24,6 +48,51 @@
 from horao.auth.basic_auth import BasicAuthBackend
 from horao.auth.peer import PeerAuthBackend
 
+LoggingInstrumentor().instrument(set_logging_format=True)
+
+resource = Resource.create(
+    {
+        "service.name": "horao",
+        "service.instance.id": os.uname().nodename,
+    }
+)
+
+if "OLTP_COLLECTOR_URL" in os.environ:
+    oltp_url = os.getenv("OLTP_COLLECTOR_URL")
+    oltp_insecure = bool(os.getenv("OLTP_INSECURE", False))
+    oltp_http = bool(os.getenv("OLTP_HTTP", False))
+    trace_provider = TracerProvider(resource=resource)
+    processor = BatchSpanProcessor(
+        OTLPSpanExporter(
+            endpoint=f"{oltp_url}/v1/traces",
+            insecure=oltp_insecure,
+        )
+        if oltp_http
+        else OTLPSpanExporter(
+            endpoint=f"{oltp_url}",
+            insecure=oltp_insecure,
+        )
+    )
+    trace_provider.add_span_processor(processor)
+    trace.set_tracer_provider(trace_provider)
+
+    reader = PeriodicExportingMetricReader(
+        (
+            OTLPMetricExporter(
+                endpoint=f"{oltp_url}/v1/metrics",
+                insecure=oltp_insecure,
+            )
+            if oltp_http
+            else OTLPMetricExporter(
+                endpoint=f"{oltp_url}",
+                insecure=oltp_insecure,
+            )
+        ),
+    )
+    meter_provider = MeterProvider(resource=resource, metric_readers=[reader])
+    metrics.set_meter_provider(meter_provider)
+
+
 schemas = SchemaGenerator(
     {"openapi": "3.0.0", "info": {"title": "HORAO API", "version": "1.0"}}
 )
@@ -53,9 +122,11 @@ async def docs(request):
 
 
 def init_api() -> Starlette:
+    logger = logging.getLogger("horao.init")
+    logger.debug("initializing horao")
     cors = os.getenv("CORS", "*")
     if cors == "*":
-        logging.warning("CORS is set to *")
+        logger.warning("CORS is set to *")
     routes = [
         Route("/ping", endpoint=horao.api.alive_controller.is_alive, methods=["GET"]),
         Route(
@@ -75,6 +146,13 @@ def init_api() -> Starlette:
         middleware.append(
             Middleware(AuthenticationMiddleware, backend=BasicAuthBackend())
         )
-    return Starlette(
-        routes=routes, middleware=middleware, debug=bool(os.getenv("DEBUG", False))
+    if os.getenv("TELEMETRY", "ON") == "OFF":
+        logger.warning("Telemetry is turned off")
+        return Starlette(
+            routes=routes, middleware=middleware, debug=bool(os.getenv("DEBUG", False))
+        )
+    return StarletteInstrumentor.instrument_app(
+        Starlette(
+            routes=routes, middleware=middleware, debug=bool(os.getenv("DEBUG", False))
+        )
     )

horao/main.py

@@ -15,6 +15,7 @@ def main():
         port=8081,
         log_level="debug",
         reload=True,
+        factory=True,
     )
 
 

otel-config.yaml

@@ -0,0 +1,13 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+exporters:
+  debug:
+    verbosity: detailed
+service:
+  pipelines:
+    logs:
+      receivers: [ otlp ]
+      exporters: [ debug ]