Filter out health check traces (#232)

config/otel-collector-config.yaml

@@ -19,6 +19,19 @@ exporters:
 
 processors:
   batch:
+  tail_sampling:
+    policies: [
+      {
+        name: filter_http_url,
+        type: string_attribute,
+        string_attribute: {
+          key: http.route,
+          values: [ "/health/ready" ],
+          enabled_regex_matching: true,
+          invert_match: true
+        }
+      }
+    ]
 
 extensions:
   health_check:
@@ -32,7 +45,7 @@ service:
   pipelines:
     traces:
       receivers: [otlp]
-      processors: [batch]
+      processors: [batch, tail_sampling]
       exporters: [zipkin, logging]
     metrics:
       receivers: [otlp]

src/nv_ingest/api/main.py

@@ -13,6 +13,7 @@
 
 from opentelemetry import trace
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
@@ -22,7 +23,8 @@
 from .v1.ingest import router as IngestApiRouter
 
 # Set up the tracer provider and add a processor for exporting traces
-trace.set_tracer_provider(TracerProvider())
+resource = Resource(attributes={"service.name": "nv-ingest"})
+trace.set_tracer_provider(TracerProvider(resource=resource))
 tracer = trace.get_tracer(__name__)
 
 otel_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "otel-collector:4317")

src/nv_ingest/modules/telemetry/otel_tracer.py

@@ -83,7 +83,7 @@ def collect_timestamps(message):
             is_remote=True,
             trace_flags=TraceFlags(0x01),
         )
-        parent_ctx = trace.set_span_in_context(NonRecordingSpan(span_context))
+        parent_ctx = trace.set_span_in_context(span_context)
         parent_span = tracer.start_span(job_id, context=parent_ctx, start_time=start_time)
 
         create_span_with_timestamps(tracer, parent_span, message)