Parse OTLP logs in doc processor (#4807)

* Rename `OtlpTrace*` to `OtlpTraces*` * Parse OTLP logs in doc processor
quickwit-oss · Mar 29, 2024 · 24950c0 · 24950c0
1 parent 8f21026
commit 24950c0
Show file tree

Hide file tree

Showing 8 changed files with 479 additions and 190 deletions.
diff --git a/config/tutorials/otel-traces/kafka-source.yaml b/config/tutorials/otel-traces/kafka-source.yaml
@@ -1,7 +1,7 @@
 version: 0.7
 source_id: kafka-source
 source_type: kafka
-input_format: otlp_trace_proto
+input_format: otlp_traces_proto
 params:
   topic: otlp_spans
   client_params:

diff --git a/quickwit/quickwit-common/src/rate_limited_tracing.rs b/quickwit/quickwit-common/src/rate_limited_tracing.rs
@@ -86,7 +86,7 @@ macro_rules! rate_limited_warn {
 }
 #[macro_export]
 macro_rules! rate_limited_error {
-    ($unit:literal=$limit:literal, $($args:tt)*) => {
+    ($unit:ident=$limit:literal, $($args:tt)*) => {
         $crate::rate_limited_tracing::rate_limited_tracing!(error, $unit=$limit, $($args)*)
     };
 }

diff --git a/quickwit/quickwit-config/src/source_config/mod.rs b/quickwit/quickwit-config/src/source_config/mod.rs
@@ -182,9 +182,17 @@ impl TestableForRegression for SourceConfig {
 pub enum SourceInputFormat {
     #[default]
     Json,
-    OtlpTraceJson,
-    #[serde(alias = "otlp_trace_proto")]
-    OtlpTraceProtobuf,
+    OtlpLogsJson,
+    #[serde(alias = "otlp_logs_proto")]
+    OtlpLogsProtobuf,
+    #[serde(alias = "otlp_trace_json")]
+    OtlpTracesJson,
+    #[serde(
+        alias = "otlp_trace_proto",
+        alias = "otlp_trace_protobuf",
+        alias = "otlp_traces_proto"
+    )]
+    OtlpTracesProtobuf,
     #[serde(alias = "plain")]
     PlainText,
 }

diff --git a/quickwit/quickwit-config/src/source_config/serialize.rs b/quickwit/quickwit-config/src/source_config/serialize.rs
@@ -110,7 +110,10 @@ impl SourceConfigForSerialization {
         if let Some(transform_config) = &self.transform {
             if matches!(
                 self.input_format,
-                SourceInputFormat::OtlpTraceJson | SourceInputFormat::OtlpTraceProtobuf
+                SourceInputFormat::OtlpLogsJson
+                    | SourceInputFormat::OtlpLogsProtobuf
+                    | SourceInputFormat::OtlpTracesJson
+                    | SourceInputFormat::OtlpTracesProtobuf
             ) {
                 bail!("VRL transforms are not supported for OTLP input formats");
             }