sending logs via otel-collector -- dropped_log_records

[nbari]

This is my otel-collector config:

---
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:

  attributes/dev:
    actions:
      - key: deployment.environment
        value: development
        action: insert

exporters:
  otlp/quickwit:
    endpoint: 10.10.0.83:7281
    tls:
      insecure: true

service:
  pipelines:
    logs:
      receivers:
        - otlp
      processors:
        - batch
      exporters:
        - otlp/quickwit

    traces:
      receivers:
        - otlp
      processors:
        - batch
        - attributes/dev
      exporters:
        - otlp/quickwit

in the otel-collector I see this error:

warn        [email protected]/otlp.go:126        Partial success response        {"kind": "exporter", "data_type": "logs", "name": "otlp/quickwit", "message": "", "dropped_log_records": 10}

In quickwit I get this error:

ERROR ingest_logs:parse_logs: quickwit_opentelemetry::otlp::logs: skipping record

If I try to send logs via HTTP Instead of dropped_log_records being 10 is 2, this is the code I am using:

use anyhow::Result;
use opentelemetry::{global, trace::TracerProvider, KeyValue};
use opentelemetry_appender_tracing::layer;
use opentelemetry_otlp::WithExportConfig;
use opentelemetry_sdk::{runtime::Tokio, trace::Config, Resource};
use std::time::Duration;
use tracing_opentelemetry::OpenTelemetryLayer;
use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter, Registry};

/// Start the telemetry layer
/// # Errors
/// Will return an error if the telemetry layer fails to start
pub fn init(verbosity_level: tracing::Level) -> Result<()> {
    let tracer_provider = opentelemetry_otlp::new_pipeline()
        .tracing()
        .with_exporter(
            opentelemetry_otlp::new_exporter()
                .tonic()
                .with_timeout(Duration::from_secs(3)),
        )
        .with_trace_config(Config::default().with_resource(Resource::new(vec![
            KeyValue::new("service.name", env!("CARGO_PKG_NAME")),
            KeyValue::new("service.version", env!("CARGO_PKG_VERSION")),
        ])))
        .install_batch(Tokio)?;

    let tracer = tracer_provider
        .tracer_builder(env!("CARGO_PKG_NAME"))
        .with_version(env!("CARGO_PKG_VERSION"))
        .build();

    global::set_tracer_provider(tracer_provider);

    let otel_trace_layer = OpenTelemetryLayer::new(tracer);

    let logger = opentelemetry_otlp::new_pipeline()
        .logging()
        .with_resource(Resource::new(vec![KeyValue::new(
            env!("CARGO_PKG_NAME"),
            env!("CARGO_PKG_VERSION"),
        )]))
        .with_exporter(opentelemetry_otlp::new_exporter().tonic())
        .install_batch(Tokio)?;

    let otel_logs_layer = layer::OpenTelemetryTracingBridge::new(&logger);

    let fmt_layer = fmt::layer()
        .with_file(true)
        .with_line_number(true)
        .with_thread_ids(false)
        .with_thread_names(false)
        .with_target(false)
        .json();

    // RUST_LOG=
    let env_filter = EnvFilter::builder()
        .with_default_directive(verbosity_level.into())
        .from_env_lossy();

    let subscriber = Registry::default()
        .with(fmt_layer)
        .with(otel_trace_layer)
        .with(otel_logs_layer)
        .with(env_filter);

    Ok(tracing::subscriber::set_global_default(subscriber)?)
}

Any idea of what could it be wrong ?

[fmassot]

@nbari thanks for the report.

Unfortunately, the log error you got is not explicit. When looking at the code, this means that the field time_unix_nano of the log record is set to 0. Can you confirm that your log records don't contain the field time_unix_nano or that this field is set to 0?

[nbari]

Hi @fmassot I will check because I don't see indeed the time_unix_nano set I think there is a problem with how I am using logs, does it make any difference using GRPC or HTTP for the logs?