Merge branch 'main' into ordered-results

.github/dependabot.yml

@@ -7,6 +7,21 @@ updates:
     directory: '/'
     schedule:
       interval: 'weekly'
+    open-pull-requests-limit: 10
+    groups:
+      opentelemetry-otel:
+        patterns:
+          - go.opentelemetry.io/otel/*
+      opentelemetry-contrib:
+        patterns:
+          - go.opentelemetry.io/contrib/*
+          - github.com/open-telemetry/opentelemetry-collector-contrib/*
+      opentelemetry-collector:
+        patterns:
+          - go.opentelemetry.io/collector/*
+      k8s.io:
+        patterns:
+          - k8s.io/*
 
   - package-ecosystem: 'docker'
     directory: '/'

.github/workflows/milestoned_to_project.yml

@@ -19,7 +19,7 @@ jobs:
           app-id: ${{secrets.APP_ID}}
           private-key: ${{secrets.APP_PRIVATE_KEY}}
           owner: ${{ github.repository_owner }}
-      - uses: actions/[email protected].0
+      - uses: actions/[email protected].2
         with:
           project-url: https://github.com/orgs/grafana/projects/135
           github-token: ${{ steps.get-github-app-token.outputs.token }}

.github/workflows/sbom-report.yml

@@ -14,7 +14,7 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Anchore SBOM Action
-      uses: anchore/[email protected].4
+      uses: anchore/[email protected].5
       with:
          artifact-name: ${{ github.event.repository.name }}-spdx.json
 

CHANGELOG.md

@@ -1,8 +1,5 @@
 ## main / unreleased
-* [ENHANCEMENT] The span multiplier now also sources its value from the resource attributes. [#4210](https://github.com/grafana/tempo/pull/4210)
-* [FEATURE] Export cost attribution usage metrics from distributor [#4162](https://github.com/grafana/tempo/pull/4162) (@mdisibio)
-* [ENHANCEMENT] Changed log level from INFO to DEBUG for the TempoDB Find operation using traceId to reduce excessive/unwanted logs in log search. [#4179](https://github.com/grafana/tempo/pull/4179) (@Aki0x137)
-* [ENHANCEMENT] Pushdown collection of results from generators in the querier [#4119](https://github.com/grafana/tempo/pull/4119) (@electron0zero)
+* [CHANGE] **BREAKING CHANGE** Change the AWS Lambda serverless build tooling output from "main" to "bootstrap". Refer to https://aws.amazon.com/blogs/compute/migrating-aws-lambda-functions-from-the-go1-x-runtime-to-the-custom-runtime-on-amazon-linux-2/ for migration steps [#3852](https://github.com/grafana/tempo/pull/3852) (@zatlodan)
 * [CHANGE] Add throughput and SLO metrics in the tags and tag values endpoints [#4148](https://github.com/grafana/tempo/pull/4148) (@electron0zero)
 * [CHANGE] tempo-cli: add support for /api/v2/traces endpoint [#4127](https://github.com/grafana/tempo/pull/4127) (@electron0zero)
   **BREAKING CHANGE** The `tempo-cli` now uses the `/api/v2/traces` endpoint by default, 
@@ -15,8 +12,15 @@
 * [CHANGE] Update Go to 1.23.1 [#4146](https://github.com/grafana/tempo/pull/4146) [#4147](https://github.com/grafana/tempo/pull/4147) (@javiermolinar)
 * [CHANGE] TraceQL: Add range condition for byte predicates [#4198](https://github.com/grafana/tempo/pull/4198) (@ie-pham)
 * [CHANGE] Return 422 for TRACE_TOO_LARGE queries [#4160](https://github.com/grafana/tempo/pull/4160) (@zalegrala)
+* [CHANGE] Upgrade OTEL sdk to reduce allocs [#4243](https://github.com/grafana/tempo/pull/4243) (@joe-elliott)
+* [CHANGE] Tighten file permissions [#4251](https://github.com/grafana/tempo/pull/4251) (@zalegrala)
 * [FEATURE] Discarded span logging `log_discarded_spans` [#3957](https://github.com/grafana/tempo/issues/3957) (@dastrobu)
 * [FEATURE] TraceQL support for instrumentation scope [#3967](https://github.com/grafana/tempo/pull/3967) (@ie-pham)
+* [FEATURE] Export cost attribution usage metrics from distributor [#4162](https://github.com/grafana/tempo/pull/4162) (@mdisibio)
+* [FEATURE] TraceQL metrics: avg_over_time [#4073](https://github.com/grafana/tempo/pull/4073) (@javiermolinar)
+* [ENHANCEMENT] Changed log level from INFO to DEBUG for the TempoDB Find operation using traceId to reduce excessive/unwanted logs in log search. [#4179](https://github.com/grafana/tempo/pull/4179) (@Aki0x137)
+* [ENHANCEMENT] Pushdown collection of results from generators in the querier [#4119](https://github.com/grafana/tempo/pull/4119) (@electron0zero)
+* [ENHANCEMENT] The span multiplier now also sources its value from the resource attributes. [#4210](https://github.com/grafana/tempo/pull/4210)
 * [ENHANCEMENT] TraceQL: Attribute iterators collect matched array values [#3867](https://github.com/grafana/tempo/pull/3867) (@electron0zero, @stoewer)
 * [ENHANCEMENT] Allow returning partial traces that exceed the MaxBytes limit for V2 [#3941](https://github.com/grafana/tempo/pull/3941) (@javiermolinar)
 * [ENHANCEMENT] Added new middleware to validate request query values [#3993](https://github.com/grafana/tempo/pull/3993) (@javiermolinar)
@@ -26,6 +30,7 @@
 * [ENHANCEMENT] Reduce allocs related to marshalling dedicated columns repeatedly in the query frontend. [#4007](https://github.com/grafana/tempo/pull/4007) (@joe-elliott)
 * [ENHANCEMENT] Improve performance of TraceQL queries [#4114](https://github.com/grafana/tempo/pull/4114) (@mdisibio)
 * [ENHANCEMENT] Improve performance of TraceQL queries [#4163](https://github.com/grafana/tempo/pull/4163) (@mdisibio) 
+* [ENHANCEMENT] Reduce memory usage of classic histograms in the span-metrics and service-graphs processors [#4232](https://github.com/grafana/tempo/pull/4232) (@mdisibio)
 * [ENHANCEMENT] Implement simple Fetch by key for cache items  [#4032](https://github.com/grafana/tempo/pull/4032) (@javiermolinar)
 * [ENHANCEMENT] Replace Grafana Agent example by Grafana Alloy[#4030](https://github.com/grafana/tempo/pull/4030) (@javiermolinar)
 * [ENHANCEMENT] Support exporting internal Tempo traces via OTLP exporter when `use_otel_tracer` is enabled. Use the OpenTelemetry SDK environment variables to configure the span exporter. [#4028](https://github.com/grafana/tempo/pull/4028) (@andreasgerstmayr)
@@ -42,11 +47,18 @@
 * [ENHANCEMENT] Speedup DistinctValue collector and exit early for ingesters [#4104](https://github.com/grafana/tempo/pull/4104) (@electron0zero)
 * [ENHANCEMENT] Add disk caching in ingester SearchTagValuesV2 for completed blocks [#4069](https://github.com/grafana/tempo/pull/4069) (@electron0zero)
 * [ENHANCEMENT] Added most_recent=true query hint to TraceQL to return most recent results. [#4238](https://github.com/grafana/tempo/pull/4238) (@joe-elliott)
+* [ENHANCEMENT] chore: remove gofakeit dependency [#4274](https://github.com/grafana/tempo/pull/4274) (@javiermolinar)
+* [ENHANCEMENT] Add a max flush attempts and metric to the metrics generator [#4254](https://github.com/grafana/tempo/pull/4254) (@joe-elliott)
+* [ENHANCEMENT] Collection of query-frontend changes to reduce allocs. [#4242]https://github.com/grafana/tempo/pull/4242 (@joe-elliott)
+* [ENHANCEMENT] Added `insecure-skip-verify` option in tempo-cli to skip SSL certificate validation when connecting to the S3 backend. [#44236](https://github.com/grafana/tempo/pull/4259) (@faridtmammadov)
 * [BUGFIX] Replace hedged requests roundtrips total with a counter. [#4063](https://github.com/grafana/tempo/pull/4063) [#4078](https://github.com/grafana/tempo/pull/4078) (@galalen)
 * [BUGFIX] Metrics generators: Correctly drop from the ring before stopping ingestion to reduce drops during a rollout. [#4101](https://github.com/grafana/tempo/pull/4101) (@joe-elliott)
 * [BUGFIX] Correctly handle 400 Bad Request and 404 Not Found in gRPC streaming [#4144](https://github.com/grafana/tempo/pull/4144) (@mapno)
 * [BUGFIX] Pushes a 0 to classic histogram's counter when the series is new to allow Prometheus to start from a non-null value. [#4140](https://github.com/grafana/tempo/pull/4140) (@mapno)
 * [BUGFIX] Fix counter samples being downsampled by backdate to the previous minute the initial sample when the series is new [#44236](https://github.com/grafana/tempo/pull/4236) (@javiermolinar)
+* [BUGFIX] Skip computing exemplars for instant queries. [#4204](https://github.com/grafana/tempo/pull/4204) (@javiermolinar)
+* [BUGFIX] Gave context to orphaned spans related to various maintenance processes. [#4260](https://github.com/grafana/tempo/pull/4260) (@joe-elliott)
+* [BUGFIX] Utilize S3Pass and S3User parameters in tempo-cli options, which were previously unused in the code. [#44236](https://github.com/grafana/tempo/pull/4259) (@faridtmammadov)
 
 # v2.6.1
 

CODEOWNERS

@@ -11,15 +11,15 @@
 # In each subsection folders are ordered first by depth, then alphabetically.
 # This should make it easy to add new rules without breaking existing ones.
 
-* @joe-elliott @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar
+* @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar
 
-/docs/ @knylander-grafana @joe-elliott @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+/docs/ @knylander-grafana @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer
 
-/.github/backport.yml           @jdbaldry @joe-elliott @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
-/.github/update-make-docs.yml   @jdbaldry @joe-elliott @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
-/.github/website-next.yml       @jdbaldry @joe-elliott @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
-/.github/website-versioned.yml  @jdbaldry @joe-elliott @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
-/docs/docs.mk                   @jdbaldry @knylander-grafana @joe-elliott @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
-/docs/make-docs                 @jdbaldry @knylander-grafana @joe-elliott @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
-/docs/Makefile                  @jdbaldry @knylander-grafana @joe-elliott @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
-/docs/variables.mk              @jdbaldry @knylander-grafana @joe-elliott @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+/.github/backport.yml           @jdbaldry @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+/.github/update-make-docs.yml   @jdbaldry @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+/.github/website-next.yml       @jdbaldry @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+/.github/website-versioned.yml  @jdbaldry @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+/docs/docs.mk                   @jdbaldry @knylander-grafana @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+/docs/make-docs                 @jdbaldry @knylander-grafana @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+/docs/Makefile                  @jdbaldry @knylander-grafana @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+/docs/variables.mk              @jdbaldry @knylander-grafana @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer

GOVERNANCE.md

@@ -51,7 +51,7 @@ The current team members are:
 - Javi Molina - [javiermolinar](https://github.com/javiermolinar) ([Grafana Labs](https://grafana.com/))
 - Joe Elliott - [joe-elliott](https://github.com/joe-elliott) ([Grafana Labs](https://grafana.com/))
 - Kim Nylander - [knylander-grafana](https://github.com/knylander-grafana) ([Grafana Labs](https://grafana.com/))
-- Koenraad Verheyden - [kvrhdn](https://github.com/kvrhdn) ([Grafana Labs](https://grafana.com/))
+- Yuna Verheyden - [yvhrdn](https://github.com/yvhrdn) ([Grafana Labs](https://grafana.com/))
 - Mario Rodriguez - [mapno](https://github.com/mapno) ([Grafana Labs](https://grafana.com/))
 - Marty Disibio - [mdisibio](https://github.com/mdisibio) ([Grafana Labs](https://grafana.com/))
 - Adrian Stoewer - [stoewer](https://github.com/stoewer) ([Grafana Labs](https://grafana.com/))

MAINTAINERS.md

@@ -3,7 +3,7 @@
 * @javiermolinar
 * @joe-elliott
 * @knylander-grafana
-* @kvrhdn
+* @yvrhdn
 * @mapno
 * @mdisibio
 * @stoewer

Makefile

@@ -287,11 +287,13 @@ gen-traceql-local: ## Generate traceq local
 gen-parquet-query:  ## Generate Parquet query 
 	go run ./pkg/parquetquerygen/predicates.go > ./pkg/parquetquery/predicates.gen.go
 
+##@ Tempo tools
 ### Check vendored and generated files are up to date
 .PHONY: vendor-check
 vendor-check: gen-proto update-mod gen-traceql gen-parquet-query ## Keep up to date vendorized files
 	git diff --exit-code -- **/go.sum **/go.mod vendor/ pkg/tempopb/ pkg/traceql/
 
+
 ### Tidy dependencies for tempo and tempo-serverless modules
 .PHONY: update-mod 
 update-mod: tools-update-mod ## Update module
@@ -325,7 +327,7 @@ docs-test:
 
 ##@ jsonnet
 .PHONY: jsonnet jsonnet-check jsonnet-test
-jsonnet: tools-image ##  Generate jsonnet
+jsonnet: tools-image ## Generate jsonnet
 	$(TOOLS_CMD) $(MAKE) -C operations/jsonnet-compiled/util gen
 
 jsonnet-check: tools-image ## Check jsonnet

cmd/tempo-cli/cmd-gen-bloom.go

@@ -29,7 +29,7 @@ type forEachRecord func(id common.ID) error
 
 func ReplayBlockAndDoForEachRecord(meta *backend.BlockMeta, filepath string, forEach forEachRecord) error {
 	// replay file to extract records
-	f, err := os.OpenFile(filepath, os.O_RDONLY, 0o644)
+	f, err := os.OpenFile(filepath, os.O_RDONLY, 0o600)
 	if err != nil {
 		return err
 	}

cmd/tempo-cli/cmd-gen-index.go

@@ -24,7 +24,7 @@ type indexCmd struct {
 func ReplayBlockAndGetRecords(meta *backend.BlockMeta, filepath string) ([]v2.Record, error, error) {
 	var replayError error
 	// replay file to extract records
-	f, err := os.OpenFile(filepath, os.O_RDONLY, 0o644)
+	f, err := os.OpenFile(filepath, os.O_RDONLY, 0o600)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -152,7 +152,7 @@ func (cmd *indexCmd) Run(ctx *globalOptions) error {
 
 	// get index file with records
 	indexFilePath := cmd.backendOptions.Bucket + cmd.TenantID + "/" + cmd.BlockID + "/" + indexFilename
-	indexFile, err := os.OpenFile(indexFilePath, os.O_RDONLY, 0o644)
+	indexFile, err := os.OpenFile(indexFilePath, os.O_RDONLY, 0o600)
 	if err != nil {
 		fmt.Println("error opening index file")
 		return err
@@ -166,7 +166,7 @@ func (cmd *indexCmd) Run(ctx *globalOptions) error {
 
 	// data reader
 	dataFilePath := cmd.backendOptions.Bucket + cmd.TenantID + "/" + cmd.BlockID + "/" + dataFilename
-	dataFile, err := os.OpenFile(dataFilePath, os.O_RDONLY, 0o644)
+	dataFile, err := os.OpenFile(dataFilePath, os.O_RDONLY, 0o600)
 	if err != nil {
 		fmt.Println("error opening data file")
 		return err

cmd/tempo-cli/cmd-migrate-overrides-config.go

@@ -68,7 +68,7 @@ func (cmd *migrateOverridesConfigCmd) Run(*globalOptions) error {
 	}
 
 	if cmd.ConfigDest != "" {
-		if err := os.WriteFile(cmd.ConfigDest, configBytes, 0o644); err != nil {
+		if err := os.WriteFile(cmd.ConfigDest, configBytes, 0o600); err != nil {
 			return fmt.Errorf("failed to write config file: %w", err)
 		}
 	} else {
@@ -90,7 +90,7 @@ func (cmd *migrateOverridesConfigCmd) Run(*globalOptions) error {
 	}
 
 	if cmd.OverridesDest != "" {
-		if err := os.WriteFile(cmd.OverridesDest, overridesBytes, 0o644); err != nil {
+		if err := os.WriteFile(cmd.OverridesDest, overridesBytes, 0o600); err != nil {
 			return fmt.Errorf("failed to write overrides file: %w", err)
 		}
 	} else {

cmd/tempo-cli/main.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/grafana/dskit/flagext"
+
 	"github.com/alecthomas/kong"
 	"gopkg.in/yaml.v2"
 
@@ -30,9 +32,10 @@ type backendOptions struct {
 	Backend string `help:"backend to connect to (s3/gcs/local/azure), optional, overrides backend in config file" enum:",s3,gcs,local,azure" default:""`
 	Bucket  string `help:"bucket (or path on local backend) to scan, optional, overrides bucket in config file"`
 
-	S3Endpoint string `name:"s3-endpoint" help:"s3 endpoint (s3.dualstack.us-east-2.amazonaws.com), optional, overrides endpoint in config file"`
-	S3User     string `name:"s3-user" help:"s3 username, optional, overrides username in config file"`
-	S3Pass     string `name:"s3-pass" help:"s3 password, optional, overrides password in config file"`
+	S3Endpoint         string `name:"s3-endpoint" help:"s3 endpoint (s3.dualstack.us-east-2.amazonaws.com), optional, overrides endpoint in config file"`
+	S3User             string `name:"s3-user" help:"s3 username, optional, overrides username in config file"`
+	S3Pass             string `name:"s3-pass" help:"s3 password, optional, overrides password in config file"`
+	InsecureSkipVerify bool   `name:"insecure-skip-verify" help:"skip TLS verification, only applies to S3 and GCS" default:"false"`
 }
 
 var cli struct {
@@ -131,6 +134,17 @@ func loadBackend(b *backendOptions, g *globalOptions) (backend.Reader, backend.W
 		cfg.StorageConfig.Trace.Azure.ContainerName = b.Bucket
 	}
 
+	cfg.StorageConfig.Trace.S3.InsecureSkipVerify = b.InsecureSkipVerify
+	cfg.StorageConfig.Trace.GCS.Insecure = b.InsecureSkipVerify
+
+	if b.S3User != "" {
+		cfg.StorageConfig.Trace.S3.AccessKey = b.S3User
+	}
+
+	if b.S3Pass != "" {
+		cfg.StorageConfig.Trace.S3.SecretKey = flagext.SecretWithValue(b.S3Pass)
+	}
+
 	if b.S3Endpoint != "" {
 		cfg.StorageConfig.Trace.S3.Endpoint = b.S3Endpoint
 	}

cmd/tempo-serverless/Makefile

@@ -33,13 +33,14 @@ build-docker-lambda-test:
 	$(IN_LAMBDA) CGO_ENABLED=0 go build -o ./lambda
 	$(IN_LAMBDA) docker build -f ./Dockerfile -t tempo-serverless-lambda .
 
-# lambda zips expect a compiled executable in root. the filename "main" is important
-# as that should the handler config option in aws
+# Lambda zips expect a compiled executable in the root. The filename "bootstrap" is important here.
+# The new AWS Lambda runtime expects an executable with the name "bootstrap" to be provided, the "handler" configuration is ignored when using the new runtime.
+# See https://aws.amazon.com/blogs/compute/migrating-aws-lambda-functions-from-the-go1-x-runtime-to-the-custom-runtime-on-amazon-linux-2/ for more info.
 .PHONY: build-lambda-zip
 build-lambda-zip:
-	$(IN_LAMBDA) CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o main
-	$(IN_LAMBDA) zip tempo-serverless-$(VERSION).zip main
-	$(IN_LAMBDA) rm main
+	$(IN_LAMBDA) CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bootstrap
+	$(IN_LAMBDA) zip tempo-serverless-$(VERSION).zip bootstrap
+	$(IN_LAMBDA) rm bootstrap
 
 .PHONY: test
 test:

cmd/tempo-serverless/cloud-run/go.mod

@@ -113,9 +113,9 @@ require (
 	go.opentelemetry.io/collector/semconv v0.105.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect
-	go.opentelemetry.io/otel v1.30.0 // indirect
-	go.opentelemetry.io/otel/metric v1.30.0 // indirect
-	go.opentelemetry.io/otel/trace v1.30.0 // indirect
+	go.opentelemetry.io/otel v1.31.0 // indirect
+	go.opentelemetry.io/otel/metric v1.31.0 // indirect
+	go.opentelemetry.io/otel/trace v1.31.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect