pyroscope python profiling (#5445)

CHANGELOG.md

@@ -44,6 +44,8 @@ Main (unreleased)
 
 - Update version of River to support raw strings in flow using a backtick. (@erikbaranowski)
 
+- Added support for python profiling to `pyroscope.ebpf` component. (@korniltsev)
+
 ### Bugfixes
 
 - Fixed an issue where `loki.process` validation for stage `metric.counter` was

component/pyroscope/ebpf/args.go

@@ -22,4 +22,5 @@ type Arguments struct {
 	CollectUserProfile   bool                   `river:"collect_user_profile,attr,optional"`
 	CollectKernelProfile bool                   `river:"collect_kernel_profile,attr,optional"`
 	Demangle             string                 `river:"demangle,attr,optional"`
+	PythonEnabled        bool                   `river:"python_enabled,attr,optional"`
 }

component/pyroscope/ebpf/ebpf_linux.go

@@ -1,4 +1,4 @@
-//go:build linux
+//go:build (linux && arm64) || (linux && amd64)
 
 package ebpf
 
@@ -84,6 +84,7 @@ func defaultArguments() Arguments {
 		CollectKernelProfile: true,
 		TargetsOnly:          true,
 		Demangle:             "none",
+		PythonEnabled:        true,
 	}
 }
 
@@ -118,7 +119,7 @@ func (c *Component) Run(ctx context.Context) error {
 				return nil
 			case newArgs := <-c.argsUpdate:
 				c.args = newArgs
-				c.targetFinder.Update(targetsOptionFromArgs(c.args))
+				c.session.UpdateTargets(targetsOptionFromArgs(c.args))
 				c.metrics.targetsActive.Set(float64(len(c.targetFinder.DebugInfo())))
 				err := c.session.Update(convertSessionOptions(c.args, c.metrics))
 				if err != nil {
@@ -160,7 +161,7 @@ func (c *Component) collectProfiles() error {
 	c.metrics.profilingSessionsTotal.Inc()
 	level.Debug(c.options.Logger).Log("msg", "ebpf  collectProfiles")
 	args := c.args
-	builders := pprof.NewProfileBuilders(args.SampleRate)
+	builders := pprof.NewProfileBuilders(int64(args.SampleRate))
 	err := c.session.CollectProfiles(func(target *sd.Target, stack []string, value uint64, pid uint32) {
 		labelsHash, labels := target.Labels()
 		builder := builders.BuilderForTarget(labelsHash, labels)
@@ -232,6 +233,8 @@ func convertSessionOptions(args Arguments, ms *metrics) ebpfspy.SessionOptions {
 		CollectUser:   args.CollectUserProfile,
 		CollectKernel: args.CollectKernelProfile,
 		SampleRate:    args.SampleRate,
+		PythonEnabled: args.PythonEnabled,
+		Metrics:       ms.ebpfMetrics,
 		CacheOptions: symtab.CacheOptions{
 			SymbolOptions: symtab.SymbolOptions{
 				GoTableFallback: false,
@@ -249,7 +252,6 @@ func convertSessionOptions(args Arguments, ms *metrics) ebpfspy.SessionOptions {
 				Size:       args.SameFileCacheSize,
 				KeepRounds: args.CacheRounds,
 			},
-			Metrics: ms.symtabMetrics,
 		},
 	}
 }

component/pyroscope/ebpf/ebpf_linux_test.go

@@ -1,4 +1,4 @@
-//go:build linux
+//go:build (linux && arm64) || (linux && amd64)
 
 package ebpf
 
@@ -41,6 +41,10 @@ func (m *mockSession) Update(options ebpfspy.SessionOptions) error {
 	return nil
 }
 
+func (m *mockSession) UpdateTargets(_ sd.TargetsOptions) {
+
+}
+
 func (m *mockSession) CollectProfiles(f func(target *sd.Target, stack []string, value uint64, pid uint32)) error {
 	m.collected++
 	if m.collectError != nil {
@@ -109,7 +113,7 @@ func TestContextShutdown(t *testing.T) {
 		{"a", "b", "c"},
 		{"q", "w", "e"},
 	}
-	session.dataTarget, _ = sd.NewTarget("cid", map[string]string{"service_name": "foo"})
+	session.dataTarget = sd.NewTarget("cid", 0, map[string]string{"service_name": "foo"})
 	var g run.Group
 	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*1))
 	defer cancel()

component/pyroscope/ebpf/ebpf_placeholder.go

@@ -1,4 +1,4 @@
-//go:build !linux
+//go:build linux && !arm64 && !amd64
 
 package ebpf
 

component/pyroscope/ebpf/metrics.go

@@ -5,7 +5,7 @@
 package ebpf
 
 import (
-	"github.com/grafana/pyroscope/ebpf/symtab"
+	ebpfmetrics "github.com/grafana/pyroscope/ebpf/metrics"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -16,7 +16,7 @@ type metrics struct {
 	pprofsTotal                   *prometheus.CounterVec
 	pprofBytesTotal               *prometheus.CounterVec
 	pprofSamplesTotal             *prometheus.CounterVec
-	symtabMetrics                 *symtab.Metrics
+	ebpfMetrics                   *ebpfmetrics.Metrics
 }
 
 func newMetrics(reg prometheus.Registerer) *metrics {
@@ -45,7 +45,7 @@ func newMetrics(reg prometheus.Registerer) *metrics {
 			Name: "pyroscope_ebpf_pprof_samples_total",
 			Help: "Total number of pprof profiles collected by the ebpf component",
 		}, []string{"service_name"}),
-		symtabMetrics: symtab.NewMetrics(reg),
+		ebpfMetrics: ebpfmetrics.New(reg),
 	}
 
 	if reg != nil {

docs/sources/flow/reference/components/pyroscope.ebpf.md

@@ -1,8 +1,8 @@
 ---
 aliases:
-- /docs/grafana-cloud/agent/flow/reference/components/pyroscope.ebpf/
-- /docs/grafana-cloud/monitor-infrastructure/agent/flow/reference/components/pyroscope.ebpf/
-- /docs/grafana-cloud/monitor-infrastructure/integrations/agent/flow/reference/components/pyroscope.ebpf/
+  - /docs/grafana-cloud/agent/flow/reference/components/pyroscope.ebpf/
+  - /docs/grafana-cloud/monitor-infrastructure/agent/flow/reference/components/pyroscope.ebpf/
+  - /docs/grafana-cloud/monitor-infrastructure/integrations/agent/flow/reference/components/pyroscope.ebpf/
 canonical: https://grafana.com/docs/agent/latest/flow/reference/components/pyroscope.ebpf/
 labels:
   stage: beta
@@ -54,7 +54,7 @@ values.
 | `collect_user_profile`    | `bool`                   | A flag to enable/disable collection of userspace profiles                           | true    | no       |       
 | `collect_kernel_profile`  | `bool`                   | A flag to enable/disable collection of kernelspace profiles                         | true    | no       |       
 | `demangle`                | `string`                 | C++ demangle mode. Available options are: `none`, `simplified`, `templates`, `full` | `none`  | no       |
-
+| `python_enabled`          | `bool`                   | A flag to enable/disable python profiling                                           | true    | no       |
 
 ## Exported fields
 
@@ -136,7 +136,8 @@ with a `.gnu_debuglink` set to `libc.so.6.debug` and a build ID `0123456789abcde
 
 ### Dealing with unknown symbols
 
-Unknown symbols in the profiles you’ve collected indicate that the profiler couldn't access an ELF file associated with a given address in the trace.
+Unknown symbols in the profiles you’ve collected indicate that the profiler couldn't access an ELF file associated with
+a given address in the trace.
 
 This can occur for several reasons:
 
@@ -163,7 +164,7 @@ strip elf -o elf.stripped
 objcopy --add-gnu-debuglink=elf.debug elf.stripped elf.debuglink
 ```
 
-For system libraries, ensure that debug symbols are installed. On Ubuntu, for example, you can install debug symbols 
+For system libraries, ensure that debug symbols are installed. On Ubuntu, for example, you can install debug symbols
 for `libc` by executing:
 
 ```bash
@@ -172,7 +173,8 @@ apt install libc6-dbg
 
 ### Understanding flat stack traces
 
-If your profiles show many shallow stack traces, typically 1-2 frames deep, your binary might have been compiled without frame pointers.
+If your profiles show many shallow stack traces, typically 1-2 frames deep, your binary might have been compiled without
+frame pointers.
 
 To compile your code with frame pointers, include the `-fno-omit-frame-pointer` flag in your compiler options.
 
@@ -190,7 +192,8 @@ Interpreted methods will display the interpreter function’s name rather than t
 
 In the following example, performance profiles are collected from pods on the same node, discovered using
 `discovery.kubernetes`. Pod selection relies on the `HOSTNAME` environment variable, which is a pod name if the agent is
-used as a Grafana agent helm chart. The `service_name` label is set to `{__meta_kubernetes_namespace}/{__meta_kubernetes_pod_container_name}` from kubernetes meta labels.
+used as a Grafana agent helm chart. The `service_name` label is set
+to `{__meta_kubernetes_namespace}/{__meta_kubernetes_pod_container_name}` from kubernetes meta labels.
 
 ```river
 discovery.kubernetes "all_pods" {

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.18.44
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.34.1
 	github.com/bmatcuk/doublestar v1.3.4
-	github.com/bufbuild/connect-go v1.9.0
+	github.com/bufbuild/connect-go v1.10.0
 	github.com/buger/jsonparser v1.1.1
 	github.com/burningalchemist/sql_exporter v0.0.0-20221222155641-2ff59aa75200
 	github.com/cespare/xxhash/v2 v2.2.0
@@ -59,7 +59,7 @@ require (
 	github.com/grafana/loki v1.6.2-0.20231004111112-07cbef92268a
 	github.com/grafana/pyroscope-go/godeltaprof v0.1.3
 	github.com/grafana/pyroscope/api v0.2.0
-	github.com/grafana/pyroscope/ebpf v0.2.3
+	github.com/grafana/pyroscope/ebpf v0.3.0
 	github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db
 	github.com/grafana/river v0.2.0
 	github.com/grafana/snowflake-prometheus-exporter v0.0.0-20221213150626-862cad8e9538

go.sum

@@ -429,8 +429,8 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dR
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/boynux/squid-exporter v1.10.5-0.20230618153315-c1fae094e18e h1:C1vYe728vM2FpXaICJuDRt5zgGyRdMmUGYnVfM7WcLY=
 github.com/boynux/squid-exporter v1.10.5-0.20230618153315-c1fae094e18e/go.mod h1:8NpZERGK+R9DGuZqqsKfnf2qI/rh7yBT8End29IvgNA=
-github.com/bufbuild/connect-go v1.9.0 h1:JIgAeNuFpo+SUPfU19Yt5TcWlznsN5Bv10/gI/6Pjoc=
-github.com/bufbuild/connect-go v1.9.0/go.mod h1:CAIePUgkDR5pAFaylSMtNK45ANQjp9JvpluG20rhpV8=
+github.com/bufbuild/connect-go v1.10.0 h1:QAJ3G9A1OYQW2Jbk3DeoJbkCxuKArrvZgDt47mjdTbg=
+github.com/bufbuild/connect-go v1.10.0/go.mod h1:CAIePUgkDR5pAFaylSMtNK45ANQjp9JvpluG20rhpV8=
 github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/burningalchemist/sql_exporter v0.0.0-20221222155641-2ff59aa75200 h1:1zECtssRshqhP8+DELKyWeg8rxaRC5OO72kJQhrJOE8=
@@ -1076,8 +1076,8 @@ github.com/grafana/pyroscope-go/godeltaprof v0.1.3 h1:eunWpv1B3Z7ZK9o4499EmQGlY+
 github.com/grafana/pyroscope-go/godeltaprof v0.1.3/go.mod h1:1HSPtjU8vLG0jE9JrTdzjgFqdJ/VgN7fvxBNq3luJko=
 github.com/grafana/pyroscope/api v0.2.0 h1:TzOxL0s6SiaLEy944ZAKgHcx/JDRJXu4O8ObwkqR6p4=
 github.com/grafana/pyroscope/api v0.2.0/go.mod h1:nhH+xai9cYFgs6lMy/+L0pKj0d5yCMwji/QAiQFCP+U=
-github.com/grafana/pyroscope/ebpf v0.2.3 h1:OH7Un2x0UN998U85by4vyvImHs6mkFTo45SnO+PjHdk=
-github.com/grafana/pyroscope/ebpf v0.2.3/go.mod h1:NO9mIMKewDuohQlYaj2Q0v3miUmREjGpadz8RuA76Jw=
+github.com/grafana/pyroscope/ebpf v0.3.0 h1:7J24qd5ul786ZNDpgN8oIDbuBEXX2L829BjNNm3KVYw=
+github.com/grafana/pyroscope/ebpf v0.3.0/go.mod h1:f9gfqjtpxYs/1t7L2HskrR5kQMH64gclaCYdtZ1unKo=
 github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db h1:7aN5cccjIqCLTzedH7MZzRZt5/lsAHch6Z3L2ZGn5FA=
 github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=
 github.com/grafana/river v0.2.0 h1:VgBe/+DdYviJ1cYGUCcA8Og0FhSTjN7jdHCe/+OIFV0=