From 2a803b056d102ef4a1e894ddc954eac1eebff22b Mon Sep 17 00:00:00 2001 From: Tolya Korniltsev Date: Fri, 27 Oct 2023 15:25:17 +0700 Subject: [PATCH] pyroscope python profiling (#5445) --- CHANGELOG.md | 2 ++ component/pyroscope/ebpf/args.go | 1 + component/pyroscope/ebpf/ebpf_linux.go | 10 ++++++---- component/pyroscope/ebpf/ebpf_linux_test.go | 8 ++++++-- component/pyroscope/ebpf/ebpf_placeholder.go | 2 +- component/pyroscope/ebpf/metrics.go | 6 +++--- .../reference/components/pyroscope.ebpf.md | 19 +++++++++++-------- go.mod | 4 ++-- go.sum | 8 ++++---- 9 files changed, 36 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e3cac09a683..aa414524359d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,8 @@ Main (unreleased) - Update version of River to support raw strings in flow using a backtick. (@erikbaranowski) +- Added support for python profiling to `pyroscope.ebpf` component. (@korniltsev) + ### Bugfixes - Fixed an issue where `loki.process` validation for stage `metric.counter` was diff --git a/component/pyroscope/ebpf/args.go b/component/pyroscope/ebpf/args.go index e3f1be4f5f11..facf9129d6ba 100644 --- a/component/pyroscope/ebpf/args.go +++ b/component/pyroscope/ebpf/args.go @@ -22,4 +22,5 @@ type Arguments struct { CollectUserProfile bool `river:"collect_user_profile,attr,optional"` CollectKernelProfile bool `river:"collect_kernel_profile,attr,optional"` Demangle string `river:"demangle,attr,optional"` + PythonEnabled bool `river:"python_enabled,attr,optional"` } diff --git a/component/pyroscope/ebpf/ebpf_linux.go b/component/pyroscope/ebpf/ebpf_linux.go index 722b9f1a25ad..28910b92bce2 100644 --- a/component/pyroscope/ebpf/ebpf_linux.go +++ b/component/pyroscope/ebpf/ebpf_linux.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build (linux && arm64) || (linux && amd64) package ebpf @@ -84,6 +84,7 @@ func defaultArguments() Arguments { CollectKernelProfile: true, TargetsOnly: true, Demangle: "none", + PythonEnabled: true, } } @@ -118,7 +119,7 @@ func (c *Component) Run(ctx context.Context) error { return nil case newArgs := <-c.argsUpdate: c.args = newArgs - c.targetFinder.Update(targetsOptionFromArgs(c.args)) + c.session.UpdateTargets(targetsOptionFromArgs(c.args)) c.metrics.targetsActive.Set(float64(len(c.targetFinder.DebugInfo()))) err := c.session.Update(convertSessionOptions(c.args, c.metrics)) if err != nil { @@ -160,7 +161,7 @@ func (c *Component) collectProfiles() error { c.metrics.profilingSessionsTotal.Inc() level.Debug(c.options.Logger).Log("msg", "ebpf collectProfiles") args := c.args - builders := pprof.NewProfileBuilders(args.SampleRate) + builders := pprof.NewProfileBuilders(int64(args.SampleRate)) err := c.session.CollectProfiles(func(target *sd.Target, stack []string, value uint64, pid uint32) { labelsHash, labels := target.Labels() builder := builders.BuilderForTarget(labelsHash, labels) @@ -232,6 +233,8 @@ func convertSessionOptions(args Arguments, ms *metrics) ebpfspy.SessionOptions { CollectUser: args.CollectUserProfile, CollectKernel: args.CollectKernelProfile, SampleRate: args.SampleRate, + PythonEnabled: args.PythonEnabled, + Metrics: ms.ebpfMetrics, CacheOptions: symtab.CacheOptions{ SymbolOptions: symtab.SymbolOptions{ GoTableFallback: false, @@ -249,7 +252,6 @@ func convertSessionOptions(args Arguments, ms *metrics) ebpfspy.SessionOptions { Size: args.SameFileCacheSize, KeepRounds: args.CacheRounds, }, - Metrics: ms.symtabMetrics, }, } } diff --git a/component/pyroscope/ebpf/ebpf_linux_test.go b/component/pyroscope/ebpf/ebpf_linux_test.go index b1617e501e3b..7d91e894ed4c 100644 --- a/component/pyroscope/ebpf/ebpf_linux_test.go +++ b/component/pyroscope/ebpf/ebpf_linux_test.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build (linux && arm64) || (linux && amd64) package ebpf @@ -41,6 +41,10 @@ func (m *mockSession) Update(options ebpfspy.SessionOptions) error { return nil } +func (m *mockSession) UpdateTargets(_ sd.TargetsOptions) { + +} + func (m *mockSession) CollectProfiles(f func(target *sd.Target, stack []string, value uint64, pid uint32)) error { m.collected++ if m.collectError != nil { @@ -109,7 +113,7 @@ func TestContextShutdown(t *testing.T) { {"a", "b", "c"}, {"q", "w", "e"}, } - session.dataTarget, _ = sd.NewTarget("cid", map[string]string{"service_name": "foo"}) + session.dataTarget = sd.NewTarget("cid", 0, map[string]string{"service_name": "foo"}) var g run.Group ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*1)) defer cancel() diff --git a/component/pyroscope/ebpf/ebpf_placeholder.go b/component/pyroscope/ebpf/ebpf_placeholder.go index 59a54d9f4b20..6fd928ff496f 100644 --- a/component/pyroscope/ebpf/ebpf_placeholder.go +++ b/component/pyroscope/ebpf/ebpf_placeholder.go @@ -1,4 +1,4 @@ -//go:build !linux +//go:build linux && !arm64 && !amd64 package ebpf diff --git a/component/pyroscope/ebpf/metrics.go b/component/pyroscope/ebpf/metrics.go index d249cfa9ab99..6468c2e04cc6 100644 --- a/component/pyroscope/ebpf/metrics.go +++ b/component/pyroscope/ebpf/metrics.go @@ -5,7 +5,7 @@ package ebpf import ( - "github.com/grafana/pyroscope/ebpf/symtab" + ebpfmetrics "github.com/grafana/pyroscope/ebpf/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -16,7 +16,7 @@ type metrics struct { pprofsTotal *prometheus.CounterVec pprofBytesTotal *prometheus.CounterVec pprofSamplesTotal *prometheus.CounterVec - symtabMetrics *symtab.Metrics + ebpfMetrics *ebpfmetrics.Metrics } func newMetrics(reg prometheus.Registerer) *metrics { @@ -45,7 +45,7 @@ func newMetrics(reg prometheus.Registerer) *metrics { Name: "pyroscope_ebpf_pprof_samples_total", Help: "Total number of pprof profiles collected by the ebpf component", }, []string{"service_name"}), - symtabMetrics: symtab.NewMetrics(reg), + ebpfMetrics: ebpfmetrics.New(reg), } if reg != nil { diff --git a/docs/sources/flow/reference/components/pyroscope.ebpf.md b/docs/sources/flow/reference/components/pyroscope.ebpf.md index fb086500e4de..08dde0a41826 100644 --- a/docs/sources/flow/reference/components/pyroscope.ebpf.md +++ b/docs/sources/flow/reference/components/pyroscope.ebpf.md @@ -1,8 +1,8 @@ --- aliases: -- /docs/grafana-cloud/agent/flow/reference/components/pyroscope.ebpf/ -- /docs/grafana-cloud/monitor-infrastructure/agent/flow/reference/components/pyroscope.ebpf/ -- /docs/grafana-cloud/monitor-infrastructure/integrations/agent/flow/reference/components/pyroscope.ebpf/ + - /docs/grafana-cloud/agent/flow/reference/components/pyroscope.ebpf/ + - /docs/grafana-cloud/monitor-infrastructure/agent/flow/reference/components/pyroscope.ebpf/ + - /docs/grafana-cloud/monitor-infrastructure/integrations/agent/flow/reference/components/pyroscope.ebpf/ canonical: https://grafana.com/docs/agent/latest/flow/reference/components/pyroscope.ebpf/ labels: stage: beta @@ -54,7 +54,7 @@ values. | `collect_user_profile` | `bool` | A flag to enable/disable collection of userspace profiles | true | no | | `collect_kernel_profile` | `bool` | A flag to enable/disable collection of kernelspace profiles | true | no | | `demangle` | `string` | C++ demangle mode. Available options are: `none`, `simplified`, `templates`, `full` | `none` | no | - +| `python_enabled` | `bool` | A flag to enable/disable python profiling | true | no | ## Exported fields @@ -136,7 +136,8 @@ with a `.gnu_debuglink` set to `libc.so.6.debug` and a build ID `0123456789abcde ### Dealing with unknown symbols -Unknown symbols in the profiles you’ve collected indicate that the profiler couldn't access an ELF file associated with a given address in the trace. +Unknown symbols in the profiles you’ve collected indicate that the profiler couldn't access an ELF file associated with +a given address in the trace. This can occur for several reasons: @@ -163,7 +164,7 @@ strip elf -o elf.stripped objcopy --add-gnu-debuglink=elf.debug elf.stripped elf.debuglink ``` -For system libraries, ensure that debug symbols are installed. On Ubuntu, for example, you can install debug symbols +For system libraries, ensure that debug symbols are installed. On Ubuntu, for example, you can install debug symbols for `libc` by executing: ```bash @@ -172,7 +173,8 @@ apt install libc6-dbg ### Understanding flat stack traces -If your profiles show many shallow stack traces, typically 1-2 frames deep, your binary might have been compiled without frame pointers. +If your profiles show many shallow stack traces, typically 1-2 frames deep, your binary might have been compiled without +frame pointers. To compile your code with frame pointers, include the `-fno-omit-frame-pointer` flag in your compiler options. @@ -190,7 +192,8 @@ Interpreted methods will display the interpreter function’s name rather than t In the following example, performance profiles are collected from pods on the same node, discovered using `discovery.kubernetes`. Pod selection relies on the `HOSTNAME` environment variable, which is a pod name if the agent is -used as a Grafana agent helm chart. The `service_name` label is set to `{__meta_kubernetes_namespace}/{__meta_kubernetes_pod_container_name}` from kubernetes meta labels. +used as a Grafana agent helm chart. The `service_name` label is set +to `{__meta_kubernetes_namespace}/{__meta_kubernetes_pod_container_name}` from kubernetes meta labels. ```river discovery.kubernetes "all_pods" { diff --git a/go.mod b/go.mod index cbcd21d88878..264677166275 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.18.44 github.com/aws/aws-sdk-go-v2/service/s3 v1.34.1 github.com/bmatcuk/doublestar v1.3.4 - github.com/bufbuild/connect-go v1.9.0 + github.com/bufbuild/connect-go v1.10.0 github.com/buger/jsonparser v1.1.1 github.com/burningalchemist/sql_exporter v0.0.0-20221222155641-2ff59aa75200 github.com/cespare/xxhash/v2 v2.2.0 @@ -59,7 +59,7 @@ require ( github.com/grafana/loki v1.6.2-0.20231004111112-07cbef92268a github.com/grafana/pyroscope-go/godeltaprof v0.1.3 github.com/grafana/pyroscope/api v0.2.0 - github.com/grafana/pyroscope/ebpf v0.2.3 + github.com/grafana/pyroscope/ebpf v0.3.0 github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db github.com/grafana/river v0.2.0 github.com/grafana/snowflake-prometheus-exporter v0.0.0-20221213150626-862cad8e9538 diff --git a/go.sum b/go.sum index b12ffda786ac..428230866714 100644 --- a/go.sum +++ b/go.sum @@ -429,8 +429,8 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dR github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/boynux/squid-exporter v1.10.5-0.20230618153315-c1fae094e18e h1:C1vYe728vM2FpXaICJuDRt5zgGyRdMmUGYnVfM7WcLY= github.com/boynux/squid-exporter v1.10.5-0.20230618153315-c1fae094e18e/go.mod h1:8NpZERGK+R9DGuZqqsKfnf2qI/rh7yBT8End29IvgNA= -github.com/bufbuild/connect-go v1.9.0 h1:JIgAeNuFpo+SUPfU19Yt5TcWlznsN5Bv10/gI/6Pjoc= -github.com/bufbuild/connect-go v1.9.0/go.mod h1:CAIePUgkDR5pAFaylSMtNK45ANQjp9JvpluG20rhpV8= +github.com/bufbuild/connect-go v1.10.0 h1:QAJ3G9A1OYQW2Jbk3DeoJbkCxuKArrvZgDt47mjdTbg= +github.com/bufbuild/connect-go v1.10.0/go.mod h1:CAIePUgkDR5pAFaylSMtNK45ANQjp9JvpluG20rhpV8= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/burningalchemist/sql_exporter v0.0.0-20221222155641-2ff59aa75200 h1:1zECtssRshqhP8+DELKyWeg8rxaRC5OO72kJQhrJOE8= @@ -1076,8 +1076,8 @@ github.com/grafana/pyroscope-go/godeltaprof v0.1.3 h1:eunWpv1B3Z7ZK9o4499EmQGlY+ github.com/grafana/pyroscope-go/godeltaprof v0.1.3/go.mod h1:1HSPtjU8vLG0jE9JrTdzjgFqdJ/VgN7fvxBNq3luJko= github.com/grafana/pyroscope/api v0.2.0 h1:TzOxL0s6SiaLEy944ZAKgHcx/JDRJXu4O8ObwkqR6p4= github.com/grafana/pyroscope/api v0.2.0/go.mod h1:nhH+xai9cYFgs6lMy/+L0pKj0d5yCMwji/QAiQFCP+U= -github.com/grafana/pyroscope/ebpf v0.2.3 h1:OH7Un2x0UN998U85by4vyvImHs6mkFTo45SnO+PjHdk= -github.com/grafana/pyroscope/ebpf v0.2.3/go.mod h1:NO9mIMKewDuohQlYaj2Q0v3miUmREjGpadz8RuA76Jw= +github.com/grafana/pyroscope/ebpf v0.3.0 h1:7J24qd5ul786ZNDpgN8oIDbuBEXX2L829BjNNm3KVYw= +github.com/grafana/pyroscope/ebpf v0.3.0/go.mod h1:f9gfqjtpxYs/1t7L2HskrR5kQMH64gclaCYdtZ1unKo= github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db h1:7aN5cccjIqCLTzedH7MZzRZt5/lsAHch6Z3L2ZGn5FA= github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= github.com/grafana/river v0.2.0 h1:VgBe/+DdYviJ1cYGUCcA8Og0FhSTjN7jdHCe/+OIFV0=