Skip to content

Commit

Permalink
Extend Python version support (#2019)
Browse files Browse the repository at this point in the history 
* Extend Python version support

Signed-off-by: Kemal Akkoyun <[email protected]>

* Upgrade runtime-data

Signed-off-by: Kemal Akkoyun <[email protected]>

* Fix process relative path issue

Signed-off-by: Kemal Akkoyun <[email protected]>

* Only rely on pthread id

Signed-off-by: Kemal Akkoyun <[email protected]>

* Add proper error handling

Signed-off-by: Kemal Akkoyun <[email protected]>

---------

Signed-off-by: Kemal Akkoyun <[email protected]>
  • Loading branch information
@kakkoyun
kakkoyun authored Sep 15, 2023
1 parent 722df66 commit c2b3751
Show file tree
Hide file tree
Showing 9 changed files with 120 additions and 223 deletions.
39 changes: 13 additions & 26 deletions bpf/cpu/pyperf.bpf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ struct {
}

#define GET_OFFSETS() \
PythonVersionOffsets *offsets = bpf_map_lookup_elem(&version_specific_offsets, &state->interpreter_info.py_version); \
PythonVersionOffsets *offsets = bpf_map_lookup_elem(&version_specific_offsets, &state->interpreter_info.py_version_offset_index); \
if (offsets == NULL) { \
return 0; \
}
Expand Down Expand Up @@ -178,32 +178,19 @@ int unwind_python_stack(struct bpf_perf_event_data *ctx) {
GET_OFFSETS();

// Fetch the thread id.

// Python 3.11+ uses native_thread_id.
if (offsets->py_thread_state.native_thread_id > 0) {
u64 thread_id = 0;
LOG("offsets->py_thread_state.native_thread_id %d", offsets->py_thread_state.native_thread_id);
bpf_probe_read_user(&thread_id, sizeof(thread_id), state->thread_state + offsets->py_thread_state.native_thread_id);
LOG("thread_id %d", thread_id);
if (thread_id != tid) {
LOG("[error] thread_id %d != tid %d", thread_id, tid);
goto submit_without_unwinding;
}
} else {
LOG("offsets->py_thread_state.thread_id %d", offsets->py_thread_state.thread_id);
pthread_t pthread_id;
bpf_probe_read_user(&pthread_id, sizeof(pthread_id), state->thread_state + offsets->py_thread_state.thread_id);
LOG("pthread_id %lu", pthread_id);
// 0x10 = offsetof(tcbhead_t, self) for glibc.
pthread_t current_pthread_id;
bpf_probe_read_user(&current_pthread_id, sizeof(current_pthread_id), (void *)(tls_base + 0x10));
LOG("current_pthread_id %lu", current_pthread_id);
if (pthread_id != current_pthread_id) {
LOG("[error] pthread_id %lu != current_pthread_id %lu", pthread_id, current_pthread_id);
goto submit_without_unwinding;
}
state->current_pthread = current_pthread_id;
LOG("offsets->py_thread_state.thread_id %d", offsets->py_thread_state.thread_id);
pthread_t pthread_id;
bpf_probe_read_user(&pthread_id, sizeof(pthread_id), state->thread_state + offsets->py_thread_state.thread_id);
LOG("pthread_id %lu", pthread_id);
// 0x10 = offsetof(tcbhead_t, self) for glibc.
pthread_t current_pthread_id;
bpf_probe_read_user(&current_pthread_id, sizeof(current_pthread_id), (void *)(tls_base + 0x10));
LOG("current_pthread_id %lu", current_pthread_id);
if (pthread_id != current_pthread_id) {
LOG("[error] pthread_id %lu != current_pthread_id %lu", pthread_id, current_pthread_id);
goto submit_without_unwinding;
}
state->current_pthread = current_pthread_id;

// Get pointer to top frame from PyThreadState.

Expand Down
16 changes: 7 additions & 9 deletions bpf/cpu/pyperf.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,9 @@

typedef struct {
// u64 start_time;

// u64 interpreter_addr;
u64 thread_state_addr;

u32 py_version;
u32 py_version_offset_index;
} InterpreterInfo;

enum python_stack_status {
Expand Down Expand Up @@ -122,14 +120,14 @@ typedef struct {
u32 minor_version;
u32 patch_version;

PyObject py_object;
PyString py_string;
PyTypeObject py_type_object;
PyThreadState py_thread_state;
PyCFrame py_cframe;
PyCodeObject py_code_object;
PyFrameObject py_frame_object;
PyInterpreterState py_interpreter_state;
PyObject py_object;
PyRuntimeState py_runtime_state;
PyFrameObject py_frame_object;
PyCodeObject py_code_object;
PyString py_string;
PyThreadState py_thread_state;
PyTupleObject py_tuple_object;
PyTypeObject py_type_object;
} PythonVersionOffsets;
5 changes: 2 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
module github.com/parca-dev/parca-agent

go 1.21

toolchain go1.21.0
go 1.21.1

require (
buf.build/gen/go/prometheus/prometheus/protocolbuffers/go v1.31.0-20230726221845-41588ce133c8.1
Expand All @@ -29,6 +27,7 @@ require (
github.com/oklog/run v1.1.0
github.com/opencontainers/runtime-spec v1.1.0
github.com/parca-dev/parca v0.18.1-0.20230911134148-b79ac4f2315e
github.com/parca-dev/runtime-data v0.0.0-20230915153615-e9cfa97d7207
github.com/planetscale/vtprotobuf v0.5.0
github.com/prometheus/client_golang v1.16.0
github.com/prometheus/common v0.44.0
Expand Down
4 changes: 4 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -475,6 +475,10 @@ github.com/oracle/oci-go-sdk/v65 v65.41.1 h1:+lbosOyNiib3TGJDvLq1HwEAuFqkOjPJDIk
github.com/oracle/oci-go-sdk/v65 v65.41.1/go.mod h1:MXMLMzHnnd9wlpgadPkdlkZ9YrwQmCOmbX5kjVEJodw=
github.com/parca-dev/parca v0.18.1-0.20230911134148-b79ac4f2315e h1:EpnbLVDngNw2q3+V+G4GrlioyH26lt8QbdYZP6i7HWg=
github.com/parca-dev/parca v0.18.1-0.20230911134148-b79ac4f2315e/go.mod h1:vRD2H9Oxxbj1EFz2YyXs9GlHJIixJitDBdZu6SCGMhM=
github.com/parca-dev/runtime-data v0.0.0-20230914172248-4c59ab3f3177 h1:lVJWPCfiEY6c2PNTH8mWawv8uqGKYw4MFblGiwboISw=
github.com/parca-dev/runtime-data v0.0.0-20230914172248-4c59ab3f3177/go.mod h1:0AFERcIlkTaq5C6IVouvQvhGl0KPOfey+9XwszMAR3E=
github.com/parca-dev/runtime-data v0.0.0-20230915153615-e9cfa97d7207 h1:rDI/wVDetmh2qCBSViJtPIUZZLdwNqKXHR+8wFhJB5Y=
github.com/parca-dev/runtime-data v0.0.0-20230915153615-e9cfa97d7207/go.mod h1:0AFERcIlkTaq5C6IVouvQvhGl0KPOfey+9XwszMAR3E=
github.com/parquet-go/parquet-go v0.18.0 h1:tKWzKhgmwT24Yc45RLTbEOtUT8jfTzLTWrIUX4yG7/E=
github.com/parquet-go/parquet-go v0.18.0/go.mod h1:6pu/Ca02WRyWyF6jbY1KceESGBZMsRMSijjLbajXaG8=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
Expand Down
133 changes: 55 additions & 78 deletions pkg/profiler/cpu/maps.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ import (
"github.com/prometheus/procfs"
"golang.org/x/exp/constraints"

"github.com/parca-dev/runtime-data/pkg/python"

"github.com/parca-dev/parca-agent/pkg/buildid"
"github.com/parca-dev/parca-agent/pkg/cache"
"github.com/parca-dev/parca-agent/pkg/elfreader"
Expand Down Expand Up @@ -221,6 +223,7 @@ type bpfMaps struct {

pythonPIDToProcessInfo *bpf.BPFMap
pythonVersionSpecificOffsets *bpf.BPFMap
pythonVersionToOffsetIndex map[string]uint32

unwindShards *bpf.BPFMap
unwindTables *bpf.BPFMap
Expand Down Expand Up @@ -303,16 +306,17 @@ func initializeMaps(logger log.Logger, reg prometheus.Registerer, byteOrder bina
unwindInfoMemory := make([]byte, maxUnwindTableSize*compactUnwindRowSizeBytes)

maps := &bpfMaps{
logger: log.With(logger, "component", "bpf_maps"),
nativeModule: modules[nativeModule],
rbperfModule: modules[rbperfModule],
pyperfModule: modules[pyperfModule],
byteOrder: byteOrder,
processCache: newProcessCache(logger, reg),
mappingInfoMemory: mappingInfoMemory,
unwindInfoMemory: unwindInfoMemory,
buildIDMapping: make(map[string]uint64),
mutex: sync.Mutex{},
logger: log.With(logger, "component", "bpf_maps"),
nativeModule: modules[nativeModule],
rbperfModule: modules[rbperfModule],
pyperfModule: modules[pyperfModule],
byteOrder: byteOrder,
processCache: newProcessCache(logger, reg),
mappingInfoMemory: mappingInfoMemory,
unwindInfoMemory: unwindInfoMemory,
buildIDMapping: make(map[string]uint64),
mutex: sync.Mutex{},
pythonVersionToOffsetIndex: make(map[string]uint32),
}

if err := maps.resetInFlightBuffer(); err != nil {
Expand Down Expand Up @@ -509,8 +513,7 @@ func (m *bpfMaps) setPyperfIntepreterInfo(pid int, interpInfo pyperf.Interpreter
if m.pyperfModule == nil {
return nil
}

pidToPyData, err := m.pyperfModule.GetMap(pythonPIDToInterpreterInfoMapName)
pidToInterpreterInfo, err := m.pyperfModule.GetMap(pythonPIDToInterpreterInfoMapName)
if err != nil {
return fmt.Errorf("get map pid_to_interpreter_info: %w", err)
}
Expand All @@ -524,42 +527,50 @@ func (m *bpfMaps) setPyperfIntepreterInfo(pid int, interpInfo pyperf.Interpreter
}

pidToProcInfoKey := uint32(pid)
err = pidToPyData.Update(unsafe.Pointer(&pidToProcInfoKey), unsafe.Pointer(&buf.Bytes()[0]))
err = pidToInterpreterInfo.Update(unsafe.Pointer(&pidToProcInfoKey), unsafe.Pointer(&buf.Bytes()[0]))
if err != nil {
return fmt.Errorf("update map pid_to_interpreter_info: %w", err)
}
return nil
}

// @norelease: DRY. Move.
func (m *bpfMaps) setPyperfVersionOffsets(versionOffsets pyperf.PythonVersionOffsets) error {
func (m *bpfMaps) setPyperfVersionOffsets(versionOffsets []python.VersionOffsets) error {
if m.pyperfModule == nil {
return nil
}

versions, err := m.pyperfModule.GetMap(pythonVersionSpecificOffsetMapName)
if err != nil {
return fmt.Errorf("get map version_specific_offsets: %w", err)
}

buf := new(bytes.Buffer)
buf.Grow(int(unsafe.Sizeof(&versionOffsets)))

err = binary.Write(buf, binary.LittleEndian, &versionOffsets)
if err != nil {
return fmt.Errorf("write versionOffsets to buffer: %w", err)
if len(versionOffsets) == 0 {
return fmt.Errorf("no version offsets provided")
}

key := uint32(0)
err = versions.Update(unsafe.Pointer(&key), unsafe.Pointer(&buf.Bytes()[0]))
if err != nil {
return fmt.Errorf("update map version_specific_offsets: %w", err)
buf := new(bytes.Buffer)
i := uint32(0)
for _, v := range versionOffsets {
buf.Grow(int(unsafe.Sizeof(&v)))
err = binary.Write(buf, binary.LittleEndian, &v)
if err != nil {
level.Debug(m.logger).Log("msg", "write versionOffsets to buffer", "err", err)
continue
}
key := i
err = versions.Update(unsafe.Pointer(&key), unsafe.Pointer(&buf.Bytes()[0]))
if err != nil {
level.Debug(m.logger).Log("msg", "update map version_specific_offsets", "err", err)
continue
}
m.pythonVersionToOffsetIndex[fmt.Sprintf("%d.%d", v.MajorVersion, v.MinorVersion)] = i
i++
buf.Reset()
}
return nil
}

// TODO(javierhonduco): Add all the supported Ruby versions.
// TODO(kakkoyun): Add all the supported Python versions.
func (m *bpfMaps) setInterpreterData() error {
if m.pyperfModule == nil && m.rbperfModule == nil {
return nil
Expand Down Expand Up @@ -600,53 +611,12 @@ func (m *bpfMaps) setInterpreterData() error {
}

if m.pyperfModule != nil {
err = m.setPyperfVersionOffsets(pyperf.PythonVersionOffsets{
MajorVersion: 3,
MinorVersion: 11,
PatchVersion: 0,
PyObject: pyperf.PyObject{
ObType: 8,
},
PyString: pyperf.PyString{
Data: 48,
Size: -1,
},
PyTypeObject: pyperf.PyTypeObject{
TpName: 24,
},
PyThreadState: pyperf.PyThreadState{
Next: 8,
Interp: 16,
Frame: -1,
ThreadID: 152,
NativeThreadID: 160,
CFrame: 56,
},
PyCFrame: pyperf.PyCFrame{
CurrentFrame: 8,
},
PyInterpreterState: pyperf.PyInterpreterState{
TStateHead: 16,
},
PyRuntimeState: pyperf.PyRuntimeState{
InterpMain: 48,
},
PyFrameObject: pyperf.PyFrameObject{
FBack: 48,
FCode: 32,
FLineno: -1,
FLocalsplus: 72,
},
PyCodeObject: pyperf.PyCodeObject{
CoFilename: 112,
CoName: 120,
CoVarnames: 96,
CoFirstlineno: 72,
},
PyTupleObject: pyperf.PyTupleObject{
ObItem: 24,
},
})
versions, err := python.GetVersions()
if err != nil {
return fmt.Errorf("get python versions: %w", err)
}

err = m.setPyperfVersionOffsets(versions)
if err != nil {
return fmt.Errorf("set pyperf version offsets: %w", err)
}
Expand Down Expand Up @@ -879,10 +849,15 @@ func (m *bpfMaps) addInterpreter(pid int, interpreter runtime.Interpreter) error
}
return m.setRbperfProcessData(pid, procData)
case runtime.InterpreterPython:
i, err := m.indexForPythonVersion(interpreter.Version)
if err != nil {
return fmt.Errorf("index for python version: %w", err)
}
interpreterInfo := pyperf.InterpreterInfo{
ThreadStateAddr: interpreter.MainThreadAddress,
PyVersion: m.indexForPythonVersion(interpreter.Version),
ThreadStateAddr: interpreter.MainThreadAddress,
PyVersionOffsetIndex: i,
}
level.Debug(m.logger).Log("msg", "Python Version Offset", "pid", pid, "version_offset_index", i)
return m.setPyperfIntepreterInfo(pid, interpreterInfo)
default:
return fmt.Errorf("invalid interpreter name: %d", interpreter.Type)
Expand All @@ -894,9 +869,11 @@ func (m *bpfMaps) indexForRubyVersion(version *semver.Version) uint32 {
return 0
}

// TODO(kakkoyun): Add support for all the Python versions.
func (m *bpfMaps) indexForPythonVersion(version *semver.Version) uint32 {
return 0
func (m *bpfMaps) indexForPythonVersion(version *semver.Version) (uint32, error) {
if i, ok := m.pythonVersionToOffsetIndex[fmt.Sprintf("%d.%d", version.Major(), version.Minor())]; ok {
return i, nil
}
return 0, errors.New("unknown Python Version")
}

func (m *bpfMaps) setDebugPIDs(pids []int) error {
Expand Down
Loading

0 comments on commit c2b3751

Please sign in to comment.