Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding backoff on runc originated processes #497

Merged
merged 2 commits into from
Mar 5, 2025
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 41 additions & 8 deletions pkg/processmanager/v1/process_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"strings"
"time"

"github.com/cenkalti/backoff/v4"
"github.com/goradd/maps"
containercollection "github.com/inspektor-gadget/inspektor-gadget/pkg/container-collection"
"github.com/prometheus/procfs"
Expand Down Expand Up @@ -279,20 +280,52 @@ func (p *ProcessManager) GetProcessTreeForPID(containerID string, pid int) (apit
// If the process is runc, try to fetch the real process info.
// Intentionally we are doing this only once the process is asked for to avoid unnecessary calls to /proc and give time for the process to be created.
if strings.HasPrefix(result.Comm, runCCommPrefix) {
if process, err := p.getProcessFromProc(int(result.PID)); err == nil {
childerns := result.Children
upperLayer := result.UpperLayer
result = process
result.Children = childerns
result.UpperLayer = upperLayer
// Update the process in the tree
p.processTree.Set(result.PID, result)
if resolvedProcess, err := p.resolveRuncProcess(result); err == nil {
result = resolvedProcess
} else {
logger.L().Debug("ProcessManager - failed to resolve runc process",
helpers.Int("pid", int(result.PID)),
helpers.String("comm", result.Comm),
helpers.Error(err))
}
}

return result, nil
}

func (p *ProcessManager) resolveRuncProcess(process apitypes.Process) (apitypes.Process, error) {
err := backoff.Retry(func() error {
resolvedProcess, err := p.getProcessFromProc(int(process.PID))
if err != nil {
return err
}

if strings.HasPrefix(resolvedProcess.Comm, runCCommPrefix) {
return fmt.Errorf("runc process not resolved yet")
}

childerns := process.Children
upperLayer := process.UpperLayer
process = resolvedProcess
process.Children = childerns
process.UpperLayer = upperLayer

// Update the process in the tree
p.processTree.Set(process.PID, process)
return nil
}, backoff.NewExponentialBackOff(
backoff.WithInitialInterval(50*time.Millisecond),
backoff.WithMaxInterval(100*time.Millisecond),
backoff.WithMaxElapsedTime(2*time.Second),
))

if err != nil {
return apitypes.Process{}, fmt.Errorf("failed to resolve runc process: %v", err)
}

return process, nil
}

// ReportEvent handles process execution events from the system.
// It specifically processes execve events to track new process creations
// and updates the process tree accordingly.
Expand Down
Loading