Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: use allowed cpus from target container #256

Merged
merged 1 commit into from
Feb 5, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions extcontainer/action_stress.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// SPDX-License-Identifier: MIT
// SPDX-FileCopyrightText: 2024 Steadybit GmbH
// SPDX-FileCopyrightText: 2025 Steadybit GmbH

package extcontainer

Expand Down Expand Up @@ -92,7 +92,7 @@ func (a *stressAction) Prepare(ctx context.Context, state *StressActionState, re
return nil, err
}

readAndAdaptToContainerLimits(ctx, processInfo.CGroupPath, &opts)
readAndAdaptToContainerLimits(ctx, processInfo, &opts)

state.StressOpts = opts
state.ExecutionId = request.ExecutionId
Expand Down
39 changes: 28 additions & 11 deletions extcontainer/action_stress_limit_helpers.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// SPDX-License-Identifier: MIT
// SPDX-FileCopyrightText: 2024 Steadybit GmbH
// SPDX-FileCopyrightText: 2025 Steadybit GmbH

package extcontainer

Expand All @@ -8,39 +8,56 @@ import (
"fmt"
"github.com/kataras/iris/v12/x/mathx"
"github.com/rs/zerolog/log"
"github.com/steadybit/action-kit/go/action_kit_commons/runc"
"github.com/steadybit/action-kit/go/action_kit_commons/stress"
"github.com/steadybit/action-kit/go/action_kit_commons/utils"
"github.com/steadybit/extension-kit/extutil"
"math"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
)

var cgroupV1MemUnlimited = (math.MaxInt64 / os.Getpagesize()) * os.Getpagesize()
var osFs = osFileSystem{}

func readAndAdaptToContainerLimits(_ context.Context, cGroupPath string, opts *stress.Opts) {
func readAndAdaptToContainerLimits(_ context.Context, p runc.LinuxProcessInfo, opts *stress.Opts) {
cpuLimitInMilliCpu := -1
memLimitInBytes := -1

if isCGroupV1() {
cpuLimitInMilliCpu = readCGroupV1CpuLimit(cGroupPath, osFs)
memLimitInBytes = readCGroupV1MemLimit(cGroupPath, osFs)
cpuLimitInMilliCpu = readCGroupV1CpuLimit(p.CGroupPath, osFs)
memLimitInBytes = readCGroupV1MemLimit(p.CGroupPath, osFs)
} else {
cpuLimitInMilliCpu = readCGroupV2CpuLimit(cGroupPath, osFs)
memLimitInBytes = readCGroupV2MemLimit(cGroupPath, osFs)
cpuLimitInMilliCpu = readCGroupV2CpuLimit(p.CGroupPath, osFs)
memLimitInBytes = readCGroupV2MemLimit(p.CGroupPath, osFs)
}

if opts.CpuWorkers != nil && cpuLimitInMilliCpu >= 0 {
adaptToCpuContainerLimits(cpuLimitInMilliCpu, runtime.NumCPU(), opts)
if opts.CpuWorkers != nil {
if cpuLimitInMilliCpu >= 0 {
adaptToCpuContainerLimits(cpuLimitInMilliCpu, opts)
} else if *opts.CpuWorkers == 0 {
//there might be no limit set but the process to be restricted to certain CPUs or some CPUs programmatically turned off.
//In this case we need to read for the allowed list of CPUs for the process and pass this to the stress command as stress-ng
//always uses configured CPUs and not online CPUs
adaptToAllowedCpus(p.Pid, opts)
}
}

if opts.VmWorkers != nil && memLimitInBytes >= 0 {
adaptToMemContainerLimits(memLimitInBytes, opts)
}
}

func adaptToAllowedCpus(pid int, opts *stress.Opts) {
if cpuCount, err := utils.ReadCpusAllowedCount(fmt.Sprintf("/proc/%d/status", pid)); err != nil {
opts.CpuWorkers = extutil.Ptr(cpuCount)
} else {
log.Debug().Err(err).Msg("failed to read cpus_allowed count.")
}
}

func adaptToMemContainerLimits(memLimitInBytes int, opts *stress.Opts) {
memConsumptionInPercent := 0
if _, err := fmt.Sscanf(opts.VmBytes, "%d%%", &memConsumptionInPercent); err != nil {
Expand All @@ -55,9 +72,9 @@ func adaptToMemContainerLimits(memLimitInBytes int, opts *stress.Opts) {
log.Info().Msgf("container memory limit is %dK. Starting %d workers with memory consumption of %s each", memLimitInBytes/1024, *opts.VmWorkers, opts.VmBytes)
}

func adaptToCpuContainerLimits(cpuLimitInMilliCpu int, cpuCount int, opts *stress.Opts) {
func adaptToCpuContainerLimits(cpuLimitInMilliCpu int, opts *stress.Opts) {
cpuLoadInMillis := cpuLimitInMilliCpu * opts.CpuLoad / 100
log.Debug().Int("cpuCount", cpuCount).Int("cpuLoad", opts.CpuLoad).Int("cpuLoadInMillis", cpuLoadInMillis).Msg("adapting to container cpu limit")
log.Debug().Int("cpuLoad", opts.CpuLoad).Int("cpuLoadInMillis", cpuLoadInMillis).Msg("adapting to container cpu limit")

if *opts.CpuWorkers == 0 {
// user didn't specify the number of workers. we start as many workers as we need to reach the desired cpu consumption
Expand Down
10 changes: 4 additions & 6 deletions extcontainer/action_stress_limit_helpers_test.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
// SPDX-License-Identifier: MIT
// SPDX-FileCopyrightText: 2025 Steadybit GmbH

package extcontainer

import (
Expand All @@ -10,7 +13,6 @@ import (
func Test_adaptToCpuContainerLimits(t *testing.T) {
type args struct {
cpuLimitInMilliCpu int
cpuCount int
givenCpuWorkers int
givenCpuLoad int
}
Expand All @@ -27,7 +29,6 @@ func Test_adaptToCpuContainerLimits(t *testing.T) {
name: "worker-count not specified, desired cpu load can be handled by one worker",
args: args{
cpuLimitInMilliCpu: 200,
cpuCount: 4,
givenCpuLoad: 100,
givenCpuWorkers: 0,
},
Expand All @@ -40,7 +41,6 @@ func Test_adaptToCpuContainerLimits(t *testing.T) {
name: "worker-count not specified, desired cpu load needs multiple workers",
args: args{
cpuLimitInMilliCpu: 1500,
cpuCount: 4,
givenCpuLoad: 100,
givenCpuWorkers: 0,
},
Expand All @@ -53,7 +53,6 @@ func Test_adaptToCpuContainerLimits(t *testing.T) {
name: "worker-count not specified, desired 60% cpu fits to single worker",
args: args{
cpuLimitInMilliCpu: 1500,
cpuCount: 4,
givenCpuLoad: 60,
givenCpuWorkers: 0,
},
Expand All @@ -66,7 +65,6 @@ func Test_adaptToCpuContainerLimits(t *testing.T) {
name: "worker-count specified, desired 60% cpu is spread across workers",
args: args{
cpuLimitInMilliCpu: 1500,
cpuCount: 4,
givenCpuLoad: 60,
givenCpuWorkers: 3,
},
Expand All @@ -82,7 +80,7 @@ func Test_adaptToCpuContainerLimits(t *testing.T) {
CpuWorkers: &tt.args.givenCpuWorkers,
CpuLoad: tt.args.givenCpuLoad,
}
adaptToCpuContainerLimits(tt.args.cpuLimitInMilliCpu, tt.args.cpuCount, &opts)
adaptToCpuContainerLimits(tt.args.cpuLimitInMilliCpu, &opts)
assert.Equal(t, tt.expected.adaptedCpuWorkers, *opts.CpuWorkers)
assert.Equal(t, tt.expected.adaptedCpuLoad, opts.CpuLoad)
})
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ require (
github.com/opencontainers/runtime-spec v1.2.0
github.com/rs/zerolog v1.33.0
github.com/steadybit/action-kit/go/action_kit_api/v2 v2.9.6
github.com/steadybit/action-kit/go/action_kit_commons v1.2.17
github.com/steadybit/action-kit/go/action_kit_commons v1.2.18
github.com/steadybit/action-kit/go/action_kit_sdk v1.1.14
github.com/steadybit/action-kit/go/action_kit_test v1.3.2
github.com/steadybit/discovery-kit/go/discovery_kit_api v1.6.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -257,8 +257,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
github.com/steadybit/action-kit/go/action_kit_api/v2 v2.9.6 h1:Qci7Numf66mjCIRo7KDwHUimIxUZzq+GBfyv/1f/QCU=
github.com/steadybit/action-kit/go/action_kit_api/v2 v2.9.6/go.mod h1:ycF2RLgRsB8I/jD52aE+dKZKVru1GIEtmkcRcIR3vXk=
github.com/steadybit/action-kit/go/action_kit_commons v1.2.17 h1:NOKvnUA/iZo7mlcgSK+c8EHjxAEYCLSdyk8KVQSXmv8=
github.com/steadybit/action-kit/go/action_kit_commons v1.2.17/go.mod h1:GFLcaf/WluBIqnq+iSCEGkEqxnfB9h1I16IBR1T7kNQ=
github.com/steadybit/action-kit/go/action_kit_commons v1.2.18 h1:zg3c7b/Dl0M/odtNMbbhwVn/EbnvPZO14IPwbFy9WbY=
github.com/steadybit/action-kit/go/action_kit_commons v1.2.18/go.mod h1:GFLcaf/WluBIqnq+iSCEGkEqxnfB9h1I16IBR1T7kNQ=
github.com/steadybit/action-kit/go/action_kit_sdk v1.1.14 h1:x94RX+vh9Iyc0tS6BhiSpvknj+xE36AV0Nc3D5Yuub0=
github.com/steadybit/action-kit/go/action_kit_sdk v1.1.14/go.mod h1:Tp/klK5b7k+BCvb3JTSDZSNcnpFBaHauhndzOarnMW4=
github.com/steadybit/action-kit/go/action_kit_test v1.3.2 h1:DFDznoWEbTGv+fiGYiRaq7tq5es9VTScjrWusRAbS08=
Expand Down
Loading