feat(qrm): adding support to cpu.weight

Signed-off-by: Robin Lu <[email protected]>

cmd/katalyst-agent/app/options/qrm/cpu_plugin.go

@@ -31,6 +31,8 @@ type CPUOptions struct {
 
 	CPUDynamicPolicyOptions
 	CPUNativePolicyOptions
+
+	CPUWeightOptions
 }
 
 type CPUDynamicPolicyOptions struct {
@@ -48,6 +50,11 @@ type CPUNativePolicyOptions struct {
 	CPUAllocationOption        string
 }
 
+type CPUWeightOptions struct {
+	EnableSettingCPUWeight bool
+	CPUWeightConfigFile    string
+}
+
 func NewCPUOptions() *CPUOptions {
 	return &CPUOptions{
 		PolicyName:             "dynamic",
@@ -70,6 +77,10 @@ func NewCPUOptions() *CPUOptions {
 			EnableFullPhysicalCPUsOnly: false,
 			CPUAllocationOption:        "packed",
 		},
+		CPUWeightOptions: CPUWeightOptions{
+			EnableSettingCPUWeight: false,
+			CPUWeightConfigFile:    "",
+		},
 	}
 }
 
@@ -103,6 +114,10 @@ func (o *CPUOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 	fs.BoolVar(&o.EnableFullPhysicalCPUsOnly, "enable-full-physical-cpus-only",
 		o.EnableFullPhysicalCPUsOnly, "if set true, we will enable extra allocation restrictions to "+
 			"avoid different containers to possibly end up on the same core.")
+	fs.BoolVar(&o.EnableSettingCPUWeight, "enable-setting-cpu-weight",
+		o.EnableSettingCPUWeight, "if set true, we will enable cpu weight related features")
+	fs.StringVar(&o.CPUWeightConfigFile, "qrm-cpu-weight-config-file",
+		o.CPUWeightConfigFile, "cpu weight config file")
 }
 
 func (o *CPUOptions) ApplyTo(conf *qrmconfig.CPUQRMPluginConfig) error {
@@ -118,5 +133,7 @@ func (o *CPUOptions) ApplyTo(conf *qrmconfig.CPUQRMPluginConfig) error {
 	conf.CPUAllocationOption = o.CPUAllocationOption
 	conf.CPUNUMAHintPreferPolicy = o.CPUNUMAHintPreferPolicy
 	conf.CPUNUMAHintPreferLowThreshold = o.CPUNUMAHintPreferLowThreshold
+	conf.EnableSettingCPUWeight = o.EnableSettingCPUWeight
+	conf.CPUWeightConfigFile = o.CPUWeightConfigFile
 	return nil
 }

pkg/agent/qrm-plugins/cpu/consts/consts.go

@@ -32,6 +32,7 @@ const (
 	CheckCPUSet                = CPUPluginDynamicPolicyName + "_check_cpuset"
 	SyncCPUIdle                = CPUPluginDynamicPolicyName + "_sync_cpu_idle"
 	CommunicateWithAdvisor     = CPUPluginDynamicPolicyName + "_communicate_with_advisor"
+	SetCPUWeight               = CPUPluginDynamicPolicyName + "_set_cpu_weight"
 )
 
 const (

pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go

@@ -42,6 +42,7 @@ import (
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/cpueviction"
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state"
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/validator"
+	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/handlers/cpuweight"
 	cpuutil "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/util"
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util"
 	"github.com/kubewharf/katalyst-core/pkg/agent/utilcomponent/periodicalhandler"
@@ -133,6 +134,7 @@ type DynamicPolicy struct {
 	transitionPeriod              time.Duration
 	cpuNUMAHintPreferPolicy       string
 	cpuNUMAHintPreferLowThreshold float64
+	enableSettingCPUWeight        bool
 }
 
 func NewDynamicPolicy(agentCtx *agent.GenericContext, conf *config.Configuration,
@@ -197,6 +199,7 @@ func NewDynamicPolicy(agentCtx *agent.GenericContext, conf *config.Configuration
 		enableCPUAdvisor:              conf.CPUQRMPluginConfig.EnableCPUAdvisor,
 		cpuNUMAHintPreferPolicy:       conf.CPUQRMPluginConfig.CPUNUMAHintPreferPolicy,
 		cpuNUMAHintPreferLowThreshold: conf.CPUQRMPluginConfig.CPUNUMAHintPreferLowThreshold,
+		enableSettingCPUWeight:        conf.EnableSettingCPUWeight,
 		reservedCPUs:                  reservedCPUs,
 		extraStateFileAbsPath:         conf.ExtraStateFileAbsPath,
 		enableSyncingCPUIdle:          conf.CPUQRMPluginConfig.EnableSyncingCPUIdle,
@@ -332,6 +335,15 @@ func (p *DynamicPolicy) Start() (err error) {
 			p.cpuAdvisorSocketAbsPath, p.cpuPluginSocketAbsPath)
 	}
 
+	if p.enableSettingCPUWeight {
+		general.Infof("setCPUWeight enabled")
+		err = periodicalhandler.RegisterPeriodicalHandlerWithHealthz(cpuconsts.SetCPUWeight, general.HealthzCheckStateNotReady,
+			qrm.QRMCPUPluginPeriodicalHandlerGroupName, cpuweight.SetCPUWeight, 1800*time.Second, healthCheckTolerationTimes)
+		if err != nil {
+			general.Errorf("start %v failed,err:%v", cpuconsts.SetCPUWeight, err)
+		}
+	}
+
 	general.Infof("start dynamic policy cpu plugin with sys-advisor")
 	err = p.initAdvisorClientConn()
 	if err != nil {

pkg/agent/qrm-plugins/cpu/handlers/cpuweight/const.go

@@ -0,0 +1,21 @@
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cpuweight
+
+const (
+	metricNameCPUWeight = "async_handler_cpu_weight"
+)

pkg/agent/qrm-plugins/cpu/handlers/cpuweight/cpuweight_linux.go

@@ -0,0 +1,93 @@
+//go:build linux
+// +build linux
+
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cpuweight
+
+import (
+	"k8s.io/apimachinery/pkg/util/errors"
+
+	cpuconsts "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/consts"
+	coreconfig "github.com/kubewharf/katalyst-core/pkg/config"
+	dynamicconfig "github.com/kubewharf/katalyst-core/pkg/config/agent/dynamic"
+	"github.com/kubewharf/katalyst-core/pkg/metaserver"
+	"github.com/kubewharf/katalyst-core/pkg/metrics"
+	cgroupcm "github.com/kubewharf/katalyst-core/pkg/util/cgroup/common"
+	cgroupmgr "github.com/kubewharf/katalyst-core/pkg/util/cgroup/manager"
+	"github.com/kubewharf/katalyst-core/pkg/util/general"
+)
+
+func applyCPUWeightCgroupLevelConfig(conf *coreconfig.Configuration, emitter metrics.MetricEmitter) {
+	if conf.CPUWeightConfigFile == "" {
+		general.Errorf("CPUWeightConfigFile isn't configured")
+		return
+	}
+
+	cpuWightCgroupLevelConfigs := make(map[string]uint64)
+	err := general.LoadJsonConfig(conf.CPUWeightConfigFile, &cpuWightCgroupLevelConfigs)
+	if err != nil {
+		general.Errorf("load CPUWeightCgroupLevelConfig failed with error: %v", err)
+		return
+	}
+
+	for relativeCgPath, weight := range cpuWightCgroupLevelConfigs {
+		err := cgroupmgr.ApplyCPUWithRelativePath(relativeCgPath, &cgroupcm.CPUData{
+			Shares: weight,
+		})
+		if err != nil {
+			general.Errorf("ApplyCPUWeightWithRelativePath in relativeCgPath: %s failed with error: %v",
+				relativeCgPath, err)
+		} else {
+			general.Infof("ApplyCPUWeightWithRelativePath weight: %d in relativeCgPath: %s successfully",
+				weight, relativeCgPath)
+			_ = emitter.StoreInt64(metricNameCPUWeight, int64(weight), metrics.MetricTypeNameRaw,
+				metrics.ConvertMapToTags(map[string]string{
+					"cgPath": relativeCgPath,
+				})...)
+
+		}
+	}
+}
+
+func SetCPUWeight(conf *coreconfig.Configuration,
+	_ interface{}, _ *dynamicconfig.DynamicAgentConfiguration,
+	emitter metrics.MetricEmitter, metaServer *metaserver.MetaServer,
+) {
+	general.Infof("called")
+
+	var errList []error
+	defer func() {
+		_ = general.UpdateHealthzStateByError(cpuconsts.SetCPUWeight, errors.NewAggregate(errList))
+	}()
+
+	if conf == nil || emitter == nil || metaServer == nil {
+		general.Errorf("nil input, conf:%v, emitter:%v, metaServer:%v", conf, emitter, metaServer)
+		return
+	}
+
+	// SettingCPUWeight featuregate.
+	if !conf.EnableSettingCPUWeight {
+		general.Infof("SetCPUWeight disabled")
+		return
+	}
+
+	// checking cgroup-level cpu.weight configuration.
+	if len(conf.CPUWeightConfigFile) > 0 {
+		applyCPUWeightCgroupLevelConfig(conf, emitter)
+	}
+}

pkg/agent/qrm-plugins/cpu/handlers/cpuweight/cpuweight_unsupported.go

@@ -0,0 +1,31 @@
+//go:build !linux
+
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cpuweight
+
+import (
+	coreconfig "github.com/kubewharf/katalyst-core/pkg/config"
+	dynamicconfig "github.com/kubewharf/katalyst-core/pkg/config/agent/dynamic"
+	"github.com/kubewharf/katalyst-core/pkg/metaserver"
+	"github.com/kubewharf/katalyst-core/pkg/metrics"
+)
+
+func SetCpuWeight(conf *coreconfig.Configuration,
+	_ interface{}, _ *dynamicconfig.DynamicAgentConfiguration,
+	emitter metrics.MetricEmitter, metaServer *metaserver.MetaServer) {
+}

pkg/config/agent/qrm/cpu_plugin.go

@@ -26,6 +26,7 @@ type CPUQRMPluginConfig struct {
 
 	CPUDynamicPolicyConfig
 	CPUNativePolicyConfig
+	CPUWeightOptions
 }
 
 type CPUDynamicPolicyConfig struct {
@@ -55,6 +56,11 @@ type CPUNativePolicyConfig struct {
 	CPUAllocationOption string
 }
 
+type CPUWeightOptions struct {
+	EnableSettingCPUWeight bool
+	CPUWeightConfigFile    string
+}
+
 func NewCPUQRMPluginConfig() *CPUQRMPluginConfig {
 	return &CPUQRMPluginConfig{
 		CPUDynamicPolicyConfig: CPUDynamicPolicyConfig{},