diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index a5e4415f3..532eadfb0 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -227,6 +227,17 @@ func (p *balloons) AllocateResources(c cache.Container) error { log.Infof("not handling resources of container %s, preserving CPUs %q and memory %q", c.PrettyName(), c.GetCpusetCpus(), c.GetCpusetMems()) return nil } + + if p.bpoptions.Preserve != nil { + rule, err := p.bpoptions.Preserve.MatchContainer(c) + if err != nil { + log.Errorf("error in matching container %s to preserve conditions: %s", c, err) + } else if rule != "" { + log.Debugf("preserve container %s due to matching %s", c, rule) + return nil + } + } + log.Debug("allocating resources for container %s (request %d mCPU, limit %d mCPU)...", c.PrettyName(), p.containerRequestedMilliCpus(c.GetID()), diff --git a/config/crd/bases/config.nri_balloonspolicies.yaml b/config/crd/bases/config.nri_balloonspolicies.yaml index 16f1d3872..f672f734f 100644 --- a/config/crd/bases/config.nri_balloonspolicies.yaml +++ b/config/crd/bases/config.nri_balloonspolicies.yaml @@ -393,6 +393,54 @@ spec: overridden with the balloon type specific setting with the same name. type: boolean + preserve: + description: |- + Preserve specifies containers whose resource pinning must not be + modified by the policy. + properties: + matchExpressions: + description: MatchExpressions specifies one or more expressions. + items: + description: |- + Expression describes some runtime-evaluated condition. An expression + consist of a key, an operator and a set of values. An expressions is + evaluated against an object which implements the Evaluable interface. + Evaluating an expression consists of looking up the value for the key + in the object, then using the operator to check it agains the values + of the expression. The result is a single boolean value. An object is + said to satisfy the evaluated expression if this value is true. An + expression can contain 0, 1 or more values depending on the operator. + properties: + key: + description: Key is the expression key. + type: string + operator: + description: Op is the expression operator. + enum: + - Equals + - NotEqual + - In + - NotIn + - Exists + - NotExist + - AlwaysTrue + - Matches + - MatchesNot + - MatchesAny + - MatchesNone + type: string + values: + description: Values contains the values the key value is + evaluated against. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object reservedPoolNamespaces: description: |- ReservedPoolNamespaces is a list of namespace globs that diff --git a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml index 16f1d3872..f672f734f 100644 --- a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml +++ b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml @@ -393,6 +393,54 @@ spec: overridden with the balloon type specific setting with the same name. type: boolean + preserve: + description: |- + Preserve specifies containers whose resource pinning must not be + modified by the policy. + properties: + matchExpressions: + description: MatchExpressions specifies one or more expressions. + items: + description: |- + Expression describes some runtime-evaluated condition. An expression + consist of a key, an operator and a set of values. An expressions is + evaluated against an object which implements the Evaluable interface. + Evaluating an expression consists of looking up the value for the key + in the object, then using the operator to check it agains the values + of the expression. The result is a single boolean value. An object is + said to satisfy the evaluated expression if this value is true. An + expression can contain 0, 1 or more values depending on the operator. + properties: + key: + description: Key is the expression key. + type: string + operator: + description: Op is the expression operator. + enum: + - Equals + - NotEqual + - In + - NotIn + - Exists + - NotExist + - AlwaysTrue + - Matches + - MatchesNot + - MatchesAny + - MatchesNone + type: string + values: + description: Values contains the values the key value is + evaluated against. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object reservedPoolNamespaces: description: |- ReservedPoolNamespaces is a list of namespace globs that diff --git a/docs/resource-policy/policy/balloons.md b/docs/resource-policy/policy/balloons.md index 86f781ba7..3a46f2d7e 100644 --- a/docs/resource-policy/policy/balloons.md +++ b/docs/resource-policy/policy/balloons.md @@ -82,6 +82,23 @@ Balloons policy parameters: cause kernel to kill containers due to out-of-memory error when closest NUMA nodes do not have enough memory. In this situation consider switching this option `false`. +- `preserve` specifies containers whose resource pinning must not be + modified by the policy. + - `matchExpressions` if a container matches an expression in this + list, the policy will preserve container's resource pinning. If + there is no resource pinning, the policy will not change that + either. Example: preserve containers named "a" and "b". As a + result, the policy will not modify CPU or memory pinning of + matching containers. + ``` + ignore: + matchExpressions: + - key: name + operator: In + values: + - a + - b + ``` - `idleCPUClass` specifies the CPU class of those CPUs that do not belong to any balloon. - `reservedPoolNamespaces` is a list of namespaces (wildcards allowed) @@ -315,8 +332,16 @@ not defined, a built-in `default` balloon type is used. ## Disabling CPU or Memory Pinning of a Container Some containers may need to run on all CPUs or access all memories -without restrictions. Annotate these pods and containers to prevent -the resource policy from touching their CPU or memory pinning. +without restrictions. There are two alternatives to achieve this: +policy configuration and pod annotations. + +The resource policy will not touch allowed resources of containers +that match `preserve` criteria. See policy configuration options +above. + +Alternatively, pod annotations can opt-out all or selected containers +in the pod from CPU or memory pinning by preserving whatever existing +or non-existing pinning configuration: ```yaml cpu.preserve.resource-policy.nri.io/container.CONTAINER_NAME: "true" diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go index 04b582e63..31414a3dc 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go @@ -21,6 +21,7 @@ import ( policy "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" resmgr "github.com/containers/nri-plugins/pkg/apis/resmgr/v1alpha1" "github.com/containers/nri-plugins/pkg/cpuallocator" + "github.com/containers/nri-plugins/pkg/resmgr/cache" ) type ( @@ -78,6 +79,9 @@ type Config struct { // Reserved (CPU) resources for kube-system namespace. // +kubebuilder:validation:Required ReservedResources Constraints `json:"reservedResources"` + // Preserve specifies containers whose resource pinning must not be + // modified by the policy. + Preserve *ContainerMatchConfig `json:"preserve,omitempty"` } type CPUTopologyLevel string @@ -252,8 +256,31 @@ func (p CPUPriority) Value() cpuallocator.CPUPriority { return cpuallocator.PriorityNone } +// ContainerMatchConfig contains container matching configurations. +// +k8s:deepcopy-gen=true +type ContainerMatchConfig struct { + // MatchExpressions specifies one or more expressions. + MatchExpressions []resmgr.Expression `json:"matchExpressions,omitempty"` +} + +func (cmc *ContainerMatchConfig) MatchContainer(c cache.Container) (string, error) { + for _, expr := range cmc.MatchExpressions { + if expr.Evaluate(c) { + return expr.String(), nil + } + } + return "", nil +} + func (c *Config) Validate() error { errs := []error{} + if c.Preserve != nil { + for _, expr := range c.Preserve.MatchExpressions { + if err := expr.Validate(); err != nil { + errs = append(errs, err) + } + } + } for _, blnDef := range c.BalloonDefs { for _, expr := range blnDef.MatchExpressions { if err := expr.Validate(); err != nil { diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go index ed78429e7..38cc957ac 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go @@ -118,6 +118,11 @@ func (in *Config) DeepCopyInto(out *Config) { (*out)[key] = val } } + if in.Preserve != nil { + in, out := &in.Preserve, &out.Preserve + *out = new(ContainerMatchConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Config. @@ -129,3 +134,25 @@ func (in *Config) DeepCopy() *Config { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContainerMatchConfig) DeepCopyInto(out *ContainerMatchConfig) { + *out = *in + if in.MatchExpressions != nil { + in, out := &in.MatchExpressions, &out.MatchExpressions + *out = make([]v1alpha1.Expression, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerMatchConfig. +func (in *ContainerMatchConfig) DeepCopy() *ContainerMatchConfig { + if in == nil { + return nil + } + out := new(ContainerMatchConfig) + in.DeepCopyInto(out) + return out +} diff --git a/test/e2e/policies.test-suite/balloons/match-config.yaml b/test/e2e/policies.test-suite/balloons/match-config.yaml index 29d68649e..8ec1c1032 100644 --- a/test/e2e/policies.test-suite/balloons/match-config.yaml +++ b/test/e2e/policies.test-suite/balloons/match-config.yaml @@ -5,6 +5,15 @@ config: pinMemory: true idleCPUClass: normal allocatorTopologyBalancing: true + preserve: + matchExpressions: + - key: name + operator: In + values: + - pod1cX + - pod2c1 + - pod2c2 + - pod3cX balloonTypes: - name: special matchExpressions: diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh index 70b065532..551a6b7eb 100644 --- a/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh +++ b/test/e2e/policies.test-suite/balloons/n4c16/test11-match-expression/code.var.sh @@ -21,4 +21,13 @@ CONTCOUNT=1 create balloons-busybox report allowed verify 'len(cpus["pod1c0"]) == 1' +# pod2: run ordinary workload where pod2c1 and pod2c2 match the +# preserve matchexpression in policy configuration. +CONTCOUNT=4 create balloons-busybox +report allowed +verify 'len(cpus["pod2c0"]) == 1' \ + 'len(cpus["pod2c1"]) == 16' \ + 'len(cpus["pod2c2"]) == 16' \ + 'len(cpus["pod2c3"]) == 1' + cleanup