diff --git a/PROJECT b/PROJECT index ef0b3b2eb..82500e9e4 100644 --- a/PROJECT +++ b/PROJECT @@ -88,4 +88,11 @@ resources: kind: RedisSentinel path: redis-operator/api/v1beta1 version: v1beta1 +- group: core + kind: Pod + path: k8s.io/api/core/v1 + version: v1 + webhooks: + defaulting: true + webhookVersion: v1 version: "3" diff --git a/charts/redis-operator/templates/mutating-webhook-configuration.yaml b/charts/redis-operator/templates/mutating-webhook-configuration.yaml new file mode 100644 index 000000000..b7f715fcf --- /dev/null +++ b/charts/redis-operator/templates/mutating-webhook-configuration.yaml @@ -0,0 +1,34 @@ +{{ if .Values.redisOperator.webhook }} + +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/serving-cert +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: {{ .Release.Namespace }} + path: /mutate-core-v1-pod + failurePolicy: Fail + name: ot-mutate-pod.opstree.com + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None + objectSelector: + matchExpressions: + - key: redis_setup_type + operator: Exists + +{{ end }} \ No newline at end of file diff --git a/charts/redis-operator/values.yaml b/charts/redis-operator/values.yaml index a438be8e4..50a195c54 100644 --- a/charts/redis-operator/values.yaml +++ b/charts/redis-operator/values.yaml @@ -17,6 +17,7 @@ redisOperator: # When not specified, the operator will watch all namespaces. It can be set to a specific namespace or multiple namespaces separated by commas. watchNamespace: "" env: [] + # If you want to enable masterSlaveAntiAffinity, you need to set webhook to true. webhook: false automountServiceAccountToken: true diff --git a/example/v1beta2/redis-cluster-deploy/role-anti-affinity.yaml b/example/v1beta2/redis-cluster-deploy/role-anti-affinity.yaml new file mode 100644 index 000000000..b191888bc --- /dev/null +++ b/example/v1beta2/redis-cluster-deploy/role-anti-affinity.yaml @@ -0,0 +1,55 @@ +--- +# Redis Cluster Custom Resource Definition +# This configuration file defines a Redis Cluster setup with anti-affinity rules +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisCluster +metadata: + name: redis-cluster + annotations: + # Enable pod anti-affinity between leader and follower pods + # This ensures leader and follower pods are scheduled on different nodes + # for better high availability + # PS: you should enable operator webhook for this to work + redisclusters.redis.redis.opstreelabs.in/role-anti-affinity: "true" +spec: + clusterSize: 3 + clusterVersion: v7 + persistenceEnabled: true + podSecurityContext: + runAsUser: 1000 + fsGroup: 1000 + kubernetesConfig: + image: quay.io/opstree/redis:v7.0.12 + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 101m + memory: 128Mi + limits: + cpu: 101m + memory: 128Mi + redisExporter: + enabled: false + image: quay.io/opstree/redis-exporter:v1.44.0 + imagePullPolicy: Always + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 100m + memory: 128Mi + storage: + volumeClaimTemplate: + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + nodeConfVolume: false + nodeConfVolumeClaimTemplate: + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/main.go b/main.go index 900b121c7..eefca7401 100644 --- a/main.go +++ b/main.go @@ -29,6 +29,7 @@ import ( "github.com/OT-CONTAINER-KIT/redis-operator/pkg/controllers/redissentinel" intctrlutil "github.com/OT-CONTAINER-KIT/redis-operator/pkg/controllerutil" "github.com/OT-CONTAINER-KIT/redis-operator/pkg/k8sutils" + coreWebhook "github.com/OT-CONTAINER-KIT/redis-operator/pkg/webhook" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -39,6 +40,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) var ( @@ -172,6 +174,11 @@ func main() { setupLog.Error(err, "unable to create webhook", "webhook", "RedisSentinel") os.Exit(1) } + + wblog := ctrl.Log.WithName("webhook").WithName("PodAffiniytMutate") + mgr.GetWebhookServer().Register("/mutate-core-v1-pod", &webhook.Admission{ + Handler: coreWebhook.NewPodAffiniytMutate(mgr.GetClient(), admission.NewDecoder(scheme), wblog), + }) } // +kubebuilder:scaffold:builder diff --git a/pkg/webhook/pod_webhook.go b/pkg/webhook/pod_webhook.go new file mode 100644 index 000000000..bc5fb0d01 --- /dev/null +++ b/pkg/webhook/pod_webhook.go @@ -0,0 +1,169 @@ +/* +Copyright 2020 Opstree Solutions. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "context" + "encoding/json" + "net/http" + "reflect" + "strings" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +//+kubebuilder:webhook:path=/mutate-core-v1-pod,mutating=true,failurePolicy=fail,sideEffects=None,groups=core,resources=pods,verbs=create,versions=v1,name=mpod.kb.io,admissionReviewVersions=v1 + +// PodAntiAffiniytMutate mutate Pods +type PodAntiAffiniytMutate struct { + Client client.Client + decoder *admission.Decoder + logger logr.Logger +} + +func NewPodAffiniytMutate(c client.Client, d *admission.Decoder, log logr.Logger) admission.Handler { + return &PodAntiAffiniytMutate{ + Client: c, + decoder: d, + logger: log, + } +} + +const ( + podAnnotationsRedisClusterApp = "redis.opstreelabs.instance" + podLabelsPodName = "statefulset.kubernetes.io/pod-name" + podLabelsRedisType = "redis_setup_type" +) + +const annotationKeyEnablePodAntiAffinity = "redisclusters.redis.redis.opstreelabs.in/role-anti-affinity" + +func (v *PodAntiAffiniytMutate) Handle(ctx context.Context, req admission.Request) admission.Response { + logger := v.logger.WithValues("Request.Namespace", req.Namespace, "Request.Name", req.Name) + + pod := &corev1.Pod{} + err := v.decoder.Decode(req, pod) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + // only mutate pods that belong to redis cluster + if !v.isRedisClusterPod(pod) { + return admission.Allowed("") + } + // check if the pod anti-affinity is enabled + annotations := pod.GetAnnotations() + if annotations == nil { + return admission.Allowed("") + } + if enable, ok := annotations[annotationKeyEnablePodAntiAffinity]; !ok || enable != "true" { + logger.V(1).Info("pod anti-affinity is not enabled") + return admission.Allowed("") + } + + old := pod.DeepCopy() + + v.AddPodAntiAffinity(pod) + if !reflect.DeepEqual(old, pod) { + marshaledPod, err := json.Marshal(pod) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + + logger.Info("mutate pod with anti-affinity") + return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod) + } + + return admission.Allowed("") +} + +// PodAntiAffiniytMutate implements admission.DecoderInjector. +// A decoder will be automatically injected. + +// InjectDecoder injects the decoder. +func (v *PodAntiAffiniytMutate) InjectDecoder(d *admission.Decoder) error { + v.decoder = d + return nil +} + +func (m *PodAntiAffiniytMutate) InjectLogger(l logr.Logger) error { + m.logger = l + return nil +} + +func (v *PodAntiAffiniytMutate) AddPodAntiAffinity(pod *corev1.Pod) { + podName := pod.ObjectMeta.Name + antiLabelValue := v.getAntiAffinityValue(podName) + + if pod.Spec.Affinity == nil { + pod.Spec.Affinity = &corev1.Affinity{} + } + if pod.Spec.Affinity.PodAntiAffinity == nil { + pod.Spec.Affinity.PodAntiAffinity = &corev1.PodAntiAffinity{} + } + if pod.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + pod.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = make([]corev1.PodAffinityTerm, 0) + } + addAntiAffinity := corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: podLabelsPodName, + Operator: metav1.LabelSelectorOpIn, + Values: []string{antiLabelValue}, + }, + }, + }, + TopologyKey: "kubernetes.io/hostname", + } + + pod.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append(pod.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, addAntiAffinity) +} + +func (v *PodAntiAffiniytMutate) getPodAnnotations(pod *corev1.Pod) map[string]string { + if pod.Annotations == nil { + pod.Annotations = make(map[string]string) + } + return pod.Annotations +} + +func (v *PodAntiAffiniytMutate) isRedisClusterPod(pod *corev1.Pod) bool { + annotations := v.getPodAnnotations(pod) + if _, ok := annotations[podAnnotationsRedisClusterApp]; !ok { + return false + } + + labels := pod.GetLabels() + if _, ok := labels[podLabelsRedisType]; !ok { + return false + } + + return true +} + +func (v *PodAntiAffiniytMutate) getAntiAffinityValue(podName string) string { + if strings.Contains(podName, "follower") { + return strings.Replace(podName, "follower", "leader", -1) + } + if strings.Contains(podName, "leader") { + return strings.Replace(podName, "leader", "follower", -1) + } + return "" +} diff --git a/pkg/webhook/pod_webhook_test.go b/pkg/webhook/pod_webhook_test.go new file mode 100644 index 000000000..03ad4ef29 --- /dev/null +++ b/pkg/webhook/pod_webhook_test.go @@ -0,0 +1,113 @@ +package webhook + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "gomodules.xyz/jsonpatch/v2" + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +func TestPodAntiAffinityMutate_Handle(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + decoder := admission.NewDecoder(scheme) + logger := zap.New(zap.WriteTo(nil)) + + mutator := NewPodAffiniytMutate(fakeClient, decoder, logger) + + tests := []struct { + name string + pod *corev1.Pod + expectedPatches []jsonpatch.JsonPatchOperation + }{ + { + name: "Should mutate pod with anti-affinity", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "redis-leader-0", + Namespace: "default", + Annotations: map[string]string{ + annotationKeyEnablePodAntiAffinity: "true", + podAnnotationsRedisClusterApp: "db-01", + }, + Labels: map[string]string{ + podLabelsRedisType: "redis-cluster", + }, + }, + Spec: corev1.PodSpec{}, + }, + expectedPatches: []jsonpatch.JsonPatchOperation{ + { + Operation: "add", + Path: "/spec/affinity", + Value: map[string]interface{}{ + "podAntiAffinity": map[string]interface{}{ + "requiredDuringSchedulingIgnoredDuringExecution": []interface{}{ + map[string]interface{}{ + "labelSelector": map[string]interface{}{ + "matchExpressions": []interface{}{ + map[string]interface{}{ + "key": "statefulset.kubernetes.io/pod-name", + "operator": "In", + "values": []interface{}{"redis-follower-0"}, + }, + }, + }, + "topologyKey": "kubernetes.io/hostname", + }, + }, + }, + }, + }, + }, + }, + { + name: "Should not mutate pod without proper annotations", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "redis-follower-0", + Namespace: "default", + Annotations: map[string]string{ + podAnnotationsRedisClusterApp: "db-01", + }, + Labels: map[string]string{ + podLabelsRedisType: "redis-cluster", + }, + }, + Spec: corev1.PodSpec{}, + }, + expectedPatches: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + podBytes, err := json.Marshal(tt.pod) + assert.NoError(t, err) + + req := admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: "default", + Object: runtime.RawExtension{ + Raw: podBytes, + }, + }, + } + + resp := mutator.Handle(context.Background(), req) + assert.True(t, resp.Allowed) + assert.Equal(t, tt.expectedPatches, resp.Patches) + }) + } +}