controlplane: Use leader election for control controllers

This commit adds leader election for control controllers. Signed-off-by: Or Ozeri <[email protected]>
clusterlink-net · Jun 19, 2024 · a4dd48b · a4dd48b
1 parent 3443ecb
commit a4dd48b
Show file tree

Hide file tree

Showing 6 changed files with 62 additions and 11 deletions.
diff --git a/cmd/cl-controlplane/app/server.go b/cmd/cl-controlplane/app/server.go
@@ -165,7 +165,10 @@ func (o *Options) Run() error {
 				},
 			},
 		},
-		Scheme: scheme,
+		LeaderElection:          true,
+		LeaderElectionNamespace: namespace,
+		LeaderElectionID:        "cl-controlplane",
+		Scheme:                  scheme,
 	}
 
 	mgr, err := manager.New(config, managerOptions)

diff --git a/config/operator/rbac/role.yaml b/config/operator/rbac/role.yaml
@@ -4,6 +4,13 @@ kind: ClusterRole
 metadata:
   name: cl-operator-manager-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - update
 - apiGroups:
   - ""
   resources:
@@ -97,6 +104,14 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - update
 - apiGroups:
   - discovery.k8s.io
   resources:

diff --git a/pkg/bootstrap/platform/k8s.go b/pkg/bootstrap/platform/k8s.go
@@ -198,9 +198,15 @@ kind: ClusterRole
 metadata:
   name: cl-controlplane
 rules:
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "update"]
 - apiGroups: [""]
   resources: ["services"]
   verbs: ["get", "list", "watch", "create", "delete", "update"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "create", "update"]
 - apiGroups: ["discovery.k8s.io"]
   resources: ["endpointslices"]
   verbs: ["get", "list", "watch", "create", "delete", "update"]

diff --git a/pkg/controlplane/control/controllers.go b/pkg/controlplane/control/controllers.go
@@ -29,8 +29,9 @@ import (
 // CreateControllers creates the various k8s controllers used to update the control manager.
 func CreateControllers(mgr *Manager, controllerManager ctrl.Manager) error {
 	err := controller.AddToManager(controllerManager, &controller.Spec{
-		Name:   "control.peer",
-		Object: &v1alpha1.Peer{},
+		Name:                "control.peer",
+		Object:              &v1alpha1.Peer{},
+		NeedsLeaderElection: true,
 		AddHandler: func(ctx context.Context, object any) error {
 			mgr.AddPeer(object.(*v1alpha1.Peer))
 			return nil
@@ -44,8 +45,9 @@ func CreateControllers(mgr *Manager, controllerManager ctrl.Manager) error {
 		return err
 	}
 	err = controller.AddToManager(controllerManager, &controller.Spec{
-		Name:   "control.service",
-		Object: &v1.Service{},
+		Name:                "control.service",
+		Object:              &v1.Service{},
+		NeedsLeaderElection: true,
 		AddHandler: func(ctx context.Context, object any) error {
 			return mgr.addService(ctx, object.(*v1.Service))
 		},
@@ -58,8 +60,9 @@ func CreateControllers(mgr *Manager, controllerManager ctrl.Manager) error {
 	}
 
 	err = controller.AddToManager(controllerManager, &controller.Spec{
-		Name:   "control.export",
-		Object: &v1alpha1.Export{},
+		Name:                "control.export",
+		Object:              &v1alpha1.Export{},
+		NeedsLeaderElection: true,
 		AddHandler: func(ctx context.Context, object any) error {
 			return mgr.AddExport(ctx, object.(*v1alpha1.Export))
 		},
@@ -72,8 +75,9 @@ func CreateControllers(mgr *Manager, controllerManager ctrl.Manager) error {
 	}
 
 	err = controller.AddToManager(controllerManager, &controller.Spec{
-		Name:   "control.import",
-		Object: &v1alpha1.Import{},
+		Name:                "control.import",
+		Object:              &v1alpha1.Import{},
+		NeedsLeaderElection: true,
 		AddHandler: func(ctx context.Context, object any) error {
 			return mgr.AddImport(ctx, object.(*v1alpha1.Import))
 		},
@@ -84,8 +88,9 @@ func CreateControllers(mgr *Manager, controllerManager ctrl.Manager) error {
 	}
 
 	return controller.AddToManager(controllerManager, &controller.Spec{
-		Name:   "control.endpointslice",
-		Object: &discv1.EndpointSlice{},
+		Name:                "control.endpointslice",
+		Object:              &discv1.EndpointSlice{},
+		NeedsLeaderElection: true,
 		AddHandler: func(ctx context.Context, object any) error {
 			return mgr.addEndpointSlice(ctx, object.(*discv1.EndpointSlice))
 		},

diff --git a/pkg/operator/controller/instance_controller.go b/pkg/operator/controller/instance_controller.go
@@ -65,7 +65,9 @@ type InstanceReconciler struct {
 // +kubebuilder:rbac:groups=clusterlink.net,resources=instances,verbs=list;get;watch;update;patch
 // +kubebuilder:rbac:groups=clusterlink.net,resources=instances/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=clusterlink.net,resources=instances/finalizers,verbs=update
+// +kubebuilder:rbac:groups="",resources=events,verbs=create;update
 // +kubebuilder:rbac:groups="",resources=services;serviceaccounts,verbs=list;get;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups="coordination.k8s.io",resources=leases,verbs=get;create;update
 // +kubebuilder:rbac:groups="discovery.k8s.io",resources=endpointslices,verbs=list;get;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups="",resources=nodes,verbs=list;get;watch
 // +kubebuilder:rbac:groups="",resources=pods,verbs=list;get;watch
@@ -421,13 +423,27 @@ func (r *InstanceReconciler) createAccessControl(ctx context.Context, name, name
 			Name: name + namespace,
 		},
 		Rules: []rbacv1.PolicyRule{
+			{
+				APIGroups: []string{""},
+				Resources: []string{"events"},
+				Verbs: []string{
+					"create", "update",
+				},
+			},
 			{
 				APIGroups: []string{""},
 				Resources: []string{"services"},
 				Verbs: []string{
 					"get", "list", "watch", "create", "delete", "update",
 				},
 			},
+			{
+				APIGroups: []string{"coordination.k8s.io"},
+				Resources: []string{"leases"},
+				Verbs: []string{
+					"get", "create", "update",
+				},
+			},
 			{
 				APIGroups: []string{"discovery.k8s.io"},
 				Resources: []string{"endpointslices"},

diff --git a/pkg/util/controller/controller.go b/pkg/util/controller/controller.go
@@ -22,6 +22,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 )
 
 // Spec holds everything needed to create a controller.
@@ -30,6 +31,8 @@ type Spec struct {
 	Name string
 	// Object being watched.
 	Object client.Object
+	// NeedsLeaderElection determines if the controller needs leader election.
+	NeedsLeaderElection bool
 	// AddHandler handles object create/update.
 	AddHandler func(ctx context.Context, object any) error
 	// DeleteHandler handles object deletes.
@@ -75,5 +78,8 @@ func newReconciler(clnt client.Client, spec *Spec) *reconciler {
 func AddToManager(manager ctrl.Manager, spec *Spec) error {
 	return ctrl.NewControllerManagedBy(manager).
 		For(spec.Object).
+		WithOptions(controller.Options{
+			NeedLeaderElection: &spec.NeedsLeaderElection,
+		}).
 		Complete(newReconciler(manager.GetClient(), spec))
 }