controlplane: Store JWK keys in k8s secret

Today, the JWK is initialized by the controlplane when loading, and stored in-memory. To support multiple controlpanes with the same JWK key, we change the controlplane authz manager to ready the JWK key from a secret, instead of generating it.manager The secret is created by the (soon to be leader elected) control manager. Signed-off-by: Or Ozeri <[email protected]>
clusterlink-net · Jun 19, 2024 · f34049c · f34049c
1 parent 3443ecb
commit f34049c
Show file tree

Hide file tree

Showing 8 changed files with 268 additions and 43 deletions.
diff --git a/cmd/cl-controlplane/app/server.go b/cmd/cl-controlplane/app/server.go
@@ -163,6 +163,11 @@ func (o *Options) Run() error {
 						namespace: {},
 					},
 				},
+				&v1.Secret{}: {
+					Namespaces: map[string]cache.Config{
+						namespace: {},
+					},
+				},
 			},
 		},
 		Scheme: scheme,
@@ -177,11 +182,7 @@ func (o *Options) Run() error {
 	controlplaneServerListenAddress := fmt.Sprintf("0.0.0.0:%d", api.ListenPort)
 	grpcServer := grpc.NewServer("controlplane-grpc", controlplaneCertData.ServerConfig())
 
-	authzManager, err := authz.NewManager(mgr.GetClient(), namespace)
-	if err != nil {
-		return fmt.Errorf("cannot create authorization manager: %w", err)
-	}
-
+	authzManager := authz.NewManager(mgr.GetClient(), namespace)
 	peerCertsWatcher.AddConsumer(authzManager)
 
 	err = authz.CreateControllers(authzManager, mgr)
@@ -193,6 +194,9 @@ func (o *Options) Run() error {
 
 	controlManager := control.NewManager(mgr.GetClient(), namespace)
 	peerCertsWatcher.AddConsumer(controlManager)
+	if err := controlManager.CreateJWKSSecret(context.Background()); err != nil {
+		return fmt.Errorf("cannot create JWKS secret: %w", err)
+	}
 
 	err = control.CreateControllers(controlManager, mgr)
 	if err != nil {

diff --git a/config/operator/rbac/role.yaml b/config/operator/rbac/role.yaml
@@ -20,6 +20,16 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
+  - get
+  - list
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:

diff --git a/pkg/bootstrap/platform/k8s.go b/pkg/bootstrap/platform/k8s.go
@@ -201,6 +201,9 @@ rules:
 - apiGroups: [""]
   resources: ["services"]
   verbs: ["get", "list", "watch", "create", "delete", "update"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list", "watch", "create", "update"]
 - apiGroups: ["discovery.k8s.io"]
   resources: ["endpointslices"]
   verbs: ["get", "list", "watch", "create", "delete", "update"]

diff --git a/pkg/controlplane/authz/controllers.go b/pkg/controlplane/authz/controllers.go
@@ -93,7 +93,21 @@ func CreateControllers(mgr *Manager, controllerManager ctrl.Manager) error {
 		AddHandler: func(ctx context.Context, object any) error {
 			return nil
 		},
-		DeleteHandler: func(ctx context.Context, name types.NamespacedName) error {
+		DeleteHandler: func(_ context.Context, _ types.NamespacedName) error {
+			return nil
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	err = controller.AddToManager(controllerManager, &controller.Spec{
+		Name:   "authz.secret",
+		Object: &v1.Secret{},
+		AddHandler: func(_ context.Context, object any) error {
+			return mgr.addSecret(object.(*v1.Secret))
+		},
+		DeleteHandler: func(context.Context, types.NamespacedName) error {
 			return nil
 		},
 	})

diff --git a/pkg/controlplane/authz/manager.go b/pkg/controlplane/authz/manager.go
@@ -15,8 +15,6 @@ package authz
 
 import (
 	"context"
-	"crypto/rand"
-	"crypto/rsa"
 	"fmt"
 	"sync"
 	"time"
@@ -33,6 +31,7 @@ import (
 	"github.com/clusterlink-net/clusterlink/pkg/apis/clusterlink.net/v1alpha1"
 	cpapi "github.com/clusterlink-net/clusterlink/pkg/controlplane/api"
 	"github.com/clusterlink-net/clusterlink/pkg/controlplane/authz/connectivitypdp"
+	"github.com/clusterlink-net/clusterlink/pkg/controlplane/control"
 	"github.com/clusterlink-net/clusterlink/pkg/controlplane/peer"
 	"github.com/clusterlink-net/clusterlink/pkg/util/tls"
 )
@@ -109,6 +108,7 @@ type Manager struct {
 	ipToPod map[string]types.NamespacedName
 	podList map[types.NamespacedName]podInfo
 
+	jwksLock     sync.RWMutex
 	jwkSignKey   jwk.Key
 	jwkVerifyKey jwk.Key
 
@@ -176,6 +176,35 @@ func (m *Manager) addPod(pod *v1.Pod) {
 	}
 }
 
+// addSecret adds a new secret.
+func (m *Manager) addSecret(secret *v1.Secret) error {
+	if secret.Namespace != m.namespace || secret.Name != control.JWKSecretName {
+		return nil
+	}
+
+	privateKey, err := control.ParseJWKSSecret(secret)
+	if err != nil {
+		return fmt.Errorf("cannot parse JWKS secret: %w", err)
+	}
+
+	jwkSignKey, err := jwk.New(privateKey)
+	if err != nil {
+		return fmt.Errorf("unable to create JWK signing key: %w", err)
+	}
+
+	jwkVerifyKey, err := jwk.New(privateKey.PublicKey)
+	if err != nil {
+		return fmt.Errorf("unable to create JWK verifing key: %w", err)
+	}
+
+	m.jwksLock.Lock()
+	defer m.jwksLock.Unlock()
+	m.jwkSignKey = jwkSignKey
+	m.jwkVerifyKey = jwkVerifyKey
+
+	return nil
+}
+
 // getPodInfoByIP returns the information about the Pod with the specified IP address.
 func (m *Manager) getPodInfoByIP(ip string) *podInfo {
 	m.podLock.RLock()
@@ -292,8 +321,16 @@ func (m *Manager) authorizeEgress(ctx context.Context, req *egressAuthorizationR
 func (m *Manager) parseAuthorizationHeader(token string) (string, error) {
 	m.logger.Debug("Parsing access token.")
 
+	m.jwksLock.RLock()
+	jwkVerifyKey := m.jwkVerifyKey
+	m.jwksLock.RUnlock()
+
+	if jwkVerifyKey == nil {
+		return "", fmt.Errorf("jwk verify key undefined")
+	}
+
 	parsedToken, err := jwt.ParseString(
-		token, jwt.WithVerify(cpapi.JWTSignatureAlgorithm, m.jwkVerifyKey), jwt.WithValidate(true))
+		token, jwt.WithVerify(cpapi.JWTSignatureAlgorithm, jwkVerifyKey), jwt.WithValidate(true))
 	if err != nil {
 		return "", err
 	}
@@ -369,8 +406,16 @@ func (m *Manager) authorizeIngress(
 		return nil, fmt.Errorf("unable to generate access token: %w", err)
 	}
 
+	m.jwksLock.RLock()
+	jwkSignKey := m.jwkSignKey
+	m.jwksLock.RUnlock()
+
+	if jwkSignKey == nil {
+		return nil, fmt.Errorf("jwk sign key undefined")
+	}
+
 	// sign access token
-	signed, err := jwt.Sign(token, cpapi.JWTSignatureAlgorithm, m.jwkSignKey)
+	signed, err := jwt.Sign(token, cpapi.JWTSignatureAlgorithm, jwkSignKey)
 	if err != nil {
 		return nil, fmt.Errorf("unable to sign access token: %w", err)
 	}
@@ -411,34 +456,15 @@ func (m *Manager) SetPeerCertificates(peerTLS *tls.ParsedCertData, _ *tls.RawCer
 }
 
 // NewManager returns a new authorization manager.
-func NewManager(cl client.Client, namespace string) (*Manager, error) {
-	// generate RSA key-pair for JWT signing
-	// TODO: instead of generating, read from k8s secret
-	rsaKey, err := rsa.GenerateKey(rand.Reader, 2048)
-	if err != nil {
-		return nil, fmt.Errorf("unable to generate RSA keys: %w", err)
-	}
-
-	jwkSignKey, err := jwk.New(rsaKey)
-	if err != nil {
-		return nil, fmt.Errorf("unable to create JWK signing key: %w", err)
-	}
-
-	jwkVerifyKey, err := jwk.New(rsaKey.PublicKey)
-	if err != nil {
-		return nil, fmt.Errorf("unable to create JWK verifing key: %w", err)
-	}
-
+func NewManager(cl client.Client, namespace string) *Manager {
 	return &Manager{
 		client:          cl,
 		namespace:       namespace,
 		connectivityPDP: connectivitypdp.NewPDP(),
 		loadBalancer:    NewLoadBalancer(),
 		peerClient:      make(map[string]*peer.Client),
-		jwkSignKey:      jwkSignKey,
-		jwkVerifyKey:    jwkVerifyKey,
 		ipToPod:         make(map[string]types.NamespacedName),
 		podList:         make(map[types.NamespacedName]podInfo),
 		logger:          logrus.WithField("component", "controlplane.authz.manager"),
-	}, nil
+	}
 }
diff --git a/pkg/controlplane/control/controllers.go b/pkg/controlplane/control/controllers.go
@@ -61,7 +61,7 @@ func CreateControllers(mgr *Manager, controllerManager ctrl.Manager) error {
 		Name:   "control.export",
 		Object: &v1alpha1.Export{},
 		AddHandler: func(ctx context.Context, object any) error {
-			return mgr.AddExport(ctx, object.(*v1alpha1.Export))
+			return mgr.addExport(ctx, object.(*v1alpha1.Export))
 		},
 		DeleteHandler: func(ctx context.Context, name types.NamespacedName) error {
 			return nil
@@ -75,9 +75,21 @@ func CreateControllers(mgr *Manager, controllerManager ctrl.Manager) error {
 		Name:   "control.import",
 		Object: &v1alpha1.Import{},
 		AddHandler: func(ctx context.Context, object any) error {
-			return mgr.AddImport(ctx, object.(*v1alpha1.Import))
+			return mgr.addImport(ctx, object.(*v1alpha1.Import))
 		},
-		DeleteHandler: mgr.DeleteImport,
+		DeleteHandler: mgr.deleteImport,
+	})
+	if err != nil {
+		return err
+	}
+
+	err = controller.AddToManager(controllerManager, &controller.Spec{
+		Name:   "control.secret",
+		Object: &v1.Secret{},
+		AddHandler: func(ctx context.Context, object any) error {
+			return mgr.addSecret(ctx, object.(*v1.Secret))
+		},
+		DeleteHandler: mgr.deleteSecret,
 	})
 	if err != nil {
 		return err