From 93abbadc292626a8e68515c2765d5034decc9780 Mon Sep 17 00:00:00 2001 From: Or Ozeri Date: Tue, 18 Jun 2024 14:25:42 +0300 Subject: [PATCH] controlplane: Store JWK keys in k8s secret Today, the JWK is initialized by the controlplane when loading, and stored in-memory. To support multiple controlpanes with the same JWK key, we change the controlplane authz manager to ready the JWK key from a secret, instead of generating it.manager The secret is created by the (soon to be leader elected) control manager. Signed-off-by: Or Ozeri --- cmd/cl-controlplane/app/server.go | 14 +- config/operator/rbac/role.yaml | 10 ++ pkg/bootstrap/platform/k8s.go | 3 + pkg/controlplane/authz/controllers.go | 16 +- pkg/controlplane/authz/manager.go | 76 +++++--- pkg/controlplane/control/controllers.go | 18 +- pkg/controlplane/control/manager.go | 166 +++++++++++++++++- .../controller/instance_controller.go | 8 + 8 files changed, 268 insertions(+), 43 deletions(-) diff --git a/cmd/cl-controlplane/app/server.go b/cmd/cl-controlplane/app/server.go index 6ddf49920..59711b2c9 100644 --- a/cmd/cl-controlplane/app/server.go +++ b/cmd/cl-controlplane/app/server.go @@ -163,6 +163,11 @@ func (o *Options) Run() error { namespace: {}, }, }, + &v1.Secret{}: { + Namespaces: map[string]cache.Config{ + namespace: {}, + }, + }, }, }, LeaderElection: true, @@ -180,11 +185,7 @@ func (o *Options) Run() error { controlplaneServerListenAddress := fmt.Sprintf("0.0.0.0:%d", api.ListenPort) grpcServer := grpc.NewServer("controlplane-grpc", controlplaneCertData.ServerConfig()) - authzManager, err := authz.NewManager(mgr.GetClient(), namespace) - if err != nil { - return fmt.Errorf("cannot create authorization manager: %w", err) - } - + authzManager := authz.NewManager(mgr.GetClient(), namespace) peerCertsWatcher.AddConsumer(authzManager) err = authz.CreateControllers(authzManager, mgr) @@ -196,6 +197,9 @@ func (o *Options) Run() error { controlManager := control.NewManager(mgr.GetClient(), namespace) peerCertsWatcher.AddConsumer(controlManager) + if err := controlManager.CreateJWKSSecret(context.Background()); err != nil { + return fmt.Errorf("cannot create JWKS secret: %w", err) + } err = control.CreateControllers(controlManager, mgr) if err != nil { diff --git a/config/operator/rbac/role.yaml b/config/operator/rbac/role.yaml index 0d526dd67..3b42b3e67 100644 --- a/config/operator/rbac/role.yaml +++ b/config/operator/rbac/role.yaml @@ -27,6 +27,16 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - get + - list + - update + - watch - apiGroups: - "" resources: diff --git a/pkg/bootstrap/platform/k8s.go b/pkg/bootstrap/platform/k8s.go index 3642190bf..efc84ec0e 100644 --- a/pkg/bootstrap/platform/k8s.go +++ b/pkg/bootstrap/platform/k8s.go @@ -207,6 +207,9 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "create", "update"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch", "create", "update"] - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] verbs: ["get", "list", "watch", "create", "delete", "update"] diff --git a/pkg/controlplane/authz/controllers.go b/pkg/controlplane/authz/controllers.go index 1e972acec..6a8debd4e 100644 --- a/pkg/controlplane/authz/controllers.go +++ b/pkg/controlplane/authz/controllers.go @@ -93,7 +93,21 @@ func CreateControllers(mgr *Manager, controllerManager ctrl.Manager) error { AddHandler: func(ctx context.Context, object any) error { return nil }, - DeleteHandler: func(ctx context.Context, name types.NamespacedName) error { + DeleteHandler: func(_ context.Context, _ types.NamespacedName) error { + return nil + }, + }) + if err != nil { + return err + } + + err = controller.AddToManager(controllerManager, &controller.Spec{ + Name: "authz.secret", + Object: &v1.Secret{}, + AddHandler: func(_ context.Context, object any) error { + return mgr.addSecret(object.(*v1.Secret)) + }, + DeleteHandler: func(context.Context, types.NamespacedName) error { return nil }, }) diff --git a/pkg/controlplane/authz/manager.go b/pkg/controlplane/authz/manager.go index 28ac357f0..94a64074d 100644 --- a/pkg/controlplane/authz/manager.go +++ b/pkg/controlplane/authz/manager.go @@ -15,8 +15,6 @@ package authz import ( "context" - "crypto/rand" - "crypto/rsa" "fmt" "sync" "time" @@ -33,6 +31,7 @@ import ( "github.com/clusterlink-net/clusterlink/pkg/apis/clusterlink.net/v1alpha1" cpapi "github.com/clusterlink-net/clusterlink/pkg/controlplane/api" "github.com/clusterlink-net/clusterlink/pkg/controlplane/authz/connectivitypdp" + "github.com/clusterlink-net/clusterlink/pkg/controlplane/control" "github.com/clusterlink-net/clusterlink/pkg/controlplane/peer" "github.com/clusterlink-net/clusterlink/pkg/util/tls" ) @@ -109,6 +108,7 @@ type Manager struct { ipToPod map[string]types.NamespacedName podList map[types.NamespacedName]podInfo + jwksLock sync.RWMutex jwkSignKey jwk.Key jwkVerifyKey jwk.Key @@ -176,6 +176,35 @@ func (m *Manager) addPod(pod *v1.Pod) { } } +// addSecret adds a new secret. +func (m *Manager) addSecret(secret *v1.Secret) error { + if secret.Namespace != m.namespace || secret.Name != control.JWKSecretName { + return nil + } + + privateKey, err := control.ParseJWKSSecret(secret) + if err != nil { + return fmt.Errorf("cannot parse JWKS secret: %w", err) + } + + jwkSignKey, err := jwk.New(privateKey) + if err != nil { + return fmt.Errorf("unable to create JWK signing key: %w", err) + } + + jwkVerifyKey, err := jwk.New(privateKey.PublicKey) + if err != nil { + return fmt.Errorf("unable to create JWK verifing key: %w", err) + } + + m.jwksLock.Lock() + defer m.jwksLock.Unlock() + m.jwkSignKey = jwkSignKey + m.jwkVerifyKey = jwkVerifyKey + + return nil +} + // getPodInfoByIP returns the information about the Pod with the specified IP address. func (m *Manager) getPodInfoByIP(ip string) *podInfo { m.podLock.RLock() @@ -292,8 +321,16 @@ func (m *Manager) authorizeEgress(ctx context.Context, req *egressAuthorizationR func (m *Manager) parseAuthorizationHeader(token string) (string, error) { m.logger.Debug("Parsing access token.") + m.jwksLock.RLock() + jwkVerifyKey := m.jwkVerifyKey + m.jwksLock.RUnlock() + + if jwkVerifyKey == nil { + return "", fmt.Errorf("jwk verify key undefined") + } + parsedToken, err := jwt.ParseString( - token, jwt.WithVerify(cpapi.JWTSignatureAlgorithm, m.jwkVerifyKey), jwt.WithValidate(true)) + token, jwt.WithVerify(cpapi.JWTSignatureAlgorithm, jwkVerifyKey), jwt.WithValidate(true)) if err != nil { return "", err } @@ -369,8 +406,16 @@ func (m *Manager) authorizeIngress( return nil, fmt.Errorf("unable to generate access token: %w", err) } + m.jwksLock.RLock() + jwkSignKey := m.jwkSignKey + m.jwksLock.RUnlock() + + if jwkSignKey == nil { + return nil, fmt.Errorf("jwk sign key undefined") + } + // sign access token - signed, err := jwt.Sign(token, cpapi.JWTSignatureAlgorithm, m.jwkSignKey) + signed, err := jwt.Sign(token, cpapi.JWTSignatureAlgorithm, jwkSignKey) if err != nil { return nil, fmt.Errorf("unable to sign access token: %w", err) } @@ -411,34 +456,15 @@ func (m *Manager) SetPeerCertificates(peerTLS *tls.ParsedCertData, _ *tls.RawCer } // NewManager returns a new authorization manager. -func NewManager(cl client.Client, namespace string) (*Manager, error) { - // generate RSA key-pair for JWT signing - // TODO: instead of generating, read from k8s secret - rsaKey, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - return nil, fmt.Errorf("unable to generate RSA keys: %w", err) - } - - jwkSignKey, err := jwk.New(rsaKey) - if err != nil { - return nil, fmt.Errorf("unable to create JWK signing key: %w", err) - } - - jwkVerifyKey, err := jwk.New(rsaKey.PublicKey) - if err != nil { - return nil, fmt.Errorf("unable to create JWK verifing key: %w", err) - } - +func NewManager(cl client.Client, namespace string) *Manager { return &Manager{ client: cl, namespace: namespace, connectivityPDP: connectivitypdp.NewPDP(), loadBalancer: NewLoadBalancer(), peerClient: make(map[string]*peer.Client), - jwkSignKey: jwkSignKey, - jwkVerifyKey: jwkVerifyKey, ipToPod: make(map[string]types.NamespacedName), podList: make(map[types.NamespacedName]podInfo), logger: logrus.WithField("component", "controlplane.authz.manager"), - }, nil + } } diff --git a/pkg/controlplane/control/controllers.go b/pkg/controlplane/control/controllers.go index 035cd7f7a..807cf327f 100644 --- a/pkg/controlplane/control/controllers.go +++ b/pkg/controlplane/control/controllers.go @@ -64,7 +64,7 @@ func CreateControllers(mgr *Manager, controllerManager ctrl.Manager) error { Object: &v1alpha1.Export{}, NeedsLeaderElection: true, AddHandler: func(ctx context.Context, object any) error { - return mgr.AddExport(ctx, object.(*v1alpha1.Export)) + return mgr.addExport(ctx, object.(*v1alpha1.Export)) }, DeleteHandler: func(ctx context.Context, name types.NamespacedName) error { return nil @@ -79,9 +79,21 @@ func CreateControllers(mgr *Manager, controllerManager ctrl.Manager) error { Object: &v1alpha1.Import{}, NeedsLeaderElection: true, AddHandler: func(ctx context.Context, object any) error { - return mgr.AddImport(ctx, object.(*v1alpha1.Import)) + return mgr.addImport(ctx, object.(*v1alpha1.Import)) }, - DeleteHandler: mgr.DeleteImport, + DeleteHandler: mgr.deleteImport, + }) + if err != nil { + return err + } + + err = controller.AddToManager(controllerManager, &controller.Spec{ + Name: "control.secret", + Object: &v1.Secret{}, + AddHandler: func(ctx context.Context, object any) error { + return mgr.addSecret(ctx, object.(*v1.Secret)) + }, + DeleteHandler: mgr.deleteSecret, }) if err != nil { return err diff --git a/pkg/controlplane/control/manager.go b/pkg/controlplane/control/manager.go index 256ed9f85..4453f7677 100644 --- a/pkg/controlplane/control/manager.go +++ b/pkg/controlplane/control/manager.go @@ -14,7 +14,13 @@ package control import ( + "bytes" "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/pem" "reflect" "strconv" "strings" @@ -51,6 +57,10 @@ const ( // endpoint slice labels. LabelDPEndpointSliceName = "clusterlink.net/dataplane-endpointslice-name" + + // JWK secret. + JWKSecretName = "jwk" + JWKSecretKeyName = "key" ) type exportServiceNotExistError struct { @@ -144,8 +154,8 @@ type Manager struct { logger *logrus.Entry } -// AddImport adds a listening socket for an imported remote service. -func (m *Manager) AddImport(ctx context.Context, imp *v1alpha1.Import) (err error) { +// addImport adds a listening socket for an imported remote service. +func (m *Manager) addImport(ctx context.Context, imp *v1alpha1.Import) (err error) { m.logger.Infof("Adding import '%s/%s'.", imp.Namespace, imp.Name) targetPortValidCond := &metav1.Condition{ @@ -262,8 +272,8 @@ func (m *Manager) AddImport(ctx context.Context, imp *v1alpha1.Import) (err erro return m.addImportService(ctx, imp, userService) } -// DeleteImport removes the listening socket of a previously imported service. -func (m *Manager) DeleteImport(ctx context.Context, name types.NamespacedName) error { +// deleteImport removes the listening socket of a previously imported service. +func (m *Manager) deleteImport(ctx context.Context, name types.NamespacedName) error { m.logger.Infof("Deleting import '%s/%s'.", name.Namespace, name.Name) // delete user service @@ -287,8 +297,8 @@ func (m *Manager) DeleteImport(ctx context.Context, name types.NamespacedName) e return errors.Join(errs...) } -// AddExport defines a new route target for ingress dataplane connections. -func (m *Manager) AddExport(ctx context.Context, export *v1alpha1.Export) (err error) { +// addExport defines a new route target for ingress dataplane connections. +func (m *Manager) addExport(ctx context.Context, export *v1alpha1.Export) (err error) { m.logger.Infof("Adding export '%s/%s'.", export.Namespace, export.Name) defer func() { @@ -372,6 +382,71 @@ func (m *Manager) deleteService(ctx context.Context, name types.NamespacedName) return m.checkExportService(ctx, name) } +// addSecret adds a new secret. +func (m *Manager) addSecret(ctx context.Context, secret *v1.Secret) error { + return m.checkJWKSecret(ctx, types.NamespacedName{ + Namespace: secret.Namespace, + Name: secret.Name, + }) +} + +// deleteSecret deletes a secret. +func (m *Manager) deleteSecret(ctx context.Context, name types.NamespacedName) error { + return m.checkJWKSecret(ctx, name) +} + +func (m *Manager) checkJWKSecret(ctx context.Context, name types.NamespacedName) error { + if name.Namespace != m.namespace || name.Name != JWKSecretName { + return nil + } + + secretName := types.NamespacedName{ + Namespace: m.namespace, + Name: JWKSecretName, + } + + var secret v1.Secret + create := true + if err := m.client.Get(ctx, secretName, &secret); err == nil { + if !k8serrors.IsNotFound(err) { + return err + } + + m.logger.Info("No JWKS secret defined.") + } else { + if _, err := ParseJWKSSecret(&secret); err == nil { + return nil + } + + create = false + m.logger.Warnf("Bad JWKS secret: %v.", err) + } + + m.logger.Info("Generating new JWKS secret.") + jwksSecretData, err := generateJWKSecret() + if err != nil { + return fmt.Errorf("cannot generate JWK secret: %w", err) + } + + newSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: JWKSecretName, + Namespace: m.namespace, + }, + Data: map[string][]byte{ + JWKSecretKeyName: jwksSecretData, + }, + } + + if create { + m.logger.Infof("Creating JWK Secret.") + return m.client.Create(ctx, newSecret) + } + + m.logger.Infof("Updating JWK Secret.") + return m.client.Update(ctx, newSecret) +} + // addEndpointSlice adds a dataplane / import endpoint slices. func (m *Manager) addEndpointSlice(ctx context.Context, endpointSlice *discv1.EndpointSlice) error { if endpointSlice.Labels[discv1.LabelServiceName] == dpapp.Name && endpointSlice.Namespace == m.namespace { @@ -445,7 +520,7 @@ func (m *Manager) checkExportService(ctx context.Context, name types.NamespacedN return nil } - return m.AddExport(ctx, &export) + return m.addExport(ctx, &export) } func (m *Manager) checkImportService(ctx context.Context, name types.NamespacedName) error { @@ -455,7 +530,7 @@ func (m *Manager) checkImportService(ctx context.Context, name types.NamespacedN return err } } else { - if err := m.AddImport(ctx, &imp); err != nil { + if err := m.addImport(ctx, &imp); err != nil { return err } } @@ -477,7 +552,7 @@ func (m *Manager) checkImportService(ctx context.Context, name types.NamespacedN return err } } else { - return m.AddImport(ctx, &imp) + return m.addImport(ctx, &imp) } return nil @@ -773,6 +848,55 @@ func (m *Manager) deleteImportEndpointSlices(ctx context.Context, imp types.Name return nil } +// CreateJWKSSecret creates the JWKS secret if it does not exist. +func (m *Manager) CreateJWKSSecret(ctx context.Context) error { + jwksSecretData, err := generateJWKSecret() + if err != nil { + return fmt.Errorf("cannot generate JWKS secret: %w", err) + } + + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: JWKSecretName, + Namespace: m.namespace, + }, + Data: map[string][]byte{ + JWKSecretKeyName: jwksSecretData, + }, + } + + m.logger.Info("Trying to create JWKS secret.") + if err := m.client.Create(ctx, secret); err != nil { + if !k8serrors.IsAlreadyExists(err) { + return err + } + + m.logger.Info("JWKS secret already exists.") + } + + return nil +} + +func generateJWKSecret() ([]byte, error) { + rsaKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, fmt.Errorf("unable to generate JWK key: %w", err) + } + + // PEM encode private key + keyPEM := new(bytes.Buffer) + err = pem.Encode(keyPEM, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(rsaKey), + }) + + if err != nil { + return nil, fmt.Errorf("cannot encode JWK key: %w", err) + } + + return []byte(base64.StdEncoding.EncodeToString(keyPEM.Bytes())), nil +} + func checkServiceLabels(service *v1.Service, importName types.NamespacedName) bool { if managedBy, ok := service.Labels[LabelManagedBy]; !ok || managedBy != AppName { return false @@ -887,6 +1011,30 @@ func endpointSliceChanged(endpointSlice1, endpointSlice2 *discv1.EndpointSlice) return false } +func ParseJWKSSecret(secret *v1.Secret) (*rsa.PrivateKey, error) { + keyBase64, ok := secret.Data[JWKSecretKeyName] + if !ok { + return nil, fmt.Errorf("secret missing %s key", JWKSecretKeyName) + } + + keyPEM, err := base64.StdEncoding.DecodeString(string(keyBase64)) + if err != nil { + return nil, fmt.Errorf("cannot base64 decode key") + } + + keyBlock, _ := pem.Decode(keyPEM) + if keyBlock == nil { + return nil, fmt.Errorf("key is not in PEM format") + } + + privateKey, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes) + if err != nil { + return nil, fmt.Errorf("error parsing private key: %w", err) + } + + return privateKey, nil +} + // NewManager returns a new control manager. func NewManager(cl client.Client, namespace string) *Manager { logger := logrus.WithField("component", "controlplane.control.manager") diff --git a/pkg/operator/controller/instance_controller.go b/pkg/operator/controller/instance_controller.go index 9cb592a36..f3b128268 100644 --- a/pkg/operator/controller/instance_controller.go +++ b/pkg/operator/controller/instance_controller.go @@ -66,6 +66,7 @@ type InstanceReconciler struct { // +kubebuilder:rbac:groups=clusterlink.net,resources=instances/status,verbs=get;update;patch // +kubebuilder:rbac:groups=clusterlink.net,resources=instances/finalizers,verbs=update // +kubebuilder:rbac:groups="",resources=events,verbs=create;update +// +kubebuilder:rbac:groups="",resources=secrets,verbs=list;get;watch;create;update // +kubebuilder:rbac:groups="",resources=services;serviceaccounts,verbs=list;get;watch;create;update;patch;delete // +kubebuilder:rbac:groups="coordination.k8s.io",resources=leases,verbs=get;create;update // +kubebuilder:rbac:groups="discovery.k8s.io",resources=endpointslices,verbs=list;get;watch;create;update;patch;delete @@ -430,6 +431,13 @@ func (r *InstanceReconciler) createAccessControl(ctx context.Context, name, name "create", "update", }, }, + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{ + "get", "list", "watch", "create", "update", + }, + }, { APIGroups: []string{""}, Resources: []string{"services"},