From f8d861f641df3efc0b7b00fa9697ea0cd7c25c28 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Wed, 16 Sep 2020 15:39:30 -0700 Subject: [PATCH] Add missing ClusterRoleBindings (#329) * Update k3s to allow white-labeling k3s controller RBAC * Add missing ClusterRoleBindings Several critical ClusterRoleBindings were being inherited from k3s's rolebindings.yaml. Now that we have purged all k3s content, we need to provide them locally. Signed-off-by: Brad Davidson --- go.mod | 2 +- go.sum | 4 +- pkg/rke2/clusterrole.go | 90 +++++++++++++++++++++++++++++++ pkg/rke2/clusterrole_templates.go | 70 ++++++++++++++++++++++++ pkg/rke2/psp.go | 49 ++++++++--------- pkg/rke2/rke2.go | 1 + 6 files changed, 187 insertions(+), 29 deletions(-) create mode 100644 pkg/rke2/clusterrole.go create mode 100644 pkg/rke2/clusterrole_templates.go diff --git a/go.mod b/go.mod index d2c4db15cb..fcee850a0c 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/google/go-containerregistry v0.0.0-20190617215043-876b8855d23c github.com/pkg/errors v0.9.1 github.com/rancher/helm-controller v0.7.3 - github.com/rancher/k3s v1.19.1-rc1.0.20200915034458-a08e998bc5dd + github.com/rancher/k3s v1.19.1-rc2.0.20200916010251-ae5519c0472e github.com/rancher/wrangler v0.6.1 github.com/sirupsen/logrus v1.6.0 github.com/urfave/cli v1.22.2 diff --git a/go.sum b/go.sum index b975074640..dc59da89af 100644 --- a/go.sum +++ b/go.sum @@ -695,8 +695,8 @@ github.com/rancher/go-powershell v0.0.0-20200701184732-233247d45373 h1:BePi97poJ github.com/rancher/go-powershell v0.0.0-20200701184732-233247d45373/go.mod h1:Vz8oLnHgttpo/aZrTpjbcpZEDzzElqNau2zmorToY0E= github.com/rancher/helm-controller v0.7.3 h1:WTQHcNF2vl9w6Xd1eBtXDe0JUsYMFFstqX9ghGhI5Ac= github.com/rancher/helm-controller v0.7.3/go.mod h1:ZylsxIMGNADRPRNW+NiBWhrwwks9vnKLQiCHYWb6Bi0= -github.com/rancher/k3s v1.19.1-rc1.0.20200915034458-a08e998bc5dd h1:LG+mI6JH7AC/AdX39B5hNn7amjiBMU/xvry+ev2Y9jo= -github.com/rancher/k3s v1.19.1-rc1.0.20200915034458-a08e998bc5dd/go.mod h1:KZ7cVGFco3Q8FfQGynHIZZ7MYVE+yPdk6TZB9s9YqWk= +github.com/rancher/k3s v1.19.1-rc2.0.20200916010251-ae5519c0472e h1:Bngz9JRrYfVDBf+J1Ih9/iDaeiwITZCM59Ob1ldJ4kA= +github.com/rancher/k3s v1.19.1-rc2.0.20200916010251-ae5519c0472e/go.mod h1:KZ7cVGFco3Q8FfQGynHIZZ7MYVE+yPdk6TZB9s9YqWk= github.com/rancher/kine v0.4.0 h1:1IhWy3TzjExG8xnj46eyUEWdzqNAD1WrgL4eEBKm6Uc= github.com/rancher/kine v0.4.0/go.mod h1:IImtCJ68AIkE+VY/kUI0NkyJL5q5WzO8QvMsSXqbrpA= github.com/rancher/kubernetes v1.19.0-k3s1 h1:TPFj4qlQgZ2E9xE/ScFLtBQoymxPNCXAYHJpSZYRW3o= diff --git a/pkg/rke2/clusterrole.go b/pkg/rke2/clusterrole.go new file mode 100644 index 0000000000..d92b009a2e --- /dev/null +++ b/pkg/rke2/clusterrole.go @@ -0,0 +1,90 @@ +package rke2 + +import ( + "context" + "fmt" + + "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// setClusterRoles applies common clusterroles and clusterrolebindings that are critical +// to the function of internal controllers. +func setClusterRoles() func(context.Context, <-chan struct{}, string) error { + return func(ctx context.Context, apiServerReady <-chan struct{}, kubeConfigAdmin string) error { + go func() { + <-apiServerReady + logrus.Info("Applying Cluster Role Bindings") + + cs, err := newClient(kubeConfigAdmin, nil) + if err != nil { + logrus.Fatalf("clusterrole: new k8s client: %s", err.Error()) + } + + if err := setKubeletAPIServerRoleBinding(ctx, cs); err != nil { + logrus.Fatalf("psp: set kubeletAPIServerRoleBinding: %s", err.Error()) + } + + if err := setTunnelControllerRoleBinding(ctx, cs); err != nil { + logrus.Fatalf("psp: set tunnelControllerRoleBinding: %s", err.Error()) + } + + logrus.Info("Cluster Role Bindings applied successfully") + }() + return nil + } +} + +// setKubeletAPIServerRoleBinding creates the clusterrolebinding that grants the apiserver full access to the kubelet API +func setKubeletAPIServerRoleBinding(ctx context.Context, cs *kubernetes.Clientset) error { + // check if clusterrolebinding exists + if _, err := cs.RbacV1().ClusterRoleBindings().Get(ctx, kubeletAPIServerRoleBindingName, metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + logrus.Infof("Setting Cluster RoleBinding: %s", kubeletAPIServerRoleBindingName) + + tmpl := fmt.Sprintf(kubeletAPIServerRoleBindingTemplate, kubeletAPIServerRoleBindingName) + if err := deployClusterRoleBindingFromYaml(ctx, cs, tmpl); err != nil { + return err + } + } else { + return err + } + } + return nil +} + +// setTunnelControllerRoleBinding creates the clusterrole and clusterrolebinding used by internal controllers +// such as the agent tunnel controller +func setTunnelControllerRoleBinding(ctx context.Context, cs *kubernetes.Clientset) error { + // check if clusterrole exists + if _, err := cs.RbacV1().ClusterRoles().Get(ctx, tunnelControllerRoleName, metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + logrus.Infof("Setting Cluster Role: %s", tunnelControllerRoleName) + + tmpl := fmt.Sprintf(tunnelControllerRoleTemplate, tunnelControllerRoleName) + if err := deployClusterRoleFromYaml(ctx, cs, tmpl); err != nil { + return err + } + } else { + return err + } + } + + // check if clusterrolebinding exists + if _, err := cs.RbacV1().ClusterRoleBindings().Get(ctx, tunnelControllerRoleName, metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + logrus.Infof("Setting Cluster RoleBinding: %s", tunnelControllerRoleName) + + tmpl := fmt.Sprintf(tunnelControllerRoleBindingTemplate, tunnelControllerRoleName, tunnelControllerRoleName) + if err := deployClusterRoleBindingFromYaml(ctx, cs, tmpl); err != nil { + return err + } + } else { + return err + } + } + + return nil +} diff --git a/pkg/rke2/clusterrole_templates.go b/pkg/rke2/clusterrole_templates.go new file mode 100644 index 0000000000..8997019052 --- /dev/null +++ b/pkg/rke2/clusterrole_templates.go @@ -0,0 +1,70 @@ +package rke2 + +const ( + kubeletAPIServerRoleBindingName = "kube-apiserver-kubelet-admin" + tunnelControllerRoleName = "system:rke2-controller" +) + +const kubeletAPIServerRoleBindingTemplate = `apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: %s +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kubelet-api-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: kube-apiserver +` + +const tunnelControllerRoleTemplate = `apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: %s +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - namespaces + verbs: + - list + - watch +- apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - endpoints + - pods + verbs: + - list + - get + - watch +` + +const tunnelControllerRoleBindingTemplate = `apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:k3s-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: %s +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: User + name: %s +` diff --git a/pkg/rke2/psp.go b/pkg/rke2/psp.go index a6ff118f6e..bd0b6f7403 100644 --- a/pkg/rke2/psp.go +++ b/pkg/rke2/psp.go @@ -153,9 +153,9 @@ func setSystemUnrestricted(ctx context.Context, cs *kubernetes.Clientset, ns *v1 // if they do, delete them. func setPSPs() func(context.Context, <-chan struct{}, string) error { return func(ctx context.Context, apiServerReady <-chan struct{}, kubeConfigAdmin string) error { - logrus.Info("Applying PSP's...") go func() { <-apiServerReady + logrus.Info("Applying Pod Security Policies") cs, err := newClient(kubeConfigAdmin, nil) if err != nil { @@ -164,7 +164,7 @@ func setPSPs() func(context.Context, <-chan struct{}, string) error { ns, err := cs.CoreV1().Namespaces().Get(ctx, metav1.NamespaceSystem, metav1.GetOptions{}) if err != nil { - logrus.Fatalf("psp: get kube-system namespace: %s", err.Error()) + logrus.Fatalf("psp: get namespace %s: %s", metav1.NamespaceSystem, err.Error()) } if ns.Annotations == nil { ns.Annotations = make(map[string]string) @@ -211,19 +211,19 @@ func setPSPs() func(context.Context, <-chan struct{}, string) error { // check if role binding exists and delete it _, err = cs.RbacV1().ClusterRoleBindings().Get(ctx, globalRestrictedRoleBindingName, metav1.GetOptions{}) if err != nil && !apierrors.IsNotFound(err) { - logrus.Fatalf("psp: get clusterrole: %s", err.Error()) + logrus.Fatalf("psp: get clusterrolebinding: %s", err.Error()) } if err != nil { switch { case apierrors.IsAlreadyExists(err): logrus.Infof("Deleting clusterRole binding: %s", globalRestrictedRoleBindingName) if err := cs.RbacV1().ClusterRoleBindings().Delete(ctx, globalRestrictedRoleBindingName, metav1.DeleteOptions{}); err != nil { - logrus.Fatalf("psp: delete clusterrole binding: %s", err.Error()) + logrus.Fatalf("psp: delete clusterrolebinding: %s", err.Error()) } case apierrors.IsNotFound(err): break default: - logrus.Fatalf("psp: get clusterrole binding: %s", err.Error()) + logrus.Fatalf("psp: get clusterrolebinding: %s", err.Error()) } } ns.Annotations[namespaceAnnotationGlobalRestricted] = cisAnnotationValue @@ -233,7 +233,7 @@ func setPSPs() func(context.Context, <-chan struct{}, string) error { if apierrors.IsNotFound(err) { tmpl := fmt.Sprintf(nodeClusterRoleBindingTemplate, globalUnrestrictedRoleName) if err := deployClusterRoleBindingFromYaml(ctx, cs, tmpl); err != nil { - logrus.Fatalf("psp: deploy psp: %s", err.Error()) + logrus.Fatalf("psp: deploy clusterrolebinding: %s", err.Error()) } } else { logrus.Fatalf("psp: get clusterrole binding: %s", err.Error()) @@ -244,7 +244,7 @@ func setPSPs() func(context.Context, <-chan struct{}, string) error { if _, err := cs.PolicyV1beta1().PodSecurityPolicies().Get(ctx, globalRestrictedPSPName, metav1.GetOptions{}); err != nil { tmpl := fmt.Sprintf(globalRestrictedPSPTemplate, globalRestrictedPSPName) if err := deployPodSecurityPolicyFromYaml(ctx, cs, tmpl); err != nil { - logrus.Fatalf("psp: delete clusterrole: %s", err.Error()) + logrus.Fatalf("psp: deploy psp: %s", err.Error()) } } if _, err := cs.RbacV1().ClusterRoles().Get(ctx, globalRestrictedRoleName, metav1.GetOptions{}); err != nil { @@ -261,10 +261,10 @@ func setPSPs() func(context.Context, <-chan struct{}, string) error { if apierrors.IsNotFound(err) { tmpl := fmt.Sprintf(globalRestrictedRoleBindingTemplate, globalRestrictedRoleBindingName, globalRestrictedRoleName) if err := deployClusterRoleBindingFromYaml(ctx, cs, tmpl); err != nil { - logrus.Fatalf("psp: deploy clusterrole binding: %s", err.Error()) + logrus.Fatalf("psp: deploy clusterrolebinding: %s", err.Error()) } } else { - logrus.Fatalf("psp: get clusterrole binding: %s", err.Error()) + logrus.Fatalf("psp: get clusterrolebinding: %s", err.Error()) } } ns.Annotations[namespaceAnnotationGlobalRestricted] = cisAnnotationValue @@ -309,12 +309,12 @@ func setPSPs() func(context.Context, <-chan struct{}, string) error { case apierrors.IsAlreadyExists(err): logrus.Infof("Deleting clusterRoleBinding: %s", globalUnrestrictedRoleBindingName) if err := cs.RbacV1().ClusterRoleBindings().Delete(ctx, globalUnrestrictedRoleBindingName, metav1.DeleteOptions{}); err != nil { - logrus.Fatalf("psp: delete clusterrole binding: %s", err.Error()) + logrus.Fatalf("psp: delete clusterrolebinding: %s", err.Error()) } case apierrors.IsNotFound(err): break default: - logrus.Fatalf("psp: get clusterrole binding: %s", err.Error()) + logrus.Fatalf("psp: get clusterrolebinding: %s", err.Error()) } } ns.Annotations[namespaceAnnotationGlobalUnrestricted] = cisAnnotationValue @@ -324,36 +324,34 @@ func setPSPs() func(context.Context, <-chan struct{}, string) error { if apierrors.IsNotFound(err) { tmpl := fmt.Sprintf(nodeClusterRoleBindingTemplate, globalRestrictedRoleName) if err := deployClusterRoleBindingFromYaml(ctx, cs, tmpl); err != nil { - logrus.Fatalf("psp: deploy psp: %s", err.Error()) + logrus.Fatalf("psp: deploy clusterrolebinding: %s", err.Error()) } } else { - logrus.Fatalf("psp: get clusterrole binding: %s", err.Error()) + logrus.Fatalf("psp: get clusterrolebinding: %s", err.Error()) } } } if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - if _, err := cs.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{}); err != nil { - if apierrors.IsConflict(err) { - if err := updateNamespaceRef(ctx, cs, ns); err != nil { - return err - } - } + if err := updateNamespaceRef(ctx, cs, ns); err != nil { return err } - return nil + + _, err := cs.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{}) + return err }); err != nil { logrus.Fatalf("psp: update namespace: %s - %s", ns.Name, err.Error()) } - logrus.Info("Applying PSP's complete") + logrus.Info("Pod Security Policies applied successfully") }() return nil } } -// updateNamespaceRef receives a value of type v1.Namespace pointer -// and updates that value to point to a newly retrieve value in -// the event a conflict error is returned. +// updateNamespaceRef retrieves the most recent revision of Namespace ns, copies over any annotations from +// the passed revision of the Namespace to the most recent revision, and updates the pointer to refer to the +// most recent revision. This get/change/update pattern is required to alter an object +// that may have changed since it was retrieved. func updateNamespaceRef(ctx context.Context, cs *kubernetes.Clientset, ns *v1.Namespace) error { logrus.Info("updating namespace: " + ns.Name) newNS, err := cs.CoreV1().Namespaces().Get(ctx, ns.Name, metav1.GetOptions{}) @@ -363,8 +361,7 @@ func updateNamespaceRef(ctx context.Context, cs *kubernetes.Clientset, ns *v1.Na if newNS.Annotations == nil { newNS.Annotations = make(map[string]string, len(ns.Annotations)) } - // copy any annotations that need to be written that - // may not have been written yet. + // copy annotations, since we may have changed them for k, v := range ns.Annotations { newNS.Annotations[k] = v } diff --git a/pkg/rke2/rke2.go b/pkg/rke2/rke2.go index 49ff2e92e0..347491217a 100644 --- a/pkg/rke2/rke2.go +++ b/pkg/rke2/rke2.go @@ -42,6 +42,7 @@ func Server(clx *cli.Context, cfg Config) error { cmds.ServerConfig.StartupHooks = append(cmds.ServerConfig.StartupHooks, setPSPs(), setNetworkPolicies(), + setClusterRoles(), ) return server.Run(clx)