From e406d6ed6338014905544bb6bf3749976f62967d Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Mon, 26 Feb 2024 20:38:37 +0000 Subject: [PATCH] Refactor netpol creation and add two new netpols Eliminate copy-pasted netpol creation functions in favor of iterating over templates; Add new netpols for snapshot validation webhook and metrics-server Signed-off-by: Brad Davidson --- pkg/rke2/np.go | 380 +++++++++++++------------------------ pkg/rke2/serviceaccount.go | 5 +- pkg/rke2/spc.go | 5 +- 3 files changed, 133 insertions(+), 257 deletions(-) diff --git a/pkg/rke2/np.go b/pkg/rke2/np.go index e96342761a..1617b43217 100644 --- a/pkg/rke2/np.go +++ b/pkg/rke2/np.go @@ -4,50 +4,47 @@ import ( "context" "fmt" "sync" - "time" "github.com/k3s-io/k3s/pkg/cli/cmds" + "github.com/k3s-io/k3s/pkg/util" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/networking/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/transport" "k8s.io/client-go/util/retry" ) const ( - namespaceAnnotationNetworkPolicy = "np.rke2.io" - namespaceAnnotationNetworkDNSPolicy = "np.rke2.io/dns" - namespaceAnnotationNetworkIngressPolicy = "np.rke2.io/ingress" - namespaceAnnotationNetworkWebhookPolicy = "np.rke2.io/ingress-webhook" - - defaultNetworkPolicyName = "default-network-policy" - defaultNetworkDNSPolicyName = "default-network-dns-policy" - defaultNetworkIngressPolicyName = "default-network-ingress-policy" - defaultNetworkWebhookPolicyName = "default-network-ingress-webhook-policy" - - defaultTimeout = 30 cisAnnotationValue = "resolved" ) -// networkPolicy specifies a base level network policy applied -// to the 3 primary namespace: kube-system, kube-public, and default. -// This policy only allows for intra-namespace traffic. -var networkPolicy = v1.NetworkPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultNetworkPolicyName, - }, - Spec: v1.NetworkPolicySpec{ - PodSelector: metav1.LabelSelector{}, // empty to match all pods - PolicyTypes: []v1.PolicyType{ - v1.PolicyTypeIngress, - }, - Ingress: []v1.NetworkPolicyIngressRule{ +var ( + tcp = corev1.ProtocolTCP + udp = corev1.ProtocolUDP +) + +type policyTemplate struct { + name string + annotationKey string + podSelector metav1.LabelSelector + ingress []v1.NetworkPolicyIngressRule +} + +// defaultNamespacePolicies contains a list of policies that are applied to all core namespaces. +var defaultNamespacePolicies = []policyTemplate{ + { + // default-network-policy is a base level network policy applied + // to the 3 primary namespaces: kube-system, kube-public, and default. + // This policy only allows for intra-namespace traffic. + name: "default-network-policy", + annotationKey: "np.rke2.io", + podSelector: metav1.LabelSelector{}, // empty to match all pods + ingress: []v1.NetworkPolicyIngressRule{ { From: []v1.NetworkPolicyPeer{ { @@ -59,27 +56,14 @@ var networkPolicy = v1.NetworkPolicy{ }, } -var ( - tcp = corev1.ProtocolTCP - udp = corev1.ProtocolUDP -) - -// networkDNSPolicy allows for all DNS traffic -// into the kube-system namespace. -var networkDNSPolicy = v1.NetworkPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultNetworkDNSPolicyName, - }, - Spec: v1.NetworkPolicySpec{ - PodSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "k8s-app": "kube-dns", - }, - }, - PolicyTypes: []v1.PolicyType{ - v1.PolicyTypeIngress, - }, - Ingress: []v1.NetworkPolicyIngressRule{ +// defaultKubeSystemPolicies is a list of policies that are applied to the kube-system namespace. +var defaultKubeSystemPolicies = []policyTemplate{ + { + // allows DNS traffic into the coredns pods + name: "default-network-dns-policy", + annotationKey: "np.rke2.io/dns", + podSelector: metav1.LabelSelector{MatchLabels: labels.Set{"k8s-app": "kube-dns"}}, + ingress: []v1.NetworkPolicyIngressRule{ { Ports: []v1.NetworkPolicyPort{ { @@ -97,26 +81,13 @@ var networkDNSPolicy = v1.NetworkPolicy{ }, }, }, - Egress: []v1.NetworkPolicyEgressRule{}, }, -} - -// networkIngressPolicy allows for all http and https traffic -// into the kube-system namespace to the ingress controller pods. -var networkIngressPolicy = v1.NetworkPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultNetworkIngressPolicyName, - }, - Spec: v1.NetworkPolicySpec{ - PodSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app.kubernetes.io/name": "rke2-ingress-nginx", - }, - }, - PolicyTypes: []v1.PolicyType{ - v1.PolicyTypeIngress, - }, - Ingress: []v1.NetworkPolicyIngressRule{ + { + // allows for all http and https traffic into the kube-system namespace to the ingress controller pods + name: "default-network-ingress-policy", + annotationKey: "np.rke2.io/ingress", + podSelector: metav1.LabelSelector{MatchLabels: labels.Set{"app.kubernetes.io/name": "rke2-ingress-nginx"}}, + ingress: []v1.NetworkPolicyIngressRule{ { Ports: []v1.NetworkPolicyPort{ { @@ -136,47 +107,88 @@ var networkIngressPolicy = v1.NetworkPolicy{ }, }, }, - Egress: []v1.NetworkPolicyEgressRule{}, }, -} - -// networkWebhookPolicy allows for https traffic -// into the kube-system namespace to the ingress controller webhook. -var networkWebhookPolicy = v1.NetworkPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaultNetworkWebhookPolicyName, - }, - Spec: v1.NetworkPolicySpec{ - PodSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app.kubernetes.io/name": "rke2-ingress-nginx", + { + // allows for https traffic into the to the ingress controller webhook + name: "default-network-ingress-webhook-policy", + annotationKey: "np.rke2.io/ingress-webhook", + podSelector: metav1.LabelSelector{MatchLabels: labels.Set{"app.kubernetes.io/name": "rke2-ingress-nginx"}}, + ingress: []v1.NetworkPolicyIngressRule{ + { + Ports: []v1.NetworkPolicyPort{ + { + Protocol: &tcp, + Port: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "webhook", + }, + }, + }, }, }, - PolicyTypes: []v1.PolicyType{ - v1.PolicyTypeIngress, + }, + { + // allows for https traffic into the CSI snapshot validation webhook + name: "default-network-snapshot-validation-webhook-policy", + annotationKey: "np.rke2.io/snapshot-validation-webhook", + podSelector: metav1.LabelSelector{MatchLabels: labels.Set{"app.kubernetes.io/name": "rke2-snapshot-validation-webhook"}}, + ingress: []v1.NetworkPolicyIngressRule{ + { + Ports: []v1.NetworkPolicyPort{ + { + Protocol: &tcp, + Port: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "https", + }, + }, + }, + }, }, - Ingress: []v1.NetworkPolicyIngressRule{ + }, + { + // allows for https traffic into the to the metrics server + name: "default-network-metrics-server-policy", + annotationKey: "np.rke2.io/metrics-server", + podSelector: metav1.LabelSelector{MatchLabels: labels.Set{"app": "rke2-metrics-server"}}, + ingress: []v1.NetworkPolicyIngressRule{ { Ports: []v1.NetworkPolicyPort{ { Protocol: &tcp, Port: &intstr.IntOrString{ Type: intstr.String, - StrVal: "webhook", + StrVal: "https", }, }, }, }, }, - Egress: []v1.NetworkPolicyEgressRule{}, }, } -// setNetworkPolicy applies the default network policy for the given namespace and updates +// policyFromTemplate returns a full v1.NetworkPolicy for the provided policy template +func policyFromTemplate(template policyTemplate) *v1.NetworkPolicy { + return &v1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: template.name, + Annotations: map[string]string{template.annotationKey: cisAnnotationValue}, + }, + Spec: v1.NetworkPolicySpec{ + PodSelector: template.podSelector, + PolicyTypes: []v1.PolicyType{ + v1.PolicyTypeIngress, + }, + Ingress: template.ingress, + }, + } +} + +// setNetworkPolicies applies the provided network policy templates to the given namespace and updates // the given namespaces' annotation. First, the namespaces' annotation is checked for existence. // If the annotation exists, we move on. If the annotation doesn't exist, we check to see if the // policy exists. If it does, we delete it, and create the new default policy. -func setNetworkPolicy(ctx context.Context, namespace string, cs *kubernetes.Clientset) error { +func setPoliciesFromTemplates(ctx context.Context, cs kubernetes.Interface, templates []policyTemplate, namespace string) error { ns, err := cs.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) if err != nil { return fmt.Errorf("networkPolicy: get %s - %w", namespace, err) @@ -184,152 +196,34 @@ func setNetworkPolicy(ctx context.Context, namespace string, cs *kubernetes.Clie if ns.Annotations == nil { ns.Annotations = make(map[string]string) } - if _, ok := ns.Annotations[namespaceAnnotationNetworkPolicy]; !ok { - if _, err := cs.NetworkingV1().NetworkPolicies(namespace).Get(ctx, defaultNetworkPolicyName, metav1.GetOptions{}); err == nil { - if err := cs.NetworkingV1().NetworkPolicies(namespace).Delete(ctx, defaultNetworkPolicyName, metav1.DeleteOptions{}); err != nil { - if !apierrors.IsNotFound(err) { - return err - } - } - } - if _, err := cs.NetworkingV1().NetworkPolicies(namespace).Create(ctx, &networkPolicy, metav1.CreateOptions{}); err != nil { - if !apierrors.IsAlreadyExists(err) { - return err - } - } - ns.Annotations[namespaceAnnotationNetworkPolicy] = cisAnnotationValue - - if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - if _, err := cs.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{}); err != nil { - if apierrors.IsConflict(err) { - return updateNamespaceRef(ctx, cs, ns) + for _, template := range templates { + if _, ok := ns.Annotations[template.annotationKey]; !ok { + if _, err := cs.NetworkingV1().NetworkPolicies(namespace).Get(ctx, template.name, metav1.GetOptions{}); err == nil { + if err := cs.NetworkingV1().NetworkPolicies(namespace).Delete(ctx, template.name, metav1.DeleteOptions{}); err != nil { + if !apierrors.IsNotFound(err) { + return err + } } - return err } - return nil - }); err != nil { - logrus.Fatalf("networkPolicy: update namespace: %s - %s", ns.Name, err.Error()) - } - } - return nil -} - -// setNetworkDNSPolicy sets the default DNS policy allowing the -// required DNS traffic. -func setNetworkDNSPolicy(ctx context.Context, cs *kubernetes.Clientset) error { - ns, err := cs.CoreV1().Namespaces().Get(ctx, metav1.NamespaceSystem, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("networkPolicy: get %s - %w", metav1.NamespaceSystem, err) - } - if ns.Annotations == nil { - ns.Annotations = make(map[string]string) - } - if _, ok := ns.Annotations[namespaceAnnotationNetworkDNSPolicy]; !ok { - if _, err := cs.NetworkingV1().NetworkPolicies(metav1.NamespaceSystem).Get(ctx, defaultNetworkDNSPolicyName, metav1.GetOptions{}); err == nil { - if err := cs.NetworkingV1().NetworkPolicies(metav1.NamespaceSystem).Delete(ctx, defaultNetworkDNSPolicyName, metav1.DeleteOptions{}); err != nil { - if !apierrors.IsNotFound(err) { + if _, err := cs.NetworkingV1().NetworkPolicies(namespace).Create(ctx, policyFromTemplate(template), metav1.CreateOptions{}); err != nil { + if !apierrors.IsAlreadyExists(err) { return err } } - } - if _, err := cs.NetworkingV1().NetworkPolicies(metav1.NamespaceSystem).Create(ctx, &networkDNSPolicy, metav1.CreateOptions{}); err != nil { - if !apierrors.IsAlreadyExists(err) { - return err - } - } - ns.Annotations[namespaceAnnotationNetworkDNSPolicy] = cisAnnotationValue - - if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - if _, err := cs.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{}); err != nil { - if apierrors.IsConflict(err) { - return updateNamespaceRef(ctx, cs, ns) - } - return err - } - return nil - }); err != nil { - logrus.Fatalf("networkPolicy: update namespace: %s - %s", ns.Name, err.Error()) - } - } - return nil -} - -// setNetworkIngressPolicy sets the default Ingress policy allowing the -// required HTTP/HTTPS traffic to ingress nginx pods. -func setNetworkIngressPolicy(ctx context.Context, cs *kubernetes.Clientset) error { - ns, err := cs.CoreV1().Namespaces().Get(ctx, metav1.NamespaceSystem, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("networkPolicy: get %s - %w", metav1.NamespaceSystem, err) - } - if ns.Annotations == nil { - ns.Annotations = make(map[string]string) - } - if _, ok := ns.Annotations[namespaceAnnotationNetworkIngressPolicy]; !ok { - if _, err := cs.NetworkingV1().NetworkPolicies(metav1.NamespaceSystem).Get(ctx, defaultNetworkIngressPolicyName, metav1.GetOptions{}); err == nil { - if err := cs.NetworkingV1().NetworkPolicies(metav1.NamespaceSystem).Delete(ctx, defaultNetworkIngressPolicyName, metav1.DeleteOptions{}); err != nil { - if !apierrors.IsNotFound(err) { - return err - } - } - } - if _, err := cs.NetworkingV1().NetworkPolicies(metav1.NamespaceSystem).Create(ctx, &networkIngressPolicy, metav1.CreateOptions{}); err != nil { - if !apierrors.IsAlreadyExists(err) { - return err - } - } - ns.Annotations[namespaceAnnotationNetworkIngressPolicy] = cisAnnotationValue - - if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - if _, err := cs.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{}); err != nil { - if apierrors.IsConflict(err) { - return updateNamespaceRef(ctx, cs, ns) - } - return err - } - return nil - }); err != nil { - logrus.Fatalf("networkPolicy: update namespace: %s - %s", ns.Name, err.Error()) - } - } - return nil -} + ns.Annotations[template.annotationKey] = cisAnnotationValue -// setNetworkWebhookPolicy sets the default Ingress Webhook policy allowing the -// required webhook traffic to ingress nginx pods. -func setNetworkWebhookPolicy(ctx context.Context, cs *kubernetes.Clientset) error { - ns, err := cs.CoreV1().Namespaces().Get(ctx, metav1.NamespaceSystem, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("networkPolicy: get %s - %w", metav1.NamespaceSystem, err) - } - if ns.Annotations == nil { - ns.Annotations = make(map[string]string) - } - if _, ok := ns.Annotations[namespaceAnnotationNetworkWebhookPolicy]; !ok { - if _, err := cs.NetworkingV1().NetworkPolicies(metav1.NamespaceSystem).Get(ctx, defaultNetworkWebhookPolicyName, metav1.GetOptions{}); err == nil { - if err := cs.NetworkingV1().NetworkPolicies(metav1.NamespaceSystem).Delete(ctx, defaultNetworkWebhookPolicyName, metav1.DeleteOptions{}); err != nil { - if !apierrors.IsNotFound(err) { + if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if _, err := cs.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{}); err != nil { + if apierrors.IsConflict(err) { + return updateNamespaceRef(ctx, cs, ns) + } return err } + return nil + }); err != nil { + logrus.Fatalf("Failed to apply network policy %s to namespace %s: %v", template.name, ns.Name, err) } } - if _, err := cs.NetworkingV1().NetworkPolicies(metav1.NamespaceSystem).Create(ctx, &networkWebhookPolicy, metav1.CreateOptions{}); err != nil { - if !apierrors.IsAlreadyExists(err) { - return err - } - } - ns.Annotations[namespaceAnnotationNetworkWebhookPolicy] = cisAnnotationValue - - if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - if _, err := cs.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{}); err != nil { - if apierrors.IsConflict(err) { - return updateNamespaceRef(ctx, cs, ns) - } - return err - } - return nil - }); err != nil { - logrus.Fatalf("networkPolicy: update namespace: %s - %s", ns.Name, err.Error()) - } } return nil } @@ -348,27 +242,20 @@ func setNetworkPolicies(cisMode bool, namespaces []string) cmds.StartupHook { go func() { defer wg.Done() <-args.APIServerReady - cs, err := newClient(args.KubeConfigSupervisor, nil) + cs, err := util.GetClientSet(args.KubeConfigSupervisor) if err != nil { - logrus.Fatalf("networkPolicy: new k8s client: %s", err.Error()) + logrus.Fatalf("np: new k8s client: %v", err) } for _, namespace := range namespaces { - if err := setNetworkPolicy(ctx, namespace, cs); err != nil { + if err := setPoliciesFromTemplates(ctx, cs, defaultNamespacePolicies, namespace); err != nil { logrus.Fatal(err) } + if namespace == metav1.NamespaceSystem { + if err := setPoliciesFromTemplates(ctx, cs, defaultKubeSystemPolicies, namespace); err != nil { + logrus.Fatal(err) + } + } } - if err := setNetworkDNSPolicy(ctx, cs); err != nil { - logrus.Fatal(err) - } - - if err := setNetworkIngressPolicy(ctx, cs); err != nil { - logrus.Fatal(err) - } - - if err := setNetworkWebhookPolicy(ctx, cs); err != nil { - logrus.Fatal(err) - } - logrus.Info("Applying network policies complete") }() @@ -380,8 +267,8 @@ func setNetworkPolicies(cisMode bool, namespaces []string) cmds.StartupHook { // the passed revision of the Namespace to the most recent revision, and updates the pointer to refer to the // most recent revision. This get/change/update pattern is required to alter an object // that may have changed since it was retrieved. -func updateNamespaceRef(ctx context.Context, cs *kubernetes.Clientset, ns *corev1.Namespace) error { - logrus.Info("updating namespace: " + ns.Name) +func updateNamespaceRef(ctx context.Context, cs kubernetes.Interface, ns *corev1.Namespace) error { + logrus.Infof("Updating namespace %s to apply network policy annotations", ns.Name) newNS, err := cs.CoreV1().Namespaces().Get(ctx, ns.Name, metav1.GetOptions{}) if err != nil { return err @@ -396,16 +283,3 @@ func updateNamespaceRef(ctx context.Context, cs *kubernetes.Clientset, ns *corev *ns = *newNS return nil } - -// newClient create a new Kubernetes client from configuration. -func newClient(kubeConfigPath string, k8sWrapTransport transport.WrapperFunc) (*kubernetes.Clientset, error) { - config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) - if err != nil { - return nil, err - } - if k8sWrapTransport != nil { - config.WrapTransport = k8sWrapTransport - } - config.Timeout = time.Second * defaultTimeout - return kubernetes.NewForConfig(config) -} diff --git a/pkg/rke2/serviceaccount.go b/pkg/rke2/serviceaccount.go index 41f4d2cb43..4282c06ed8 100644 --- a/pkg/rke2/serviceaccount.go +++ b/pkg/rke2/serviceaccount.go @@ -6,6 +6,7 @@ import ( "time" "github.com/k3s-io/k3s/pkg/cli/cmds" + "github.com/k3s-io/k3s/pkg/util" "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" @@ -74,9 +75,9 @@ func restrictServiceAccounts(cisMode bool, namespaces []string) cmds.StartupHook go func() { defer wg.Done() <-args.APIServerReady - cs, err := newClient(args.KubeConfigSupervisor, nil) + cs, err := util.GetClientSet(args.KubeConfigSupervisor) if err != nil { - logrus.Fatalf("serviceAccount: new k8s client: %s", err.Error()) + logrus.Fatalf("serviceAccount: new k8s client: %v", err) } nps := append(namespaces, "kube-node-lease") for _, namespace := range nps { diff --git a/pkg/rke2/spc.go b/pkg/rke2/spc.go index ee48fc7452..4180418f89 100644 --- a/pkg/rke2/spc.go +++ b/pkg/rke2/spc.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/k3s-io/k3s/pkg/cli/cmds" + "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -33,9 +34,9 @@ func cleanupStaticPodsOnSelfDelete(dataDir string) cmds.StartupHook { go func() { defer wg.Done() <-args.APIServerReady - cs, err := newClient(args.KubeConfigSupervisor, nil) + cs, err := util.GetClientSet(args.KubeConfigSupervisor) if err != nil { - logrus.Fatalf("spc: new k8s client: %s", err.Error()) + logrus.Fatalf("spc: new k8s client: %v", err) } go watchForSelfDelete(ctx, dataDir, cs) }()