[release-1.27] version bumps and backports for 2024-06 release cycle

Dockerfile

@@ -114,7 +114,7 @@ RUN rm -vf /charts/*.sh /charts/*.md /charts/chart_versions.yaml
 # must be placed in bin/ of the file image and subdirectories of bin/ will be flattened during installation.
 # This means bin/foo/bar will become bin/bar when rke2 installs this to the host
 FROM rancher/hardened-kubernetes:v1.27.14-rke2r1-build20240515 AS kubernetes
-FROM rancher/hardened-containerd:v1.7.11-k3s2-build20231211 AS containerd
+FROM rancher/hardened-containerd:v1.7.17-k3s1-build20240531 AS containerd
 FROM rancher/hardened-crictl:v1.26.1-build20230606 AS crictl
 FROM rancher/hardened-runc:v1.1.12-build20240201 AS runc
 

Dockerfile.windows

@@ -38,7 +38,7 @@ RUN curl -sL https://raw.githubusercontent.com/golangci/golangci-lint/master/ins
 WORKDIR /source
 # End Dapper stuff
 
-FROM rancher/hardened-containerd:v1.7.11-k3s2-build20231211-amd64-windows AS containerd
+FROM rancher/hardened-containerd:v1.7.17-k3s1-build20240531-amd64-windows AS containerd
 FROM build as windows-runtime-collect
 ARG KUBERNETES_VERSION=dev
 

bundle/bin/rke2-killall.sh

@@ -31,7 +31,7 @@ killtree() {
 }
 
 getshims() {
-    ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'rke2/data/[^/]*/bin/containerd-shim' | cut -f1
+    COLUMNS=2147483647 ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'rke2/data/[^/]*/bin/containerd-shim' | cut -f1
 }
 
 do_unmount_and_remove() {

bundle/bin/rke2-uninstall.sh

@@ -105,7 +105,7 @@ uninstall_remove_files()
             rm -f /etc/fapolicyd/rules.d/80-rke2.rules
         fi
         fagenrules --load
-        systemctl restart fapolicyd
+        systemctl try-restart fapolicyd
     fi
 }
 

charts/build-chart.sh

@@ -38,6 +38,7 @@ metadata:
   namespace: "${CHART_NAMESPACE:="kube-system"}"
   annotations:
     helm.cattle.io/chart-url: "${CHART_URL}"
+    rke2.cattle.io/inject-cluster-config: "true"
 spec:
   bootstrap: ${CHART_BOOTSTRAP:=false}
   chartContent: $(base64 -w0 < "${CHART_TMP}")

developer-docs/updating_rke2_charts.md

@@ -19,8 +19,8 @@ the controllers part of `rke2 server` binary.
 
 ## How to update a chart?
 
-Before going into updating the chart, note a chart update normally means updating the images that this chart is consumin
-g. In general, rke2 is consuming hardened images that are built using a FIPS compliant process. In other words, **do not
+Before going into updating the chart, note a chart update normally means updating the images that this chart is consuming.
+In general, rke2 is consuming hardened images that are built using a FIPS compliant process. In other words, **do not
  use the upstream images**. Instead, refer to the Github project building that image and use the code of the upstream
 project to yield a hardened image. The Github projects building hardened images are under our [github rancher](https://github.com/rancher/)
 and start with the name `image-build-`, for example: [image-build-coredns](https://github.com/rancher/image-build-coredns).

pkg/bootstrap/bootstrap.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"strconv"
 	"strings"
 
 	"github.com/google/go-containerregistry/pkg/authn"
@@ -20,6 +21,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/agent"
 	daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/util"
+	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
 	"github.com/rancher/rke2/pkg/images"
 	"github.com/rancher/wharfie/pkg/credentialprovider/plugin"
@@ -33,8 +35,11 @@ import (
 )
 
 var (
-	releasePattern = regexp.MustCompile("^v[0-9]")
-	helmChartGVK   = helmv1.SchemeGroupVersion.WithKind("HelmChart")
+	releasePattern      = regexp.MustCompile("^v[0-9]")
+	helmChartGVK        = helmv1.SchemeGroupVersion.WithKind("HelmChart")
+	injectAnnotationKey = version.Program + ".cattle.io/inject-cluster-config"
+	injectEnvKey        = version.ProgramUpper + "_INJECT_CLUSTER_CONFIG"
+	injectDefault       = true
 )
 
 // binDirForDigest returns the path to dataDir/data/refDigest/bin.
@@ -373,6 +378,11 @@ OBJECTS:
 			continue
 		}
 
+		// Ignore object if injection is disabled via annotation or default setting
+		if !isInjectEnabled(unst) {
+			continue
+		}
+
 		var contentChanged bool
 		content := unst.UnstructuredContent()
 
@@ -429,3 +439,19 @@ OBJECTS:
 	logrus.Infof("Updated manifest %s to set cluster configuration values", fileName)
 	return nil
 }
+
+func isInjectEnabled(obj *unstructured.Unstructured) bool {
+	if v, ok := obj.GetAnnotations()[injectAnnotationKey]; ok {
+		if b, err := strconv.ParseBool(v); err == nil {
+			return b
+		}
+	}
+	return getInjectDefault()
+}
+
+func getInjectDefault() bool {
+	if b, err := strconv.ParseBool(os.Getenv(injectEnvKey)); err == nil {
+		return b
+	}
+	return injectDefault
+}

pkg/cli/cmds/agent.go

@@ -54,6 +54,8 @@ var (
 		"selinux":                           copyFlag,
 		"lb-server-port":                    copyFlag,
 		"airgap-extra-registry":             copyFlag,
+		"bind-address":                      copyFlag,
+		"enable-pprof":                      copyFlag,
 	})
 	deprecatedFlags = []cli.Flag{
 		&cli.StringFlag{

pkg/cli/cmds/server.go

@@ -66,6 +66,7 @@ var (
 		"token-file":                        copyFlag,
 		"write-kubeconfig":                  copyFlag,
 		"write-kubeconfig-mode":             copyFlag,
+		"write-kubeconfig-group":            copyFlag,
 		"kube-apiserver-arg":                copyFlag,
 		"etcd-arg":                          copyFlag,
 		"kube-scheduler-arg":                copyFlag,
@@ -105,6 +106,7 @@ var (
 		"container-runtime-endpoint":        copyFlag,
 		"disable-default-registry-endpoint": copyFlag,
 		"embedded-registry":                 copyFlag,
+		"supervisor-metrics":                copyFlag,
 		"image-service-endpoint":            dropFlag,
 		"pause-image":                       dropFlag,
 		"default-runtime":                   copyFlag,

pkg/controllers/cisnetworkpolicy/cleanup.go

@@ -0,0 +1,73 @@
+package cisnetworkpolicy
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"github.com/k3s-io/k3s/pkg/server"
+	coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
+	"github.com/sirupsen/logrus"
+	core "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+var (
+	finalizerKey = "wrangler.cattle.io/cisnetworkpolicy-node"
+)
+
+// Cleanup removes the OnRemove finalizer from any nodes.
+// This must be done to clean up from any previously registered OnRemove handlers that are currently disabled.
+func Cleanup(ctx context.Context, sc *server.Context) error {
+	return unregister(ctx, sc.Core.Core().V1().Node())
+}
+
+func unregister(ctx context.Context, nodes coreclient.NodeController) error {
+	logrus.Debugf("CISNetworkPolicyController: Removing controller hooks for NetworkPolicy %s", flannelHostNetworkPolicyName)
+	go wait.PollImmediateUntilWithContext(ctx, time.Second*30, func(_ context.Context) (bool, error) {
+		nodesList, err := nodes.List(metav1.ListOptions{})
+		if err != nil {
+			logrus.Warnf("CISNetworkPolicyController: failed to list nodes: %v", err)
+			return false, nil
+		}
+		for _, node := range nodesList.Items {
+			for _, finalizer := range node.ObjectMeta.Finalizers {
+				if finalizer == finalizerKey {
+					if err := removeFinalizer(nodes, node); err != nil {
+						logrus.Warnf("CISNetworkPolicyController: failed to remove finalizer from node %s: %v", node.Name, err)
+						return false, nil
+					}
+					break
+				}
+			}
+		}
+		return true, nil
+	})
+	return nil
+}
+
+func removeFinalizer(nodes coreclient.NodeController, node core.Node) error {
+	newFinalizers := []string{}
+	finalizers := node.ObjectMeta.Finalizers
+	for k, v := range finalizers {
+		if v != finalizerKey {
+			continue
+		}
+		newFinalizers = append(finalizers[:k], finalizers[k+1:]...)
+	}
+	patch := []map[string]interface{}{
+		{
+			"op":    "replace",
+			"value": newFinalizers,
+			"path":  "/metadata/finalizers",
+		},
+	}
+	b, err := json.Marshal(patch)
+	if err != nil {
+		return err
+	}
+	_, err = nodes.Patch(node.Name, types.JSONPatchType, b)
+	return err
+}

pkg/rke2/np.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/util"
@@ -15,6 +16,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/util/retry"
 )
@@ -221,7 +223,7 @@ func setPoliciesFromTemplates(ctx context.Context, cs kubernetes.Interface, temp
 				}
 				return nil
 			}); err != nil {
-				logrus.Fatalf("Failed to apply network policy %s to namespace %s: %v", template.name, ns.Name, err)
+				return fmt.Errorf("failed to apply network policy %s to namespace %s: %v", template.name, ns.Name, err)
 			}
 		}
 	}
@@ -238,27 +240,32 @@ func setNetworkPolicies(cisMode bool, namespaces []string) cmds.StartupHook {
 			return nil
 		}
 
-		logrus.Info("Applying network policies...")
 		go func() {
 			defer wg.Done()
 			<-args.APIServerReady
 			cs, err := util.GetClientSet(args.KubeConfigSupervisor)
 			if err != nil {
 				logrus.Fatalf("np: new k8s client: %v", err)
 			}
-			for _, namespace := range namespaces {
-				if err := setPoliciesFromTemplates(ctx, cs, defaultNamespacePolicies, namespace); err != nil {
-					logrus.Fatal(err)
-				}
-				if namespace == metav1.NamespaceSystem {
-					if err := setPoliciesFromTemplates(ctx, cs, defaultKubeSystemPolicies, namespace); err != nil {
-						logrus.Fatal(err)
+
+			go wait.PollImmediateInfiniteWithContext(ctx, 5*time.Second, func(ctx context.Context) (bool, error) {
+				logrus.Info("Applying network policies...")
+				for _, namespace := range namespaces {
+					if err := setPoliciesFromTemplates(ctx, cs, defaultNamespacePolicies, namespace); err != nil {
+						logrus.Errorf("Network policy apply failed, will retry: %v", err)
+						return false, nil
+					}
+					if namespace == metav1.NamespaceSystem {
+						if err := setPoliciesFromTemplates(ctx, cs, defaultKubeSystemPolicies, namespace); err != nil {
+							logrus.Errorf("Network policy apply failed, will retry: %v", err)
+							return false, nil
+						}
 					}
 				}
-			}
-			logrus.Info("Applying network policies complete")
+				logrus.Info("Applying network policies complete")
+				return true, nil
+			})
 		}()
-
 		return nil
 	}
 }

pkg/rke2/rke2.go

@@ -118,6 +118,8 @@ func Server(clx *cli.Context, cfg Config) error {
 	cnis := clx.StringSlice("cni")
 	if cisMode && (len(cnis) == 0 || slice.ContainsString(cnis, "canal")) {
 		leaderControllers = append(leaderControllers, cisnetworkpolicy.Controller)
+	} else {
+		leaderControllers = append(leaderControllers, cisnetworkpolicy.Cleanup)
 	}
 
 	return server.RunWithControllers(clx, leaderControllers, rawServer.CustomControllers{})

scripts/build-binary

@@ -30,7 +30,7 @@ VERSION_FLAGS="
     -X ${K3S_PKG}/pkg/version.Version=${VERSION}
     -X ${K3S_PKG}/pkg/version.UpstreamGolang=${VERSION_GOLANG}
     -X ${RKE2_PKG}/pkg/images.DefaultRegistry=${REGISTRY}
-    -X ${RKE2_PKG}/pkg/images.DefaultEtcdImage=rancher/hardened-etcd:${ETCD_VERSION}-build20240418
+    -X ${RKE2_PKG}/pkg/images.DefaultEtcdImage=rancher/hardened-etcd:${ETCD_VERSION}-build20240531
     -X ${RKE2_PKG}/pkg/images.DefaultKubernetesImage=rancher/hardened-kubernetes:${KUBERNETES_IMAGE_TAG}
     -X ${RKE2_PKG}/pkg/images.DefaultPauseImage=rancher/mirrored-pause:${PAUSE_VERSION}
     -X ${RKE2_PKG}/pkg/images.DefaultRuntimeImage=${REPO}/${PROG}-runtime:${DOCKERIZED_VERSION}

scripts/build-images

@@ -16,10 +16,10 @@ xargs -n1 -t docker image pull --quiet << EOF >> build/images-core.txt
     ${REGISTRY}/rancher/hardened-coredns:v1.11.1-build20240305
     ${REGISTRY}/rancher/hardened-cluster-autoscaler:v1.8.10-build20240124
     ${REGISTRY}/rancher/hardened-dns-node-cache:1.22.28-build20240125
-    ${REGISTRY}/rancher/hardened-etcd:${ETCD_VERSION}-build20240418
+    ${REGISTRY}/rancher/hardened-etcd:${ETCD_VERSION}-build20240531
     ${REGISTRY}/rancher/hardened-k8s-metrics-server:v0.7.1-build20240401
     ${REGISTRY}/rancher/hardened-addon-resizer:1.8.20-build20240410
-    ${REGISTRY}/rancher/klipper-helm:v0.8.3-build20240228
+    ${REGISTRY}/rancher/klipper-helm:v0.8.4-build20240523
     ${REGISTRY}/rancher/klipper-lb:v0.4.7
     ${REGISTRY}/rancher/mirrored-pause:${PAUSE_VERSION}
     ${REGISTRY}/rancher/mirrored-ingress-nginx-kube-webhook-certgen:v20230312-helm-chart-4.5.2-28-g66a760794

scripts/build-windows-binary

@@ -31,7 +31,7 @@ VERSION_FLAGS="
     -X ${K3S_PKG}/pkg/version.Version=${VERSION}
     -X ${K3S_PKG}/pkg/version.UpstreamGolang=${VERSION_GOLANG}
     -X ${RKE2_PKG}/pkg/images.DefaultRegistry=${REGISTRY}
-    -X ${RKE2_PKG}/pkg/images.DefaultEtcdImage=rancher/hardened-etcd:${ETCD_VERSION}-build20240418
+    -X ${RKE2_PKG}/pkg/images.DefaultEtcdImage=rancher/hardened-etcd:${ETCD_VERSION}-build20240531
     -X ${RKE2_PKG}/pkg/images.DefaultKubernetesImage=${REPO}/hardened-kubernetes:${KUBERNETES_IMAGE_TAG}
     -X ${RKE2_PKG}/pkg/images.DefaultPauseImage=rancher/pause:${PAUSE_VERSION}
     -X ${RKE2_PKG}/pkg/images.DefaultRuntimeImage=${REPO}/${PROG}-runtime:${DOCKERIZED_VERSION}-${GOOS}-${GOARCH}

scripts/version.sh

@@ -33,7 +33,7 @@ RELEASE=${PROG}.${PLATFORM}
 # hardcode versions unless set specifically
 KUBERNETES_VERSION=${KUBERNETES_VERSION:-v1.27.14}
 KUBERNETES_IMAGE_TAG=${KUBERNETES_IMAGE_TAG:-v1.27.14-rke2r1-build20240515}
-ETCD_VERSION=${ETCD_VERSION:-v3.5.9-k3s1}
+ETCD_VERSION=${ETCD_VERSION:-v3.5.13-k3s1}
 PAUSE_VERSION=${PAUSE_VERSION:-3.6}
 CCM_VERSION=${CCM_VERSION:-v1.29.3-build20240515}