Add kine support

Signed-off-by: Vitor Savian <[email protected]>

pkg/cli/cmds/server.go

@@ -73,10 +73,10 @@ var (
 		"kube-controller-manager-arg":       copyFlag,
 		"kube-cloud-controller-manager-arg": copyFlag,
 		"kube-cloud-controller-arg":         dropFlag, // deprecated version of kube-cloud-controller-manager-arg
-		"datastore-endpoint":                dropFlag,
-		"datastore-cafile":                  dropFlag,
-		"datastore-certfile":                dropFlag,
-		"datastore-keyfile":                 dropFlag,
+		"datastore-endpoint":                copyFlag,
+		"datastore-cafile":                  copyFlag,
+		"datastore-certfile":                copyFlag,
+		"datastore-keyfile":                 copyFlag,
 		"default-local-storage-path":        dropFlag,
 		"disable": {
 			Usage: "(components) Do not deploy packaged components and delete any deployed components (valid items: " + strings.Join(DisableItems, ", ") + ")",

pkg/podexecutor/staticpod.go

@@ -109,19 +109,20 @@ type StaticPodConfig struct {
 	ControlPlaneProbeConfs
 	ControlPlaneEnv
 	ControlPlaneMounts
-	ManifestsDir    string
-	ImagesDir       string
-	Resolver        *images.Resolver
-	CloudProvider   *CloudProviderConfig
-	DataDir         string
-	AuditPolicyFile string
-	PSAConfigFile   string
-	KubeletPath     string
-	RuntimeEndpoint string
-	KubeProxyChan   chan struct{}
-	CISMode         bool
-	DisableETCD     bool
-	IsServer        bool
+	ManifestsDir     string
+	ImagesDir        string
+	Resolver         *images.Resolver
+	CloudProvider    *CloudProviderConfig
+	DataDir          string
+	AuditPolicyFile  string
+	PSAConfigFile    string
+	KubeletPath      string
+	RuntimeEndpoint  string
+	KubeProxyChan    chan struct{}
+	CISMode          bool
+	DisableETCD      bool
+	ExternalDatabase bool
+	IsServer         bool
 
 	stopKubelet context.CancelFunc
 }
@@ -264,6 +265,11 @@ func (s *StaticPodConfig) APIServer(_ context.Context, etcdReady <-chan struct{}
 		case "--advertise-port", "--basic-auth-file":
 			// This is an option k3s adds that does not exist upstream
 			args = append(args[:i], args[i+1:]...)
+		case "--etcd-servers":
+			if s.ExternalDatabase {
+				args = append(args[:i], args[i+1:]...)
+				args = append([]string{"--etcd-servers=" + "unixs://" + filepath.Join(s.DataDir, "server", "kine.sock")}, args...)
+			}
 		case "--audit-log-path":
 			auditLogFile = value
 		case "--kubelet-preferred-address-types":
@@ -309,47 +315,56 @@ func (s *StaticPodConfig) APIServer(_ context.Context, etcdReady <-chan struct{}
 	if !s.DisableETCD {
 		files = append(files, etcdNameFile(s.DataDir))
 	}
+
+	sockets := []string{}
+	if s.ExternalDatabase {
+		sockets = append(sockets, kineSock(s.DataDir))
+	}
+
 	dirs := onlyExisting(ssldirs)
 	if auditLogFile != "" && auditLogFile != "-" {
 		dirs = append(dirs, filepath.Dir(auditLogFile))
 		excludeFiles = append(excludeFiles, auditLogFile)
 	}
 
+	apiServerArgs := staticpod.Args{
+		Command:       "kube-apiserver",
+		Args:          args,
+		Image:         image,
+		Dirs:          dirs,
+		CISMode:       s.CISMode,
+		CPURequest:    s.ControlPlaneResources.KubeAPIServerCPURequest,
+		CPULimit:      s.ControlPlaneResources.KubeAPIServerCPULimit,
+		MemoryRequest: s.ControlPlaneResources.KubeAPIServerMemoryRequest,
+		MemoryLimit:   s.ControlPlaneResources.KubeAPIServerMemoryLimit,
+		ExtraEnv:      s.ControlPlaneEnv.KubeAPIServer,
+		ExtraMounts:   s.ControlPlaneMounts.KubeAPIServer,
+		ProbeConfs:    s.ControlPlaneProbeConfs.KubeAPIServer,
+		Sockets:       sockets,
+		Files:         files,
+		ExcludeFiles:  excludeFiles,
+		HealthExec: []string{
+			"kubectl",
+			"get",
+			"--server=https://localhost:6443/",
+			"--client-certificate=" + s.DataDir + "/server/tls/client-kube-apiserver.crt",
+			"--client-key=" + s.DataDir + "/server/tls/client-kube-apiserver.key",
+			"--certificate-authority=" + s.DataDir + "/server/tls/server-ca.crt",
+			"--raw=/livez",
+		},
+		ReadyExec: []string{
+			"kubectl",
+			"get",
+			"--server=https://localhost:6443/",
+			"--client-certificate=" + s.DataDir + "/server/tls/client-kube-apiserver.crt",
+			"--client-key=" + s.DataDir + "/server/tls/client-kube-apiserver.key",
+			"--certificate-authority=" + s.DataDir + "/server/tls/server-ca.crt",
+			"--raw=/readyz",
+		},
+	}
+
 	return after(etcdReady, func() error {
-		return staticpod.Run(s.ManifestsDir, staticpod.Args{
-			Command:       "kube-apiserver",
-			Args:          args,
-			Image:         image,
-			Dirs:          dirs,
-			CISMode:       s.CISMode,
-			CPURequest:    s.ControlPlaneResources.KubeAPIServerCPURequest,
-			CPULimit:      s.ControlPlaneResources.KubeAPIServerCPULimit,
-			MemoryRequest: s.ControlPlaneResources.KubeAPIServerMemoryRequest,
-			MemoryLimit:   s.ControlPlaneResources.KubeAPIServerMemoryLimit,
-			ExtraEnv:      s.ControlPlaneEnv.KubeAPIServer,
-			ExtraMounts:   s.ControlPlaneMounts.KubeAPIServer,
-			ProbeConfs:    s.ControlPlaneProbeConfs.KubeAPIServer,
-			Files:         files,
-			ExcludeFiles:  excludeFiles,
-			HealthExec: []string{
-				"kubectl",
-				"get",
-				"--server=https://localhost:6443/",
-				"--client-certificate=" + s.DataDir + "/server/tls/client-kube-apiserver.crt",
-				"--client-key=" + s.DataDir + "/server/tls/client-kube-apiserver.key",
-				"--certificate-authority=" + s.DataDir + "/server/tls/server-ca.crt",
-				"--raw=/livez",
-			},
-			ReadyExec: []string{
-				"kubectl",
-				"get",
-				"--server=https://localhost:6443/",
-				"--client-certificate=" + s.DataDir + "/server/tls/client-kube-apiserver.crt",
-				"--client-key=" + s.DataDir + "/server/tls/client-kube-apiserver.key",
-				"--certificate-authority=" + s.DataDir + "/server/tls/server-ca.crt",
-				"--raw=/readyz",
-			},
-		})
+		return staticpod.Run(s.ManifestsDir, apiServerArgs)
 	})
 }
 
@@ -368,6 +383,12 @@ func (s *StaticPodConfig) Scheduler(_ context.Context, apiReady <-chan struct{},
 	if !s.DisableETCD {
 		files = append(files, etcdNameFile(s.DataDir))
 	}
+
+	sockets := []string{}
+	if s.ExternalDatabase {
+		sockets = append(sockets, kineSock(s.DataDir))
+	}
+
 	args = append(permitPortSharingFlag, args...)
 	return after(apiReady, func() error {
 		return staticpod.Run(s.ManifestsDir, staticpod.Args{
@@ -385,6 +406,7 @@ func (s *StaticPodConfig) Scheduler(_ context.Context, apiReady <-chan struct{},
 			ExtraMounts:   s.ControlPlaneMounts.KubeScheduler,
 			ProbeConfs:    s.ControlPlaneProbeConfs.KubeScheduler,
 			Files:         files,
+			Sockets:       sockets,
 		})
 	})
 }
@@ -433,6 +455,12 @@ func (s *StaticPodConfig) ControllerManager(_ context.Context, apiReady <-chan s
 	if !s.DisableETCD {
 		files = append(files, etcdNameFile(s.DataDir))
 	}
+
+	sockets := []string{}
+	if s.ExternalDatabase {
+		sockets = append(sockets, kineSock(s.DataDir))
+	}
+
 	return after(apiReady, func() error {
 		extraArgs := []string{
 			"--flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins",
@@ -455,6 +483,7 @@ func (s *StaticPodConfig) ControllerManager(_ context.Context, apiReady <-chan s
 			ExtraMounts:   s.ControlPlaneMounts.KubeControllerManager,
 			ProbeConfs:    s.ControlPlaneProbeConfs.KubeControllerManager,
 			Files:         files,
+			Sockets:       sockets,
 		})
 	})
 }
@@ -686,6 +715,10 @@ func chownr(path string, uid, gid int) error {
 	})
 }
 
+func kineSock(dataDir string) string {
+	return filepath.Join(dataDir, "server", "kine.sock")
+}
+
 func etcdNameFile(dataDir string) string {
 	return filepath.Join(dataDir, "server", "db", "etcd", "name")
 }

pkg/rke2/rke2.go

@@ -154,17 +154,18 @@ func setup(clx *cli.Context, cfg Config, isServer bool) error {
 	}
 
 	// check for missing db name file on a server running etcd, indicating we're rejoining after cluster reset on a different node
-	if _, err := os.Stat(etcdNameFile(dataDir)); err != nil && os.IsNotExist(err) && isServer && !clx.Bool("disable-etcd") {
+	if _, err := os.Stat(etcdNameFile(dataDir)); err != nil && os.IsNotExist(err) && isServer && !clx.Bool("disable-etcd") && !clx.IsSet("datastore-endpoint") {
 		clusterReset = true
 	}
 
 	disabledItems := map[string]bool{
 		"cloud-controller-manager": !isServer || forceRestart || clx.Bool("disable-cloud-controller"),
-		"etcd":                     !isServer || forceRestart || clx.Bool("disable-etcd"),
+		"etcd":                     !isServer || forceRestart || clx.Bool("disable-etcd") || clx.IsSet("datastore-endpoint"),
 		"kube-apiserver":           !isServer || forceRestart || clx.Bool("disable-apiserver"),
 		"kube-controller-manager":  !isServer || forceRestart || clx.Bool("disable-controller-manager"),
 		"kube-scheduler":           !isServer || forceRestart || clx.Bool("disable-scheduler"),
 	}
+
 	// adding force restart file when cluster reset restore path is passed
 	if clusterResetRestorePath != "" {
 		forceRestartFile := ForceRestartFile(dataDir)
@@ -175,6 +176,7 @@ func setup(clx *cli.Context, cfg Config, isServer bool) error {
 			return err
 		}
 	}
+
 	return removeDisabledPods(dataDir, containerRuntimeEndpoint, disabledItems, clusterReset)
 }
 

pkg/rke2/rke2_linux.go

@@ -57,10 +57,17 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta
 		return nil, err
 	}
 
-	agentManifestsDir := filepath.Join(dataDir, "agent", config.DefaultPodManifestPath)
-	agentImagesDir := filepath.Join(dataDir, "agent", "images")
+	// Verify if the user are using a external database
+	// and then remove the etcd from the static pod
+	ExternalDatabase := false
+	if cmds.ServerConfig.DatastoreEndpoint != "" {
+		cmds.ServerConfig.ClusterInit = false
+		ExternalDatabase = true
+	}
 
 	managed.RegisterDriver(&etcd.ETCD{})
+	agentManifestsDir := filepath.Join(dataDir, "agent", config.DefaultPodManifestPath)
+	agentImagesDir := filepath.Join(dataDir, "agent", "images")
 
 	if clx.IsSet("cloud-provider-config") || clx.IsSet("cloud-provider-name") {
 		if clx.IsSet("node-external-ip") {
@@ -115,6 +122,7 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta
 	if err != nil {
 		return nil, err
 	}
+
 	// Adding PSAs
 	podSecurityConfigFile := clx.String("pod-security-admission-config-file")
 	if podSecurityConfigFile == "" {
@@ -142,6 +150,7 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta
 		RuntimeEndpoint:        containerRuntimeEndpoint,
 		KubeProxyChan:          make(chan struct{}),
 		DisableETCD:            clx.Bool("disable-etcd"),
+		ExternalDatabase:       ExternalDatabase,
 		IsServer:               isServer,
 		ControlPlaneResources:  *controlPlaneResources,
 		ControlPlaneProbeConfs: *controlPlaneProbeConfs,
@@ -155,7 +164,7 @@ func parseControlPlaneResources(cfg Config) (*podexecutor.ControlPlaneResources,
 	// resources is a map of the component (kube-apiserver, kube-controller-manager, etc.) to a map[string]*string,
 	// where the key of the downstream map is the `cpu-request`, `cpu-limit`, `memory-request`, or `memory-limit` and
 	// the value corresponds to a pointer to the component resources array
-	var resources = map[string]map[string]*string{
+	resources := map[string]map[string]*string{
 		KubeAPIServer: {
 			CPURequest:    &controlPlaneResources.KubeAPIServerCPURequest,
 			CPULimit:      &controlPlaneResources.KubeAPIServerCPULimit,
@@ -195,7 +204,7 @@ func parseControlPlaneResources(cfg Config) (*podexecutor.ControlPlaneResources,
 	}
 
 	// defaultResources contains a map of default resources for each component, used if not explicitly configured.
-	var defaultResources = map[string]map[string]string{
+	defaultResources := map[string]map[string]string{
 		KubeAPIServer: {
 			CPURequest:    "250m",
 			MemoryRequest: "1024Mi",
@@ -222,7 +231,7 @@ func parseControlPlaneResources(cfg Config) (*podexecutor.ControlPlaneResources,
 		},
 	}
 
-	var parsedRequestsLimits = make(map[string]string)
+	parsedRequestsLimits := make(map[string]string)
 
 	for _, requests := range cfg.ControlPlaneResourceRequests {
 		for _, rawRequest := range strings.Split(requests, ",") {
@@ -262,7 +271,7 @@ func parseControlPlaneProbeConfs(cfg Config) (*podexecutor.ControlPlaneProbeConf
 	var controlPlaneProbes podexecutor.ControlPlaneProbeConfs
 	// probes is a map of the component (kube-apiserver, kube-controller-manager, etc.) probe type, and setting, where
 	// the value corresponds to a pointer to the component probes array.
-	var probes = map[string]map[string]map[string]*int32{
+	probes := map[string]map[string]map[string]*int32{
 		KubeAPIServer: {
 			Liveness: {
 				InitialDelaySeconds: &controlPlaneProbes.KubeAPIServer.Liveness.InitialDelaySeconds,
@@ -386,7 +395,7 @@ func parseControlPlaneProbeConfs(cfg Config) (*podexecutor.ControlPlaneProbeConf
 	}
 
 	// defaultProbeConf contains a map of default probe settings for each type, used if not explicitly configured.
-	var defaultProbeConf = map[string]map[string]int32{
+	defaultProbeConf := map[string]map[string]int32{
 		// https://github.com/kubernetes/kubernetes/blob/v1.24.0/cmd/kubeadm/app/util/staticpod/utils.go#L246
 		Liveness: {
 			InitialDelaySeconds: 10,
@@ -410,7 +419,7 @@ func parseControlPlaneProbeConfs(cfg Config) (*podexecutor.ControlPlaneProbeConf
 		},
 	}
 
-	var parsedProbeConf = make(map[string]int32)
+	parsedProbeConf := make(map[string]int32)
 
 	for _, conf := range cfg.ControlPlaneProbeConf {
 		for _, rawConf := range strings.Split(conf, ",") {

pkg/staticpod/staticpod.go

@@ -28,8 +28,13 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+type typeVolume string
+
 const (
-	extraMountPrefix = "extra-mount"
+	extraMountPrefix            = "extra-mount"
+	socket           typeVolume = "socket"
+	dir              typeVolume = "dir"
+	file             typeVolume = "file"
 )
 
 type ProbeConf struct {
@@ -51,6 +56,7 @@ type Args struct {
 	Image           name.Reference
 	Dirs            []string
 	Files           []string
+	Sockets         []string
 	CISMode         bool // CIS requires that the manifest be saved with 600 permissions
 	ExcludeFiles    []string
 	HealthExec      []string
@@ -278,22 +284,33 @@ func pod(args Args) (*v1.Pod, error) {
 		}
 	}
 
-	addVolumes(p, args.Dirs, true)
-	addVolumes(p, args.Files, false)
+	addVolumes(p, args.Sockets, socket)
+	addVolumes(p, args.Dirs, dir)
+	addVolumes(p, args.Files, file)
 
 	addExtraMounts(p, args.ExtraMounts)
 	addExtraEnv(p, args.ExtraEnv)
 
 	return p, nil
 }
 
-func addVolumes(p *v1.Pod, src []string, dir bool) {
+func addVolumes(p *v1.Pod, src []string, volume typeVolume) {
 	var (
-		prefix     = "dir"
-		sourceType = v1.HostPathDirectoryOrCreate
-		readOnly   = false
+		prefix     string
+		sourceType v1.HostPathType
+		readOnly   bool
 	)
-	if !dir {
+
+	switch volume {
+	case dir:
+		prefix = "dir"
+		sourceType = v1.HostPathDirectoryOrCreate
+		readOnly = false
+	case socket:
+		prefix = "sock"
+		sourceType = v1.HostPathSocket
+		readOnly = false
+	default:
 		prefix = "file"
 		sourceType = v1.HostPathFile
 		readOnly = true
@@ -319,9 +336,7 @@ func addVolumes(p *v1.Pod, src []string, dir bool) {
 }
 
 func addExtraMounts(p *v1.Pod, extraMounts []string) {
-	var (
-		sourceType = v1.HostPathDirectoryOrCreate
-	)
+	sourceType := v1.HostPathDirectoryOrCreate
 
 	for i, rawMount := range extraMounts {
 		mount := strings.Split(rawMount, ":")