k8sd: Add logic to set and get the config that the cluster was bootra…

…pped with

src/k8s/go.mod

@@ -7,7 +7,7 @@ toolchain go1.23.4
 require (
 	dario.cat/mergo v1.0.0
 	github.com/canonical/go-dqlite/v2 v2.0.0
-	github.com/canonical/k8s-snap-api v1.0.17
+	github.com/canonical/k8s-snap-api v1.0.18-0.20250214142145-772178e23c39
 	github.com/canonical/lxd v0.0.0-20250113143058-52441d41dab7
 	github.com/canonical/microcluster/v2 v2.1.1-0.20250127104725-631889214b18
 	github.com/go-logr/logr v1.4.2

src/k8s/go.sum

@@ -53,8 +53,8 @@ github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXe
 github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
 github.com/canonical/go-dqlite/v2 v2.0.0 h1:RNFcFVhHMh70muKKErbW35rSzqmAFswheHdAgxW0Ddw=
 github.com/canonical/go-dqlite/v2 v2.0.0/go.mod h1:IaIC8u4Z1UmPjuAqPzA2r83YMaMHRLoKZdHKI5uHCJI=
-github.com/canonical/k8s-snap-api v1.0.17 h1:r+xr+eQflaP+fadIH2RfBcAyF3Q4UFV9FtJ6TnBFm/k=
-github.com/canonical/k8s-snap-api v1.0.17/go.mod h1:LDPoIYCeYnfgOFrwVPJ/4edGU264w7BB7g0GsVi36AY=
+github.com/canonical/k8s-snap-api v1.0.18-0.20250214142145-772178e23c39 h1:86HHK+7E6nVlApYtWMlKGHebVh9SyuCwTr4bFPHgejQ=
+github.com/canonical/k8s-snap-api v1.0.18-0.20250214142145-772178e23c39/go.mod h1:LDPoIYCeYnfgOFrwVPJ/4edGU264w7BB7g0GsVi36AY=
 github.com/canonical/lxd v0.0.0-20250113143058-52441d41dab7 h1:lZCOt9/1KowNdnWXjfA1/51Uj7+R0fKtByos9EVrYn4=
 github.com/canonical/lxd v0.0.0-20250113143058-52441d41dab7/go.mod h1:4Ssm3YxIz8wyazciTLDR9V0aR2GPlGIHb+S0182T5pA=
 github.com/canonical/microcluster/v2 v2.1.1-0.20250127104725-631889214b18 h1:h5VJaUnE4gAKPolBTJ11HMRTEN5JyA+oR4gHkoK//6o=

src/k8s/pkg/k8sd/api/cluster_config.go

@@ -10,6 +10,7 @@ import (
 	"github.com/canonical/k8s/pkg/k8sd/database"
 	databaseutil "github.com/canonical/k8s/pkg/k8sd/database/util"
 	"github.com/canonical/k8s/pkg/k8sd/types"
+	snaputil "github.com/canonical/k8s/pkg/snap/util"
 	"github.com/canonical/k8s/pkg/utils"
 	"github.com/canonical/lxd/lxd/response"
 	"github.com/canonical/microcluster/v2/state"
@@ -63,3 +64,29 @@ func (e *Endpoints) getClusterConfig(s state.State, r *http.Request) response.Re
 		Config: config.ToUserFacing(),
 	})
 }
+
+func (e *Endpoints) getClusterBootstrapConfig(s state.State, r *http.Request) response.Response {
+	config, err := databaseutil.GetClusterBootstrapConfig(r.Context(), s)
+	if err != nil {
+		return response.InternalError(fmt.Errorf("failed to get cluster bootstrap config: %w", err))
+	}
+
+	var nodeTaints *[]string
+	snap := e.provider.Snap()
+	isWorker, err := snaputil.IsWorker(snap)
+	if err != nil {
+		return response.InternalError(fmt.Errorf("failed to check if node is a worker: %w", err))
+	}
+
+	if isWorker {
+		nodeTaints = config.Kubelet.WorkerTaints
+	} else {
+		nodeTaints = config.Kubelet.ControlPlaneTaints
+	}
+
+	return response.SyncResponse(true, &apiv1.GetClusterBootstrapConfigResponse{
+		ClusterConfig: config.ToUserFacing(),
+		Datastore:     config.Datastore.ToUserFacing(),
+		NodeTaints:    nodeTaints,
+	})
+}

src/k8s/pkg/k8sd/api/endpoints.go

@@ -114,6 +114,12 @@ func (e *Endpoints) Endpoints() []rest.Endpoint {
 			Put:  rest.EndpointAction{Handler: e.putClusterConfig, AccessHandler: e.restrictWorkers},
 			Get:  rest.EndpointAction{Handler: e.getClusterConfig, AccessHandler: e.restrictWorkers},
 		},
+		// Get the config that the cluster was bootstrapped with.
+		{
+			Name: "ClusterBootstrapConfig",
+			Path: apiv1.GetClusterBootstrapConfigRPC,
+			Get:  rest.EndpointAction{Handler: e.getClusterBootstrapConfig},
+		},
 		// Kubernetes auth tokens and token review webhook for kube-apiserver
 		{
 			Name:   "KubernetesAuthTokens",

src/k8s/pkg/k8sd/app/hooks_bootstrap.go

@@ -205,6 +205,15 @@ func (a *App) onBootstrapWorkerNode(ctx context.Context, s state.State, encodedT
 		return fmt.Errorf("failed to generate kube-proxy kubeconfig: %w", err)
 	}
 
+	// NOTE(Hue): This is how the taints are set for the worker nodes in the charm.
+	// https://github.com/canonical/k8s-operator/blob/bd9ebbda153053f9bfd6e66a93d2afb629a6cfd8/charms/worker/k8s/src/config/extra_args.py#L89
+	var taints []string
+	if taintsStr, ok := joinConfig.ExtraNodeKubeletArgs["--register-with-taints"]; ok {
+		if taintsStr != nil {
+			taints = strings.Split(*taintsStr, ",")
+		}
+	}
+
 	// Write worker node configuration to dqlite
 	//
 	// Worker nodes only use a subset of the ClusterConfig struct. At the moment, these are:
@@ -231,6 +240,16 @@ func (a *App) onBootstrapWorkerNode(ctx context.Context, s state.State, encodedT
 		Annotations: response.Annotations,
 	}
 
+	if len(taints) > 0 {
+		cfg.Kubelet = types.Kubelet{
+			// NOTE(Hue): We set the worker taints here so that the charm
+			// can later prevent the user from changing these taints through charm config.
+			// These taints for the worker nodes are set by the `bootstrap-node-taints` charm config.
+			// https://github.com/canonical/k8s-operator/blob/bd9ebbda153053f9bfd6e66a93d2afb629a6cfd8/charms/worker/charmcraft.yaml#L67
+			WorkerTaints: utils.Pointer(taints),
+		}
+	}
+
 	serviceConfigs := types.K8sServiceConfigs{
 		ExtraNodeKubeletArgs:   joinConfig.ExtraNodeKubeletArgs,
 		ExtraNodeKubeProxyArgs: joinConfig.ExtraNodeKubeProxyArgs,
@@ -241,6 +260,16 @@ func (a *App) onBootstrapWorkerNode(ctx context.Context, s state.State, encodedT
 		return fmt.Errorf("pre-init checks failed for worker node: %w", err)
 	}
 
+	// Write cluster bootstrap configuration to dqlite
+	if err := s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error {
+		if err := database.SetClusterBootstrapConfig(ctx, tx, cfg); err != nil {
+			return fmt.Errorf("failed to set cluster bootstrap config: %w", err)
+		}
+		return nil
+	}); err != nil {
+		return fmt.Errorf("database transaction to set cluster bootstrap config failed: %w", err)
+	}
+
 	if err := s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error {
 		if _, err := database.SetClusterConfig(ctx, tx, cfg); err != nil {
 			return fmt.Errorf("failed to write cluster configuration: %w", err)
@@ -254,7 +283,7 @@ func (a *App) onBootstrapWorkerNode(ctx context.Context, s state.State, encodedT
 	if err := setup.Containerd(snap, joinConfig.ExtraNodeContainerdConfig, joinConfig.ExtraNodeContainerdArgs); err != nil {
 		return fmt.Errorf("failed to configure containerd: %w", err)
 	}
-	if err := setup.KubeletWorker(snap, s.Name(), nodeIP, response.ClusterDNS, response.ClusterDomain, response.CloudProvider, joinConfig.ExtraNodeKubeletArgs); err != nil {
+	if err := setup.KubeletWorker(snap, s.Name(), nodeIP, response.ClusterDNS, response.ClusterDomain, response.CloudProvider, joinConfig.ExtraNodeKubeletArgs, taints); err != nil {
 		return fmt.Errorf("failed to configure kubelet: %w", err)
 	}
 	if err := setup.KubeProxy(ctx, snap, s.Name(), response.PodCIDR, localhostAddress, joinConfig.ExtraNodeKubeProxyArgs); err != nil {
@@ -477,6 +506,16 @@ func (a *App) onBootstrapControlPlane(ctx context.Context, s state.State, bootst
 		return fmt.Errorf("failed to write extra node config files: %w", err)
 	}
 
+	// Write cluster bootstrap configuration to dqlite
+	if err := s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error {
+		if err := database.SetClusterBootstrapConfig(ctx, tx, cfg); err != nil {
+			return fmt.Errorf("failed to set cluster bootstrap config: %w", err)
+		}
+		return nil
+	}); err != nil {
+		return fmt.Errorf("database transaction to set cluster bootstrap config failed: %w", err)
+	}
+
 	// Write cluster configuration to dqlite
 	if err := s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error {
 		if _, err := database.SetClusterConfig(ctx, tx, cfg); err != nil {

src/k8s/pkg/k8sd/database/cluster_config.go

@@ -10,9 +10,22 @@ import (
 	"github.com/canonical/microcluster/v2/cluster"
 )
 
-var clusterConfigsStmts = map[string]int{
-	"insert-v1alpha2": MustPrepareStatement("cluster-configs", "insert-v1alpha2.sql"),
-	"select-v1alpha2": MustPrepareStatement("cluster-configs", "select-v1alpha2.sql"),
+const (
+	clusterConfigsDir string = "cluster-configs"
+)
+
+type clusterConfigsStmtsSchema struct {
+	insertV1alpha2          int
+	insertBootstrapV1alpha2 int
+	selectV1alpha2          int
+	selectBootstrapV1alpha2 int
+}
+
+var clusterConfigsStmts = clusterConfigsStmtsSchema{
+	insertV1alpha2:          MustPrepareStatement(clusterConfigsDir, "insert-v1alpha2.sql"),
+	insertBootstrapV1alpha2: MustPrepareStatement(clusterConfigsDir, "insert-bootstrap-v1alpha2.sql"),
+	selectV1alpha2:          MustPrepareStatement(clusterConfigsDir, "select-v1alpha2.sql"),
+	selectBootstrapV1alpha2: MustPrepareStatement(clusterConfigsDir, "select-bootstrap-v1alpha2.sql"),
 }
 
 // SetClusterConfig updates the cluster configuration with any non-empty values that are set.
@@ -32,7 +45,7 @@ func SetClusterConfig(ctx context.Context, tx *sql.Tx, new types.ClusterConfig)
 	if err != nil {
 		return types.ClusterConfig{}, fmt.Errorf("failed to encode cluster config: %w", err)
 	}
-	insertTxStmt, err := cluster.Stmt(tx, clusterConfigsStmts["insert-v1alpha2"])
+	insertTxStmt, err := cluster.Stmt(tx, clusterConfigsStmts.insertV1alpha2)
 	if err != nil {
 		return types.ClusterConfig{}, fmt.Errorf("failed to prepare insert statement: %w", err)
 	}
@@ -42,9 +55,30 @@ func SetClusterConfig(ctx context.Context, tx *sql.Tx, new types.ClusterConfig)
 	return config, nil
 }
 
+// SetClusterBootstrapConfig sets the cluster bootstrap configuration.
+// SetClusterBootstrapConfig will ignore the insertion command if the configuration is already set.
+// For workers, SetClusterBootstrapConfig sets the config that the worker was joined (bootstrapped) with.
+func SetClusterBootstrapConfig(ctx context.Context, tx *sql.Tx, config types.ClusterConfig) error {
+	b, err := json.Marshal(config)
+	if err != nil {
+		return fmt.Errorf("failed to encode cluster bootstrap config: %w", err)
+	}
+
+	insertTxStmt, err := cluster.Stmt(tx, clusterConfigsStmts.insertBootstrapV1alpha2)
+	if err != nil {
+		return fmt.Errorf("failed to prepare insert bootstrap config statement: %w", err)
+	}
+
+	if _, err := insertTxStmt.ExecContext(ctx, string(b)); err != nil {
+		return fmt.Errorf("failed to insert v1alpha2 bootstrap config: %w", err)
+	}
+
+	return nil
+}
+
 // GetClusterConfig retrieves the cluster configuration from the database.
 func GetClusterConfig(ctx context.Context, tx *sql.Tx) (types.ClusterConfig, error) {
-	txStmt, err := cluster.Stmt(tx, clusterConfigsStmts["select-v1alpha2"])
+	txStmt, err := cluster.Stmt(tx, clusterConfigsStmts.selectV1alpha2)
 	if err != nil {
 		return types.ClusterConfig{}, fmt.Errorf("failed to prepare statement: %w", err)
 	}
@@ -64,3 +98,27 @@ func GetClusterConfig(ctx context.Context, tx *sql.Tx) (types.ClusterConfig, err
 
 	return clusterConfig, nil
 }
+
+// GetClusterBootstrapConfig retrieves the cluster bootstrap configuration from the database.
+// For workers, GetClusterBootstrapConfig returns the config that the worker was joined (bootstrapped) with.
+func GetClusterBootstrapConfig(ctx context.Context, tx *sql.Tx) (types.ClusterConfig, error) {
+	txStmt, err := cluster.Stmt(tx, clusterConfigsStmts.selectBootstrapV1alpha2)
+	if err != nil {
+		return types.ClusterConfig{}, fmt.Errorf("failed to prepare get bootstrap config statement: %w", err)
+	}
+
+	var s string
+	if err := txStmt.QueryRowContext(ctx).Scan(&s); err != nil {
+		if err == sql.ErrNoRows {
+			return types.ClusterConfig{}, nil
+		}
+		return types.ClusterConfig{}, fmt.Errorf("failed to retrieve v1alpha2 bootstrap config: %w", err)
+	}
+
+	var clusterConfig types.ClusterConfig
+	if err := json.Unmarshal([]byte(s), &clusterConfig); err != nil {
+		return types.ClusterConfig{}, fmt.Errorf("failed to parse v1alpha2 bootstrap config: %w", err)
+	}
+
+	return clusterConfig, nil
+}

src/k8s/pkg/k8sd/database/cluster_config_test.go

@@ -11,6 +11,7 @@ import (
 	testenv "github.com/canonical/k8s/pkg/utils/microcluster"
 	"github.com/canonical/microcluster/v2/state"
 	. "github.com/onsi/gomega"
+	"k8s.io/utils/ptr"
 )
 
 func TestClusterConfig(t *testing.T) {
@@ -119,5 +120,43 @@ func TestClusterConfig(t *testing.T) {
 			})
 			g.Expect(err).To(Not(HaveOccurred()))
 		})
+
+		t.Run("SetBootstrapConfig", func(t *testing.T) {
+			g := NewWithT(t)
+			expBootstrapConfig := types.ClusterConfig{}
+			expBootstrapConfig.SetDefaults()
+
+			err := s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error {
+				return database.SetClusterBootstrapConfig(context.Background(), tx, expBootstrapConfig)
+			})
+			g.Expect(err).To(Not(HaveOccurred()))
+
+			err = s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error {
+				bootstrapConfig, err := database.GetClusterBootstrapConfig(ctx, tx)
+				g.Expect(bootstrapConfig).To(Equal(expBootstrapConfig))
+				return err
+			})
+			g.Expect(err).To(Not(HaveOccurred()))
+
+			newConfig := expBootstrapConfig
+			// Toggle the network enabled field as an example of a change
+			if *newConfig.Network.Enabled {
+				newConfig.Network.Enabled = ptr.To(false)
+			} else {
+				newConfig.Network.Enabled = ptr.To(true)
+			}
+
+			err = s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error {
+				return database.SetClusterBootstrapConfig(context.Background(), tx, newConfig)
+			})
+			g.Expect(err).To(Not(HaveOccurred()))
+
+			err = s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error {
+				bootstrapConfig, err := database.GetClusterBootstrapConfig(ctx, tx)
+				g.Expect(bootstrapConfig).To(Equal(expBootstrapConfig))
+				return err
+			})
+			g.Expect(err).To(Not(HaveOccurred()))
+		})
 	})
 }

src/k8s/pkg/k8sd/database/sql/queries/cluster-configs/insert-bootstrap-v1alpha2.sql

@@ -0,0 +1,4 @@
+INSERT OR IGNORE INTO
+    cluster_configs(key, value)
+VALUES
+    ("bootstrap-v1alpha2", ?);

src/k8s/pkg/k8sd/database/sql/queries/cluster-configs/select-bootstrap-v1alpha2.sql

@@ -0,0 +1,6 @@
+SELECT
+    c.value
+FROM
+    cluster_configs AS c
+WHERE
+    c.key = "bootstrap-v1alpha2"

src/k8s/pkg/k8sd/database/util/cluster.go

@@ -27,3 +27,20 @@ func GetClusterConfig(ctx context.Context, state state.State) (types.ClusterConf
 
 	return clusterConfig, nil
 }
+
+// GetClusterBootstrapConfig is a convenience wrapper around the database call to get the cluster bootstrap config.
+func GetClusterBootstrapConfig(ctx context.Context, state state.State) (types.ClusterConfig, error) {
+	var config types.ClusterConfig
+
+	if err := state.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error {
+		var err error
+		if config, err = database.GetClusterBootstrapConfig(ctx, tx); err != nil {
+			return fmt.Errorf("failed to get cluster bootstrap config from database: %w", err)
+		}
+		return nil
+	}); err != nil {
+		return types.ClusterConfig{}, fmt.Errorf("failed to perform get cluster bootstrap config transaction: %w", err)
+	}
+
+	return config, nil
+}

src/k8s/pkg/k8sd/setup/kubelet.go

@@ -41,8 +41,8 @@ func KubeletControlPlane(snap snap.Snap, hostname string, nodeIP net.IP, cluster
 }
 
 // KubeletWorker configures kubelet on a worker node.
-func KubeletWorker(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, extraArgs map[string]*string) error {
-	return kubelet(snap, hostname, nodeIP, clusterDNS, clusterDomain, cloudProvider, nil, kubeletWorkerLabels, extraArgs)
+func KubeletWorker(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, extraArgs map[string]*string, taints []string) error {
+	return kubelet(snap, hostname, nodeIP, clusterDNS, clusterDomain, cloudProvider, taints, kubeletWorkerLabels, extraArgs)
 }
 
 // kubelet configures kubelet on the local node.