feat(k8sd)!: introduce support for storing helm charts on microcluste…

…r db

- Added a table to the microcluster database for storing helm charts.
- Added queries and helper functions for inserting and selecting helm charts.
- Added the ChartLoader interface and updated the helm client to enable loading helm charts in different ways.
- Added the database based ChartLoader to fetch charts from microcluser.
- Added the embed based ChartLoader to fetch charts from the embedded charts, used in testing.
- Added a helm chart controller that synchronizes available charts in the snap to the microcluster database.
- Moved helm charts from `k8s/manifests` into respective feature directories and embedded the charts using go embed package.
- Packed helm charts that are in directory form to tarballs.
- Updated feature controller to create the helm char with the database loader and updated features to consume a helm client.
- Fixed the bug of ports still being in use, in the testing util `WithState` by adding a retry loop for microcluster bootstrapping.
- Updated InstallableChart type to include the name and version of the chart.

src/k8s/cmd/k8sd/k8sd.go

@@ -17,6 +17,7 @@ var rootCmdOpts struct {
 	pprofAddress                        string
 	disableNodeConfigController         bool
 	disableNodeLabelController          bool
+	disableHelmChartController          bool
 	disableControlPlaneConfigController bool
 	disableFeatureController            bool
 	disableUpdateNodeConfigController   bool
@@ -54,6 +55,7 @@ func NewRootCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
 				PprofAddress:                        rootCmdOpts.pprofAddress,
 				DisableNodeConfigController:         rootCmdOpts.disableNodeConfigController,
 				DisableNodeLabelController:          rootCmdOpts.disableNodeLabelController,
+				DisableHelmChartController:          rootCmdOpts.disableHelmChartController,
 				DisableControlPlaneConfigController: rootCmdOpts.disableControlPlaneConfigController,
 				DisableUpdateNodeConfigController:   rootCmdOpts.disableUpdateNodeConfigController,
 				DisableFeatureController:            rootCmdOpts.disableFeatureController,
@@ -85,6 +87,7 @@ func NewRootCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
 	cmd.PersistentFlags().StringVar(&rootCmdOpts.pprofAddress, "pprof-address", "", "Listen address for pprof endpoints, e.g. \"127.0.0.1:4217\"")
 	cmd.PersistentFlags().BoolVar(&rootCmdOpts.disableNodeConfigController, "disable-node-config-controller", false, "Disable the Node Config Controller")
 	cmd.PersistentFlags().BoolVar(&rootCmdOpts.disableNodeLabelController, "disable-node-label-controller", false, "Disable the Node Label Controller")
+	cmd.PersistentFlags().BoolVar(&rootCmdOpts.disableHelmChartController, "disable-helm-chart-controller", false, "Disable the Helm Chart Controller")
 	cmd.PersistentFlags().BoolVar(&rootCmdOpts.disableControlPlaneConfigController, "disable-control-plane-config-controller", false, "Disable the Control Plane Config Controller")
 	cmd.PersistentFlags().BoolVar(&rootCmdOpts.disableUpdateNodeConfigController, "disable-update-node-config-controller", false, "Disable the Update Node Config Controller")
 	cmd.PersistentFlags().BoolVar(&rootCmdOpts.disableFeatureController, "disable-feature-controller", false, "Disable the Feature Controller")

src/k8s/pkg/client/helm/chart.go

@@ -2,13 +2,15 @@ package helm
 
 // InstallableChart describes a chart that can be deployed on a running cluster.
 type InstallableChart struct {
-	// Name is the install name of the chart.
+	// Name is the name of the chart.
 	Name string
 
-	// Namespace is the namespace to install the chart.
-	Namespace string
+	// Version is the version of the chart.
+	Version string
 
-	// ManifestPath is the path to the chart's manifest, typically relative to "$SNAP/k8s/manifests".
-	// TODO(neoaggelos): this should be a *chart.Chart, and we should use the "embed" package to load it when building k8sd.
-	ManifestPath string
+	// InstallName is the install name of the chart, used as the release name in helm.
+	InstallName string
+
+	// InstallNamespace is the namespace to install the chart into.
+	InstallNamespace string
 }

src/k8s/pkg/client/helm/client.go

@@ -6,29 +6,27 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"path/filepath"
 
 	"github.com/canonical/k8s/pkg/log"
 	"helm.sh/helm/v3/pkg/action"
-	"helm.sh/helm/v3/pkg/chart/loader"
 	"helm.sh/helm/v3/pkg/storage/driver"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 )
 
 // client implements Client using Helm.
 type client struct {
+	chartLoader      ChartLoader
 	restClientGetter func(string) genericclioptions.RESTClientGetter
-	manifestsBaseDir string
 }
 
 // ensure *client implements Client.
 var _ Client = &client{}
 
 // NewClient creates a new client.
-func NewClient(manifestsBaseDir string, restClientGetter func(string) genericclioptions.RESTClientGetter) *client {
+func NewClient(restClientGetter func(string) genericclioptions.RESTClientGetter, chartLoader ChartLoader) *client {
 	return &client{
 		restClientGetter: restClientGetter,
-		manifestsBaseDir: manifestsBaseDir,
+		chartLoader:      chartLoader,
 	}
 }
 
@@ -46,7 +44,7 @@ func (h *client) newActionConfiguration(ctx context.Context, namespace string) (
 
 // Apply implements the Client interface.
 func (h *client) Apply(ctx context.Context, c InstallableChart, desired State, values map[string]any) (bool, error) {
-	cfg, err := h.newActionConfiguration(ctx, c.Namespace)
+	cfg, err := h.newActionConfiguration(ctx, c.InstallNamespace)
 	if err != nil {
 		return false, fmt.Errorf("failed to create action configuration: %w", err)
 	}
@@ -56,10 +54,10 @@ func (h *client) Apply(ctx context.Context, c InstallableChart, desired State, v
 
 	// get the latest Helm release with the specified name
 	get := action.NewGet(cfg)
-	release, err := get.Run(c.Name)
+	release, err := get.Run(c.InstallName)
 	if err != nil {
 		if !errors.Is(err, driver.ErrReleaseNotFound) {
-			return false, fmt.Errorf("failed to get status of release %s: %w", c.Name, err)
+			return false, fmt.Errorf("failed to get status of release %s: %w", c.InstallName, err)
 		}
 		isInstalled = false
 	} else {
@@ -73,46 +71,46 @@ func (h *client) Apply(ctx context.Context, c InstallableChart, desired State, v
 		return false, nil
 	case !isInstalled && desired == StateUpgradeOnly:
 		// there is no release installed, this is an error
-		return false, fmt.Errorf("cannot upgrade %s as it is not installed", c.Name)
+		return false, fmt.Errorf("cannot upgrade %s as it is not installed", c.InstallName)
 	case !isInstalled && desired == StatePresent:
 		// there is no release installed, so we must run an install action
 		install := action.NewInstall(cfg)
-		install.ReleaseName = c.Name
-		install.Namespace = c.Namespace
+		install.ReleaseName = c.InstallName
+		install.Namespace = c.InstallNamespace
 		install.CreateNamespace = true
 
-		chart, err := loader.Load(filepath.Join(h.manifestsBaseDir, c.ManifestPath))
+		chart, err := h.chartLoader.Load(ctx, c)
 		if err != nil {
-			return false, fmt.Errorf("failed to load manifest for %s: %w", c.Name, err)
+			return false, fmt.Errorf("failed to load manifest for %s: %w", c.InstallName, err)
 		}
 
 		if _, err := install.RunWithContext(ctx, chart, values); err != nil {
-			return false, fmt.Errorf("failed to install %s: %w", c.Name, err)
+			return false, fmt.Errorf("failed to install %s: %w", c.InstallName, err)
 		}
 		return true, nil
 	case isInstalled && desired != StateDeleted:
 		// there is already a release installed, so we must run an upgrade action
 		upgrade := action.NewUpgrade(cfg)
-		upgrade.Namespace = c.Namespace
+		upgrade.Namespace = c.InstallNamespace
 		upgrade.ResetThenReuseValues = true
 
-		chart, err := loader.Load(filepath.Join(h.manifestsBaseDir, c.ManifestPath))
+		chart, err := h.chartLoader.Load(ctx, c)
 		if err != nil {
-			return false, fmt.Errorf("failed to load manifest for %s: %w", c.Name, err)
+			return false, fmt.Errorf("failed to load manifest for %s: %w", c.InstallName, err)
 		}
 
-		release, err := upgrade.RunWithContext(ctx, c.Name, chart, values)
+		release, err := upgrade.RunWithContext(ctx, c.InstallName, chart, values)
 		if err != nil {
-			return false, fmt.Errorf("failed to upgrade %s: %w", c.Name, err)
+			return false, fmt.Errorf("failed to upgrade %s: %w", c.InstallName, err)
 		}
 
 		// oldConfig and release.Config are the previous and current values. they are compared by checking their respective JSON, as that is good enough for our needs of comparing unstructured map[string]any data.
 		return !jsonEqual(oldConfig, release.Config), nil
 	case isInstalled && desired == StateDeleted:
 		// run an uninstall action
 		uninstall := action.NewUninstall(cfg)
-		if _, err := uninstall.Run(c.Name); err != nil {
-			return false, fmt.Errorf("failed to uninstall %s: %w", c.Name, err)
+		if _, err := uninstall.Run(c.InstallName); err != nil {
+			return false, fmt.Errorf("failed to uninstall %s: %w", c.InstallName, err)
 		}
 
 		return true, nil

src/k8s/pkg/client/helm/interface.go

@@ -1,6 +1,10 @@
 package helm
 
-import "context"
+import (
+	"context"
+
+	"helm.sh/helm/v3/pkg/chart"
+)
 
 // Client handles the lifecycle of charts (manifests + config) on the cluster.
 type Client interface {
@@ -11,3 +15,9 @@ type Client interface {
 	// Apply returns an error in case of failure.
 	Apply(ctx context.Context, f InstallableChart, desired State, values map[string]any) (bool, error)
 }
+
+// ChartLoader handles the loading of charts from various sources.
+type ChartLoader interface {
+	// Load loads a chart
+	Load(ctx context.Context, f InstallableChart) (*chart.Chart, error)
+}

src/k8s/pkg/client/helm/loader/db.go

@@ -0,0 +1,46 @@
+package loader
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"fmt"
+
+	"github.com/canonical/k8s/pkg/client/helm"
+	"github.com/canonical/k8s/pkg/k8sd/database"
+	"github.com/canonical/k8s/pkg/k8sd/types"
+	"github.com/canonical/microcluster/v2/state"
+	"helm.sh/helm/v3/pkg/chart"
+	"helm.sh/helm/v3/pkg/chart/loader"
+)
+
+// DatabaseLoader is a helm chart loader that loads charts from the microcluster database.
+type databaseLoader struct {
+	s state.State
+}
+
+// NewDatabaseLoader creates a new database loader.
+func NewDatabaseLoader(s state.State) *databaseLoader {
+	return &databaseLoader{
+		s: s,
+	}
+}
+
+// Load loads a helm chart from the microcluster database by name and version.
+func (l *databaseLoader) Load(ctx context.Context, f helm.InstallableChart) (*chart.Chart, error) {
+	var chartEntry *types.HelmChartEntry
+	if err := l.s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error {
+		var err error
+		chartEntry, err = database.GetHelmChart(ctx, tx, f.Name, f.Version)
+		return err
+	}); err != nil {
+		return nil, fmt.Errorf("failed to get helm chart: %w", err)
+	}
+
+	chart, err := loader.LoadArchive(bytes.NewReader(chartEntry.Contents))
+	if err != nil {
+		return nil, fmt.Errorf("failed to load helm chart: %w", err)
+	}
+
+	return chart, nil
+}

src/k8s/pkg/client/helm/loader/embed.go

@@ -0,0 +1,40 @@
+package loader
+
+import (
+	"bytes"
+	"context"
+	"embed"
+	"fmt"
+	"path/filepath"
+
+	"github.com/canonical/k8s/pkg/client/helm"
+	"helm.sh/helm/v3/pkg/chart"
+	"helm.sh/helm/v3/pkg/chart/loader"
+)
+
+// embedLoader is a helm chart loader that loads charts from the embedded filesystem.
+type embedLoader struct {
+	chartFS *embed.FS
+}
+
+// NewEmbedLoader creates a new embed loader.
+func NewEmbedLoader(chartFS *embed.FS) *embedLoader {
+	return &embedLoader{
+		chartFS: chartFS,
+	}
+}
+
+// Load loads a helm chart from the filesystem by name and version.
+func (l *embedLoader) Load(_ context.Context, f helm.InstallableChart) (*chart.Chart, error) {
+	chartBytes, err := l.chartFS.ReadFile(filepath.Join("charts", fmt.Sprintf("%s-%s.tgz", f.Name, f.Version)))
+	if err != nil {
+		return nil, fmt.Errorf("failed to read helm chart: %w", err)
+	}
+
+	chart, err := loader.LoadArchive(bytes.NewReader(chartBytes))
+	if err != nil {
+		return nil, fmt.Errorf("failed to load helm chart: %w", err)
+	}
+
+	return chart, nil
+}

src/k8s/pkg/k8sd/app/app.go

@@ -45,6 +45,8 @@ type Config struct {
 	DisableFeatureController bool
 	// DisableCSRSigningController is a bool flag to disable csrsigning controller.
 	DisableCSRSigningController bool
+	// DisableHelmChartController is a bool flag to disable helm chart controller.
+	DisableHelmChartController bool
 	// DrainConnectionsTimeout is the amount of time to allow for all connections to drain when shutting down.
 	DrainConnectionsTimeout time.Duration
 }
@@ -66,6 +68,7 @@ type App struct {
 	nodeLabelController          *controllers.NodeLabelController
 	controlPlaneConfigController *controllers.ControlPlaneConfigurationController
 	csrsigningController         *csrsigning.Controller
+	helmChartController          *controllers.HelmChartController
 
 	// updateNodeConfigController
 	triggerUpdateNodeConfigControllerCh chan struct{}
@@ -116,6 +119,15 @@ func New(cfg Config) (*App, error) {
 		log.L().Info("node-config-controller disabled via config")
 	}
 
+	if !cfg.DisableHelmChartController {
+		app.helmChartController = controllers.NewHelmChartController(
+			cfg.Snap,
+			app.readyWg.Wait,
+		)
+	} else {
+		log.L().Info("helm-chart-controller disabled via config")
+	}
+
 	if !cfg.DisableNodeLabelController {
 		app.nodeLabelController = controllers.NewNodeLabelController(
 			cfg.Snap,

src/k8s/pkg/k8sd/app/hooks_start.go

@@ -24,6 +24,11 @@ func (a *App) onStart(ctx context.Context, s state.State) error {
 		}
 	}()
 
+	// start helm chart controller
+	if a.helmChartController != nil {
+		go a.helmChartController.Run(ctx, s)
+	}
+
 	// start node config controller
 	if a.nodeConfigController != nil {
 		go a.nodeConfigController.Run(ctx, func(ctx context.Context) (*rsa.PublicKey, error) {

src/k8s/pkg/k8sd/charts/charts.go

@@ -0,0 +1,24 @@
+package charts
+
+import (
+	"embed"
+)
+
+// registeredCharts is a list of filesystems that contain charts for features used by k8s-snap.
+var registeredCharts []*embed.FS
+
+// Charts returns the list of registered charts.
+func Charts() []*embed.FS {
+	if registeredCharts == nil {
+		return nil
+	}
+	chartFSList := make([]*embed.FS, len(registeredCharts))
+	copy(chartFSList, registeredCharts)
+	return chartFSList
+}
+
+// Register charts that are used by k8s-snap.
+// Register is used by the `init()` method in individual packages.
+func Register(charts *embed.FS) {
+	registeredCharts = append(registeredCharts, charts)
+}