From b87e05977fa784ee512aaa61018d244b52622ec8 Mon Sep 17 00:00:00 2001 From: Artem Bortnikov Date: Fri, 4 Oct 2024 14:48:32 +0300 Subject: [PATCH] naive bios management job implementation Signed-off-by: Artem Bortnikov --- cmd/jobbios/main.go | 77 +++++++++++ internal/{controller => bmcutils}/bmcutils.go | 7 +- internal/controller/bmc_controller.go | 17 +-- internal/controller/bmc_controller_test.go | 5 +- internal/controller/endpoint_controller.go | 12 +- internal/controller/server_controller.go | 11 +- internal/controller/serverbios_controller.go | 6 + internal/job/bios_bmc.go | 129 ++++++++++++++++++ 8 files changed, 239 insertions(+), 25 deletions(-) create mode 100644 cmd/jobbios/main.go rename internal/{controller => bmcutils}/bmcutils.go (97%) create mode 100644 internal/job/bios_bmc.go diff --git a/cmd/jobbios/main.go b/cmd/jobbios/main.go new file mode 100644 index 0000000..6136321 --- /dev/null +++ b/cmd/jobbios/main.go @@ -0,0 +1,77 @@ +package main + +import ( + "flag" + "os" + + metalv1alpha1 "github.com/ironcore-dev/metal-operator/api/v1alpha1" + "github.com/ironcore-dev/metal-operator/internal/job" + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(metalv1alpha1.AddToScheme(scheme)) + // +kubebuilder:scaffold:scheme +} + +func main() { + var ( + jobTypeString string + serverBIOSRef string + insecure bool + ) + + pflag.StringVar(&jobTypeString, "job-type", "", "job type") + pflag.StringVar(&serverBIOSRef, "server-bios-ref", "", "server bios ref") + pflag.BoolVar(&insecure, "insecure", true, "use insecure connection to BMC") + + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + if jobTypeString == "" { + setupLog.Error(nil, "job type is required") + os.Exit(1) + } + + if serverBIOSRef == "" { + setupLog.Error(nil, "server bios ref is required") + os.Exit(1) + } + + config, err := rest.InClusterConfig() + if err != nil { + setupLog.Error(err, "unable to get in cluster config") + os.Exit(1) + } + kubeClient, err := client.New(config, client.Options{ + Scheme: scheme, + }) + if err != nil { + setupLog.Error(err, "unable to create kubernetes client") + os.Exit(1) + } + + executor := job.New(kubeClient) + ctx := ctrl.SetupSignalHandler() + if err = executor.Run(ctx, jobTypeString, serverBIOSRef); err != nil { + os.Exit(1) + } +} diff --git a/internal/controller/bmcutils.go b/internal/bmcutils/bmcutils.go similarity index 97% rename from internal/controller/bmcutils.go rename to internal/bmcutils/bmcutils.go index f3d8a41..8f408e4 100644 --- a/internal/controller/bmcutils.go +++ b/internal/bmcutils/bmcutils.go @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and IronCore contributors // SPDX-License-Identifier: Apache-2.0 -package controller +package bmcutils import ( "context" @@ -10,11 +10,10 @@ import ( metalv1alpha1 "github.com/ironcore-dev/metal-operator/api/v1alpha1" "github.com/ironcore-dev/metal-operator/bmc" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) -const DefaultKubeNamespace = "default" - func GetBMCClientForServer(ctx context.Context, c client.Client, server *metalv1alpha1.Server, insecure bool) (bmc.BMC, error) { if server.Spec.BMCRef != nil { b := &metalv1alpha1.BMC{} @@ -94,7 +93,7 @@ func CreateBMCClient(ctx context.Context, c client.Client, insecure bool, bmcPro if err != nil { return nil, fmt.Errorf("failed to get credentials from BMC secret: %w", err) } - bmcClient, err = bmc.NewRedfishKubeBMCClient(ctx, bmcAddress, username, password, true, c, DefaultKubeNamespace) + bmcClient, err = bmc.NewRedfishKubeBMCClient(ctx, bmcAddress, username, password, true, c, metav1.NamespaceDefault) if err != nil { return nil, fmt.Errorf("failed to create Redfish client: %w", err) } diff --git a/internal/controller/bmc_controller.go b/internal/controller/bmc_controller.go index cba09a4..ea7a087 100644 --- a/internal/controller/bmc_controller.go +++ b/internal/controller/bmc_controller.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "github.com/ironcore-dev/metal-operator/internal/bmcutils" "k8s.io/apimachinery/pkg/api/errors" "github.com/go-logr/logr" @@ -29,11 +30,11 @@ type BMCReconciler struct { Insecure bool } -//+kubebuilder:rbac:groups=metal.ironcore.dev,resources=endpoints,verbs=get;list;watch -//+kubebuilder:rbac:groups=metal.ironcore.dev,resources=bmcsecrets,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=metal.ironcore.dev,resources=bmcs,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=metal.ironcore.dev,resources=bmcs/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=metal.ironcore.dev,resources=bmcs/finalizers,verbs=update +// +kubebuilder:rbac:groups=metal.ironcore.dev,resources=endpoints,verbs=get;list;watch +// +kubebuilder:rbac:groups=metal.ironcore.dev,resources=bmcsecrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=metal.ironcore.dev,resources=bmcs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=metal.ironcore.dev,resources=bmcs/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=metal.ironcore.dev,resources=bmcs/finalizers,verbs=update // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -102,7 +103,7 @@ func (r *BMCReconciler) updateBMCStatusDetails(ctx context.Context, log logr.Log return fmt.Errorf("failed to patch IP and MAC address status: %w", err) } - bmcClient, err := GetBMCClientFromBMC(ctx, r.Client, bmcObj, r.Insecure) + bmcClient, err := bmcutils.GetBMCClientFromBMC(ctx, r.Client, bmcObj, r.Insecure) if err != nil { return fmt.Errorf("failed to create BMC client: %w", err) } @@ -132,7 +133,7 @@ func (r *BMCReconciler) updateBMCStatusDetails(ctx context.Context, log logr.Log } func (r *BMCReconciler) discoverServers(ctx context.Context, log logr.Logger, bmcObj *metalv1alpha1.BMC) error { - bmcClient, err := GetBMCClientFromBMC(ctx, r.Client, bmcObj, r.Insecure) + bmcClient, err := bmcutils.GetBMCClientFromBMC(ctx, r.Client, bmcObj, r.Insecure) if err != nil { return fmt.Errorf("failed to create BMC client: %w", err) } @@ -149,7 +150,7 @@ func (r *BMCReconciler) discoverServers(ctx context.Context, log logr.Logger, bm Kind: "Server", }, ObjectMeta: metav1.ObjectMeta{ - Name: GetServerNameFromBMCandIndex(i, bmcObj), + Name: bmcutils.GetServerNameFromBMCandIndex(i, bmcObj), }, Spec: metalv1alpha1.ServerSpec{ UUID: s.UUID, diff --git a/internal/controller/bmc_controller_test.go b/internal/controller/bmc_controller_test.go index cb7fac2..36674ce 100644 --- a/internal/controller/bmc_controller_test.go +++ b/internal/controller/bmc_controller_test.go @@ -5,6 +5,7 @@ package controller import ( metalv1alpha1 "github.com/ironcore-dev/metal-operator/api/v1alpha1" + "github.com/ironcore-dev/metal-operator/internal/bmcutils" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -51,7 +52,7 @@ var _ = Describe("BMC Controller", func() { By("Ensuring that the Server resource will be removed") server := &metalv1alpha1.Server{ ObjectMeta: metav1.ObjectMeta{ - Name: GetServerNameFromBMCandIndex(0, bmc), + Name: bmcutils.GetServerNameFromBMCandIndex(0, bmc), }, } DeferCleanup(k8sClient.Delete, server) @@ -84,7 +85,7 @@ var _ = Describe("BMC Controller", func() { By("Ensuring that the Server resource has been created") server := &metalv1alpha1.Server{ ObjectMeta: metav1.ObjectMeta{ - Name: GetServerNameFromBMCandIndex(0, bmc), + Name: bmcutils.GetServerNameFromBMCandIndex(0, bmc), }, } Eventually(Object(server)).Should(SatisfyAll( diff --git a/internal/controller/endpoint_controller.go b/internal/controller/endpoint_controller.go index 034677c..053cfc7 100644 --- a/internal/controller/endpoint_controller.go +++ b/internal/controller/endpoint_controller.go @@ -34,11 +34,11 @@ type EndpointReconciler struct { Insecure bool } -//+kubebuilder:rbac:groups=metal.ironcore.dev,resources=bmcs,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=metal.ironcore.dev,resources=bmcsecrets,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=metal.ironcore.dev,resources=endpoints,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=metal.ironcore.dev,resources=endpoints/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=metal.ironcore.dev,resources=endpoints/finalizers,verbs=update +// +kubebuilder:rbac:groups=metal.ironcore.dev,resources=bmcs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=metal.ironcore.dev,resources=bmcsecrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=metal.ironcore.dev,resources=endpoints,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=metal.ironcore.dev,resources=endpoints/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=metal.ironcore.dev,resources=endpoints/finalizers,verbs=update func (r *EndpointReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := ctrl.LoggerFrom(ctx) @@ -125,7 +125,7 @@ func (r *EndpointReconciler) reconcile(ctx context.Context, log logr.Logger, end case metalv1alpha1.ProtocolRedfishKube: log.V(1).Info("Creating client for a kube test BMC") bmcAddress := fmt.Sprintf("%s://%s:%d", r.getProtocol(), endpoint.Spec.IP, m.Port) - bmcClient, err := bmc.NewRedfishKubeBMCClient(ctx, bmcAddress, m.DefaultCredentials[0].Username, m.DefaultCredentials[0].Password, true, r.Client, DefaultKubeNamespace) + bmcClient, err := bmc.NewRedfishKubeBMCClient(ctx, bmcAddress, m.DefaultCredentials[0].Username, m.DefaultCredentials[0].Password, true, r.Client, metav1.NamespaceDefault) if err != nil { return ctrl.Result{}, fmt.Errorf("failed to create BMC client: %w", err) } diff --git a/internal/controller/server_controller.go b/internal/controller/server_controller.go index dc48a0b..f9205bf 100644 --- a/internal/controller/server_controller.go +++ b/internal/controller/server_controller.go @@ -13,6 +13,7 @@ import ( "time" "github.com/ironcore-dev/metal-operator/bmc" + "github.com/ironcore-dev/metal-operator/internal/bmcutils" "k8s.io/apimachinery/pkg/util/wait" "github.com/go-logr/logr" @@ -361,7 +362,7 @@ func (r *ServerReconciler) updateServerStatus(ctx context.Context, log logr.Logg log.V(1).Info("Server has no BMC connection configured") return nil } - bmcClient, err := GetBMCClientForServer(ctx, r.Client, server, r.Insecure) + bmcClient, err := bmcutils.GetBMCClientForServer(ctx, r.Client, server, r.Insecure) if err != nil { return fmt.Errorf("failed to create BMC client: %w", err) } @@ -538,7 +539,7 @@ func (r *ServerReconciler) pxeBootServer(ctx context.Context, log logr.Logger, s return fmt.Errorf("can only PXE boot server with valid BMC ref or inline BMC configuration") } - bmcClient, err := GetBMCClientForServer(ctx, r.Client, server, r.Insecure) + bmcClient, err := bmcutils.GetBMCClientForServer(ctx, r.Client, server, r.Insecure) defer func() { if bmcClient != nil { bmcClient.Logout() @@ -629,7 +630,7 @@ func (r *ServerReconciler) ensureServerPowerState(ctx context.Context, log logr. return nil } - bmcClient, err := GetBMCClientForServer(ctx, r.Client, server, r.Insecure) + bmcClient, err := bmcutils.GetBMCClientForServer(ctx, r.Client, server, r.Insecure) defer func() { if bmcClient != nil { bmcClient.Logout() @@ -751,7 +752,7 @@ func (r *ServerReconciler) applyBootOrder(ctx context.Context, log logr.Logger, log.V(1).Info("Server has no BMC connection configured") return nil } - bmcClient, err := GetBMCClientForServer(ctx, r.Client, server, r.Insecure) + bmcClient, err := bmcutils.GetBMCClientForServer(ctx, r.Client, server, r.Insecure) if err != nil { return fmt.Errorf("failed to create BMC client: %w", err) } @@ -785,7 +786,7 @@ func (r *ServerReconciler) handleAnnotionOperations(ctx context.Context, log log if !ok { return false, nil } - bmcClient, err := GetBMCClientForServer(ctx, r.Client, server, r.Insecure) + bmcClient, err := bmcutils.GetBMCClientForServer(ctx, r.Client, server, r.Insecure) if err != nil { return false, fmt.Errorf("failed to create BMC client: %w", err) } diff --git a/internal/controller/serverbios_controller.go b/internal/controller/serverbios_controller.go index 7c4e802..74c7592 100644 --- a/internal/controller/serverbios_controller.go +++ b/internal/controller/serverbios_controller.go @@ -6,6 +6,7 @@ package controller import ( "context" "fmt" + "strconv" "time" "github.com/go-logr/logr" @@ -29,6 +30,7 @@ type ServerBIOSReconciler struct { client.Client Scheme *runtime.Scheme + Insecure bool // todo: need to decide how to provide jobs' configuration to controller JobNamespace string JobImage string @@ -283,6 +285,10 @@ func (r *ServerBIOSReconciler) createJob( Name: "SERVER_BIOS_REF", Value: serverBIOS.Name, }, + { + Name: "INSECURE", + Value: strconv.FormatBool(r.Insecure), + }, }, }, }, diff --git a/internal/job/bios_bmc.go b/internal/job/bios_bmc.go new file mode 100644 index 0000000..c6e9d49 --- /dev/null +++ b/internal/job/bios_bmc.go @@ -0,0 +1,129 @@ +package job + +import ( + "context" + "fmt" + + metalv1alpha1 "github.com/ironcore-dev/metal-operator/api/v1alpha1" + "github.com/ironcore-dev/metal-operator/internal/bmcutils" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type BmcBIOSExecutor struct { + client.Client + Insecure bool +} + +func New(c client.Client) *BmcBIOSExecutor { + return &BmcBIOSExecutor{ + Client: c, + } +} + +func (e *BmcBIOSExecutor) Run(ctx context.Context, jobTypeString, serverBIOSRef string) error { + jobType := metalv1alpha1.JobType(jobTypeString) + switch jobType { + case metalv1alpha1.UpdateBIOSVersionJobType: + return e.UpdateBIOSVersion(ctx, serverBIOSRef) + case metalv1alpha1.ScanBIOSVersionJobType: + return e.ScanBIOSVersionSettings(ctx, serverBIOSRef) + case metalv1alpha1.ApplyBIOSSettingsJobType: + return e.ApplyBIOSSettings(ctx, serverBIOSRef) + default: + return fmt.Errorf("unknown job type: %s", jobTypeString) + } +} + +func (e *BmcBIOSExecutor) ApplyBIOSSettings(ctx context.Context, serverBIOSRef string) error { + serverBIOS, server, err := e.getObjects(ctx, serverBIOSRef) + if err != nil { + return err + } + bmcClient, err := bmcutils.GetBMCClientForServer(ctx, e.Client, server, e.Insecure) + if err != nil { + return err + } + defer bmcClient.Logout() + + diff := make(map[string]string) + for k, v := range serverBIOS.Spec.BIOS.Settings { + if v == serverBIOS.Status.BIOS.Settings[k] { + continue + } + diff[k] = v + } + reset, err := bmcClient.SetBiosAttributes(server.Spec.UUID, diff) + if err != nil { + return err + } + if reset { + if err = e.patchServerCondition(ctx, server); err != nil { + return fmt.Errorf("failed to patch Server status: %w", err) + } + } + + serverBIOSBase := serverBIOS.DeepCopy() + serverBIOS.Status.BIOS.Settings = serverBIOS.Spec.BIOS.Settings + serverBIOS.Status.RunningJob = metalv1alpha1.RunningJobRef{} + return e.Patch(ctx, serverBIOSBase, client.MergeFrom(serverBIOS)) +} + +func (e *BmcBIOSExecutor) ScanBIOSVersionSettings(ctx context.Context, serverBIOSRef string) error { + serverBIOS, server, err := e.getObjects(ctx, serverBIOSRef) + if err != nil { + return err + } + bmcClient, err := bmcutils.GetBMCClientForServer(ctx, e.Client, server, e.Insecure) + if err != nil { + return err + } + currentBIOSVersion, err := bmcClient.GetBiosVersion(server.Spec.UUID) + if err != nil { + return err + } + attributes := make([]string, 0) + for k := range serverBIOS.Spec.BIOS.Settings { + attributes = append(attributes, k) + } + currentSettings, err := bmcClient.GetBiosAttributeValues(server.Spec.UUID, attributes) + if err != nil { + return err + } + serverBIOSBase := serverBIOS.DeepCopy() + serverBIOS.Status.BIOS.Version = currentBIOSVersion + serverBIOS.Status.BIOS.Settings = currentSettings + serverBIOS.Status.LastScanTime = metav1.Now() + serverBIOS.Status.RunningJob = metalv1alpha1.RunningJobRef{} + return e.Patch(ctx, serverBIOSBase, client.MergeFrom(serverBIOS)) +} + +func (e *BmcBIOSExecutor) UpdateBIOSVersion(ctx context.Context, serverBIOSRef string) error { + // todo: implement me + return nil +} + +func (e *BmcBIOSExecutor) getObjects(ctx context.Context, serverBIOSRef string) (*metalv1alpha1.ServerBIOS, *metalv1alpha1.Server, error) { + serverBIOS := &metalv1alpha1.ServerBIOS{} + if err := e.Get(ctx, client.ObjectKey{Name: serverBIOSRef}, serverBIOS); err != nil { + return nil, nil, err + } + server := &metalv1alpha1.Server{} + if err := e.Get(ctx, client.ObjectKey{Name: serverBIOS.Spec.ServerRef.Name}, server); err != nil { + return nil, nil, err + } + return serverBIOS, server, nil +} + +func (e *BmcBIOSExecutor) patchServerCondition(ctx context.Context, server *metalv1alpha1.Server) error { + serverBase := server.DeepCopy() + changed := meta.SetStatusCondition(&server.Status.Conditions, metav1.Condition{ + Type: "RebootNeeded", + }) + if changed { + return e.Status().Patch(ctx, serverBase, client.MergeFrom(server)) + + } + return nil +}