From c36d5b99a0cf8f55a9061ab7b622fda64887afc8 Mon Sep 17 00:00:00 2001 From: Dmitry Kartsev Date: Sun, 24 Dec 2023 18:05:41 +0200 Subject: [PATCH] add status configmap generator --- charts/cnvrg-all-in-one/Chart.yaml | 2 +- cmd/copctl/cmd/create/kubecerts.go | 11 +- cmd/copctl/cmd/create/utils.go | 30 ----- cmd/copctl/cmd/create/webhook.go | 9 +- cmd/copctl/cmd/get/get.go | 11 ++ cmd/copctl/cmd/get/status.go | 194 +++++++++++++++++++++++++++++ cmd/copctl/cmd/root.go | 3 + cmd/copctl/cmd/start/admission.go | 22 +--- cmd/copctl/utils/utils.go | 53 ++++++++ pkg/admission/domain.go | 3 + 10 files changed, 277 insertions(+), 61 deletions(-) delete mode 100644 cmd/copctl/cmd/create/utils.go create mode 100644 cmd/copctl/cmd/get/get.go create mode 100644 cmd/copctl/cmd/get/status.go create mode 100644 cmd/copctl/utils/utils.go diff --git a/charts/cnvrg-all-in-one/Chart.yaml b/charts/cnvrg-all-in-one/Chart.yaml index 88d0598f..c23f74ba 100644 --- a/charts/cnvrg-all-in-one/Chart.yaml +++ b/charts/cnvrg-all-in-one/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 appVersion: 5.0.0 description: A cnvrg.io AI:OS chart for K8s -name: cnvrg-all-in-one +name: cnvrg-mlops type: application version: 5.0.0 dependencies: diff --git a/cmd/copctl/cmd/create/kubecerts.go b/cmd/copctl/cmd/create/kubecerts.go index 94dec70a..0151b976 100644 --- a/cmd/copctl/cmd/create/kubecerts.go +++ b/cmd/copctl/cmd/create/kubecerts.go @@ -8,6 +8,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "github.com/AccessibleAI/cnvrg-operator/cmd/copctl/utils" "github.com/spf13/cobra" "github.com/spf13/viper" "go.uber.org/zap" @@ -47,7 +48,7 @@ var kubeCertsCmd = &cobra.Command{ if viper.GetBool("override") { clean(viper.GetString("certs-dir"), viper.GetString("common-name")) } - pkey := privateKey() + pkey := utils.PrivateKey() approveCsr( createCsr( csrPem( @@ -64,7 +65,7 @@ var kubeCertsCmd = &cobra.Command{ func clean(certsDir string, commonName string) { zap.S().Info("cleaning up exiting certs") - err := clientset(). + err := utils.Clientset(). CertificatesV1(). CertificateSigningRequests(). Delete(context.Background(), commonName, metav1.DeleteOptions{}) @@ -127,7 +128,7 @@ func createCsr(csrPem *bytes.Buffer, commonName string) *certsv1.CertificateSign //signerName := "kubernetes.io/kube-apiserver-client-kubelet" //signerName := "kubernetes.io/kube-apiserver-client" signerName := "kubernetes.io/kubelet-serving" - csr, err := clientset().CertificatesV1(). + csr, err := utils.Clientset().CertificatesV1(). CertificateSigningRequests(). Create(context.Background(), &certsv1.CertificateSigningRequest{ ObjectMeta: metav1.ObjectMeta{Name: commonName}, @@ -159,7 +160,7 @@ func approveCsr(csr *certsv1.CertificateSigningRequest) { LastTransitionTime: metav1.Now(), }) - if _, err := clientset().CertificatesV1(). + if _, err := utils.Clientset().CertificatesV1(). CertificateSigningRequests(). UpdateApproval(context.Background(), csr.ObjectMeta.Name, csr, metav1.UpdateOptions{}); err != nil { zap.S().Fatal(err) @@ -167,7 +168,7 @@ func approveCsr(csr *certsv1.CertificateSigningRequest) { } func fetchCertificateFromCsr(commonName string) []byte { - csr, err := clientset(). + csr, err := utils.Clientset(). CertificatesV1(). CertificateSigningRequests(). Get(context.Background(), commonName, metav1.GetOptions{}) diff --git a/cmd/copctl/cmd/create/utils.go b/cmd/copctl/cmd/create/utils.go deleted file mode 100644 index 5e907cdb..00000000 --- a/cmd/copctl/cmd/create/utils.go +++ /dev/null @@ -1,30 +0,0 @@ -package create - -import ( - "crypto/rand" - "crypto/rsa" - "go.uber.org/zap" - "k8s.io/client-go/kubernetes" - "sigs.k8s.io/controller-runtime/pkg/client/config" -) - -func clientset() *kubernetes.Clientset { - rc, err := config.GetConfig() - if err != nil { - zap.S().Fatalf("unable to construct K8s configs, err: %s", err.Error()) - } - cltset, err := kubernetes.NewForConfig(rc) - if err != nil { - zap.S().Fatalf("err: %s, unable to connect to K8s", err.Error()) - } - return cltset -} - -func privateKey() *rsa.PrivateKey { - zap.S().Info("generating private key") - privKey, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - zap.S().Fatal(err) - } - return privKey -} diff --git a/cmd/copctl/cmd/create/webhook.go b/cmd/copctl/cmd/create/webhook.go index 52b52df6..6ff1a398 100644 --- a/cmd/copctl/cmd/create/webhook.go +++ b/cmd/copctl/cmd/create/webhook.go @@ -8,6 +8,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "github.com/AccessibleAI/cnvrg-operator/cmd/copctl/utils" "github.com/AccessibleAI/cnvrg-operator/pkg/admission" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -69,7 +70,7 @@ func (h *Webhook) run() { h.clean() } // get key for CA - cakey := privateKey() + cakey := utils.PrivateKey() // create CA certificate ca, caPEM := h.createCA(cakey) // create certificate and key for server @@ -141,7 +142,7 @@ func (h *Webhook) serverCrtAndKey(ca *x509.Certificate, cakey *rsa.PrivateKey) ( KeyUsage: x509.KeyUsageDigitalSignature, } - serverKey := privateKey() + serverKey := utils.PrivateKey() certBytes, err := x509.CreateCertificate(rand.Reader, serverCrt, ca, &serverKey.PublicKey, cakey) if err != nil { zap.S().Fatalf("error creating server certificate, err: %s ", err.Error()) @@ -189,7 +190,7 @@ func (h *Webhook) createMutatingWebhookCfg(hookCfg *admissionv1.MutatingWebhookC zap.S().Infof("creating webhook: %s", hookCfg.Name) - err := clientset(). + err := utils.Clientset(). AdmissionregistrationV1(). MutatingWebhookConfigurations(). Delete(context.Background(), hookCfg.Name, metav1.DeleteOptions{}) @@ -198,7 +199,7 @@ func (h *Webhook) createMutatingWebhookCfg(hookCfg *admissionv1.MutatingWebhookC zap.S().Fatalf("error deleting webhook: %s, err: %s ", hookCfg.Name, err.Error()) } - if _, err := clientset(). + if _, err := utils.Clientset(). AdmissionregistrationV1(). MutatingWebhookConfigurations(). Create(context.Background(), hookCfg, metav1.CreateOptions{}); err != nil { diff --git a/cmd/copctl/cmd/get/get.go b/cmd/copctl/cmd/get/get.go new file mode 100644 index 00000000..367d2374 --- /dev/null +++ b/cmd/copctl/cmd/get/get.go @@ -0,0 +1,11 @@ +package get + +import "github.com/spf13/cobra" + +var ( + Cmd = &cobra.Command{ + Use: "get", + Aliases: []string{"g"}, + Short: "Get copctl resources", + } +) diff --git a/cmd/copctl/cmd/get/status.go b/cmd/copctl/cmd/get/status.go new file mode 100644 index 00000000..bb083d04 --- /dev/null +++ b/cmd/copctl/cmd/get/status.go @@ -0,0 +1,194 @@ +package get + +import ( + "context" + "encoding/json" + "fmt" + mlopsv1 "github.com/AccessibleAI/cnvrg-operator/api/v1" + "github.com/AccessibleAI/cnvrg-operator/cmd/copctl/utils" + "github.com/AccessibleAI/cnvrg-shim/apis/metacloud/v1alpha1" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "go.uber.org/zap" + "istio.io/istio/pkg/config/protocol" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "os" + "os/signal" + "sigs.k8s.io/controller-runtime/pkg/client" + "syscall" + "time" +) + +type Status struct { + NamespacedName types.NamespacedName + StatusConfigmapName string + Interval time.Duration +} + +func init() { + statusCmd.PersistentFlags().StringP("namespace", "", "", "current namespace") + statusCmd.PersistentFlags().StringP("name", "", "", "name of the CnvrgApp CR") + statusCmd.PersistentFlags().StringP("status-configmap", "", "service-instance-status", "the status cm name") + statusCmd.PersistentFlags().IntP("interval", "i", 1, "status generation interval") + + viper.BindPFlag("namespace", statusCmd.PersistentFlags().Lookup("namespace")) + viper.BindPFlag("name", statusCmd.PersistentFlags().Lookup("name")) + viper.BindPFlag("status-configmap", statusCmd.PersistentFlags().Lookup("status-configmap")) + viper.BindPFlag("interval", statusCmd.PersistentFlags().Lookup("interval")) + + Cmd.AddCommand(statusCmd) +} + +var statusCmd = &cobra.Command{ + Use: "status", + Aliases: []string{"s"}, + Short: "Get cnvrg app status", + Run: func(cmd *cobra.Command, args []string) { + + NewStatus( + viper.GetString("namespace"), + viper.GetString("name"), + viper.GetString("status-configmap"), + viper.GetInt("interval"), + ).run() + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + for { + select { + case s := <-sigCh: + zap.S().Infof("signal: %s, shutting down", s) + zap.S().Info("bye bye 👋") + os.Exit(0) + } + } + + }, +} + +func NewStatus(ns, name, statusConfigmap string, interval int) *Status { + if name == "" || ns == "" { + zap.S().Fatal("name and namespace must be set") + } + return &Status{ + NamespacedName: types.NamespacedName{Namespace: ns, Name: name}, + StatusConfigmapName: statusConfigmap, + Interval: time.Second * time.Duration(interval), + } +} + +func (s *Status) loadCnvrgApp(cap *mlopsv1.CnvrgApp) error { + return utils. + Kubecrudclient(). + Get(context.Background(), + s.NamespacedName, + cap, + []client.GetOption{}..., + ) + +} + +func (s *Status) run() { + for { + cnvrgAppInstance := &mlopsv1.CnvrgApp{} + // fetch cnvrgapp instance + if err := s.loadCnvrgApp(cnvrgAppInstance); err != nil { + zap.S().Errorf("failed to fetch cnvrgapp instance, err: %s", err.Error()) + continue + } + + siStatus, err := json.Marshal(s.generateStatus(cnvrgAppInstance)) + if err != nil { + zap.S().Errorf("failed to marshal service instance, err: %s", err.Error()) + continue + } + + if err := s.writeStatus(siStatus); err != nil { + zap.S().Errorf("failed to create status configmap, err: %s", err.Error()) + continue + } + zap.S().Info("service instance configmap updated") + time.Sleep(s.Interval) + } +} + +func (s *Status) writeStatus(payload []byte) error { + + // get status cm + cm, err := utils. + Clientset(). + CoreV1(). + ConfigMaps(s.NamespacedName.Namespace). + Get(context.Background(), s.StatusConfigmapName, v1.GetOptions{}) + + // if not found create it + if errors.IsNotFound(err) { + // construct configmap + statusCm := &corev1.ConfigMap{ + ObjectMeta: v1.ObjectMeta{ + Namespace: s.NamespacedName.Namespace, + Name: s.StatusConfigmapName, + }, + Data: map[string]string{"serviceInstanceStatus": string(payload)}, + } + //create configmap + _, err = utils. + Clientset(). + CoreV1(). + ConfigMaps(s.NamespacedName.Namespace). + Create(context.Background(), statusCm, v1.CreateOptions{}) + + return err + + } else if err != nil { + // error fetching configmap + return err + } + + // status configmap exists, update it + cm.Data = map[string]string{"serviceInstanceStatus": string(payload)} + _, err = utils. + Clientset(). + CoreV1(). + ConfigMaps(s.NamespacedName.Namespace). + Update(context.Background(), cm, v1.UpdateOptions{}) + + return err + +} + +func (s *Status) generateStatus(cnvrgApp *mlopsv1.CnvrgApp) *v1alpha1.ServiceInstanceStatus { + + port := 80 + proto := protocol.HTTP + status := v1alpha1.StatusHealthy + + if cnvrgApp.Spec.Networking.HTTPS.Enabled { + proto = protocol.HTTPS + port = 443 + } + + if cnvrgApp.Status.Status != "READY" { + status = v1alpha1.StatusReconciling + } + + return &v1alpha1.ServiceInstanceStatus{ + Status: status, + Sins: []v1alpha1.Sins{ + { + Name: cnvrgApp.Name, + IngressEndpoints: []v1alpha1.IngressEndpoint{ + { + Protocol: proto, + Address: []string{fmt.Sprintf("app.%s", cnvrgApp.Spec.ClusterDomain)}, + Port: uint32(port), + }, + }, + }, + }, + } + +} diff --git a/cmd/copctl/cmd/root.go b/cmd/copctl/cmd/root.go index ad3fdfdc..e3fbd3dc 100644 --- a/cmd/copctl/cmd/root.go +++ b/cmd/copctl/cmd/root.go @@ -2,6 +2,7 @@ package cmd import ( "github.com/AccessibleAI/cnvrg-operator/cmd/copctl/cmd/create" + "github.com/AccessibleAI/cnvrg-operator/cmd/copctl/cmd/get" "github.com/AccessibleAI/cnvrg-operator/cmd/copctl/cmd/start" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -29,6 +30,7 @@ func init() { cobra.OnInitialize(initConfig) RootCmd.AddCommand(start.Cmd) RootCmd.AddCommand(create.Cmd) + RootCmd.AddCommand(get.Cmd) } func initConfig() { @@ -41,6 +43,7 @@ func initConfig() { func initZapLog() { config := zap.NewDevelopmentConfig() //config := zap.NewProductionConfig() + config.EncoderConfig.StacktraceKey = "" config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder config.EncoderConfig.TimeKey = "timestamp" config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder diff --git a/cmd/copctl/cmd/start/admission.go b/cmd/copctl/cmd/start/admission.go index 35f6798e..2eae1cbc 100644 --- a/cmd/copctl/cmd/start/admission.go +++ b/cmd/copctl/cmd/start/admission.go @@ -2,7 +2,6 @@ package start import ( "crypto/tls" - "fmt" "github.com/AccessibleAI/cnvrg-operator/pkg/admission" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -10,30 +9,11 @@ import ( "net/http" ) -type PlatformType string - -const ( - CAGPPlatform PlatformType = "cagp" -) - func init() { - admissionCtrlCmd.PersistentFlags().StringP("platform", "", "cagp", - fmt.Sprintf("one of: %s|", CAGPPlatform)) - admissionCtrlCmd.PersistentFlags().StringP("domain-pool", "", "zerossl", "domain pool to use when deploying on CAGP") - admissionCtrlCmd.PersistentFlags().StringP("domain-claim", "", "cnvrg-domain-claim", "domain claim to create") - admissionCtrlCmd.PersistentFlags().StringP("gateway-name", "", "cnvrg-gateway", "gateway to create") - admissionCtrlCmd.PersistentFlags().StringP("namespace", "", "cnvrg", "namespace") - admissionCtrlCmd.PersistentFlags().StringP("reg-user", "", "reg-user", "registry user") - admissionCtrlCmd.PersistentFlags().StringP("reg-password", "", "reg-password", "registry password") + admissionCtrlCmd.PersistentFlags().StringP("crt", "", "", "path to certificate file") admissionCtrlCmd.PersistentFlags().StringP("key", "", "", "path to key file") - viper.BindPFlag("platform", admissionCtrlCmd.PersistentFlags().Lookup("platform")) - viper.BindPFlag("domain-pool", admissionCtrlCmd.PersistentFlags().Lookup("domain-pool")) - viper.BindPFlag("domain-claim", admissionCtrlCmd.PersistentFlags().Lookup("domain-claim")) - viper.BindPFlag("namespace", admissionCtrlCmd.PersistentFlags().Lookup("namespace")) - viper.BindPFlag("reg-user", admissionCtrlCmd.PersistentFlags().Lookup("reg-user")) - viper.BindPFlag("reg-password", admissionCtrlCmd.PersistentFlags().Lookup("reg-password")) viper.BindPFlag("crt", admissionCtrlCmd.PersistentFlags().Lookup("crt")) viper.BindPFlag("key", admissionCtrlCmd.PersistentFlags().Lookup("key")) diff --git a/cmd/copctl/utils/utils.go b/cmd/copctl/utils/utils.go new file mode 100644 index 00000000..ff5c29bf --- /dev/null +++ b/cmd/copctl/utils/utils.go @@ -0,0 +1,53 @@ +package utils + +import ( + "crypto/rand" + "crypto/rsa" + mlopsv1 "github.com/AccessibleAI/cnvrg-operator/api/v1" + "go.uber.org/zap" + v1core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +func Clientset() *kubernetes.Clientset { + rc, err := config.GetConfig() + if err != nil { + zap.S().Fatalf("unable to construct K8s configs, err: %s", err.Error()) + } + cltset, err := kubernetes.NewForConfig(rc) + if err != nil { + zap.S().Fatalf("err: %s, unable to connect to K8s", err.Error()) + } + return cltset +} + +func Kubecrudclient() client.Client { + + rc, err := config.GetConfig() + if err != nil { + zap.S().Fatal(err) + } + + scheme := runtime.NewScheme() + utilruntime.Must(v1core.AddToScheme(scheme)) + utilruntime.Must(mlopsv1.AddToScheme(scheme)) + + cc, err := client.New(rc, client.Options{Scheme: scheme}) + if err != nil { + zap.S().Fatal(err) + } + return cc +} + +func PrivateKey() *rsa.PrivateKey { + zap.S().Info("generating private key") + privKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + zap.S().Fatal(err) + } + return privKey +} diff --git a/pkg/admission/domain.go b/pkg/admission/domain.go index 992e3c27..4de3e8ad 100644 --- a/pkg/admission/domain.go +++ b/pkg/admission/domain.go @@ -78,6 +78,9 @@ func (h *AICloudDomainHandler) HookCfg(ns, svc string, caBundle []byte) *admissi Webhooks: []admissionv1.MutatingWebhook{ { Name: hookName, + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"name": ns}, + }, ClientConfig: admissionv1.WebhookClientConfig{ Service: &admissionv1.ServiceReference{ Namespace: ns,