Skip to content

Commit

Permalink
add status configmap generator
Browse files Browse the repository at this point in the history
  • Loading branch information
Dimss committed Dec 24, 2023
1 parent 61601f0 commit c36d5b9
Show file tree
Hide file tree
Showing 10 changed files with 277 additions and 61 deletions.
2 changes: 1 addition & 1 deletion charts/cnvrg-all-in-one/Chart.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: v2
appVersion: 5.0.0
description: A cnvrg.io AI:OS chart for K8s
name: cnvrg-all-in-one
name: cnvrg-mlops
type: application
version: 5.0.0
dependencies:
Expand Down
11 changes: 6 additions & 5 deletions cmd/copctl/cmd/create/kubecerts.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"github.com/AccessibleAI/cnvrg-operator/cmd/copctl/utils"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"go.uber.org/zap"
Expand Down Expand Up @@ -47,7 +48,7 @@ var kubeCertsCmd = &cobra.Command{
if viper.GetBool("override") {
clean(viper.GetString("certs-dir"), viper.GetString("common-name"))
}
pkey := privateKey()
pkey := utils.PrivateKey()
approveCsr(
createCsr(
csrPem(
Expand All @@ -64,7 +65,7 @@ var kubeCertsCmd = &cobra.Command{

func clean(certsDir string, commonName string) {
zap.S().Info("cleaning up exiting certs")
err := clientset().
err := utils.Clientset().
CertificatesV1().
CertificateSigningRequests().
Delete(context.Background(), commonName, metav1.DeleteOptions{})
Expand Down Expand Up @@ -127,7 +128,7 @@ func createCsr(csrPem *bytes.Buffer, commonName string) *certsv1.CertificateSign
//signerName := "kubernetes.io/kube-apiserver-client-kubelet"
//signerName := "kubernetes.io/kube-apiserver-client"
signerName := "kubernetes.io/kubelet-serving"
csr, err := clientset().CertificatesV1().
csr, err := utils.Clientset().CertificatesV1().
CertificateSigningRequests().
Create(context.Background(), &certsv1.CertificateSigningRequest{
ObjectMeta: metav1.ObjectMeta{Name: commonName},
Expand Down Expand Up @@ -159,15 +160,15 @@ func approveCsr(csr *certsv1.CertificateSigningRequest) {
LastTransitionTime: metav1.Now(),
})

if _, err := clientset().CertificatesV1().
if _, err := utils.Clientset().CertificatesV1().
CertificateSigningRequests().
UpdateApproval(context.Background(), csr.ObjectMeta.Name, csr, metav1.UpdateOptions{}); err != nil {
zap.S().Fatal(err)
}
}

func fetchCertificateFromCsr(commonName string) []byte {
csr, err := clientset().
csr, err := utils.Clientset().
CertificatesV1().
CertificateSigningRequests().
Get(context.Background(), commonName, metav1.GetOptions{})
Expand Down
30 changes: 0 additions & 30 deletions cmd/copctl/cmd/create/utils.go

This file was deleted.

9 changes: 5 additions & 4 deletions cmd/copctl/cmd/create/webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"github.com/AccessibleAI/cnvrg-operator/cmd/copctl/utils"
"github.com/AccessibleAI/cnvrg-operator/pkg/admission"
"github.com/spf13/cobra"
"github.com/spf13/viper"
Expand Down Expand Up @@ -69,7 +70,7 @@ func (h *Webhook) run() {
h.clean()
}
// get key for CA
cakey := privateKey()
cakey := utils.PrivateKey()
// create CA certificate
ca, caPEM := h.createCA(cakey)
// create certificate and key for server
Expand Down Expand Up @@ -141,7 +142,7 @@ func (h *Webhook) serverCrtAndKey(ca *x509.Certificate, cakey *rsa.PrivateKey) (
KeyUsage: x509.KeyUsageDigitalSignature,
}

serverKey := privateKey()
serverKey := utils.PrivateKey()
certBytes, err := x509.CreateCertificate(rand.Reader, serverCrt, ca, &serverKey.PublicKey, cakey)
if err != nil {
zap.S().Fatalf("error creating server certificate, err: %s ", err.Error())
Expand Down Expand Up @@ -189,7 +190,7 @@ func (h *Webhook) createMutatingWebhookCfg(hookCfg *admissionv1.MutatingWebhookC

zap.S().Infof("creating webhook: %s", hookCfg.Name)

err := clientset().
err := utils.Clientset().
AdmissionregistrationV1().
MutatingWebhookConfigurations().
Delete(context.Background(), hookCfg.Name, metav1.DeleteOptions{})
Expand All @@ -198,7 +199,7 @@ func (h *Webhook) createMutatingWebhookCfg(hookCfg *admissionv1.MutatingWebhookC
zap.S().Fatalf("error deleting webhook: %s, err: %s ", hookCfg.Name, err.Error())
}

if _, err := clientset().
if _, err := utils.Clientset().
AdmissionregistrationV1().
MutatingWebhookConfigurations().
Create(context.Background(), hookCfg, metav1.CreateOptions{}); err != nil {
Expand Down
11 changes: 11 additions & 0 deletions cmd/copctl/cmd/get/get.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package get

import "github.com/spf13/cobra"

var (
Cmd = &cobra.Command{
Use: "get",
Aliases: []string{"g"},
Short: "Get copctl resources",
}
)
194 changes: 194 additions & 0 deletions cmd/copctl/cmd/get/status.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
package get

import (
"context"
"encoding/json"
"fmt"
mlopsv1 "github.com/AccessibleAI/cnvrg-operator/api/v1"
"github.com/AccessibleAI/cnvrg-operator/cmd/copctl/utils"
"github.com/AccessibleAI/cnvrg-shim/apis/metacloud/v1alpha1"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"go.uber.org/zap"
"istio.io/istio/pkg/config/protocol"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"os"
"os/signal"
"sigs.k8s.io/controller-runtime/pkg/client"
"syscall"
"time"
)

type Status struct {
NamespacedName types.NamespacedName
StatusConfigmapName string
Interval time.Duration
}

func init() {
statusCmd.PersistentFlags().StringP("namespace", "", "", "current namespace")
statusCmd.PersistentFlags().StringP("name", "", "", "name of the CnvrgApp CR")
statusCmd.PersistentFlags().StringP("status-configmap", "", "service-instance-status", "the status cm name")
statusCmd.PersistentFlags().IntP("interval", "i", 1, "status generation interval")

viper.BindPFlag("namespace", statusCmd.PersistentFlags().Lookup("namespace"))
viper.BindPFlag("name", statusCmd.PersistentFlags().Lookup("name"))
viper.BindPFlag("status-configmap", statusCmd.PersistentFlags().Lookup("status-configmap"))
viper.BindPFlag("interval", statusCmd.PersistentFlags().Lookup("interval"))

Cmd.AddCommand(statusCmd)
}

var statusCmd = &cobra.Command{
Use: "status",
Aliases: []string{"s"},
Short: "Get cnvrg app status",
Run: func(cmd *cobra.Command, args []string) {

NewStatus(
viper.GetString("namespace"),
viper.GetString("name"),
viper.GetString("status-configmap"),
viper.GetInt("interval"),
).run()

sigCh := make(chan os.Signal, 1)
signal.Notify(sigCh, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
for {
select {
case s := <-sigCh:
zap.S().Infof("signal: %s, shutting down", s)
zap.S().Info("bye bye 👋")
os.Exit(0)
}
}

},
}

func NewStatus(ns, name, statusConfigmap string, interval int) *Status {
if name == "" || ns == "" {
zap.S().Fatal("name and namespace must be set")
}
return &Status{
NamespacedName: types.NamespacedName{Namespace: ns, Name: name},
StatusConfigmapName: statusConfigmap,
Interval: time.Second * time.Duration(interval),
}
}

func (s *Status) loadCnvrgApp(cap *mlopsv1.CnvrgApp) error {
return utils.
Kubecrudclient().
Get(context.Background(),
s.NamespacedName,
cap,
[]client.GetOption{}...,
)

}

func (s *Status) run() {
for {
cnvrgAppInstance := &mlopsv1.CnvrgApp{}
// fetch cnvrgapp instance
if err := s.loadCnvrgApp(cnvrgAppInstance); err != nil {
zap.S().Errorf("failed to fetch cnvrgapp instance, err: %s", err.Error())
continue
}

siStatus, err := json.Marshal(s.generateStatus(cnvrgAppInstance))
if err != nil {
zap.S().Errorf("failed to marshal service instance, err: %s", err.Error())
continue
}

if err := s.writeStatus(siStatus); err != nil {
zap.S().Errorf("failed to create status configmap, err: %s", err.Error())
continue
}
zap.S().Info("service instance configmap updated")
time.Sleep(s.Interval)
}
}

func (s *Status) writeStatus(payload []byte) error {

// get status cm
cm, err := utils.
Clientset().
CoreV1().
ConfigMaps(s.NamespacedName.Namespace).
Get(context.Background(), s.StatusConfigmapName, v1.GetOptions{})

// if not found create it
if errors.IsNotFound(err) {
// construct configmap
statusCm := &corev1.ConfigMap{
ObjectMeta: v1.ObjectMeta{
Namespace: s.NamespacedName.Namespace,
Name: s.StatusConfigmapName,
},
Data: map[string]string{"serviceInstanceStatus": string(payload)},
}
//create configmap
_, err = utils.
Clientset().
CoreV1().
ConfigMaps(s.NamespacedName.Namespace).
Create(context.Background(), statusCm, v1.CreateOptions{})

return err

} else if err != nil {
// error fetching configmap
return err
}

// status configmap exists, update it
cm.Data = map[string]string{"serviceInstanceStatus": string(payload)}
_, err = utils.
Clientset().
CoreV1().
ConfigMaps(s.NamespacedName.Namespace).
Update(context.Background(), cm, v1.UpdateOptions{})

return err

}

func (s *Status) generateStatus(cnvrgApp *mlopsv1.CnvrgApp) *v1alpha1.ServiceInstanceStatus {

port := 80
proto := protocol.HTTP
status := v1alpha1.StatusHealthy

if cnvrgApp.Spec.Networking.HTTPS.Enabled {
proto = protocol.HTTPS
port = 443
}

if cnvrgApp.Status.Status != "READY" {
status = v1alpha1.StatusReconciling
}

return &v1alpha1.ServiceInstanceStatus{
Status: status,
Sins: []v1alpha1.Sins{
{
Name: cnvrgApp.Name,
IngressEndpoints: []v1alpha1.IngressEndpoint{
{
Protocol: proto,
Address: []string{fmt.Sprintf("app.%s", cnvrgApp.Spec.ClusterDomain)},
Port: uint32(port),
},
},
},
},
}

}
3 changes: 3 additions & 0 deletions cmd/copctl/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package cmd

import (
"github.com/AccessibleAI/cnvrg-operator/cmd/copctl/cmd/create"
"github.com/AccessibleAI/cnvrg-operator/cmd/copctl/cmd/get"
"github.com/AccessibleAI/cnvrg-operator/cmd/copctl/cmd/start"
"github.com/spf13/cobra"
"github.com/spf13/viper"
Expand Down Expand Up @@ -29,6 +30,7 @@ func init() {
cobra.OnInitialize(initConfig)
RootCmd.AddCommand(start.Cmd)
RootCmd.AddCommand(create.Cmd)
RootCmd.AddCommand(get.Cmd)
}

func initConfig() {
Expand All @@ -41,6 +43,7 @@ func initConfig() {
func initZapLog() {
config := zap.NewDevelopmentConfig()
//config := zap.NewProductionConfig()
config.EncoderConfig.StacktraceKey = ""
config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
config.EncoderConfig.TimeKey = "timestamp"
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
Expand Down
Loading

0 comments on commit c36d5b9

Please sign in to comment.