diff --git a/agent/container/api/agent.gen.go b/agent/container/api/agent.gen.go index 3b9beab7..322dbc64 100644 --- a/agent/container/api/agent.gen.go +++ b/agent/container/api/agent.gen.go @@ -27,6 +27,9 @@ type ServerInterface interface { // Post Dockerhub artifactory events // (POST /event/docker/hub) PostEventDockerHub(c *gin.Context) + // Post Jfrog Container Registry webhook events + // (POST /event/jfrog/container) + PostEventJfrogContainer(c *gin.Context) // Kubernetes readiness and liveness probe endpoint // (GET /status) GetStatus(c *gin.Context) @@ -71,6 +74,16 @@ func (siw *ServerInterfaceWrapper) PostEventDockerHub(c *gin.Context) { siw.Handler.PostEventDockerHub(c) } +// PostEventJfrogContainer operation middleware +func (siw *ServerInterfaceWrapper) PostEventJfrogContainer(c *gin.Context) { + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + } + + siw.Handler.PostEventJfrogContainer(c) +} + // GetStatus operation middleware func (siw *ServerInterfaceWrapper) GetStatus(c *gin.Context) { @@ -116,6 +129,8 @@ func RegisterHandlersWithOptions(router *gin.Engine, si ServerInterface, options router.POST(options.BaseURL+"/event/docker/hub", wrapper.PostEventDockerHub) + router.POST(options.BaseURL+"/event/jfrog/container", wrapper.PostEventJfrogContainer) + router.GET(options.BaseURL+"/status", wrapper.GetStatus) return router @@ -124,13 +139,14 @@ func RegisterHandlersWithOptions(router *gin.Engine, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/6SSQWscMQyF/4rQebqzaW9zC01oQwoJ2d5CDh5buyMyaxtJnrJd5r8Xz9K0pCUJ7ckI", - "3nv6JOuIPu1zihRNsTvODXLcJuyOGEi9cDZOETv8mKI5jiTQC4cdwU2mCHeXm69wfnsFmsnzlr1b5A0a", - "20iv2zbPbBOJnvqdrdarNc4NpkzRZcYOP6zWqzNsMDsbKiu2LvO7kPxS7MjqkzLJknYVsMNPZOeZL6qk", - "QSHNKSot8vfr9Z9D3lzjPDeoZb93csAOv7AapG1lVciSJg4UoD+ADQRKMrGnOq3bKXb3mEs/sseHGtLS", - "RNFa970Itf7nGmrPnPQvqLdJ7bJazqvjaW//xl3DYAmCXz9wRztWkwN8o35I6REWQn2ZPyT/SNIOpX8D", - "+sUi/lz6/6A+ZQylByfGW+ctyeEVVDVn5cUr2JwUb8HS4j2pbssITzHPQK9LTxLJSEHIBY6kCi4GGHmi", - "pciSegKKISeO9ju38OSMKniNJKknj939EYuM2GGL88P8IwAA//90yrlhlQMAAA==", + "H4sIAAAAAAAC/6yTQW/TQBCF/8pqziZO4eZbRCsoRWrVcKt6WK/HyVBnZzUzaxQi/3e0jiiooBIKJ2uk", + "995882wfIPAuccRoCs1hqoBiz9AcoEMNQsmIIzTwlqN5iiiuFeo26K4TRnd7sf7kVjeXThMG6in4WV6B", + "kQ34Z9v6iW1E0eO+s8VysYSpAk4YfSJo4M1iuTiDCpK3bWGF2id61XGYhw1aeXBCmdMuO2jgHdoq0XmR", + "VCCoiaPiLH+9XP565PUVTFMFmnc7L3to4COpOe4Lq7okPFKHnWv3zrboFGWkgOVav1Fo7iDldqAA9yWk", + "xhGj1f5rFqzD9xrKzsT6G9QbVrsollVxPPb2Mu4S5uYg9+MN3OKG1GTvvmC7ZX5wM6E+z99xeECpt7k9", + "Af18Fr/P7T9QHzO2uXVejHofjGV/EurnXnjzV1V/KI7/UfUc9PKq1bzlZ7/i9VFxCqHmEFC1z4N7jHnC", + "fJVblIiG6gR9RxFVnY+dG2jEeUjCLTqMXWKK9jO30OgNC3iJRCm/LDR3B8gyQAM1TPfTtwAAAP//tJ+y", + "/VUEAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/agent/container/openapi.yaml b/agent/container/openapi.yaml index bcd76a59..c873ba14 100755 --- a/agent/container/openapi.yaml +++ b/agent/container/openapi.yaml @@ -44,5 +44,12 @@ paths: responses: '200': description: OK - + /event/jfrog/container: + post: + tags: + - public + summary: Post Jfrog Container Registry webhook events + responses: + '200': + description: OK # oapi-codegen -config ./cfg.yaml ./openapi.yaml diff --git a/agent/container/pkg/handler/api_handler.go b/agent/container/pkg/handler/api_handler.go index 5e5e2834..37525ff5 100755 --- a/agent/container/pkg/handler/api_handler.go +++ b/agent/container/pkg/handler/api_handler.go @@ -35,6 +35,8 @@ func (ah *APIHandler) BindRequest(r *gin.Engine) { apiGroup.GET("/status", ah.GetStatus) apiGroup.POST("/event/docker/hub", ah.PostEventDockerHub) apiGroup.POST("/event/azure/container", ah.PostEventAzureContainer) + apiGroup.POST("/event/jfrog/container", ah.PostEventJfrogContainer) + } } diff --git a/agent/container/pkg/handler/jfrog_container.go b/agent/container/pkg/handler/jfrog_container.go new file mode 100644 index 00000000..5655026c --- /dev/null +++ b/agent/container/pkg/handler/jfrog_container.go @@ -0,0 +1,45 @@ +package handler + +import ( + "encoding/json" + "errors" + "io" + "log" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/intelops/kubviz/model" +) + +var ErrInvalidPayloads = errors.New("invalid or malformed jfrog Container Registry webhook payload") + +func (ah *APIHandler) PostEventJfrogContainer(c *gin.Context) { + defer func() { + _, _ = io.Copy(io.Discard, c.Request.Body) + _ = c.Request.Body.Close() + }() + payload, err := io.ReadAll(c.Request.Body) + if err != nil || len(payload) == 0 { + log.Printf("%v: %v", ErrReadingBody, err) + c.Status(http.StatusBadRequest) + return + } + + var pushEvent model.JfrogContainerPushEventPayload + err = json.Unmarshal(payload, &pushEvent) + if err != nil { + log.Printf("%v: %v", ErrInvalidPayloads, err) + c.JSON(http.StatusBadRequest, gin.H{"error": "Bad Request"}) + return + } + + log.Printf("Received event from jfrog Container Registry: %v", pushEvent) + + err = ah.conn.Publish(payload, "Jfrog_Container_Registry") + if err != nil { + log.Printf("%v: %v", ErrPublishToNats, err) + c.Status(http.StatusInternalServerError) + return + } + c.Status(http.StatusOK) +} diff --git a/agent/kubviz/k8smetrics_agent.go b/agent/kubviz/k8smetrics_agent.go index 9201a8fd..317ff3a5 100644 --- a/agent/kubviz/k8smetrics_agent.go +++ b/agent/kubviz/k8smetrics_agent.go @@ -53,8 +53,16 @@ var ( //for local testing provide the location of kubeconfig // inside the civo file paste your kubeconfig // uncomment this line from Dockerfile.Kubviz (COPY --from=builder /workspace/civo /etc/myapp/civo) - cluster_conf_loc string = os.Getenv("CONFIG_LOCATION") - schedulingIntervalStr string = os.Getenv("SCHEDULING_INTERVAL") + cluster_conf_loc string = os.Getenv("CONFIG_LOCATION") + schedulingIntervalStr string = os.Getenv("SCHEDULING_INTERVAL") + enableScheduling string = os.Getenv("ENABLE_SCHEDULING") + outdatedIntervalStr string = os.Getenv("OUTDATED_INTERVAL") + preUpgradeIntervalStr string = os.Getenv("PRE_UPGRADE_INTERVAL") + getAllResourcesIntervalStr string = os.Getenv("GET_ALL_RESOURCES_INTERVAL") + rakkessIntervalStr string = os.Getenv("RAKKESS_INTERVAL") + getclientIntervalStr string = os.Getenv("GETCLIENT_INTERVAL") + trivyIntervalStr string = os.Getenv("TRIVY_INTERVAL") + kubescoreIntervalStr string = os.Getenv("KUBESCORE_INTERVAL") ) func runTrivyScans(config *rest.Config, js nats.JetStreamContext, wg *sync.WaitGroup, trivyImagescanChan, trivySbomcanChan, trivyK8sMetricsChan chan error) { @@ -160,41 +168,96 @@ func main() { }() wg.Add(6) // Initialize the WaitGroup for the seven goroutines // ... start other goroutines ... - go outDatedImages(config, js, &wg, outdatedErrChan) - go KubePreUpgradeDetector(config, js, &wg, kubePreUpgradeChan) - go GetAllResources(config, js, &wg, getAllResourceChan) - go RakeesOutput(config, js, &wg, RakeesErrChan) - go getK8sEvents(clientset) - // Run these functions sequentially within a single goroutine using the wrapper function - go runTrivyScans(config, js, &wg, trivyImagescanChan, trivySbomcanChan, trivyK8sMetricsChan) - go RunKubeScore(clientset, js, &wg, kubescoreMetricsChan) - wg.Wait() - // once the go routines completes we will close the error channels - close(outdatedErrChan) - close(kubePreUpgradeChan) - close(getAllResourceChan) - // close(clusterMetricsChan) - close(kubescoreMetricsChan) - close(trivyImagescanChan) - close(trivySbomcanChan) - close(trivyK8sMetricsChan) - close(RakeesErrChan) - // Signal that all other goroutines have finished - doneChan <- true - close(doneChan) + if enableScheduling == "true" { + + // ... read other intervals ... + + // Convert interval strings to time.Duration + outdatedInterval, _ := time.ParseDuration(outdatedIntervalStr) + if err != nil { + log.Fatalf("Failed to parse SCHEDULING_INTERVAL for outdated: %v", err) + } + preUpgradeInterval, _ := time.ParseDuration(preUpgradeIntervalStr) + if err != nil { + log.Fatalf("Failed to parse SCHEDULING_INTERVAL for preupgrade: %v", err) + } + getAllResourcesInterval, _ := time.ParseDuration(getAllResourcesIntervalStr) + if err != nil { + log.Fatalf("Failed to parse SCHEDULING_INTERVAL for allresource: %v", err) + } + rakkessInterval, _ := time.ParseDuration(rakkessIntervalStr) + if err != nil { + log.Fatalf("Failed to parse SCHEDULING_INTERVAL for Rakkess: %v", err) + } + getclientInterval, _ := time.ParseDuration(getclientIntervalStr) + if err != nil { + log.Fatalf("Failed to parse SCHEDULING_INTERVAL for Rakkess: %v", err) + } + trivyInterval, _ := time.ParseDuration(trivyIntervalStr) + if err != nil { + log.Fatalf("Failed to parse SCHEDULING_INTERVAL for Rakkess: %v", err) + } + kubescoreInterval, _ := time.ParseDuration(kubescoreIntervalStr) + if err != nil { + log.Fatalf("Failed to parse SCHEDULING_INTERVAL for Rakkess: %v", err) + } + // ... convert other intervals ... + s := gocron.NewScheduler(time.UTC) + + s.Every(outdatedInterval).Do(outDatedImages, config, js, &wg, outdatedErrChan) + s.Every(preUpgradeInterval).Do(KubePreUpgradeDetector, config, js, &wg, kubePreUpgradeChan) + s.Every(getAllResourcesInterval).Do(GetAllResources, config, js, &wg, getAllResourceChan) + s.Every(rakkessInterval).Do(RakeesOutput, config, js, &wg, RakeesErrChan) + s.Every(getclientInterval).Do(getK8sClient, clientset) + s.Every(trivyInterval).Do(runTrivyScans, js, &wg, trivyImagescanChan, trivySbomcanChan, trivyK8sMetricsChan) + s.Every(kubescoreInterval).Do(RunKubeScore, clientset, js, &wg, kubescoreMetricsChan) + + // once the go routines completes we will close the error channels + s.StartBlocking() + // ... call other functions ... + } else { + + outDatedImages(config, js, &wg, outdatedErrChan) + KubePreUpgradeDetector(config, js, &wg, kubePreUpgradeChan) + GetAllResources(config, js, &wg, getAllResourceChan) + RakeesOutput(config, js, &wg, RakeesErrChan) + getK8sEvents(clientset) + // Run these functions sequentially within a single goroutine using the wrapper function + runTrivyScans(config, js, &wg, trivyImagescanChan, trivySbomcanChan, trivyK8sMetricsChan) + RunKubeScore(clientset, js, &wg, kubescoreMetricsChan) + + wg.Wait() + // once the go routines completes we will close the error channels + close(outdatedErrChan) + close(kubePreUpgradeChan) + close(getAllResourceChan) + // close(clusterMetricsChan) + close(kubescoreMetricsChan) + close(trivyImagescanChan) + close(trivySbomcanChan) + close(trivyK8sMetricsChan) + close(RakeesErrChan) + // Signal that all other goroutines have finished + doneChan <- true + close(doneChan) + + } } collectAndPublishMetrics() - if schedulingIntervalStr == "" { - schedulingIntervalStr = "20m" // Default value, e.g., 20 minutes - } - schedulingInterval, err := time.ParseDuration(schedulingIntervalStr) - if err != nil { - log.Fatalf("Failed to parse SCHEDULING_INTERVAL: %v", err) + if enableScheduling == "false" { + if schedulingIntervalStr == "" { + schedulingIntervalStr = "20m" // Default value, e.g., 20 minutes + } + schedulingInterval, err := time.ParseDuration(schedulingIntervalStr) + if err != nil { + log.Fatalf("Failed to parse SCHEDULING_INTERVAL: %v", err) + } + s := gocron.NewScheduler(time.UTC) + s.Every(schedulingInterval).Do(collectAndPublishMetrics) // Run immediately and then at the scheduled interval + s.StartBlocking() } - s := gocron.NewScheduler(time.UTC) - s.Every(schedulingInterval).Do(collectAndPublishMetrics) // Run immediately and then at the scheduled interval - s.StartBlocking() // Blocks the main function -} + +} // Blocks the main function // publishMetrics publishes stream of events // with subject "METRICS.created" diff --git a/agent/kubviz/kube_score.go b/agent/kubviz/kube_score.go index f7a2b838..89485afe 100644 --- a/agent/kubviz/kube_score.go +++ b/agent/kubviz/kube_score.go @@ -1,67 +1,59 @@ package main import ( - "context" "encoding/json" + "log" + exec "os/exec" + "sync" + "github.com/google/uuid" "github.com/intelops/kubviz/constants" "github.com/intelops/kubviz/model" "github.com/nats-io/nats.go" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/zegl/kube-score/renderer/json_v2" "k8s.io/client-go/kubernetes" - "log" - exec "os/exec" - "sync" ) func RunKubeScore(clientset *kubernetes.Clientset, js nats.JetStreamContext, wg *sync.WaitGroup, errCh chan error) { defer wg.Done() + var report []json_v2.ScoredObject + cmd := "kubectl api-resources --verbs=list --namespaced -o name | xargs -n1 -I{} sh -c \"kubectl get {} --all-namespaces -oyaml && echo ---\" | kube-score score - -o json" + log.Printf("Command: %#v,", cmd) + out, err := executeCommand(cmd) - nsList, err := clientset.CoreV1(). - Namespaces(). - List(context.Background(), metav1.ListOptions{}) if err != nil { - log.Println("Error occurred while getting client set for kube-score: ", err) - return + log.Printf("Error scanning image %s:", err) } - log.Printf("Namespace size: %d", len(nsList.Items)) - for _, n := range nsList.Items { - log.Printf("Publishing kube-score recommendations for namespace: %s\n", n.Name) - publish(n.Name, js, errCh) + err = json.Unmarshal([]byte(out), &report) + + if err != nil { + log.Printf("Error occurred while Unmarshalling json: %v", err) } + publishKubescoreMetrics(report, js, errCh) } -func publish(ns string, js nats.JetStreamContext, errCh chan error) { - cmd := "kubectl api-resources --verbs=list --namespaced -o name | xargs -n1 -I{} sh -c \"kubectl get {} -n " + ns + " -oyaml && echo ---\" | kube-score score - " - log.Printf("Command: %#v,", cmd) - out, err := executeCommand(cmd) - if err != nil { - log.Println("Error occurred while running kube-score: ", err) - errCh <- err +func publishKubescoreMetrics(report []json_v2.ScoredObject, js nats.JetStreamContext, errCh chan error) { + metrics := model.KubeScoreRecommendations{ + ID: uuid.New().String(), + ClusterName: ClusterName, + Report: report, } - err = publishKubescoreMetrics(uuid.New().String(), ns, out, js) + metricsJson, err := json.Marshal(metrics) if err != nil { + log.Printf("Error marshaling metrics to JSON: %v", err) errCh <- err + return } - errCh <- nil -} -func publishKubescoreMetrics(id string, ns string, recommendations string, js nats.JetStreamContext) error { - metrics := model.KubeScoreRecommendations{ - ID: id, - Namespace: ns, - Recommendations: recommendations, - ClusterName: ClusterName, - } - metricsJson, _ := json.Marshal(metrics) - _, err := js.Publish(constants.KUBESCORE_SUBJECT, metricsJson) + _, err = js.Publish(constants.KUBESCORE_SUBJECT, metricsJson) if err != nil { - return err + log.Printf("error occures while publish %v", err) + errCh <- err + return } - log.Printf("Recommendations with ID:%s has been published\n", id) - log.Printf("Recommendations :%#v", recommendations) - return nil + log.Printf("KubeScore report with ID:%s has been published\n", metrics.ID) + errCh <- nil } func executeCommand(command string) (string, error) { diff --git a/client/pkg/clickhouse/db_client.go b/client/pkg/clickhouse/db_client.go index 3d728993..9590822a 100644 --- a/client/pkg/clickhouse/db_client.go +++ b/client/pkg/clickhouse/db_client.go @@ -41,6 +41,8 @@ type DBInterface interface { RetrieveKubvizEvent() ([]model.DbEvent, error) InsertContainerEventDockerHub(model.DockerHubBuild) InsertContainerEventAzure(model.AzureContainerPushEventPayload) + InsertContainerEventJfrog(model.JfrogContainerPushEventPayload) + InsertContainerEventGithub(string) InsertGitCommon(metrics model.GitCommonAttribute, statement dbstatement.DBStatement) error Close() @@ -72,7 +74,9 @@ func NewDBClient(conf *config.Config) (DBInterface, error) { return nil, err } - tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, trivyTableImage,trivySbomTable, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, azureContainerPushEventTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)} + tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, trivyTableImage, trivySbomTable, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, azureContainerPushEventTable, jfrogContainerPushEventTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)} + //tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, trivyTableImage, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, azureContainerPushEventTable, jfrogContainerPushEventTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)} + for _, table := range tables { if err = splconn.Exec(context.Background(), string(table)); err != nil { return nil, err @@ -131,7 +135,47 @@ func (c *DBClient) InsertContainerEventAzure(pushEvent model.AzureContainerPushE log.Fatal(err) } } +func (c *DBClient) InsertContainerEventJfrog(pushEvent model.JfrogContainerPushEventPayload) { + var ( + tx, _ = c.conn.Begin() + stmt, _ = tx.Prepare(string(InsertJfrogContainerPushEvent)) + ) + defer stmt.Close() + registryURL := pushEvent.Data.Path + repositoryName := pushEvent.Data.Name + tag := pushEvent.Data.Tag + + if tag == "" { + tag = "latest" + } + imageName := registryURL + "/" + repositoryName + ":" + tag + size := pushEvent.Data.Size + shaID := pushEvent.Data.SHA256 + // Marshaling the pushEvent into a JSON string + pushEventJSON, err := json.Marshal(pushEvent) + if err != nil { + log.Printf("Error while marshaling Azure Container Registry payload: %v", err) + return + } + + if _, err := stmt.Exec( + registryURL, + repositoryName, + tag, + imageName, + string(pushEventJSON), + pushEvent.Domain, + pushEvent.EventType, + size, + shaID, + ); err != nil { + log.Fatal(err) + } + if err := tx.Commit(); err != nil { + log.Fatal(err) + } +} func (c *DBClient) InsertRakeesMetrics(metrics model.RakeesMetrics) { var ( tx, _ = c.conn.Begin() @@ -311,24 +355,48 @@ func (c *DBClient) InsertContainerEvent(event string) { } func (c *DBClient) InsertKubeScoreMetrics(metrics model.KubeScoreRecommendations) { - var ( - tx, _ = c.conn.Begin() - stmt, _ = tx.Prepare(InsertKubeScore) - ) - defer stmt.Close() - if _, err := stmt.Exec( - metrics.ID, - metrics.Namespace, - metrics.ClusterName, - metrics.Recommendations, - ); err != nil { + tx, err := c.conn.Begin() + if err != nil { + log.Fatal(err) + } + defer tx.Rollback() + + stmt, err := tx.Prepare(InsertKubeScore) + if err != nil { log.Fatal(err) } + defer stmt.Close() + + for _, result := range metrics.Report { + for _, check := range result.Checks { + for _, comments := range check.Comments { + + if _, err := stmt.Exec( + metrics.ID, + metrics.ClusterName, + result.ObjectName, + result.TypeMeta.Kind, + result.TypeMeta.APIVersion, + result.ObjectMeta.Name, + result.ObjectMeta.Namespace, + check.Check.TargetType, + comments.Description, + comments.Path, + comments.Summary, + result.FileName, + result.FileRow, + ); err != nil { + log.Println("Error while inserting KubeScore metrics:", err) + } + } + + } + } + // Commit the transaction after the loop finishes. if err := tx.Commit(); err != nil { log.Fatal(err) } } - func (c *DBClient) InsertTrivyMetrics(metrics model.Trivy) { for _, finding := range metrics.Report.Findings { for _, result := range finding.Results { @@ -406,12 +474,12 @@ func (c *DBClient) InsertTrivyImageMetrics(metrics model.TrivyImage) { metrics.ID, metrics.ClusterName, metrics.Report.ArtifactName, - // metrics.Report.Metadata.Size, - // metrics.Report.Metadata.OS.Name, - // metrics.Report.Metadata.ImageID, - // strings.Join(metrics.Report.Metadata.DiffIDs, ","), - // strings.Join(metrics.Report.Metadata.RepoTags, ","), - // strings.Join(metrics.Report.Metadata.RepoDigests, ","), + metrics.Report.Metadata.Size, + metrics.Report.Metadata.OS.Name, + metrics.Report.Metadata.ImageID, + strings.Join(metrics.Report.Metadata.DiffIDs, ","), + strings.Join(metrics.Report.Metadata.RepoTags, ","), + strings.Join(metrics.Report.Metadata.RepoDigests, ","), vulnerability.VulnerabilityID, vulnerability.PkgID, vulnerability.PkgName, diff --git a/client/pkg/clickhouse/statements.go b/client/pkg/clickhouse/statements.go index 037c8004..d6919be2 100644 --- a/client/pkg/clickhouse/statements.go +++ b/client/pkg/clickhouse/statements.go @@ -71,14 +71,24 @@ CREATE TABLE IF NOT EXISTS outdated_images ( VersionsBehind Int64 ) engine=File(TabSeparated) ` -const kubescoreTable DBStatement = ` - CREATE TABLE IF NOT EXISTS kubescore ( - id UUID, - namespace String, - cluster_name String, - recommendations String - ) engine=File(TabSeparated) - ` +const kubescoreTable DBStatement = ` +CREATE TABLE IF NOT EXISTS kubescore ( + id UUID, + clustername String, + object_name String, + kind String, + apiVersion String, + name String, + namespace String, + target_type String, + description String, + path String, + summary String, + file_name String, + file_row BIGINT +) engine=File(TabSeparated) +` + const trivyTableVul DBStatement = ` CREATE TABLE IF NOT EXISTS trivy_vul ( id UUID, @@ -124,6 +134,12 @@ const trivyTableImage DBStatement = ` id UUID, cluster_name String, artifact_name String, + metadata_size BIGINT, + metadata_osname String, + metadata_imageid String, + metadata_diffid String, + metadata_repotags String, + metadata_repodigest String, vul_id String, vul_pkg_id String, vul_pkg_name String, @@ -157,6 +173,19 @@ const azureContainerPushEventTable DBStatement = ` SHAID String ) engine=File(TabSeparated) ` +const jfrogContainerPushEventTable DBStatement = ` + CREATE TABLE IF NOT EXISTS azurecontainerpush ( + RegistryURL String, + RepositoryName String, + Tag String, + ImageName String, + Event String, + Domain String, + EventType String, + Size Int32, + SHAID String + ) engine=File(TabSeparated) + ` const trivySbomTable DBStatement = ` CREATE TABLE IF NOT EXISTS trivysbom ( id UUID, @@ -193,9 +222,10 @@ const InsertKubvizEvent DBStatement = "INSERT INTO events (ClusterName, Id, Even const clickhouseExperimental DBStatement = `SET allow_experimental_object_type=1;` const containerDockerhubTable DBStatement = `CREATE table IF NOT EXISTS container_dockerhub(event JSON) ENGINE = MergeTree ORDER BY tuple();` const containerGithubTable DBStatement = `CREATE table IF NOT EXISTS container_github(event JSON) ENGINE = MergeTree ORDER BY tuple();` -const InsertKubeScore string = "INSERT INTO kubescore (id, namespace, cluster_name, recommendations) VALUES (?, ?, ?, ?)" +const InsertKubeScore string = "INSERT INTO kubescore(id,clustername,object_name,kind,apiVersion,name,namespace,target_type,description,path,summary,file_name,file_row) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)" const InsertTrivyVul string = "INSERT INTO trivy_vul (id, cluster_name, namespace, kind, name, vul_id, vul_vendor_ids, vul_pkg_id, vul_pkg_name, vul_pkg_path, vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?. ?)" -const InsertTrivyImage string = "INSERT INTO trivyimage (id, cluster_name, artifact_name, vul_id, vul_pkg_id, vul_pkg_name, vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES ( ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" +const InsertTrivyImage string = "INSERT INTO trivyimage (id, cluster_name, artifact_name,metadata_size ,metadata_osname ,metadata_imageid ,metadata_diffid ,metadata_repotags ,metadata_repodigest, vul_id, vul_pkg_id, vul_pkg_name, vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES ( ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?,?,?,?,?,?)" const InsertTrivyMisconfig string = "INSERT INTO trivy_misconfig (id, cluster_name, namespace, kind, name, misconfig_id, misconfig_avdid, misconfig_type, misconfig_title, misconfig_desc, misconfig_msg, misconfig_query, misconfig_resolution, misconfig_severity, misconfig_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?. ?, ?)" const InsertAzureContainerPushEvent DBStatement = "INSERT INTO azurecontainerpush (RegistryURL, RepositoryName, Tag, ImageName, Event, Timestamp, Size, SHAID) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" const InsertTrivySbom string = "INSERT INTO trivysbom (id, schema, bom_format,spec_version,serial_number, version, metadata_timestamp,metatool_vendor,metatool_name,metatool_version,component_bom_ref,component_type,component_name,component_version,component_property_name,component_property_value,component_hash_alg,component_hash_content,component_license_exp,component_purl,dependency_ref) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" +const InsertJfrogContainerPushEvent DBStatement = "INSERT INTO jfrogcontainerpush (RegistryURL, RepositoryName, Tag, ImageName, Event, Domain,EventType, Size, SHAID) VALUES (?, ?, ?, ?, ?, ?, ?, ?,?)" diff --git a/client/pkg/clients/container_client.go b/client/pkg/clients/container_client.go index 3688a102..f8bb4eb7 100644 --- a/client/pkg/clients/container_client.go +++ b/client/pkg/clients/container_client.go @@ -59,6 +59,16 @@ func (n *NATSContext) SubscribeContainerNats(conn clickhouse.DBInterface) { // Extract the necessary information from pushEvent and insert into ClickHouse conn.InsertContainerEventAzure(pushEvent) log.Println("Inserted Azure Container Registry metrics:", string(msg.Data)) + } else if repoName == "Jfrog_Container_Registry" { + var pushEvent model.JfrogContainerPushEventPayload + err := json.Unmarshal(msg.Data, &pushEvent) + if err != nil { + log.Printf("Error while unmarshaling Azure Container Registry payload: %v", err) + return + } + // Extract the necessary information from pushEvent and insert into ClickHouse + conn.InsertContainerEventJfrog(pushEvent) + log.Println("Inserted Azure Container Registry metrics:", string(msg.Data)) } }, nats.Durable(string(containerConsumer)), nats.ManualAck()) diff --git a/go.mod b/go.mod index 8188081c..962956a1 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/nats-io/nats.go v1.27.1 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 - github.com/vijeyash1/go-github-container v1.0.0 + github.com/zegl/kube-score v1.17.0 golang.org/x/term v0.10.0 k8s.io/api v0.27.3 k8s.io/apimachinery v0.27.3 @@ -48,7 +48,7 @@ require ( github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/fatih/color v1.14.1 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-errors/errors v1.4.2 // indirect diff --git a/go.sum b/go.sum index babc6d30..b9a43f61 100644 --- a/go.sum +++ b/go.sum @@ -118,8 +118,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= -github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= -github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fernet/fernet-go v0.0.0-20180830025343-9eac43b88a5e/go.mod h1:2H9hjfbpSMHwY503FclkV/lZTBh2YlOmLLSda12uL8c= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= @@ -440,8 +440,6 @@ github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95 github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= -github.com/vijeyash1/go-github-container v1.0.0 h1:SWtzxwGFFSCn8UB27IMcCbQ9xg1l6sQgk3pW2aD0fsQ= -github.com/vijeyash1/go-github-container v1.0.0/go.mod h1:yljHpWvbjXtjy48MXoBonmrTBUYNk8iA0cACfyU0Om4= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= @@ -457,6 +455,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zclconf/go-cty v1.10.0 h1:mp9ZXQeIcN8kAwuqorjH+Q+njbJKjLrvB2yIh4q7U+0= github.com/zclconf/go-cty-yaml v1.0.2 h1:dNyg4QLTrv2IfJpm7Wtxi55ed5gLGOlPrZ6kMd51hY0= +github.com/zegl/kube-score v1.17.0 h1:vedzK0pm5yOb1ocm5gybMNYsJRG8iTAatbo3LFIWbUc= +github.com/zegl/kube-score v1.17.0/go.mod h1:0pt4Lt36uTKPiCQbXQFow29eaAbgMLI9RoESjBoGSq0= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8= diff --git a/model/jfrogcontainer.go b/model/jfrogcontainer.go new file mode 100644 index 00000000..728ad6f5 --- /dev/null +++ b/model/jfrogcontainer.go @@ -0,0 +1,19 @@ +package model + +type JfrogContainerPushEventPayload struct { + Domain string `json:"domain"` + EventType string `json:"event_type"` + Data struct { + RepoKey string `json:"repo_key"` + Path string `json:"path"` + Name string `json:"name"` + SHA256 string `json:"sha256"` + Size int `json:"size"` + ImageName string `json:"image_name"` + Tag string `json:"tag"` + Platforms []interface{} `json:"platforms"` + } `json:"data"` + SubscriptionKey string `json:"subscription_key"` + JPDOrigin string `json:"jpd_origin"` + Source string `json:"source"` +} diff --git a/model/kubescore.go b/model/kubescore.go index 918ada85..fb63f747 100644 --- a/model/kubescore.go +++ b/model/kubescore.go @@ -1,8 +1,9 @@ package model +import "github.com/zegl/kube-score/renderer/json_v2" + type KubeScoreRecommendations struct { - ID string - Namespace string - Recommendations string - ClusterName string + ID string + ClusterName string + Report []json_v2.ScoredObject } diff --git a/steps-to-test.txt b/steps-to-test.txt index 66e17f5a..5cc2485c 100644 --- a/steps-to-test.txt +++ b/steps-to-test.txt @@ -33,4 +33,4 @@ docker tag ubuntu:latest localhost:5001/ubuntu:v1 docker push localhost:5001/ubuntu:v1 -# test commit +# test commits