From dfb0e04d3ebd65da27a6b581df146e7f64c1fec6 Mon Sep 17 00:00:00 2001 From: sg Date: Thu, 3 Oct 2024 17:01:41 +0100 Subject: [PATCH] implement #394 - add a custom key-value pair enricher --- .../enrichers/custom-annotation/README.md | 5 + .../enrichers/custom-annotation/main.go | 84 ++++++++++ .../enrichers/custom-annotation/main_test.go | 143 ++++++++++++++++++ .../enrichers/custom-annotation/task.yaml | 35 +++++ components/enrichers/test_utils.go | 76 ++++++++++ .../kustomization.yaml | 12 ++ .../pipelinerun.yaml | 24 +++ 7 files changed, 379 insertions(+) create mode 100644 components/enrichers/custom-annotation/README.md create mode 100644 components/enrichers/custom-annotation/main.go create mode 100644 components/enrichers/custom-annotation/main_test.go create mode 100644 components/enrichers/custom-annotation/task.yaml create mode 100644 examples/pipelines/custom-annotations-enricher-project/kustomization.yaml create mode 100644 examples/pipelines/custom-annotations-enricher-project/pipelinerun.yaml diff --git a/components/enrichers/custom-annotation/README.md b/components/enrichers/custom-annotation/README.md new file mode 100644 index 000000000..3469a9aec --- /dev/null +++ b/components/enrichers/custom-annotation/README.md @@ -0,0 +1,5 @@ +# Custom Annotation Enricher + +This enricher adds a set of custom annotations to every finding +Useful for development purposes and for tagging results +Can be used multiple times to add multiple annotations diff --git a/components/enrichers/custom-annotation/main.go b/components/enrichers/custom-annotation/main.go new file mode 100644 index 000000000..e389d5fdf --- /dev/null +++ b/components/enrichers/custom-annotation/main.go @@ -0,0 +1,84 @@ +// Package main of the codeowners enricher +// handles enrichment of individual issues with +// the groups/usernames listed in the github repository +// CODEOWNERS files. +// Owners are matched against the "target" field of the issue +package main + +import ( + "flag" + "log" + "log/slog" + "strings" + + apiv1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components/enrichers" +) + +var ( + defaultName = "custom-annotation" + annotations string + name string +) + +func enrichIssue(i *apiv1.Issue) (*apiv1.EnrichedIssue, error) { + enrichedIssue := apiv1.EnrichedIssue{} + annotationParts := strings.Split(annotations, ",") + issueAnnotations := map[string]string{} + for _, part := range annotationParts { + kv := strings.Split(part, ":") + if len(kv) == 2 { + issueAnnotations[kv[0]] = kv[1] + continue + } + slog.Info("could not add", slog.String("annotation", part)) + } + enrichedIssue = apiv1.EnrichedIssue{ + RawIssue: i, + Annotations: issueAnnotations, + } + return &enrichedIssue, nil +} + +func run() error { + res, err := enrichers.LoadData() + if err != nil { + return err + } + if annotations == "" { + slog.Info("annotations is empty") + } + for _, r := range res { + slog.Info("processing results for ", slog.Any("scan", r.ScanInfo)) + enrichedIssues := []*apiv1.EnrichedIssue{} + for _, i := range r.GetIssues() { + eI, err := enrichIssue(i) + if err != nil { + slog.Error(err.Error()) + continue + } + enrichedIssues = append(enrichedIssues, eI) + } + + err := enrichers.WriteData(&apiv1.EnrichedLaunchToolResponse{ + OriginalResults: r, + Issues: enrichedIssues, + }, strings.ReplaceAll(name, " ", "-")) + if err != nil { + return err + } + } + return nil +} + +func main() { + flag.StringVar(&annotations, "annotations", enrichers.LookupEnvOrString("ANNOTATIONS", ""), "what are the annotations this enricher will add to the issues") + flag.StringVar(&name, "annotation-name", enrichers.LookupEnvOrString("NAME", defaultName), "what is the name this enricher will masquerade as") + + if err := enrichers.ParseFlags(); err != nil { + log.Fatal(err) + } + if err := run(); err != nil { + log.Fatal(err) + } +} diff --git a/components/enrichers/custom-annotation/main_test.go b/components/enrichers/custom-annotation/main_test.go new file mode 100644 index 000000000..e4311c199 --- /dev/null +++ b/components/enrichers/custom-annotation/main_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" + + draconv1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components/enrichers" +) + +func TestHandlesZeroFindings(t *testing.T) { + indir, outdir := enrichers.SetupIODirs(t) + mockLaunchToolResponses := enrichers.GetEmptyLaunchToolResponse(t) + for i, r := range mockLaunchToolResponses { + // Write sample enriched responses to indir + encodedProto, err := proto.Marshal(r) + require.NoError(t, err) + rwPermission600 := os.FileMode(0o600) + require.NoError(t, os.WriteFile(fmt.Sprintf("%s/input_%d_%s.tagged.pb", indir, i, r.ToolName), encodedProto, rwPermission600)) + } + + // Run the enricher + enrichers.SetReadPathForTests(indir) + enrichers.SetWritePathForTests(outdir) + require.NoError(t, run()) + + // Check there is something in our output directory + files, err := os.ReadDir(outdir) + require.NoError(t, err) + require.NotEmpty(t, files) + require.Len(t, files, 4) + + // Check that both of them are EnrichedLaunchToolResponse + // and their Issue property is an empty list + for _, f := range files { + if strings.HasSuffix(f.Name(), ".raw.pb") { + continue + } + + encodedProto, err := os.ReadFile(fmt.Sprintf("%s/%s", outdir, f.Name())) + require.NoError(t, err) + output := &draconv1.EnrichedLaunchToolResponse{} + require.NoError(t, proto.Unmarshal(encodedProto, output)) + require.Empty(t, output.Issues) + } +} + +func TestHandlesFindings(t *testing.T) { + indir, outdir := enrichers.SetupIODirs(t) + annotations = "this is annotation key 1:annotation key 2,anno3:4" + name = "enricherName" + + mockLaunchToolResponses := enrichers.GetLaunchToolResponse(t) + for i, r := range mockLaunchToolResponses { + // Write sample enriched responses to indir + encodedProto, err := proto.Marshal(r) + require.NoError(t, err) + rwPermission600 := os.FileMode(0o600) + require.NoError(t, os.WriteFile(fmt.Sprintf("%s/input_%d_%s.tagged.pb", indir, i, r.ToolName), encodedProto, rwPermission600)) + } + + // Run the enricher + enrichers.SetReadPathForTests(indir) + enrichers.SetWritePathForTests(outdir) + require.NoError(t, run()) + + // Check there is something in our output directory + files, err := os.ReadDir(outdir) + require.NoError(t, err) + require.NotEmpty(t, files) + require.Len(t, files, 4) + + // Check that both of them are EnrichedLaunchToolResponse + // and their Issue property is not an empty list + expected := []*draconv1.EnrichedLaunchToolResponse{ + { + OriginalResults: mockLaunchToolResponses[0], + Issues: []*draconv1.EnrichedIssue{ + { + RawIssue: mockLaunchToolResponses[0].Issues[0], + Annotations: map[string]string{ + "this is annotation key 1": "annotation key 2", + "anno3": "4", + }, + }, + { + RawIssue: mockLaunchToolResponses[0].Issues[1], + Annotations: map[string]string{ + "this is annotation key 1": "annotation key 2", + "anno3": "4", + }, + }, + }, + }, + { + OriginalResults: mockLaunchToolResponses[1], + Issues: []*draconv1.EnrichedIssue{ + { + RawIssue: mockLaunchToolResponses[1].Issues[0], + Annotations: map[string]string{ + "this is annotation key 1": "annotation key 2", + "anno3": "4", + }, + }, + { + RawIssue: mockLaunchToolResponses[1].Issues[1], + Annotations: map[string]string{ + "this is annotation key 1": "annotation key 2", + "anno3": "4", + }, + }, + }, + }, + } + var actual draconv1.EnrichedLaunchToolResponse + for _, f := range files { + if strings.HasSuffix(f.Name(), ".raw.pb") { + continue + } + encodedProto, err := os.ReadFile(fmt.Sprintf("%s/%s", outdir, f.Name())) + require.NoError(t, err) + + require.NoError(t, proto.Unmarshal(encodedProto, &actual)) + if actual.OriginalResults.ToolName == expected[0].OriginalResults.ToolName { + if !proto.Equal(&actual, expected[0]) { + t.Log(cmp.Diff(&actual, expected[0], protocmp.Transform())) + t.Fatal("Actual does not match expected") + } + } else { + if !proto.Equal(&actual, expected[1]) { + t.Log(cmp.Diff(&actual, expected[1], protocmp.Transform())) + t.Fatal("Actual does not match expected") + } + } + } +} diff --git a/components/enrichers/custom-annotation/task.yaml b/components/enrichers/custom-annotation/task.yaml new file mode 100644 index 000000000..9f3c66d4d --- /dev/null +++ b/components/enrichers/custom-annotation/task.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: enricher-custom-annotation + labels: + v1.dracon.ocurity.com/component: enricher +spec: + description: Identifies a code owner for each finding. + params: + - name: enricher-custom-annotation-annotations + type: string + default: "" + description: "a comma separated list of key:value pairs" + - name: enricher-custom-annotation-name + type: string + default: "custom-annotation" + description: "the name to masquerade as, useful when running multiple instances" + workspaces: + - name: output + description: The workspace where we can output results + steps: + - name: run-enricher + imagePullPolicy: IfNotPresent + image: '{{ default "ghcr.io/ocurity/dracon" .Values.image.registry }}/components/enrichers/custom-annotation:{{ .Chart.AppVersion }}' + command: ["/app/components/enrichers/custom-annotation/custom-annotation"] + env: + - name: READ_PATH + value: $(workspaces.output.path)/.dracon/producers + - name: WRITE_PATH + value: "$(workspaces.output.path)/.dracon/enrichers/custom-annotation" + - name: ANNOTATIONS + value: "$(params.enricher-custom-annotation-annotations)" + - name: NAME + value: "$(params.enricher-custom-annotation-name)" diff --git a/components/enrichers/test_utils.go b/components/enrichers/test_utils.go index 1a82005fa..2db44b72d 100644 --- a/components/enrichers/test_utils.go +++ b/components/enrichers/test_utils.go @@ -33,3 +33,79 @@ func GetEmptyLaunchToolResponse(_ *testing.T) []*draconv1.LaunchToolResponse { }, } } + +// GetEmptyLaunchToolResponse returns a slice of LaunchToolResponse with no issues +func GetLaunchToolResponse(_ *testing.T) []*draconv1.LaunchToolResponse { + code := `this + is + some + code` + return []*draconv1.LaunchToolResponse{ + { + ToolName: "tool1", + Issues: []*draconv1.Issue{ + { + Target: "file:/a/b/c/d.php:1-2", + Type: "sometype", + Title: "this is a title", + Severity: draconv1.Severity_SEVERITY_CRITICAL, + Cvss: 1.0, + Confidence: draconv1.Confidence_CONFIDENCE_CRITICAL, + Description: "this is a handy dandy description", + Source: "this is a source", + Cve: "CVE-2020-123", + Uuid: "d9681ae9-223b-4df8-a422-7b29bb917a36", + Cwe: []int32{123}, + ContextSegment: &code, + }, + { + Target: "file:/a/b/c/d.go:2-3", + Type: "sometype1", + Title: "this is a title1", + Severity: draconv1.Severity_SEVERITY_CRITICAL, + Cvss: 1.0, + Confidence: draconv1.Confidence_CONFIDENCE_CRITICAL, + Description: "this is a handy dandy description1", + Source: "this is a source1", + Cve: "CVE-2020-124", + Uuid: "a9681ae9-223b-4df8-a422-7b29bb917a36", + Cwe: []int32{123}, + ContextSegment: &code, + }, + }, + }, + { + ToolName: "tool2", + Issues: []*draconv1.Issue{ + { + Target: "file:/a/b/c/d.py:1-2", + Type: "sometype", + Title: "this is a title", + Severity: draconv1.Severity_SEVERITY_CRITICAL, + Cvss: 1.0, + Confidence: draconv1.Confidence_CONFIDENCE_CRITICAL, + Description: "this is a handy dandy description", + Source: "this is a source", + Cve: "CVE-2020-123", + Uuid: "q9681ae9-223b-4df8-a422-7b29bb917a36", + Cwe: []int32{123}, + ContextSegment: &code, + }, + { + Target: "file:/a/b/c/d.py:2-3", + Type: "sometype1", + Title: "this is a title1", + Severity: draconv1.Severity_SEVERITY_CRITICAL, + Cvss: 1.0, + Confidence: draconv1.Confidence_CONFIDENCE_CRITICAL, + Description: "this is a handy dandy description1", + Source: "this is a source1", + Cve: "CVE-2020-124", + Uuid: "w9681ae9-223b-4df8-a422-7b29bb917a36", + Cwe: []int32{123}, + ContextSegment: &code, + }, + }, + }, + } +} diff --git a/examples/pipelines/custom-annotations-enricher-project/kustomization.yaml b/examples/pipelines/custom-annotations-enricher-project/kustomization.yaml new file mode 100644 index 000000000..2912c2e95 --- /dev/null +++ b/examples/pipelines/custom-annotations-enricher-project/kustomization.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -annotation-project +components: + - pkg:helm/dracon-oss-components/base + - pkg:helm/dracon-oss-components/git-clone + - pkg:helm/dracon-oss-components/producer-golang-gosec + - pkg:helm/dracon-oss-components/producer-aggregator + - pkg:helm/dracon-oss-components/enricher-custom-annotation + - pkg:helm/dracon-oss-components/enricher-aggregator + - pkg:helm/dracon-oss-components/consumer-stdout-json diff --git a/examples/pipelines/custom-annotations-enricher-project/pipelinerun.yaml b/examples/pipelines/custom-annotations-enricher-project/pipelinerun.yaml new file mode 100644 index 000000000..323af3548 --- /dev/null +++ b/examples/pipelines/custom-annotations-enricher-project/pipelinerun.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + generateName: dracon-annotation-project- +spec: + pipelineRef: + name: dracon-annotation-project + params: + - name: git-clone-url + value: https://github.com/sqreen/go-dvwa.git + - name: enricher-custom-annotation-annotations + value: "foo:bar,a:b,1:2" + - name: enricher-custom-annotation-name + value: "bar" + workspaces: + - name: output + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi