From 3b6aa7b51c1705e6ec14093bdb9f4b01bd5492ba Mon Sep 17 00:00:00 2001 From: sg Date: Thu, 3 Oct 2024 16:57:51 +0100 Subject: [PATCH 1/3] fix issue #393 by adjusting the default workspace for the source-code --- components/producers/producer.go | 10 +++++----- components/producers/producer_test.go | 22 ++++++++++++---------- pkg/sarif/sarif.go | 3 ++- pkg/sarif/sarif_test.go | 4 ++-- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/components/producers/producer.go b/components/producers/producer.go index 5d884c45e..d92f1dcb5 100644 --- a/components/producers/producer.go +++ b/components/producers/producer.go @@ -37,7 +37,7 @@ var ( ) const ( - sourceDir = "/workspace/output" + SourceDir = "/workspace/output/source-code/" ) var fileTargetPattern = regexp.MustCompile(`^(.*?:.*?):(.*)$`) @@ -102,9 +102,9 @@ func WriteDraconOut( source := getSource() cleanIssues := []*draconapiv1.Issue{} for _, iss := range issues { - iss.Description = strings.ReplaceAll(iss.Description, sourceDir, ".") - iss.Title = strings.ReplaceAll(iss.Title, sourceDir, ".") - iss.Target = strings.ReplaceAll(iss.Target, sourceDir, ".") + iss.Description = strings.ReplaceAll(iss.Description, SourceDir, "") + iss.Title = strings.ReplaceAll(iss.Title, SourceDir, "") + iss.Target = strings.ReplaceAll(iss.Target, SourceDir, "") iss.Source = source cleanIssues = append(cleanIssues, iss) slog.Debug(fmt.Sprintf("found issue: %+v\n", iss)) @@ -135,7 +135,7 @@ func WriteDraconOut( } func getSource() string { - sourceMetaPath := filepath.Join(sourceDir, ".source.dracon") + sourceMetaPath := filepath.Join(SourceDir, ".source.dracon") _, err := os.Stat(sourceMetaPath) if os.IsNotExist(err) { return "unknown" diff --git a/components/producers/producer_test.go b/components/producers/producer_test.go index 3b55ad7ad..c427642e0 100644 --- a/components/producers/producer_test.go +++ b/components/producers/producer_test.go @@ -3,6 +3,7 @@ package producers import ( "fmt" "os" + "path" "testing" "time" @@ -59,9 +60,9 @@ func TestWriteDraconOut(t *testing.T) { "dracon-test", []*v1.Issue{ { - Target: "/workspace/output/foobar", - Title: "/workspace/output/barfoo", - Description: "/workspace/output/example.yaml", + Target: path.Join(SourceDir, "foobar"), + Title: path.Join(SourceDir, "barfoo"), + Description: path.Join(SourceDir, "example.yaml"), Cve: "123-321", }, }, @@ -74,13 +75,14 @@ func TestWriteDraconOut(t *testing.T) { res := v1.LaunchToolResponse{} require.NoError(t, proto.Unmarshal(pBytes, &res)) - assert.Equal(t, "dracon-test", res.GetToolName()) - assert.Equal(t, "./foobar", res.GetIssues()[0].GetTarget()) - assert.Equal(t, "./barfoo", res.GetIssues()[0].GetTitle()) - assert.Equal(t, "./example.yaml", res.GetIssues()[0].GetDescription()) - assert.Equal(t, baseTime.Unix(), res.GetScanInfo().GetScanStartTime().GetSeconds()) - assert.Equal(t, "ab3d3290-cd9f-482c-97dc-ec48bdfcc4de", res.GetScanInfo().GetScanUuid()) - assert.Equal(t, "123-321", res.GetIssues()[0].GetCve()) + require.Equal(t, "dracon-test", res.GetToolName()) + require.NotEmpty(t, res.GetIssues()) + require.Equal(t, "foobar", res.GetIssues()[0].GetTarget()) + require.Equal(t, "barfoo", res.GetIssues()[0].GetTitle()) + require.Equal(t, "example.yaml", res.GetIssues()[0].GetDescription()) + require.Equal(t, baseTime.Unix(), res.GetScanInfo().GetScanStartTime().GetSeconds()) + require.Equal(t, "ab3d3290-cd9f-482c-97dc-ec48bdfcc4de", res.GetScanInfo().GetScanUuid()) + require.Equal(t, "123-321", res.GetIssues()[0].GetCve()) } func TestWriteDraconOutAppend(t *testing.T) { diff --git a/pkg/sarif/sarif.go b/pkg/sarif/sarif.go index 8969a3e7c..2bbf1efa6 100644 --- a/pkg/sarif/sarif.go +++ b/pkg/sarif/sarif.go @@ -11,6 +11,7 @@ import ( "github.com/owenrumney/go-sarif/v2/sarif" v1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components/producers" ) // DraconIssueCollection represents all the findings in a single Sarif file converted to dracon format. @@ -115,7 +116,7 @@ func FromDraconRawIssuesRun(responses []*v1.LaunchToolResponse) (*sarif.Report, } func removeDraconInternalPath(target string) string { - return strings.Replace(target, "/workspace/output", "", 1) + return strings.Replace(target, producers.SourceDir, "", 1) } func draconIssueToSarif(issue *v1.Issue, rule *sarif.ReportingDescriptor) (*sarif.Result, error) { diff --git a/pkg/sarif/sarif_test.go b/pkg/sarif/sarif_test.go index 3d1a0321d..97f8bbecb 100644 --- a/pkg/sarif/sarif_test.go +++ b/pkg/sarif/sarif_test.go @@ -356,7 +356,7 @@ func Test_draconIssueToSarif(t *testing.T) { Severity: v1.Severity_SEVERITY_INFO, Cvss: 0.0, Source: "//foo/bar:baz", - Target: "/workspace/output/foo1/bar1:baz2", + Target: "file:///workspace/output/source-code/foo1/bar1:baz2", Title: "Unit Test Title", Type: "test type", Cve: "CVE-0000-99999", @@ -364,7 +364,7 @@ func Test_draconIssueToSarif(t *testing.T) { typ := "test type" level := "note" msg := "this is a test description" - uri := "/foo1/bar1:baz2" + uri := "file://foo1/bar1:baz2" confidence := "Confidence:CONFIDENCE_INFO" source := "Source://foo/bar:baz" cve := "CVE-0000-99999" From 56a49eda59b5d298eda2291a5a282df89d786d00 Mon Sep 17 00:00:00 2001 From: sg Date: Thu, 3 Oct 2024 17:01:41 +0100 Subject: [PATCH 2/3] implement #394 - add a custom key-value pair enricher --- .../enrichers/custom-annotation/README.md | 5 + .../enrichers/custom-annotation/main.go | 81 ++++++++++ .../enrichers/custom-annotation/main_test.go | 140 ++++++++++++++++++ .../enrichers/custom-annotation/task.yaml | 35 +++++ components/enrichers/test_utils.go | 76 ++++++++++ .../kustomization.yaml | 12 ++ .../pipelinerun.yaml | 27 ++++ 7 files changed, 376 insertions(+) create mode 100644 components/enrichers/custom-annotation/README.md create mode 100644 components/enrichers/custom-annotation/main.go create mode 100644 components/enrichers/custom-annotation/main_test.go create mode 100644 components/enrichers/custom-annotation/task.yaml create mode 100644 examples/pipelines/custom-annotations-enricher-project/kustomization.yaml create mode 100644 examples/pipelines/custom-annotations-enricher-project/pipelinerun.yaml diff --git a/components/enrichers/custom-annotation/README.md b/components/enrichers/custom-annotation/README.md new file mode 100644 index 000000000..3469a9aec --- /dev/null +++ b/components/enrichers/custom-annotation/README.md @@ -0,0 +1,5 @@ +# Custom Annotation Enricher + +This enricher adds a set of custom annotations to every finding +Useful for development purposes and for tagging results +Can be used multiple times to add multiple annotations diff --git a/components/enrichers/custom-annotation/main.go b/components/enrichers/custom-annotation/main.go new file mode 100644 index 000000000..4068cc0c1 --- /dev/null +++ b/components/enrichers/custom-annotation/main.go @@ -0,0 +1,81 @@ +// Package main of the codeowners enricher +// handles enrichment of individual issues with +// the groups/usernames listed in the github repository +// CODEOWNERS files. +// Owners are matched against the "target" field of the issue +package main + +import ( + "encoding/json" + "flag" + "log" + "log/slog" + "strings" + + "github.com/go-errors/errors" + + apiv1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components/enrichers" +) + +var ( + defaultName = "custom-annotation" + annotations string + name string +) + +func enrichIssue(i *apiv1.Issue, annotations string) (*apiv1.EnrichedIssue, error) { + enrichedIssue := apiv1.EnrichedIssue{} + annotationMap := map[string]string{} + if err := json.Unmarshal([]byte(annotations), &annotationMap); err != nil { + return nil, errors.Errorf("could not unmarshall annotation object to map[string]string, err: %w", err) + } + enrichedIssue = apiv1.EnrichedIssue{ + RawIssue: i, + Annotations: annotationMap, + } + return &enrichedIssue, nil +} + +func run(name, annotations string) error { + res, err := enrichers.LoadData() + if err != nil { + return err + } + if annotations == "" { + slog.Info("annotations is empty") + } + for _, r := range res { + slog.Info("processing results for ", slog.Any("scan", r.ScanInfo)) + enrichedIssues := make([]*apiv1.EnrichedIssue, 0, len(res)) + for _, i := range r.GetIssues() { + eI, err := enrichIssue(i, annotations) + if err != nil { + slog.Error(err.Error()) + continue + } + enrichedIssues = append(enrichedIssues, eI) + } + + err := enrichers.WriteData(&apiv1.EnrichedLaunchToolResponse{ + OriginalResults: r, + Issues: enrichedIssues, + }, strings.ReplaceAll(name, " ", "-")) + if err != nil { + return err + } + } + return nil +} + +func main() { + flag.StringVar(&annotations, "annotations", enrichers.LookupEnvOrString("ANNOTATIONS", ""), "what are the annotations this enricher will add to the issues") + flag.StringVar(&name, "annotation-name", enrichers.LookupEnvOrString("NAME", defaultName), "what is the name this enricher will masquerade as") + + if err := enrichers.ParseFlags(); err != nil { + log.Fatal(err) + } + if err := run(name, annotations); err != nil { + log.Fatal(err) + } +} diff --git a/components/enrichers/custom-annotation/main_test.go b/components/enrichers/custom-annotation/main_test.go new file mode 100644 index 000000000..8b3bc8cb4 --- /dev/null +++ b/components/enrichers/custom-annotation/main_test.go @@ -0,0 +1,140 @@ +package main + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" + + draconv1 "github.com/ocurity/dracon/api/proto/v1" + "github.com/ocurity/dracon/components/enrichers" +) + +func TestHandlesZeroFindings(t *testing.T) { + indir, outdir := enrichers.SetupIODirs(t) + mockLaunchToolResponses := enrichers.GetEmptyLaunchToolResponse(t) + for i, r := range mockLaunchToolResponses { + // Write sample enriched responses to indir + encodedProto, err := proto.Marshal(r) + require.NoError(t, err) + rwPermission600 := os.FileMode(0o600) + require.NoError(t, os.WriteFile(fmt.Sprintf("%s/input_%d_%s.tagged.pb", indir, i, r.ToolName), encodedProto, rwPermission600)) + } + + // Run the enricher + enrichers.SetReadPathForTests(indir) + enrichers.SetWritePathForTests(outdir) + require.NoError(t, run("foo", `{"foo":"bar"}`)) + + // Check there is something in our output directory + files, err := os.ReadDir(outdir) + require.NoError(t, err) + require.NotEmpty(t, files) + require.Len(t, files, 4) + + // Check that both of them are EnrichedLaunchToolResponse + // and their Issue property is an empty list + for _, f := range files { + if strings.HasSuffix(f.Name(), ".raw.pb") { + continue + } + + encodedProto, err := os.ReadFile(fmt.Sprintf("%s/%s", outdir, f.Name())) + require.NoError(t, err) + output := &draconv1.EnrichedLaunchToolResponse{} + require.NoError(t, proto.Unmarshal(encodedProto, output)) + require.Empty(t, output.Issues) + } +} + +func TestHandlesFindings(t *testing.T) { + indir, outdir := enrichers.SetupIODirs(t) + annotations = `{"foo":"bar","a":"b","1":"2"}` + name = "enricherName" + + mockLaunchToolResponses := enrichers.GetLaunchToolResponse(t) + for i, r := range mockLaunchToolResponses { + // Write sample enriched responses to indir + encodedProto, err := proto.Marshal(r) + require.NoError(t, err) + rwPermission600 := os.FileMode(0o600) + require.NoError(t, os.WriteFile(fmt.Sprintf("%s/input_%d_%s.tagged.pb", indir, i, r.ToolName), encodedProto, rwPermission600)) + } + + // Run the enricher + enrichers.SetReadPathForTests(indir) + enrichers.SetWritePathForTests(outdir) + require.NoError(t, run(name, annotations)) + + // Check there is something in our output directory + files, err := os.ReadDir(outdir) + require.NoError(t, err) + require.NotEmpty(t, files) + require.Len(t, files, 4) + + // Check that both of them are EnrichedLaunchToolResponse + // and their Issue property is not an empty list + expected := map[string]*draconv1.EnrichedLaunchToolResponse{ + "tool1": { + OriginalResults: mockLaunchToolResponses[0], + Issues: []*draconv1.EnrichedIssue{ + { + RawIssue: mockLaunchToolResponses[0].Issues[0], + Annotations: map[string]string{ + "foo": "bar", + "a": "b", + "1": "2", + }, + }, + { + RawIssue: mockLaunchToolResponses[0].Issues[1], + Annotations: map[string]string{ + "foo": "bar", + "a": "b", + "1": "2", + }, + }, + }, + }, + "tool2": { + OriginalResults: mockLaunchToolResponses[1], + Issues: []*draconv1.EnrichedIssue{ + { + RawIssue: mockLaunchToolResponses[1].Issues[0], + Annotations: map[string]string{ + "foo": "bar", + "a": "b", + "1": "2", + }, + }, + { + RawIssue: mockLaunchToolResponses[1].Issues[1], + Annotations: map[string]string{ + "foo": "bar", + "a": "b", + "1": "2", + }, + }, + }, + }, + } + var actual draconv1.EnrichedLaunchToolResponse + for _, f := range files { + if strings.HasSuffix(f.Name(), ".raw.pb") { + continue + } + encodedProto, err := os.ReadFile(fmt.Sprintf("%s/%s", outdir, f.Name())) + require.NoError(t, err) + + require.NoError(t, proto.Unmarshal(encodedProto, &actual)) + if !proto.Equal(&actual, expected[actual.OriginalResults.ToolName]) { + require.True(t, proto.Equal(&actual, expected[actual.OriginalResults.ToolName]), + cmp.Diff(&actual, expected[actual.OriginalResults.ToolName], protocmp.Transform())) + } + } +} diff --git a/components/enrichers/custom-annotation/task.yaml b/components/enrichers/custom-annotation/task.yaml new file mode 100644 index 000000000..ef32514f7 --- /dev/null +++ b/components/enrichers/custom-annotation/task.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: enricher-custom-annotation + labels: + v1.dracon.ocurity.com/component: enricher +spec: + description: Adds a set of custom annotations to all issues that pass through this + params: + - name: enricher-custom-annotation-base-annotation + type: string + default: "" + description: "a comma separated list of key:value pairs" + - name: enricher-custom-annotation-name + type: string + default: "custom-annotation" + description: "the name to masquerade as, useful when running multiple instances" + workspaces: + - name: output + description: The workspace where we can output results + steps: + - name: run-enricher + imagePullPolicy: IfNotPresent + image: '{{ default "ghcr.io/ocurity/dracon" .Values.image.registry }}/components/enrichers/custom-annotation:{{ .Chart.AppVersion }}' + command: ["/app/components/enrichers/custom-annotation/custom-annotation"] + env: + - name: READ_PATH + value: $(workspaces.output.path)/.dracon/producers + - name: WRITE_PATH + value: "$(workspaces.output.path)/.dracon/enrichers/custom-annotation" + - name: ANNOTATIONS + value: "$(params.enricher-custom-annotation-base-annotation)" + - name: NAME + value: "$(params.enricher-custom-annotation-name)" diff --git a/components/enrichers/test_utils.go b/components/enrichers/test_utils.go index 1a82005fa..2db44b72d 100644 --- a/components/enrichers/test_utils.go +++ b/components/enrichers/test_utils.go @@ -33,3 +33,79 @@ func GetEmptyLaunchToolResponse(_ *testing.T) []*draconv1.LaunchToolResponse { }, } } + +// GetEmptyLaunchToolResponse returns a slice of LaunchToolResponse with no issues +func GetLaunchToolResponse(_ *testing.T) []*draconv1.LaunchToolResponse { + code := `this + is + some + code` + return []*draconv1.LaunchToolResponse{ + { + ToolName: "tool1", + Issues: []*draconv1.Issue{ + { + Target: "file:/a/b/c/d.php:1-2", + Type: "sometype", + Title: "this is a title", + Severity: draconv1.Severity_SEVERITY_CRITICAL, + Cvss: 1.0, + Confidence: draconv1.Confidence_CONFIDENCE_CRITICAL, + Description: "this is a handy dandy description", + Source: "this is a source", + Cve: "CVE-2020-123", + Uuid: "d9681ae9-223b-4df8-a422-7b29bb917a36", + Cwe: []int32{123}, + ContextSegment: &code, + }, + { + Target: "file:/a/b/c/d.go:2-3", + Type: "sometype1", + Title: "this is a title1", + Severity: draconv1.Severity_SEVERITY_CRITICAL, + Cvss: 1.0, + Confidence: draconv1.Confidence_CONFIDENCE_CRITICAL, + Description: "this is a handy dandy description1", + Source: "this is a source1", + Cve: "CVE-2020-124", + Uuid: "a9681ae9-223b-4df8-a422-7b29bb917a36", + Cwe: []int32{123}, + ContextSegment: &code, + }, + }, + }, + { + ToolName: "tool2", + Issues: []*draconv1.Issue{ + { + Target: "file:/a/b/c/d.py:1-2", + Type: "sometype", + Title: "this is a title", + Severity: draconv1.Severity_SEVERITY_CRITICAL, + Cvss: 1.0, + Confidence: draconv1.Confidence_CONFIDENCE_CRITICAL, + Description: "this is a handy dandy description", + Source: "this is a source", + Cve: "CVE-2020-123", + Uuid: "q9681ae9-223b-4df8-a422-7b29bb917a36", + Cwe: []int32{123}, + ContextSegment: &code, + }, + { + Target: "file:/a/b/c/d.py:2-3", + Type: "sometype1", + Title: "this is a title1", + Severity: draconv1.Severity_SEVERITY_CRITICAL, + Cvss: 1.0, + Confidence: draconv1.Confidence_CONFIDENCE_CRITICAL, + Description: "this is a handy dandy description1", + Source: "this is a source1", + Cve: "CVE-2020-124", + Uuid: "w9681ae9-223b-4df8-a422-7b29bb917a36", + Cwe: []int32{123}, + ContextSegment: &code, + }, + }, + }, + } +} diff --git a/examples/pipelines/custom-annotations-enricher-project/kustomization.yaml b/examples/pipelines/custom-annotations-enricher-project/kustomization.yaml new file mode 100644 index 000000000..2912c2e95 --- /dev/null +++ b/examples/pipelines/custom-annotations-enricher-project/kustomization.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +nameSuffix: -annotation-project +components: + - pkg:helm/dracon-oss-components/base + - pkg:helm/dracon-oss-components/git-clone + - pkg:helm/dracon-oss-components/producer-golang-gosec + - pkg:helm/dracon-oss-components/producer-aggregator + - pkg:helm/dracon-oss-components/enricher-custom-annotation + - pkg:helm/dracon-oss-components/enricher-aggregator + - pkg:helm/dracon-oss-components/consumer-stdout-json diff --git a/examples/pipelines/custom-annotations-enricher-project/pipelinerun.yaml b/examples/pipelines/custom-annotations-enricher-project/pipelinerun.yaml new file mode 100644 index 000000000..1b27d8c8c --- /dev/null +++ b/examples/pipelines/custom-annotations-enricher-project/pipelinerun.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + generateName: dracon-annotation-project- +spec: + pipelineRef: + name: dracon-annotation-project + params: + - name: git-clone-url + value: https://github.com/sqreen/go-dvwa.git + - name: enricher-custom-annotation-annotations + value: | + {"foo":"bar", + "a":"b", + "1":"2"} + - name: enricher-custom-annotation-name + value: "bar" + workspaces: + - name: output + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi From 88e94ca94f2f1d2ffc8b76fc5dd3d6cad82c683a Mon Sep 17 00:00:00 2001 From: sg Date: Thu, 3 Oct 2024 17:22:14 +0100 Subject: [PATCH 3/3] nit:change the golang example to use a dedicated go vulnerable web app --- examples/pipelines/golang-project/pipelinerun.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pipelines/golang-project/pipelinerun.yaml b/examples/pipelines/golang-project/pipelinerun.yaml index 0b1ff15f2..b57c7e5c1 100644 --- a/examples/pipelines/golang-project/pipelinerun.yaml +++ b/examples/pipelines/golang-project/pipelinerun.yaml @@ -8,7 +8,7 @@ spec: name: dracon-golang-project params: - name: git-clone-url - value: https://github.com/ocurity/e2e-monorepo.git + value: https://github.com/sqreen/go-dvwa.git - name: git-clone-subdirectory value: source-code workspaces: