From 3681410bcd83e621e8f038c68c28373a0f31947c Mon Sep 17 00:00:00 2001
From: andream16 <andrea@ocurity.com>
Date: Fri, 8 Nov 2024 11:09:24 +0000
Subject: [PATCH] Implementing example for go-sec pipeline.

---
 examples/pipelines/golang-project/.env        |   3 +
 examples/pipelines/golang-project/enricher.go |  39 +++
 examples/pipelines/golang-project/main.go     |  99 +++++++
 examples/pipelines/golang-project/migrate.go  |  32 +++
 examples/pipelines/golang-project/reporter.go |  30 ++
 examples/pipelines/golang-project/scanner.go  | 256 ++++++++++++++++++
 examples/pipelines/golang-project/target.go   |  46 ++++
 7 files changed, 505 insertions(+)
 create mode 100644 examples/pipelines/golang-project/.env
 create mode 100644 examples/pipelines/golang-project/enricher.go
 create mode 100644 examples/pipelines/golang-project/main.go
 create mode 100644 examples/pipelines/golang-project/migrate.go
 create mode 100644 examples/pipelines/golang-project/reporter.go
 create mode 100644 examples/pipelines/golang-project/scanner.go
 create mode 100644 examples/pipelines/golang-project/target.go

diff --git a/examples/pipelines/golang-project/.env b/examples/pipelines/golang-project/.env
new file mode 100644
index 000000000..6c0c25d44
--- /dev/null
+++ b/examples/pipelines/golang-project/.env
@@ -0,0 +1,3 @@
+SMITHY_INSTANCE_ID=8d719c1c-c569-4078-87b3-4951bd4012ee
+SMITHY_LOG_LEVEL=debug
+SMITHY_BACKEND_STORE_TYPE=local
diff --git a/examples/pipelines/golang-project/enricher.go b/examples/pipelines/golang-project/enricher.go
new file mode 100644
index 000000000..d2786f368
--- /dev/null
+++ b/examples/pipelines/golang-project/enricher.go
@@ -0,0 +1,39 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"slices"
+
+	"github.com/go-errors/errors"
+
+	ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1"
+)
+
+type (
+	customAnnotationEnricher struct{}
+
+	CustomAnnotation struct {
+		Foo string `json:"foo"`
+	}
+)
+
+func (m *customAnnotationEnricher) Annotate(
+	ctx context.Context,
+	findings []*ocsf.VulnerabilityFinding,
+) ([]*ocsf.VulnerabilityFinding, error) {
+	var newFindings = slices.Clone(findings)
+
+	for idx := range newFindings {
+		b, err := json.Marshal(CustomAnnotation{Foo: "bar"})
+		if err != nil {
+			return nil, errors.Errorf("could not json marshal custom annotation: %w", err)
+		}
+		newFindings[idx].Enrichments = append(newFindings[idx].Enrichments, &ocsf.Enrichment{
+			Name:  "custom-annotation",
+			Value: string(b),
+		})
+	}
+
+	return newFindings, nil
+}
diff --git a/examples/pipelines/golang-project/main.go b/examples/pipelines/golang-project/main.go
new file mode 100644
index 000000000..75b28bc33
--- /dev/null
+++ b/examples/pipelines/golang-project/main.go
@@ -0,0 +1,99 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/smithy-security/smithy/sdk/component"
+)
+
+const (
+	repoPath = "govwa"
+)
+
+func main() {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+
+	if err := Main(ctx); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func Main(ctx context.Context) error {
+	if err := migrate(); err != nil {
+		log.Fatalf("failed to migrate: %v", err)
+	}
+
+	defer func() {
+		if err := os.RemoveAll("smithy.db"); err != nil {
+			log.Printf("failed to remove sqlite db: %v\n", err)
+		}
+	}()
+
+	if err := os.Mkdir(repoPath, os.ModePerm); err != nil {
+		return fmt.Errorf("failed to create clone path %s: %v", repoPath, err)
+	}
+
+	defer func() {
+		if err := os.RemoveAll(repoPath); err != nil {
+			log.Printf("failed to remove clone path %s: %v\n", repoPath, err)
+		}
+	}()
+
+	gitClone, err := NewGitCloneTarget("https://github.com/0c34/govwa.git", repoPath)
+	if err != nil {
+		return fmt.Errorf("failed to create git clone target: %w", err)
+	}
+
+	goSec, err := NewGoSecScanner(repoPath)
+	if err != nil {
+		return fmt.Errorf("failed to create gosec scanner: %w", err)
+	}
+
+	var (
+		customAnnotation = &customAnnotationEnricher{}
+		jsonLogger       = &jsonReporter{}
+	)
+
+	if err := component.RunTarget(
+		ctx,
+		gitClone,
+		component.RunnerWithComponentName("git-clone"),
+	); err != nil {
+		return fmt.Errorf("target failed: %w", err)
+	}
+
+	if err := component.RunScanner(
+		ctx,
+		goSec,
+		component.RunnerWithComponentName("go-sec"),
+	); err != nil {
+		return fmt.Errorf("scanner failed: %w", err)
+	}
+
+	if err := component.RunEnricher(
+		ctx,
+		customAnnotation,
+		component.RunnerWithComponentName("custom-annotation"),
+	); err != nil {
+		return fmt.Errorf("enricher failed: %w", err)
+	}
+
+	if err := component.RunReporter(
+		ctx,
+		jsonLogger,
+		component.RunnerWithComponentName("json-logger"),
+	); err != nil {
+		return fmt.Errorf("reporter failed: %w", err)
+	}
+
+	return nil
+}
+
+func ptr[T any](v T) *T {
+	return &v
+}
diff --git a/examples/pipelines/golang-project/migrate.go b/examples/pipelines/golang-project/migrate.go
new file mode 100644
index 000000000..a5fe32f02
--- /dev/null
+++ b/examples/pipelines/golang-project/migrate.go
@@ -0,0 +1,32 @@
+package main
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func migrate() error {
+	db, err := sql.Open("sqlite3", "smithy.db")
+	if err != nil {
+		return fmt.Errorf("could not open sqlite db: %w", err)
+	}
+
+	stmt, err := db.Prepare(`
+		CREATE TABLE IF NOT EXISTS finding (
+			id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE,
+			instance_id UUID NOT NULL UNIQUE,
+			findings TEXT NOT NULL,
+			created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+			updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+		);
+	`)
+	if err != nil {
+		return fmt.Errorf("could not prepare statement for creating table: %w", err)
+	}
+
+	if _, err := stmt.Exec(); err != nil {
+		return fmt.Errorf("could not create table: %w", err)
+	}
+
+	return stmt.Close()
+}
diff --git a/examples/pipelines/golang-project/reporter.go b/examples/pipelines/golang-project/reporter.go
new file mode 100644
index 000000000..a1de1fd42
--- /dev/null
+++ b/examples/pipelines/golang-project/reporter.go
@@ -0,0 +1,30 @@
+package main
+
+import (
+	"context"
+	"log/slog"
+
+	"github.com/go-errors/errors"
+	"google.golang.org/protobuf/encoding/protojson"
+
+	"github.com/smithy-security/smithy/sdk/component"
+	ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1"
+)
+
+type jsonReporter struct{}
+
+func (j jsonReporter) Report(
+	ctx context.Context,
+	findings []*ocsf.VulnerabilityFinding,
+) error {
+	logger := component.LoggerFromContext(ctx)
+	for _, finding := range findings {
+		b, err := protojson.Marshal(finding)
+		if err != nil {
+			return errors.Errorf("could not json marshal finding: %w", err)
+		}
+		logger.Info("found finding", slog.String("finding", string(b)))
+	}
+
+	return nil
+}
diff --git a/examples/pipelines/golang-project/scanner.go b/examples/pipelines/golang-project/scanner.go
new file mode 100644
index 000000000..9146c917b
--- /dev/null
+++ b/examples/pipelines/golang-project/scanner.go
@@ -0,0 +1,256 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"log/slog"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+	"google.golang.org/protobuf/types/known/structpb"
+
+	"github.com/smithy-security/smithy/sdk/component"
+	ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1"
+)
+
+const goSecOutPath = "gosec_out.json"
+
+var (
+	confidences = map[string]*ocsf.VulnerabilityFinding_ConfidenceId{
+		"LOW":    ptr(ocsf.VulnerabilityFinding_CONFIDENCE_ID_LOW),
+		"MEDIUM": ptr(ocsf.VulnerabilityFinding_CONFIDENCE_ID_MEDIUM),
+		"HIGH":   ptr(ocsf.VulnerabilityFinding_CONFIDENCE_ID_HIGH),
+		"OTHER":  ptr(ocsf.VulnerabilityFinding_CONFIDENCE_ID_OTHER),
+	}
+	severities = map[string]ocsf.VulnerabilityFinding_SeverityId{
+		"CRITICAL":      ocsf.VulnerabilityFinding_SEVERITY_ID_CRITICAL,
+		"MEDIUM":        ocsf.VulnerabilityFinding_SEVERITY_ID_MEDIUM,
+		"LOW":           ocsf.VulnerabilityFinding_SEVERITY_ID_LOW,
+		"OTHER":         ocsf.VulnerabilityFinding_SEVERITY_ID_OTHER,
+		"FATAL":         ocsf.VulnerabilityFinding_SEVERITY_ID_FATAL,
+		"INFORMATIONAL": ocsf.VulnerabilityFinding_SEVERITY_ID_INFORMATIONAL,
+		"HIGH":          ocsf.VulnerabilityFinding_SEVERITY_ID_HIGH,
+	}
+)
+
+type (
+	goSecScanner struct {
+		repoPath       string
+		dockerTestPool *dockertest.Pool
+	}
+
+	GoSecOut struct {
+		Issues []GoSecIssue `json:"Issues"`
+	}
+
+	GoSecIssue struct {
+		Severity     string   `json:"severity"`
+		Confidence   string   `json:"confidence"`
+		Cwe          GoSecCwe `json:"cwe"`
+		RuleID       string   `json:"rule_id"`
+		Details      string   `json:"details"`
+		File         string   `json:"file"`
+		Code         string   `json:"code"`
+		Line         string   `json:"line"`
+		Column       string   `json:"column"`
+		Nosec        bool     `json:"nosec"`
+		Suppressions any      `json:"suppressions"`
+	}
+
+	GoSecCwe struct {
+		ID  string `json:"id"`
+		URL string `json:"url"`
+	}
+)
+
+func NewGoSecScanner(repoPath string) (*goSecScanner, error) {
+	if repoPath == "" {
+		return nil, errors.New("must specify a repository path")
+	}
+
+	pool, err := dockertest.NewPool("")
+	if err != nil {
+		return nil, fmt.Errorf("could not connect to docker: %w", err)
+	}
+
+	return &goSecScanner{
+		repoPath:       repoPath,
+		dockerTestPool: pool,
+	}, nil
+}
+
+func (g *goSecScanner) Transform(ctx context.Context) ([]*ocsf.VulnerabilityFinding, error) {
+	if err := g.runGoSec(ctx); err != nil {
+		return nil, fmt.Errorf("could not run gosec: %w", err)
+	}
+
+	vulns, err := g.parseVulns(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse vulns: %w", err)
+	}
+
+	return vulns, nil
+}
+
+func (g *goSecScanner) parseVulns(ctx context.Context) ([]*ocsf.VulnerabilityFinding, error) {
+	f, err := os.Open(goSecOutPath)
+	if err != nil {
+		return nil, fmt.Errorf("could not open gosec_out.json: %w", err)
+	}
+
+	defer func() {
+		if err := f.Close(); err != nil {
+			component.
+				LoggerFromContext(ctx).
+				Error(
+					"could not close gosec_out.json",
+					slog.String("err", err.Error()),
+				)
+		}
+
+		if err := os.RemoveAll(goSecOutPath); err != nil {
+			component.
+				LoggerFromContext(ctx).
+				Error(
+					"could not remove gosec_out.json",
+					slog.String("err", err.Error()),
+				)
+		}
+	}()
+
+	b, err := io.ReadAll(f)
+	if err != nil {
+		return nil, fmt.Errorf("could not read gosec_out.json: %w", err)
+	}
+
+	var out GoSecOut
+	if err := json.Unmarshal(b, &out); err != nil {
+		return nil, fmt.Errorf("could not decode gosec_out.json: %w", err)
+	}
+
+	var (
+		vulns = make([]*ocsf.VulnerabilityFinding, 0, len(out.Issues))
+		now   = time.Now().Unix()
+	)
+
+	for _, issue := range out.Issues {
+		vulns = append(vulns, &ocsf.VulnerabilityFinding{
+			ActivityId:   ocsf.VulnerabilityFinding_ACTIVITY_ID_CREATE,
+			CategoryUid:  ocsf.VulnerabilityFinding_CATEGORY_UID_FINDINGS,
+			ClassUid:     ocsf.VulnerabilityFinding_CLASS_UID_VULNERABILITY_FINDING,
+			Confidence:   &issue.Confidence,
+			ConfidenceId: confidences[issue.Confidence],
+			Count:        ptr(int32(1)),
+			FindingInfo: &ocsf.FindingInfo{
+				CreatedTime: &now,
+				DataSources: []string{
+					issue.File,
+				},
+				Desc:          &issue.Details,
+				FirstSeenTime: &now,
+				LastSeenTime:  &now,
+				ModifiedTime:  &now,
+				ProductUid:    ptr("gosec"),
+				Title:         issue.Details,
+				Uid:           issue.RuleID,
+			},
+			Message: ptr(issue.Details),
+			Resource: &ocsf.ResourceDetails{
+				Uid: ptr(
+					strings.Join([]string{
+						issue.File,
+						issue.Line,
+						issue.Column,
+					},
+						":",
+					),
+				),
+				Data: &structpb.Value{
+					Kind: &structpb.Value_StringValue{
+						StringValue: issue.Code,
+					},
+				},
+			},
+			RawData:    ptr(string(b)),
+			Severity:   &issue.Severity,
+			SeverityId: severities[issue.Severity],
+			StartTime:  &now,
+			Status:     ptr("opened"),
+			Time:       now,
+			TypeUid: int64(
+				ocsf.VulnerabilityFinding_CLASS_UID_VULNERABILITY_FINDING.Number()*
+					100 +
+					ocsf.VulnerabilityFinding_ACTIVITY_ID_CREATE.Number(),
+			),
+			Vulnerabilities: []*ocsf.Vulnerability{
+				{
+					Cwe: &ocsf.Cwe{
+						Uid:    issue.Cwe.ID,
+						SrcUrl: &issue.Cwe.URL,
+					},
+				},
+			},
+		})
+	}
+
+	return vulns, nil
+}
+
+func (g *goSecScanner) runGoSec(ctx context.Context) error {
+	p, err := filepath.Abs(".")
+	if err != nil {
+		return fmt.Errorf("could not get absolute path: %w", err)
+	}
+
+	component.
+		LoggerFromContext(ctx).
+		Info("preparing to run gosec",
+			slog.String("path", path.Join(p, g.repoPath)),
+			slog.String("output", goSecOutPath),
+		)
+
+	r, err := g.dockerTestPool.RunWithOptions(&dockertest.RunOptions{
+		Platform:   "linux/amd64",
+		Repository: "docker.io/securego/gosec",
+		Tag:        "2.15.0",
+		WorkingDir: "/workspace",
+		Cmd: []string{
+			"-r",
+			"-sort",
+			"-nosec",
+			"-fmt=json",
+			fmt.Sprintf("-out=%s", goSecOutPath),
+			"-no-fail",
+			fmt.Sprintf("./%s", g.repoPath),
+		},
+	}, func(config *docker.HostConfig) {
+		config.AutoRemove = true
+		config.Binds = []string{fmt.Sprintf("%s:/workspace", p)}
+	})
+	if err != nil {
+		return fmt.Errorf("could not start gosec container: %w", err)
+	}
+
+	if err := g.dockerTestPool.Client.Logs(docker.LogsOptions{
+		Context:      ctx,
+		Container:    r.Container.ID,
+		OutputStream: os.Stdout,
+		ErrorStream:  os.Stderr,
+		Stdout:       true,
+		Stderr:       true,
+		Follow:       true,
+	}); err != nil {
+		log.Fatalf("Could not retrieve logs: %s", err)
+	}
+
+	return nil
+}
diff --git a/examples/pipelines/golang-project/target.go b/examples/pipelines/golang-project/target.go
new file mode 100644
index 000000000..ef1872a04
--- /dev/null
+++ b/examples/pipelines/golang-project/target.go
@@ -0,0 +1,46 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+
+	"github.com/go-git/go-git/v5"
+
+	"github.com/smithy-security/smithy/sdk/component"
+)
+
+type gitCloneTarget struct {
+	repositoryURL string
+	clonePath     string
+}
+
+func NewGitCloneTarget(repositoryURL string, clonePath string) (*gitCloneTarget, error) {
+	switch {
+	case repositoryURL == "":
+		return nil, fmt.Errorf("repositoryURL is empty")
+	case clonePath == "":
+		return nil, fmt.Errorf("clonePath is empty")
+	}
+	return &gitCloneTarget{repositoryURL: repositoryURL, clonePath: clonePath}, nil
+}
+
+func (g *gitCloneTarget) Prepare(ctx context.Context) error {
+	logger := component.
+		LoggerFromContext(ctx).
+		With(slog.String("repository_url", g.repositoryURL)).
+		With(slog.String("clone_path", g.clonePath))
+
+	logger.Debug("preparing to clone repo")
+
+	if _, err := git.PlainClone(g.clonePath, false, &git.CloneOptions{
+		URL:               g.repositoryURL,
+		RecurseSubmodules: git.DefaultSubmoduleRecursionDepth,
+	}); err != nil {
+		return fmt.Errorf("could not clone repository: %w", err)
+	}
+
+	logger.Debug("successfully cloned repo")
+
+	return nil
+}