From 3e9d7db9bb26327f1a18031c1338d45ffc1f8cc9 Mon Sep 17 00:00:00 2001 From: sg Date: Fri, 13 Sep 2024 21:30:36 +0100 Subject: [PATCH] fix 349 by adding api keys to the elasticsearch consumer so that it can work with elasticsearch saas --- components/consumers/elasticsearch/main.go | 65 ++++++++++++------- .../consumers/elasticsearch/main_test.go | 57 ++++++++++++---- components/consumers/elasticsearch/task.yaml | 22 +++++-- 3 files changed, 103 insertions(+), 41 deletions(-) diff --git a/components/consumers/elasticsearch/main.go b/components/consumers/elasticsearch/main.go index 58ea61038..81c3d3c5a 100644 --- a/components/consumers/elasticsearch/main.go +++ b/components/consumers/elasticsearch/main.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "log" + "log/slog" "strings" "time" @@ -14,33 +15,28 @@ import ( "github.com/ocurity/dracon/pkg/enumtransformers" "github.com/ocurity/dracon/pkg/templating" - // TODO: Support multiple versions of ES elasticsearch "github.com/elastic/go-elasticsearch/v8" ) var ( - esUrls string - esAddrs []string - esIndex string + esUrls string + esAddrs []string + esIndex string + + esAPIKey string + esCloudID string + basicAuthUser string basicAuthPass string issueTemplate string ) -func init() { - flag.StringVar(&esUrls, "es-urls", "", "[OPTIONAL] URLs to connect to elasticsearch comma separated. Can also use env variable ELASTICSEARCH_URL") - flag.StringVar(&esIndex, "es-index", "", "the index in elasticsearch to push results to") - flag.StringVar(&basicAuthUser, "basic-auth-user", "", "[OPTIONAL] the basic auth username") - flag.StringVar(&basicAuthPass, "basic-auth-pass", "", "[OPTIONAL] the basic auth password") - flag.StringVar(&issueTemplate, "descriptionTemplate", "", "a Go Template string describing how to show Raw or Enriched issues") -} - func parseFlags() error { if err := consumers.ParseFlags(); err != nil { return err } - if len(esIndex) < 1 { - return fmt.Errorf("es-index is undefined") + if len(esIndex) == 0 { + return fmt.Errorf("esIndex '%s' is undefined", esIndex) } if len(esUrls) > 0 { for _, u := range strings.Split(esUrls, ",") { @@ -51,6 +47,19 @@ func parseFlags() error { } func main() { + flag.StringVar(&esIndex, "esIndex", "", "the index in elasticsearch to push results to") + flag.StringVar(&issueTemplate, "descriptionTemplate", "", "a Go Template string describing how to show Raw or Enriched issues") + + // es SaaS options + flag.StringVar(&esAPIKey, "esAPIKey", "", "the api key in elasticsearch to contact results to") + flag.StringVar(&esCloudID, "esCloudID", "", "the cloud id in elasticsearch to contact results to") + + // es self-hosted options + flag.StringVar(&esUrls, "esURL", "", "[OPTIONAL] URLs to connect to elasticsearch comma separated. Can also use env variable ELASTICSEARCH_URL") + flag.StringVar(&basicAuthUser, "basic-auth-user", "", "[OPTIONAL] the basic auth username") + flag.StringVar(&basicAuthPass, "basic-auth-pass", "", "[OPTIONAL] the basic auth password") + flag.Parse() + if err := parseFlags(); err != nil { log.Fatal(err) } @@ -61,15 +70,16 @@ func main() { } if consumers.Raw { - log.Print("Parsing Raw results") + slog.Debug("Parsing Raw results") responses, err := consumers.LoadToolResponse() if err != nil { log.Fatal("could not load raw results, file malformed: ", err) } + numIssues := 0 for _, res := range responses { scanStartTime := res.GetScanInfo().GetScanStartTime().AsTime() + numIssues += len(res.GetIssues()) for _, iss := range res.GetIssues() { - log.Printf("Pushing %d, issues to es \n", len(responses)) b, err := getRawIssue(scanStartTime, res, iss) if err != nil { log.Fatal("Could not parse raw issue", err) @@ -81,26 +91,29 @@ func main() { } } } + slog.Info("Pushed", "numIssues", numIssues, "issues to Elasticsearch", "") } else { log.Print("Parsing Enriched results") responses, err := consumers.LoadEnrichedToolResponse() if err != nil { log.Fatal("could not load enriched results, file malformed: ", err) } + numIssues := 0 for _, res := range responses { scanStartTime := res.GetOriginalResults().GetScanInfo().GetScanStartTime().AsTime() + numIssues += len(res.GetIssues()) for _, iss := range res.GetIssues() { b, err := getEnrichedIssue(scanStartTime, res, iss) if err != nil { log.Fatal("Could not parse enriched issue", err) } res, err := es.Index(esIndex, bytes.NewBuffer(b)) - log.Printf("%+v", res) - if err != nil { - log.Fatal("Could not push enriched issue", err) + if err != nil || res.IsError() { + log.Fatal("Could not push enriched issue", err, "received", res.StatusCode) } } } + slog.Info("Pushed", "numIssues", numIssues, "issues to Elasticsearch", "") } } @@ -194,7 +207,12 @@ func getESClient() (*elasticsearch.Client, error) { esConfig.Username = basicAuthUser esConfig.Password = basicAuthPass } - + if esAPIKey != "" { + esConfig.APIKey = esAPIKey + } + if esCloudID != "" { + esConfig.CloudID = esCloudID + } if len(esAddrs) > 0 { esConfig.Addresses = esAddrs } @@ -217,11 +235,8 @@ func getESClient() (*elasticsearch.Client, error) { if err := json.NewDecoder(res.Body).Decode(&info); err != nil { return nil, err } - switch info.Version.Number[0] { - case '8': - // noop - we support this version - default: - err = fmt.Errorf("unsupported ES Server version %s", info.Version.Number) + if info.Version.Number[0] != '8' { + return nil, fmt.Errorf("unsupported ES Server version %s", info.Version.Number) } return es, err } diff --git a/components/consumers/elasticsearch/main_test.go b/components/consumers/elasticsearch/main_test.go index f1032682d..718a5293a 100644 --- a/components/consumers/elasticsearch/main_test.go +++ b/components/consumers/elasticsearch/main_test.go @@ -11,7 +11,6 @@ import ( v1 "github.com/ocurity/dracon/api/proto/v1" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -55,21 +54,21 @@ func TestEsPushBasicAuth(t *testing.T) { if r.Method == http.MethodGet { uname, pass, ok := r.BasicAuth() - assert.Equal(t, uname, "foo") - assert.Equal(t, pass, "bar") - assert.Equal(t, ok, true) + require.Equal(t, uname, "foo") + require.Equal(t, pass, "bar") + require.Equal(t, ok, true) _, err = w.Write([]byte(info)) require.NoError(t, err) } else if r.Method == http.MethodPost { // assert non authed operation (write results to index) - assert.Equal(t, buf.String(), string(esIn)) - assert.Equal(t, r.RequestURI, "/"+esIndex+"/_doc") + require.Equal(t, buf.String(), string(esIn)) + require.Equal(t, r.RequestURI, "/"+esIndex+"/_doc") uname, pass, ok := r.BasicAuth() - assert.Equal(t, uname, "foo") - assert.Equal(t, pass, "bar") - assert.Equal(t, ok, true) + require.Equal(t, uname, "foo") + require.Equal(t, pass, "bar") + require.Equal(t, ok, true) _, err = w.Write([]byte(want)) require.NoError(t, err) @@ -99,8 +98,8 @@ func TestEsPush(t *testing.T) { _, err = w.Write([]byte(info)) } else if r.Method == http.MethodPost { // assert non authed operation (write results to index) - assert.Equal(t, buf.String(), string(esIn)) - assert.Equal(t, r.RequestURI, "/"+esIndex+"/_doc") + require.Equal(t, buf.String(), string(esIn)) + require.Equal(t, r.RequestURI, "/"+esIndex+"/_doc") _, err = w.Write([]byte(want)) } require.NoError(t, err) @@ -112,3 +111,39 @@ func TestEsPush(t *testing.T) { _, err = client.Index(esIndex, bytes.NewBuffer(esIn)) require.NoError(t, err) } +func TestEsPushAPIKey(t *testing.T) { + esIndex = "dracon-es-test" + + esStub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + buf := new(bytes.Buffer) + _, err := buf.ReadFrom(r.Body) + require.NoError(t, err) + + w.Header().Set("X-Elastic-Product", "Elasticsearch") + w.WriteHeader(http.StatusOK) + + require.Equal(t, r.Header.Get("Authorization"), "APIKey foo") + + if r.Method == http.MethodGet { + _, err = w.Write([]byte(info)) + require.NoError(t, err) + } else if r.Method == http.MethodPost { + // assert non authed operation (write results to index) + require.Equal(t, buf.String(), string(esIn)) + require.Equal(t, r.RequestURI, "/"+esIndex+"/_doc") + + _, err = w.Write([]byte(want)) + require.NoError(t, err) + } + })) + defer esStub.Close() + os.Setenv("ELASTICSEARCH_URL", esStub.URL) + + // apikey ops + esAPIKey = "foo" + esCloudID = esStub.Config.Addr + client, err := getESClient() + require.NoError(t, err) + _, err = client.Index(esIndex, bytes.NewBuffer(esIn)) + require.NoError(t, err) +} diff --git a/components/consumers/elasticsearch/task.yaml b/components/consumers/elasticsearch/task.yaml index 714af7af3..ad33e59fe 100644 --- a/components/consumers/elasticsearch/task.yaml +++ b/components/consumers/elasticsearch/task.yaml @@ -14,6 +14,18 @@ spec: - name: consumer-elasticsearch-description-template type: string default: "" + - name: consumer-elasticsearch-api-key + type: string + default: "" + - name: consumer-elasticsearch-index-name + type: string + default: "" + - name: consumer-elasticsearch-index + type: string + default: "" + - name: consumer-elasticsearch-cloud-id + type: string + default: "" workspaces: - name: output description: The workspace containing the source-code to scan. @@ -22,11 +34,11 @@ spec: imagePullPolicy: IfNotPresent image: '{{ default "ghcr.io/ocurity/dracon" .Values.image.registry }}/components/consumers/elasticsearch:{{ .Chart.AppVersion }}' command: ["/app/components/consumers/elasticsearch/elasticsearch"] - env: - - name: ELASTICSEARCH_URL - value: "$(params.consumer-elasticsearch-url)" args: [ "-in", "$(workspaces.output.path)/.dracon/enrichers/", - "-es-index", "dracon", - "-descriptionTemplate","$(params.consumer-elasticsearch-description-template)" + "-descriptionTemplate","$(params.consumer-elasticsearch-description-template)", + "-esIndex", "$(params.consumer-elasticsearch-index-name)", + "-esAPIKey", "$(params.consumer-elasticsearch-api-key)", + "-esURL", "$(params.consumer-elasticsearch-url)", + "-esCloudID", "$(params.consumer-elasticsearch-cloud-id)", ]