From 1b96e7b6a897e736f827dd84bf85a3f570d06914 Mon Sep 17 00:00:00 2001 From: andream16 Date: Thu, 5 Dec 2024 14:45:19 +0000 Subject: [PATCH 1/8] Moving local store to an accessible package. --- .../internal/storer/local/sqlite/enum_enum.go | 49 ------------ sdk/component/internal/storer/storer.go | 5 -- sdk/component/store/error.go | 6 ++ .../local/sqlite => store/local}/enum.go | 4 +- sdk/component/store/local/enum_enum.go | 49 ++++++++++++ .../sqlite/sqlite.go => store/local/store.go} | 77 +++++++++++-------- .../local/storer_test.go} | 33 ++------ sdk/component/storer.go | 22 ------ 8 files changed, 109 insertions(+), 136 deletions(-) delete mode 100644 sdk/component/internal/storer/local/sqlite/enum_enum.go delete mode 100644 sdk/component/internal/storer/storer.go create mode 100644 sdk/component/store/error.go rename sdk/component/{internal/storer/local/sqlite => store/local}/enum.go (52%) create mode 100644 sdk/component/store/local/enum_enum.go rename sdk/component/{internal/storer/local/sqlite/sqlite.go => store/local/store.go} (78%) rename sdk/component/{internal/storer/local/sqlite/sqlite_test.go => store/local/storer_test.go} (89%) delete mode 100644 sdk/component/storer.go diff --git a/sdk/component/internal/storer/local/sqlite/enum_enum.go b/sdk/component/internal/storer/local/sqlite/enum_enum.go deleted file mode 100644 index 6c66455fe..000000000 --- a/sdk/component/internal/storer/local/sqlite/enum_enum.go +++ /dev/null @@ -1,49 +0,0 @@ -// Code generated by go-enum DO NOT EDIT. -// Version: -// Revision: -// Build Date: -// Built By: - -package sqlite - -import ( - "errors" - "fmt" -) - -const ( - // ColumnNameFindings is a columnName of type findings. - ColumnNameFindings columnName = "findings" - // ColumnNameInstanceId is a columnName of type instance_id. - ColumnNameInstanceId columnName = "instance_id" - // ColumnNameUpdatedAt is a columnName of type updated_at. - ColumnNameUpdatedAt columnName = "updated_at" -) - -var ErrInvalidcolumnName = errors.New("not a valid columnName") - -// String implements the Stringer interface. -func (x columnName) String() string { - return string(x) -} - -// IsValid provides a quick way to determine if the typed value is -// part of the allowed enumerated values -func (x columnName) IsValid() bool { - _, err := ParsecolumnName(string(x)) - return err == nil -} - -var _columnNameValue = map[string]columnName{ - "findings": ColumnNameFindings, - "instance_id": ColumnNameInstanceId, - "updated_at": ColumnNameUpdatedAt, -} - -// ParsecolumnName attempts to convert a string to a columnName. -func ParsecolumnName(name string) (columnName, error) { - if x, ok := _columnNameValue[name]; ok { - return x, nil - } - return columnName(""), fmt.Errorf("%s is %w", name, ErrInvalidcolumnName) -} diff --git a/sdk/component/internal/storer/storer.go b/sdk/component/internal/storer/storer.go deleted file mode 100644 index 3eedf7a3f..000000000 --- a/sdk/component/internal/storer/storer.go +++ /dev/null @@ -1,5 +0,0 @@ -package storer - -import "github.com/go-errors/errors" - -var ErrNoFindingsFound = errors.New("no findings found") diff --git a/sdk/component/store/error.go b/sdk/component/store/error.go new file mode 100644 index 000000000..14fd80f59 --- /dev/null +++ b/sdk/component/store/error.go @@ -0,0 +1,6 @@ +package store + +import "errors" + +// ErrNoFindingsFound is returned when no findings are found. +var ErrNoFindingsFound = errors.New("no findings found") diff --git a/sdk/component/internal/storer/local/sqlite/enum.go b/sdk/component/store/local/enum.go similarity index 52% rename from sdk/component/internal/storer/local/sqlite/enum.go rename to sdk/component/store/local/enum.go index bde170686..49375d6d1 100644 --- a/sdk/component/internal/storer/local/sqlite/enum.go +++ b/sdk/component/store/local/enum.go @@ -1,6 +1,6 @@ -package sqlite +package localstore type ( // ENUM(findings, instance_id, updated_at) - columnName string + localStoreColumnName string ) diff --git a/sdk/component/store/local/enum_enum.go b/sdk/component/store/local/enum_enum.go new file mode 100644 index 000000000..1c79e08cf --- /dev/null +++ b/sdk/component/store/local/enum_enum.go @@ -0,0 +1,49 @@ +// Code generated by go-enum DO NOT EDIT. +// Version: +// Revision: +// Build Date: +// Built By: + +package localstore + +import ( + "errors" + "fmt" +) + +const ( + // LocalStoreColumnNameFindings is a localStoreColumnName of type findings. + LocalStoreColumnNameFindings localStoreColumnName = "findings" + // LocalStoreColumnNameInstanceId is a localStoreColumnName of type instance_id. + LocalStoreColumnNameInstanceId localStoreColumnName = "instance_id" + // LocalStoreColumnNameUpdatedAt is a localStoreColumnName of type updated_at. + LocalStoreColumnNameUpdatedAt localStoreColumnName = "updated_at" +) + +var ErrInvalidlocalStoreColumnName = errors.New("not a valid localStoreColumnName") + +// String implements the Stringer interface. +func (x localStoreColumnName) String() string { + return string(x) +} + +// IsValid provides a quick way to determine if the typed value is +// part of the allowed enumerated values +func (x localStoreColumnName) IsValid() bool { + _, err := ParselocalStoreColumnName(string(x)) + return err == nil +} + +var _localStoreColumnNameValue = map[string]localStoreColumnName{ + "findings": LocalStoreColumnNameFindings, + "instance_id": LocalStoreColumnNameInstanceId, + "updated_at": LocalStoreColumnNameUpdatedAt, +} + +// ParselocalStoreColumnName attempts to convert a string to a localStoreColumnName. +func ParselocalStoreColumnName(name string) (localStoreColumnName, error) { + if x, ok := _localStoreColumnNameValue[name]; ok { + return x, nil + } + return localStoreColumnName(""), fmt.Errorf("%s is %w", name, ErrInvalidlocalStoreColumnName) +} diff --git a/sdk/component/internal/storer/local/sqlite/sqlite.go b/sdk/component/store/local/store.go similarity index 78% rename from sdk/component/internal/storer/local/sqlite/sqlite.go rename to sdk/component/store/local/store.go index a05a1ab95..cb32f36c9 100644 --- a/sdk/component/internal/storer/local/sqlite/sqlite.go +++ b/sdk/component/store/local/store.go @@ -1,10 +1,11 @@ -package sqlite +package localstore import ( "context" "database/sql" "encoding/json" "fmt" + "os" "time" "github.com/go-errors/errors" @@ -12,16 +13,15 @@ import ( _ "github.com/mattn/go-sqlite3" "google.golang.org/protobuf/encoding/protojson" - "github.com/smithy-security/smithy/sdk/component/internal/storer" - "github.com/smithy-security/smithy/sdk/component/internal/uuid" + "github.com/smithy-security/smithy/sdk/component/store" + "github.com/smithy-security/smithy/sdk/component/uuid" ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1" ) -const errInvalidConstructorEmptyReason = "cannot be empty" - type ( manager struct { clock clockwork.Clock + dsn string db *sql.DB } @@ -38,7 +38,7 @@ type ( func ManagerWithClock(clock clockwork.Clock) managerOption { return func(m *manager) error { if clock == nil { - return errors.New("cannot set clock on nil clock") + return errors.New("invalid nil clock") } m.clock = clock return nil @@ -50,23 +50,14 @@ func (e ErrInvalidConstructor) Error() string { } // NewManager returns a new SQLite database manager. -func NewManager(dsn string, opts ...managerOption) (*manager, error) { - if dsn == "" { - return nil, ErrInvalidConstructor{ - argName: "db dsn", - reason: errInvalidConstructorEmptyReason, +func NewManager(opts ...managerOption) (*manager, error) { + var ( + mgr = &manager{ + clock: clockwork.NewRealClock(), + dsn: "smithy.db", } - } - - db, err := sql.Open("sqlite3", dsn) - if err != nil { - return nil, errors.Errorf("could not open sqlite db: %w", err) - } - - mgr := &manager{ - clock: clockwork.NewRealClock(), - db: db, - } + err error + ) for _, opt := range opts { if err := opt(mgr); err != nil { @@ -74,6 +65,22 @@ func NewManager(dsn string, opts ...managerOption) (*manager, error) { } } + // Create sqlite database file if not exists. + if _, err := os.Stat(mgr.dsn); err != nil { + if !os.IsNotExist(err) { + f, err := os.Create(mgr.dsn) + if err != nil { + return nil, errors.Errorf("could not create sqlite db: %w", err) + } + _ = f.Close() + } + } + + mgr.db, err = sql.Open("sqlite3", mgr.dsn) + if err != nil { + return nil, errors.Errorf("could not open sqlite db: %w", err) + } + if err := mgr.migrate(); err != nil { return nil, errors.Errorf("could not apply migrations: %w", err) } @@ -81,13 +88,14 @@ func NewManager(dsn string, opts ...managerOption) (*manager, error) { return mgr, nil } -// Validate. TODO - implement. +// Validate validates the passed finding. +// TODO: to be tackled with https://linear.app/smithy/issue/OCU-259/validate-component-input-and-output. func (m *manager) Validate(*ocsf.VulnerabilityFinding) error { return nil } // Read finds Vulnerability Findings by instanceID. -// It returns storer.ErrNoFindingsFound is not vulnerabilities were found. +// It returns ErrNoFindingsFound is not vulnerabilities were found. func (m *manager) Read(ctx context.Context, instanceID uuid.UUID) ([]*ocsf.VulnerabilityFinding, error) { stmt, err := m.db.PrepareContext(ctx, ` SELECT (findings) @@ -105,12 +113,12 @@ func (m *manager) Read(ctx context.Context, instanceID uuid.UUID) ([]*ocsf.Vulne err = stmt. QueryRowContext( ctx, - sql.Named(ColumnNameInstanceId.String(), instanceID.String()), + sql.Named(LocalStoreColumnNameInstanceId.String(), instanceID.String()), ). Scan(&jsonFindingsStr) if err != nil { if errors.Is(err, sql.ErrNoRows) { - return nil, errors.Errorf("%s: %w", instanceID.String(), storer.ErrNoFindingsFound) + return nil, errors.Errorf("%s: %w", instanceID.String(), store.ErrNoFindingsFound) } return nil, errors.Errorf("could not select findings: %w", err) } @@ -151,8 +159,8 @@ func (m *manager) Write(ctx context.Context, instanceID uuid.UUID, findings []*o defer stmt.Close() if _, err = stmt.Exec( - sql.Named(ColumnNameInstanceId.String(), instanceID.String()), - sql.Named(ColumnNameFindings.String(), jsonFindings), + sql.Named(LocalStoreColumnNameInstanceId.String(), instanceID.String()), + sql.Named(LocalStoreColumnNameFindings.String(), jsonFindings), ); err != nil { return errors.Errorf("could not insert findings: %w", err) } @@ -161,7 +169,7 @@ func (m *manager) Write(ctx context.Context, instanceID uuid.UUID, findings []*o } // Update updates existing vulnerabilities in the underlying database. -// It returns storer.ErrNoFindingsFound if the passed instanceID is not found. +// It returns ErrNoFindingsFound if the passed instanceID is not found. func (m *manager) Update(ctx context.Context, instanceID uuid.UUID, findings []*ocsf.VulnerabilityFinding) error { jsonFindings, err := m.marshalFindings(findings) if err != nil { @@ -184,9 +192,9 @@ func (m *manager) Update(ctx context.Context, instanceID uuid.UUID, findings []* defer stmt.Close() res, err := stmt.Exec( - sql.Named(ColumnNameInstanceId.String(), instanceID.String()), - sql.Named(ColumnNameUpdatedAt.String(), m.clock.Now().UTC().Format(time.RFC3339)), - sql.Named(ColumnNameFindings.String(), jsonFindings), + sql.Named(LocalStoreColumnNameInstanceId.String(), instanceID.String()), + sql.Named(LocalStoreColumnNameUpdatedAt.String(), m.clock.Now().UTC().Format(time.RFC3339)), + sql.Named(LocalStoreColumnNameFindings.String(), jsonFindings), ) if err != nil { return errors.Errorf("could not update findings: %w", err) @@ -200,7 +208,7 @@ func (m *manager) Update(ctx context.Context, instanceID uuid.UUID, findings []* return errors.Errorf( "could not update findings for instance '%s': %w", instanceID.String(), - storer.ErrNoFindingsFound, + store.ErrNoFindingsFound, ) } @@ -212,6 +220,9 @@ func (m *manager) Close(ctx context.Context) error { if err := m.db.Close(); err != nil { return errors.Errorf("could not close sqlite db: %w", err) } + if err := os.Remove(m.dsn); err != nil { + return errors.Errorf("could not remove sqlite db file: %w", err) + } return nil } diff --git a/sdk/component/internal/storer/local/sqlite/sqlite_test.go b/sdk/component/store/local/storer_test.go similarity index 89% rename from sdk/component/internal/storer/local/sqlite/sqlite_test.go rename to sdk/component/store/local/storer_test.go index 2f50dc323..7e3b62000 100644 --- a/sdk/component/internal/storer/local/sqlite/sqlite_test.go +++ b/sdk/component/store/local/storer_test.go @@ -1,8 +1,7 @@ -package sqlite_test +package localstore_test import ( "context" - "os" "testing" "time" @@ -13,29 +12,18 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/smithy-security/smithy/sdk/component" - "github.com/smithy-security/smithy/sdk/component/internal/storer" - "github.com/smithy-security/smithy/sdk/component/internal/storer/local/sqlite" - "github.com/smithy-security/smithy/sdk/component/internal/uuid" + "github.com/smithy-security/smithy/sdk/component/store" + localstore "github.com/smithy-security/smithy/sdk/component/store/local" + "github.com/smithy-security/smithy/sdk/component/uuid" ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1" ) -const ( - dbName = "smithy.db" -) - type ( - localStorer interface { - component.Closer - component.Reader - component.Updater - component.Writer - } - ManagerTestSuite struct { suite.Suite t *testing.T - manager localStorer + manager component.Storer } ) @@ -46,11 +34,7 @@ func (mts *ManagerTestSuite) SetupTest() { clock = clockwork.NewFakeClock() ) - f, err := os.Create(dbName) - require.NoError(mts.t, err) - require.NoError(mts.t, f.Close()) - - mts.manager, err = sqlite.NewManager("smithy.db", sqlite.ManagerWithClock(clock)) + mts.manager, err = localstore.NewManager(localstore.ManagerWithClock(clock)) require.NoError(mts.t, err) } @@ -58,7 +42,6 @@ func (mts *ManagerTestSuite) TearDownTest() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() require.NoError(mts.t, mts.manager.Close(ctx)) - require.NoError(mts.t, os.Remove(dbName)) } func (mts *ManagerTestSuite) TestManager() { @@ -146,7 +129,7 @@ func (mts *ManagerTestSuite) TestManager() { ctx, instanceID, ) - require.ErrorIs(t, err, storer.ErrNoFindingsFound) + require.ErrorIs(t, err, store.ErrNoFindingsFound) require.Len(mts.t, resFindings, 0) }) @@ -183,7 +166,7 @@ func (mts *ManagerTestSuite) TestManager() { require.ErrorIs( mts.t, mts.manager.Update(ctx, uuid.New(), findings), - storer.ErrNoFindingsFound, + store.ErrNoFindingsFound, ) }) diff --git a/sdk/component/storer.go b/sdk/component/storer.go deleted file mode 100644 index 5dd69145c..000000000 --- a/sdk/component/storer.go +++ /dev/null @@ -1,22 +0,0 @@ -package component - -import ( - "github.com/go-errors/errors" - - "github.com/smithy-security/smithy/sdk/component/internal/storer/local/sqlite" -) - -func isAllowedStoreType(st storeType) bool { - return st == StoreTypeLocal -} - -func newStorer(conf runnerConfigStorer) (Storer, error) { - if conf.storeType == StoreTypeLocal { - localMgr, err := sqlite.NewManager(conf.dbDSN) - if err != nil { - return nil, errors.Errorf("unable to initialize local sqlite manager: %w", err) - } - return localMgr, nil - } - return nil, errors.Errorf("curently unsupported store type: %s", conf.storeType) -} From 2b8edef086e18cffcf88b7e5a26ef95957e9959c Mon Sep 17 00:00:00 2001 From: andream16 Date: Thu, 5 Dec 2024 14:46:21 +0000 Subject: [PATCH 2/8] Moving uuid package to more accessible location. --- sdk/component/{internal => }/uuid/uuid.go | 1 + sdk/component/{internal => }/uuid/uuid_test.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) rename sdk/component/{internal => }/uuid/uuid.go (95%) rename sdk/component/{internal => }/uuid/uuid_test.go (95%) diff --git a/sdk/component/internal/uuid/uuid.go b/sdk/component/uuid/uuid.go similarity index 95% rename from sdk/component/internal/uuid/uuid.go rename to sdk/component/uuid/uuid.go index 74eed0bca..cb22bc42d 100644 --- a/sdk/component/internal/uuid/uuid.go +++ b/sdk/component/uuid/uuid.go @@ -5,6 +5,7 @@ import ( "github.com/google/uuid" ) +// Nil is a UUID zero value. var Nil = UUID(uuid.Nil) // UUID is an alias for a google uuid. diff --git a/sdk/component/internal/uuid/uuid_test.go b/sdk/component/uuid/uuid_test.go similarity index 95% rename from sdk/component/internal/uuid/uuid_test.go rename to sdk/component/uuid/uuid_test.go index fb428ec5b..d7145ea7c 100644 --- a/sdk/component/internal/uuid/uuid_test.go +++ b/sdk/component/uuid/uuid_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/smithy-security/smithy/sdk/component/internal/uuid" + "github.com/smithy-security/smithy/sdk/component/uuid" ) func TestUUID(t *testing.T) { From 1f0c8bf7b93123bab6690250caadcdfe3249aecf Mon Sep 17 00:00:00 2001 From: andream16 Date: Thu, 5 Dec 2024 14:47:19 +0000 Subject: [PATCH 3/8] Updating component interfaces and regenerating mocks. --- sdk/component/component.go | 2 +- sdk/component/component_test.go | 2 +- sdk/component/internal/mocks/component_mock.go | 2 +- sdk/tools.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/component/component.go b/sdk/component/component.go index 398650b46..c57c39cd2 100644 --- a/sdk/component/component.go +++ b/sdk/component/component.go @@ -3,7 +3,7 @@ package component import ( "context" - "github.com/smithy-security/smithy/sdk/component/internal/uuid" + "github.com/smithy-security/smithy/sdk/component/uuid" ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1" ) diff --git a/sdk/component/component_test.go b/sdk/component/component_test.go index 0c251b8bf..8c06df5ae 100644 --- a/sdk/component/component_test.go +++ b/sdk/component/component_test.go @@ -4,10 +4,10 @@ import ( "context" "testing" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/smithy-security/smithy/sdk/component" - "github.com/smithy-security/smithy/sdk/component/internal/uuid" ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1" ) diff --git a/sdk/component/internal/mocks/component_mock.go b/sdk/component/internal/mocks/component_mock.go index 73b27c87d..b00b94896 100644 --- a/sdk/component/internal/mocks/component_mock.go +++ b/sdk/component/internal/mocks/component_mock.go @@ -15,7 +15,7 @@ import ( gomock "go.uber.org/mock/gomock" - uuid "github.com/smithy-security/smithy/sdk/component/internal/uuid" + uuid "github.com/smithy-security/smithy/sdk/component/uuid" pb "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1" ) diff --git a/sdk/tools.go b/sdk/tools.go index b54935351..b41ddc8fe 100644 --- a/sdk/tools.go +++ b/sdk/tools.go @@ -9,7 +9,7 @@ import ( // ENUMs GEN //go:generate go run github.com/abice/go-enum --file ./component/enum.go ./... -//go:generate go run github.com/abice/go-enum --file ./component/internal/storer/local/sqlite/enum.go ./... +//go:generate go run github.com/abice/go-enum --file ./component/store/local/enum.go ./... // Mocks GEN //go:generate go run go.uber.org/mock/mockgen -package mocks -source component/component.go -destination component/internal/mocks/component_mock.go From 42217d9b026b17171aaa5c3cde65618bfdf83f9f Mon Sep 17 00:00:00 2001 From: andream16 Date: Fri, 13 Dec 2024 14:57:47 +0000 Subject: [PATCH 4/8] Adding utils to check for Nil pointers. --- sdk/component/internal/utils/utils.go | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 sdk/component/internal/utils/utils.go diff --git a/sdk/component/internal/utils/utils.go b/sdk/component/internal/utils/utils.go new file mode 100644 index 000000000..2ece16840 --- /dev/null +++ b/sdk/component/internal/utils/utils.go @@ -0,0 +1,7 @@ +package utils + +import "reflect" + +func IsNil(v any) bool { + return v == nil || (reflect.ValueOf(v).Kind() == reflect.Ptr && reflect.ValueOf(v).IsNil()) +} From 5e5b4a135926a523fccca720cdfa22a1ae53cd58 Mon Sep 17 00:00:00 2001 From: andream16 Date: Thu, 5 Dec 2024 14:48:35 +0000 Subject: [PATCH 5/8] Use new local store and uuid packages; leverage nil utils. --- sdk/component/conf.go | 79 ++++++------------------------ sdk/component/enricher.go | 2 +- sdk/component/enricher_test.go | 4 +- sdk/component/enum.go | 3 -- sdk/component/enum_enum.go | 31 ------------ sdk/component/filter.go | 2 +- sdk/component/filter_test.go | 4 +- sdk/component/logger.go | 4 +- sdk/component/reporter.go | 2 +- sdk/component/reporter_test.go | 4 +- sdk/component/runner.go | 11 +++++ sdk/component/scanner.go | 2 +- sdk/component/scanner_test.go | 4 +- sdk/component/store/local/store.go | 3 +- sdk/component/target_test.go | 4 +- 15 files changed, 45 insertions(+), 114 deletions(-) diff --git a/sdk/component/conf.go b/sdk/component/conf.go index c479ff7b5..258fd981d 100644 --- a/sdk/component/conf.go +++ b/sdk/component/conf.go @@ -7,14 +7,14 @@ import ( "github.com/smithy-security/pkg/env" "github.com/smithy-security/smithy/sdk" - "github.com/smithy-security/smithy/sdk/component/internal/uuid" + "github.com/smithy-security/smithy/sdk/component/internal/utils" + "github.com/smithy-security/smithy/sdk/component/uuid" ) const ( // Err reasons. - errReasonCannotBeEmpty = "cannot be empty" - errReasonUnsupportedValue = "unsupported value" - errReasonCannotBeNil = "cannot be nil" + errReasonCannotBeEmpty = "cannot be empty" + errReasonCannotBeNil = "cannot be nil" // Env vars. // -- BASE @@ -22,9 +22,6 @@ const ( envVarKeyInstanceID = "SMITHY_INSTANCE_ID" // -- LOGGING envVarKeyLoggingLogLevel = "SMITHY_LOG_LEVEL" - // -- STORE - envVarKeyBackendStoreType = "SMITHY_BACKEND_STORE_TYPE" - envVarKeyBackendStoreDSN = "SMITHY_BACKEND_STORE_DSN" ) type ( @@ -40,7 +37,7 @@ type ( // TODO: add MetricsHandler. // TODO: add TracingHandler. - storerConfig runnerConfigStorer + Storer Storer } // RunnerConfigLogging contains the configuration related with the runner logger. @@ -49,12 +46,6 @@ type ( Logger Logger } - runnerConfigStorer struct { - storeType storeType - dbDSN string - store Storer - } - // RunnerConfigOption can be used to override runner configuration defaults. // For example overriding the default logger. RunnerConfigOption func(*RunnerConfig) error @@ -100,19 +91,19 @@ func (rc *RunnerConfig) isValid() error { FieldName: "instance_id", Reason: errReasonCannotBeNil, } - case rc.Logging.Logger == nil: + case utils.IsNil(rc.Logging.Logger): return ErrInvalidRunnerConfig{ FieldName: "logger", Reason: errReasonCannotBeNil, } - case rc.PanicHandler == nil: + case utils.IsNil(rc.PanicHandler): return ErrInvalidRunnerConfig{ FieldName: "panic_handler", Reason: errReasonCannotBeNil, } - case rc.storerConfig.store == nil: + case utils.IsNil(rc.Storer): return ErrInvalidRunnerConfig{ - FieldName: "store_type", + FieldName: "store", Reason: errReasonCannotBeNil, } } @@ -123,7 +114,7 @@ func (rc *RunnerConfig) isValid() error { // RunnerWithLogger allows customising the runner logger. func RunnerWithLogger(logger Logger) RunnerOption { return func(r *runner) error { - if logger == nil { + if utils.IsNil(logger) { return ErrRunnerOption{ OptionName: "logger", Reason: errReasonCannotBeNil, @@ -163,22 +154,15 @@ func RunnerWithInstanceID(id uuid.UUID) RunnerOption { } // RunnerWithStorer can be used to customise the underlying storage. -func RunnerWithStorer(stType string, store Storer) RunnerOption { +func RunnerWithStorer(store Storer) RunnerOption { return func(r *runner) error { - switch { - case !isAllowedStoreType(storeType(stType)): - return ErrRunnerOption{ - OptionName: "store_type", - Reason: errReasonUnsupportedValue, - } - case store == nil: + if utils.IsNil(store) { return ErrRunnerOption{ OptionName: "storer", Reason: errReasonCannotBeNil, } } - r.config.storerConfig.store = store - r.config.storerConfig.storeType = StoreTypeLocal + r.config.Storer = store return nil } } @@ -224,13 +208,7 @@ func newRunnerConfig() (*RunnerConfig, error) { } // --- END - LOGGING ENV - END --- - // --- BEGIN - STORER ENV - BEGIN --- - st, err := env.GetOrDefault(envVarKeyBackendStoreType, "", env.WithDefaultOnError(true)) - if err != nil { - return nil, errors.Errorf("could not lookup environment for '%s': %w", envVarKeyBackendStoreType, err) - } - - conf := &RunnerConfig{ + return &RunnerConfig{ ComponentName: componentName, SDKVersion: sdk.Version, InstanceID: instanceID, @@ -239,32 +217,5 @@ func newRunnerConfig() (*RunnerConfig, error) { Logger: logger, }, PanicHandler: panicHandler, - } - - if st != "" { - var storageType = storeType(st) - if !isAllowedStoreType(storageType) { - return nil, errors.Errorf("invalid store type for '%s': %w", envVarKeyBackendStoreType, err) - } - - conf.storerConfig.storeType = storageType - - dbDSN, err := env.GetOrDefault( - envVarKeyBackendStoreDSN, - "smithy.db", - env.WithDefaultOnError(true), - ) - if err != nil { - return nil, errors.Errorf("could not lookup environment for '%s': %w", envVarKeyBackendStoreDSN, err) - } - - conf.storerConfig.dbDSN = dbDSN - conf.storerConfig.store, err = newStorer(conf.storerConfig) - if err != nil { - return nil, errors.Errorf("could not initialise store for '%s': %w", envVarKeyBackendStoreType, err) - } - } - // --- END - STORER ENV - END --- - - return conf, nil + }, nil } diff --git a/sdk/component/enricher.go b/sdk/component/enricher.go index 356e2ac91..66e6dd7a1 100644 --- a/sdk/component/enricher.go +++ b/sdk/component/enricher.go @@ -14,7 +14,7 @@ func RunEnricher(ctx context.Context, enricher Enricher, opts ...RunnerOption) e var ( instanceID = cfg.InstanceID logger = LoggerFromContext(ctx).With(logKeyComponentType, "enricher") - store = cfg.storerConfig.store + store = cfg.Storer ) defer func() { diff --git a/sdk/component/enricher_test.go b/sdk/component/enricher_test.go index ae259ae9a..26c362e61 100644 --- a/sdk/component/enricher_test.go +++ b/sdk/component/enricher_test.go @@ -11,7 +11,7 @@ import ( "github.com/smithy-security/smithy/sdk/component" "github.com/smithy-security/smithy/sdk/component/internal/mocks" - "github.com/smithy-security/smithy/sdk/component/internal/uuid" + "github.com/smithy-security/smithy/sdk/component/uuid" ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1" ) @@ -30,7 +30,7 @@ func runEnricherHelper( component.RunnerWithLogger(component.NewNoopLogger()), component.RunnerWithComponentName("sample-enricher"), component.RunnerWithInstanceID(instanceID), - component.RunnerWithStorer("local", store), + component.RunnerWithStorer(store), ) } diff --git a/sdk/component/enum.go b/sdk/component/enum.go index 1fbfe639f..02b38aa06 100644 --- a/sdk/component/enum.go +++ b/sdk/component/enum.go @@ -4,7 +4,4 @@ type ( // RunnerConfigLoggingLevel is used to represent log levels. // ENUM(debug, info, error, warn) RunnerConfigLoggingLevel string - - // ENUM(local) - storeType string ) diff --git a/sdk/component/enum_enum.go b/sdk/component/enum_enum.go index 9b5cf836d..67414213a 100644 --- a/sdk/component/enum_enum.go +++ b/sdk/component/enum_enum.go @@ -50,34 +50,3 @@ func ParseRunnerConfigLoggingLevel(name string) (RunnerConfigLoggingLevel, error } return RunnerConfigLoggingLevel(""), fmt.Errorf("%s is %w", name, ErrInvalidRunnerConfigLoggingLevel) } - -const ( - // StoreTypeLocal is a storeType of type local. - StoreTypeLocal storeType = "local" -) - -var ErrInvalidstoreType = errors.New("not a valid storeType") - -// String implements the Stringer interface. -func (x storeType) String() string { - return string(x) -} - -// IsValid provides a quick way to determine if the typed value is -// part of the allowed enumerated values -func (x storeType) IsValid() bool { - _, err := ParsestoreType(string(x)) - return err == nil -} - -var _storeTypeValue = map[string]storeType{ - "local": StoreTypeLocal, -} - -// ParsestoreType attempts to convert a string to a storeType. -func ParsestoreType(name string) (storeType, error) { - if x, ok := _storeTypeValue[name]; ok { - return x, nil - } - return storeType(""), fmt.Errorf("%s is %w", name, ErrInvalidstoreType) -} diff --git a/sdk/component/filter.go b/sdk/component/filter.go index 6c2c71bea..335febc99 100644 --- a/sdk/component/filter.go +++ b/sdk/component/filter.go @@ -14,7 +14,7 @@ func RunFilter(ctx context.Context, filter Filter, opts ...RunnerOption) error { var ( instanceID = cfg.InstanceID logger = LoggerFromContext(ctx).With(logKeyComponentType, "filter") - store = cfg.storerConfig.store + store = cfg.Storer ) defer func() { diff --git a/sdk/component/filter_test.go b/sdk/component/filter_test.go index 2c452f0cd..41687f12a 100644 --- a/sdk/component/filter_test.go +++ b/sdk/component/filter_test.go @@ -10,7 +10,7 @@ import ( "github.com/smithy-security/smithy/sdk/component" "github.com/smithy-security/smithy/sdk/component/internal/mocks" - "github.com/smithy-security/smithy/sdk/component/internal/uuid" + "github.com/smithy-security/smithy/sdk/component/uuid" ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1" ) @@ -29,7 +29,7 @@ func runFilterHelper( component.RunnerWithLogger(component.NewNoopLogger()), component.RunnerWithComponentName("sample-filter"), component.RunnerWithInstanceID(instanceID), - component.RunnerWithStorer("local", store), + component.RunnerWithStorer(store), ) } diff --git a/sdk/component/logger.go b/sdk/component/logger.go index a4394d713..d431895af 100644 --- a/sdk/component/logger.go +++ b/sdk/component/logger.go @@ -6,6 +6,8 @@ import ( "os" "github.com/go-errors/errors" + + "github.com/smithy-security/smithy/sdk/component/internal/utils" ) const ( @@ -85,7 +87,7 @@ func ContextWithLogger(ctx context.Context, logger Logger) context.Context { // LoggerFromContext extracts a structured logger from the context for reusability. func LoggerFromContext(ctx context.Context) Logger { logger := ctx.Value(ctxLoggerKey) - if logger == nil { + if utils.IsNil(logger) { l, _ := newDefaultLogger(RunnerConfigLoggingLevelDebug) return l } diff --git a/sdk/component/reporter.go b/sdk/component/reporter.go index 1bd40f759..6f31bef41 100644 --- a/sdk/component/reporter.go +++ b/sdk/component/reporter.go @@ -14,7 +14,7 @@ func RunReporter(ctx context.Context, reporter Reporter, opts ...RunnerOption) e var ( instanceID = cfg.InstanceID logger = LoggerFromContext(ctx).With(logKeyComponentType, "reporter") - store = cfg.storerConfig.store + store = cfg.Storer ) defer func() { diff --git a/sdk/component/reporter_test.go b/sdk/component/reporter_test.go index 8de3e7a9e..812fc7e57 100644 --- a/sdk/component/reporter_test.go +++ b/sdk/component/reporter_test.go @@ -10,7 +10,7 @@ import ( "github.com/smithy-security/smithy/sdk/component" "github.com/smithy-security/smithy/sdk/component/internal/mocks" - "github.com/smithy-security/smithy/sdk/component/internal/uuid" + "github.com/smithy-security/smithy/sdk/component/uuid" ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1" ) @@ -29,7 +29,7 @@ func runReporterHelper( component.RunnerWithLogger(component.NewNoopLogger()), component.RunnerWithComponentName("sample-reporter"), component.RunnerWithInstanceID(instanceID), - component.RunnerWithStorer("local", store), + component.RunnerWithStorer(store), ) } diff --git a/sdk/component/runner.go b/sdk/component/runner.go index 538c66dfa..2a2e9d288 100644 --- a/sdk/component/runner.go +++ b/sdk/component/runner.go @@ -7,6 +7,9 @@ import ( "github.com/go-errors/errors" "golang.org/x/sync/errgroup" + + "github.com/smithy-security/smithy/sdk/component/internal/utils" + localstore "github.com/smithy-security/smithy/sdk/component/store/local" ) type ( @@ -36,6 +39,14 @@ func newRunner(opts ...RunnerOption) (*runner, error) { } } + if utils.IsNil(r.config.Storer) { + storer, err := localstore.NewManager() + if err != nil { + return nil, errors.Errorf("could not create default storer: %w", err) + } + r.config.Storer = storer + } + if err := r.config.isValid(); err != nil { return nil, errors.Errorf("invalid configuration: %w", err) } diff --git a/sdk/component/scanner.go b/sdk/component/scanner.go index d6a15a0bb..015a1a86b 100644 --- a/sdk/component/scanner.go +++ b/sdk/component/scanner.go @@ -14,7 +14,7 @@ func RunScanner(ctx context.Context, scanner Scanner, opts ...RunnerOption) erro var ( instanceID = cfg.InstanceID logger = LoggerFromContext(ctx).With(logKeyComponentType, "scanner") - store = cfg.storerConfig.store + store = cfg.Storer ) defer func() { diff --git a/sdk/component/scanner_test.go b/sdk/component/scanner_test.go index a40f29fb8..327b8e53c 100644 --- a/sdk/component/scanner_test.go +++ b/sdk/component/scanner_test.go @@ -10,7 +10,7 @@ import ( "github.com/smithy-security/smithy/sdk/component" "github.com/smithy-security/smithy/sdk/component/internal/mocks" - "github.com/smithy-security/smithy/sdk/component/internal/uuid" + "github.com/smithy-security/smithy/sdk/component/uuid" ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1" ) @@ -29,7 +29,7 @@ func runScannerHelper( component.RunnerWithLogger(component.NewNoopLogger()), component.RunnerWithComponentName("sample-scanner"), component.RunnerWithInstanceID(instanceID), - component.RunnerWithStorer("local", storer), + component.RunnerWithStorer(storer), ) } diff --git a/sdk/component/store/local/store.go b/sdk/component/store/local/store.go index cb32f36c9..dc44d7612 100644 --- a/sdk/component/store/local/store.go +++ b/sdk/component/store/local/store.go @@ -13,6 +13,7 @@ import ( _ "github.com/mattn/go-sqlite3" "google.golang.org/protobuf/encoding/protojson" + "github.com/smithy-security/smithy/sdk/component/internal/utils" "github.com/smithy-security/smithy/sdk/component/store" "github.com/smithy-security/smithy/sdk/component/uuid" ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1" @@ -37,7 +38,7 @@ type ( // ManagerWithClock allows customising manager's clock. func ManagerWithClock(clock clockwork.Clock) managerOption { return func(m *manager) error { - if clock == nil { + if utils.IsNil(clock) { return errors.New("invalid nil clock") } m.clock = clock diff --git a/sdk/component/target_test.go b/sdk/component/target_test.go index 7950a7b9d..95bf9bc53 100644 --- a/sdk/component/target_test.go +++ b/sdk/component/target_test.go @@ -11,7 +11,7 @@ import ( "github.com/smithy-security/smithy/sdk/component" "github.com/smithy-security/smithy/sdk/component/internal/mocks" - "github.com/smithy-security/smithy/sdk/component/internal/uuid" + "github.com/smithy-security/smithy/sdk/component/uuid" ) func runTargetHelper(t *testing.T, ctx context.Context, target component.Target, store component.Storer) error { @@ -23,7 +23,7 @@ func runTargetHelper(t *testing.T, ctx context.Context, target component.Target, component.RunnerWithLogger(component.NewNoopLogger()), component.RunnerWithComponentName("sample-target"), component.RunnerWithInstanceID(uuid.New()), - component.RunnerWithStorer("local", store), + component.RunnerWithStorer(store), ) } From 764af404aef284627781595424a9ac56afc174e2 Mon Sep 17 00:00:00 2001 From: andream16 Date: Thu, 5 Dec 2024 14:48:18 +0000 Subject: [PATCH 6/8] Updating docs by deprecating environment variables. --- sdk/README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/sdk/README.md b/sdk/README.md index 97db06146..03e4cfe98 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -25,12 +25,8 @@ You can customise a component using the following environment variables: | Environment Variable | Type | Required | Default | Possible Values | |-------------------------------------|--------|-----------|--------------------------|--------------------------| | SMITHY\_COMPONENT\_NAME | string | yes | - | - | -| SMITHY\_BACKEND\_STORE\_TYPE | string | yes | - | local, test, \*remote | -| SSMITHY\_BACKEND\_STORE\_DSN | string | no | smithy.db | \* | | SMITHY\_LOG\_LEVEL | string | false | info, debug, warn, error | -For `local` development, an `SQLite` Backend Store Type will be used. - `Runners` can be supplied with `RunnerConfigOption`s to customise how a component runs. In the following example you can see how we change the component name: @@ -42,6 +38,16 @@ component.RunTarget( ) ``` +For local development, a default `SQLite` Backend Store Type will be used. This can be customised with: + +```go +component.RunTarget( + ctx, + sampleTarget{}, + component.RunnerWithStorer(//a storer), +) +``` + ### Components #### Target From 694ec348f9dd79c521033b51733a2814f6a65dd1 Mon Sep 17 00:00:00 2001 From: andream16 Date: Thu, 5 Dec 2024 15:02:24 +0000 Subject: [PATCH 7/8] Removing no longer required environment variables from local .env files. --- sdk/component/examples/enricher/.env | 1 - sdk/component/examples/filter/.env | 1 - sdk/component/examples/reporter/.env | 1 - sdk/component/examples/scanner/.env | 1 - sdk/component/examples/target/.env | 1 - 5 files changed, 5 deletions(-) diff --git a/sdk/component/examples/enricher/.env b/sdk/component/examples/enricher/.env index 159cd0e76..e50a97464 100644 --- a/sdk/component/examples/enricher/.env +++ b/sdk/component/examples/enricher/.env @@ -1,4 +1,3 @@ SMITHY_COMPONENT_NAME=sample-enricher SMITHY_INSTANCE_ID=8d719c1c-c569-4078-87b3-4951bd4012ee SMITHY_LOG_LEVEL=debug -SMITHY_BACKEND_STORE_TYPE=local diff --git a/sdk/component/examples/filter/.env b/sdk/component/examples/filter/.env index 117569576..786c781e0 100644 --- a/sdk/component/examples/filter/.env +++ b/sdk/component/examples/filter/.env @@ -1,4 +1,3 @@ SMITHY_COMPONENT_NAME=sample-filter SMITHY_INSTANCE_ID=8d719c1c-c569-4078-87b3-4951bd4012ee SMITHY_LOG_LEVEL=debug -SMITHY_BACKEND_STORE_TYPE=local diff --git a/sdk/component/examples/reporter/.env b/sdk/component/examples/reporter/.env index 4d2370a31..fd26f4d67 100644 --- a/sdk/component/examples/reporter/.env +++ b/sdk/component/examples/reporter/.env @@ -1,4 +1,3 @@ SMITHY_COMPONENT_NAME=sample-reporter SMITHY_INSTANCE_ID=8d719c1c-c569-4078-87b3-4951bd4012ee SMITHY_LOG_LEVEL=debug -SMITHY_BACKEND_STORE_TYPE=local diff --git a/sdk/component/examples/scanner/.env b/sdk/component/examples/scanner/.env index 6622f5936..9d4d331e9 100644 --- a/sdk/component/examples/scanner/.env +++ b/sdk/component/examples/scanner/.env @@ -1,4 +1,3 @@ SMITHY_COMPONENT_NAME=sample-scanner SMITHY_INSTANCE_ID=8d719c1c-c569-4078-87b3-4951bd4012ee SMITHY_LOG_LEVEL=debug -SMITHY_BACKEND_STORE_TYPE=local diff --git a/sdk/component/examples/target/.env b/sdk/component/examples/target/.env index f552ba404..6a4aa0353 100644 --- a/sdk/component/examples/target/.env +++ b/sdk/component/examples/target/.env @@ -1,4 +1,3 @@ SMITHY_COMPONENT_NAME=sample-target SMITHY_INSTANCE_ID=8d719c1c-c569-4078-87b3-4951bd4012ee SMITHY_LOG_LEVEL=debug -SMITHY_BACKEND_STORE_TYPE=local From 6eaf86b4bd461f49024d64ece446fe2152077bcb Mon Sep 17 00:00:00 2001 From: andream16 Date: Thu, 5 Dec 2024 14:47:59 +0000 Subject: [PATCH 8/8] Adding empty postgres remote store. --- .../store/remote/potsgresql/store.go | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 sdk/component/store/remote/potsgresql/store.go diff --git a/sdk/component/store/remote/potsgresql/store.go b/sdk/component/store/remote/potsgresql/store.go new file mode 100644 index 000000000..8872d4001 --- /dev/null +++ b/sdk/component/store/remote/potsgresql/store.go @@ -0,0 +1,36 @@ +package potsgresql + +import ( + "context" + + "github.com/smithy-security/smithy/sdk/component/uuid" + ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1" +) + +// TODO: implement in next PR. +type manager struct{} + +func (m manager) Close(ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (m manager) Validate(finding *ocsf.VulnerabilityFinding) error { + //TODO implement me + panic("implement me") +} + +func (m manager) Read(ctx context.Context, instanceID uuid.UUID) ([]*ocsf.VulnerabilityFinding, error) { + //TODO implement me + panic("implement me") +} + +func (m manager) Update(ctx context.Context, instanceID uuid.UUID, findings []*ocsf.VulnerabilityFinding) error { + //TODO implement me + panic("implement me") +} + +func (m manager) Write(ctx context.Context, instanceID uuid.UUID, findings []*ocsf.VulnerabilityFinding) error { + //TODO implement me + panic("implement me") +}