Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement persistent findings storage - Part 1 #492

14 changes: 10 additions & 4 deletions sdk/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,8 @@ You can customise a component using the following environment variables:
| Environment Variable | Type | Required | Default | Possible Values |
|-------------------------------------|--------|-----------|--------------------------|--------------------------|
| SMITHY\_COMPONENT\_NAME | string | yes | - | - |
| SMITHY\_BACKEND\_STORE\_TYPE | string | yes | - | local, test, \*remote |
andream16 marked this conversation as resolved.
Show resolved Hide resolved
| SSMITHY\_BACKEND\_STORE\_DSN | string | no | smithy.db | \* |
| SMITHY\_LOG\_LEVEL | string | false | info, debug, warn, error |

For `local` development, an `SQLite` Backend Store Type will be used.

`Runners` can be supplied with `RunnerConfigOption`s to customise how a component runs.
In the following example you can see how we change the component name:

Expand All @@ -42,6 +38,16 @@ component.RunTarget(
)
```

For local development, a default `SQLite` Backend Store Type will be used. This can be customised with:

```go
component.RunTarget(
ctx,
sampleTarget{},
component.RunnerWithStorer(//a storer),
)
```

### Components

#### Target
Expand Down
2 changes: 1 addition & 1 deletion sdk/component/component.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package component
import (
"context"

"github.com/smithy-security/smithy/sdk/component/internal/uuid"
"github.com/smithy-security/smithy/sdk/component/uuid"
ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1"
)

Expand Down
2 changes: 1 addition & 1 deletion sdk/component/component_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ import (
"context"
"testing"

"github.com/google/uuid"
"github.com/stretchr/testify/assert"

"github.com/smithy-security/smithy/sdk/component"
"github.com/smithy-security/smithy/sdk/component/internal/uuid"
ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1"
)

Expand Down
79 changes: 15 additions & 64 deletions sdk/component/conf.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,24 +7,21 @@ import (
"github.com/smithy-security/pkg/env"

"github.com/smithy-security/smithy/sdk"
"github.com/smithy-security/smithy/sdk/component/internal/uuid"
"github.com/smithy-security/smithy/sdk/component/internal/utils"
"github.com/smithy-security/smithy/sdk/component/uuid"
)

const (
// Err reasons.
errReasonCannotBeEmpty = "cannot be empty"
errReasonUnsupportedValue = "unsupported value"
errReasonCannotBeNil = "cannot be nil"
errReasonCannotBeEmpty = "cannot be empty"
errReasonCannotBeNil = "cannot be nil"

// Env vars.
// -- BASE
envVarKeyComponentName = "SMITHY_COMPONENT_NAME"
envVarKeyInstanceID = "SMITHY_INSTANCE_ID"
// -- LOGGING
envVarKeyLoggingLogLevel = "SMITHY_LOG_LEVEL"
// -- STORE
envVarKeyBackendStoreType = "SMITHY_BACKEND_STORE_TYPE"
envVarKeyBackendStoreDSN = "SMITHY_BACKEND_STORE_DSN"
)

type (
Expand All @@ -40,7 +37,7 @@ type (
// TODO: add MetricsHandler.
// TODO: add TracingHandler.

storerConfig runnerConfigStorer
Storer Storer
}

// RunnerConfigLogging contains the configuration related with the runner logger.
Expand All @@ -49,12 +46,6 @@ type (
Logger Logger
}

runnerConfigStorer struct {
storeType storeType
dbDSN string
store Storer
}

// RunnerConfigOption can be used to override runner configuration defaults.
// For example overriding the default logger.
RunnerConfigOption func(*RunnerConfig) error
Expand Down Expand Up @@ -100,19 +91,19 @@ func (rc *RunnerConfig) isValid() error {
FieldName: "instance_id",
Reason: errReasonCannotBeNil,
}
case rc.Logging.Logger == nil:
case utils.IsNil(rc.Logging.Logger):
return ErrInvalidRunnerConfig{
FieldName: "logger",
Reason: errReasonCannotBeNil,
}
case rc.PanicHandler == nil:
case utils.IsNil(rc.PanicHandler):
return ErrInvalidRunnerConfig{
FieldName: "panic_handler",
Reason: errReasonCannotBeNil,
}
case rc.storerConfig.store == nil:
case utils.IsNil(rc.Storer):
return ErrInvalidRunnerConfig{
FieldName: "store_type",
FieldName: "store",
Reason: errReasonCannotBeNil,
}
}
Expand All @@ -123,7 +114,7 @@ func (rc *RunnerConfig) isValid() error {
// RunnerWithLogger allows customising the runner logger.
func RunnerWithLogger(logger Logger) RunnerOption {
return func(r *runner) error {
if logger == nil {
if utils.IsNil(logger) {
return ErrRunnerOption{
OptionName: "logger",
Reason: errReasonCannotBeNil,
Expand Down Expand Up @@ -163,22 +154,15 @@ func RunnerWithInstanceID(id uuid.UUID) RunnerOption {
}

// RunnerWithStorer can be used to customise the underlying storage.
func RunnerWithStorer(stType string, store Storer) RunnerOption {
func RunnerWithStorer(store Storer) RunnerOption {
return func(r *runner) error {
switch {
case !isAllowedStoreType(storeType(stType)):
return ErrRunnerOption{
OptionName: "store_type",
Reason: errReasonUnsupportedValue,
}
case store == nil:
if utils.IsNil(store) {
return ErrRunnerOption{
OptionName: "storer",
Reason: errReasonCannotBeNil,
}
}
r.config.storerConfig.store = store
r.config.storerConfig.storeType = StoreTypeLocal
r.config.Storer = store
return nil
}
}
Expand Down Expand Up @@ -224,13 +208,7 @@ func newRunnerConfig() (*RunnerConfig, error) {
}
// --- END - LOGGING ENV - END ---

// --- BEGIN - STORER ENV - BEGIN ---
st, err := env.GetOrDefault(envVarKeyBackendStoreType, "", env.WithDefaultOnError(true))
if err != nil {
return nil, errors.Errorf("could not lookup environment for '%s': %w", envVarKeyBackendStoreType, err)
}

conf := &RunnerConfig{
return &RunnerConfig{
ComponentName: componentName,
SDKVersion: sdk.Version,
InstanceID: instanceID,
Expand All @@ -239,32 +217,5 @@ func newRunnerConfig() (*RunnerConfig, error) {
Logger: logger,
},
PanicHandler: panicHandler,
}

if st != "" {
var storageType = storeType(st)
if !isAllowedStoreType(storageType) {
return nil, errors.Errorf("invalid store type for '%s': %w", envVarKeyBackendStoreType, err)
}

conf.storerConfig.storeType = storageType

dbDSN, err := env.GetOrDefault(
envVarKeyBackendStoreDSN,
"smithy.db",
env.WithDefaultOnError(true),
)
if err != nil {
return nil, errors.Errorf("could not lookup environment for '%s': %w", envVarKeyBackendStoreDSN, err)
}

conf.storerConfig.dbDSN = dbDSN
conf.storerConfig.store, err = newStorer(conf.storerConfig)
if err != nil {
return nil, errors.Errorf("could not initialise store for '%s': %w", envVarKeyBackendStoreType, err)
}
}
// --- END - STORER ENV - END ---

return conf, nil
}, nil
}
2 changes: 1 addition & 1 deletion sdk/component/enricher.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ func RunEnricher(ctx context.Context, enricher Enricher, opts ...RunnerOption) e
var (
instanceID = cfg.InstanceID
logger = LoggerFromContext(ctx).With(logKeyComponentType, "enricher")
store = cfg.storerConfig.store
store = cfg.Storer
)

defer func() {
Expand Down
4 changes: 2 additions & 2 deletions sdk/component/enricher_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (

"github.com/smithy-security/smithy/sdk/component"
"github.com/smithy-security/smithy/sdk/component/internal/mocks"
"github.com/smithy-security/smithy/sdk/component/internal/uuid"
"github.com/smithy-security/smithy/sdk/component/uuid"
ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1"
)

Expand All @@ -30,7 +30,7 @@ func runEnricherHelper(
component.RunnerWithLogger(component.NewNoopLogger()),
component.RunnerWithComponentName("sample-enricher"),
component.RunnerWithInstanceID(instanceID),
component.RunnerWithStorer("local", store),
component.RunnerWithStorer(store),
)
}

Expand Down
3 changes: 0 additions & 3 deletions sdk/component/enum.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,4 @@ type (
// RunnerConfigLoggingLevel is used to represent log levels.
// ENUM(debug, info, error, warn)
RunnerConfigLoggingLevel string

// ENUM(local)
storeType string
)
31 changes: 0 additions & 31 deletions sdk/component/enum_enum.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion sdk/component/examples/enricher/.env
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
SMITHY_COMPONENT_NAME=sample-enricher
SMITHY_INSTANCE_ID=8d719c1c-c569-4078-87b3-4951bd4012ee
SMITHY_LOG_LEVEL=debug
SMITHY_BACKEND_STORE_TYPE=local
andream16 marked this conversation as resolved.
Show resolved Hide resolved
1 change: 0 additions & 1 deletion sdk/component/examples/filter/.env
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
SMITHY_COMPONENT_NAME=sample-filter
SMITHY_INSTANCE_ID=8d719c1c-c569-4078-87b3-4951bd4012ee
SMITHY_LOG_LEVEL=debug
SMITHY_BACKEND_STORE_TYPE=local
1 change: 0 additions & 1 deletion sdk/component/examples/reporter/.env
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
SMITHY_COMPONENT_NAME=sample-reporter
SMITHY_INSTANCE_ID=8d719c1c-c569-4078-87b3-4951bd4012ee
SMITHY_LOG_LEVEL=debug
SMITHY_BACKEND_STORE_TYPE=local
1 change: 0 additions & 1 deletion sdk/component/examples/scanner/.env
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
SMITHY_COMPONENT_NAME=sample-scanner
SMITHY_INSTANCE_ID=8d719c1c-c569-4078-87b3-4951bd4012ee
SMITHY_LOG_LEVEL=debug
SMITHY_BACKEND_STORE_TYPE=local
1 change: 0 additions & 1 deletion sdk/component/examples/target/.env
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
SMITHY_COMPONENT_NAME=sample-target
SMITHY_INSTANCE_ID=8d719c1c-c569-4078-87b3-4951bd4012ee
SMITHY_LOG_LEVEL=debug
SMITHY_BACKEND_STORE_TYPE=local
2 changes: 1 addition & 1 deletion sdk/component/filter.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ func RunFilter(ctx context.Context, filter Filter, opts ...RunnerOption) error {
var (
instanceID = cfg.InstanceID
logger = LoggerFromContext(ctx).With(logKeyComponentType, "filter")
store = cfg.storerConfig.store
store = cfg.Storer
)

defer func() {
Expand Down
4 changes: 2 additions & 2 deletions sdk/component/filter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (

"github.com/smithy-security/smithy/sdk/component"
"github.com/smithy-security/smithy/sdk/component/internal/mocks"
"github.com/smithy-security/smithy/sdk/component/internal/uuid"
"github.com/smithy-security/smithy/sdk/component/uuid"
ocsf "github.com/smithy-security/smithy/sdk/gen/com/github/ocsf/ocsf_schema/v1"
)

Expand All @@ -29,7 +29,7 @@ func runFilterHelper(
component.RunnerWithLogger(component.NewNoopLogger()),
component.RunnerWithComponentName("sample-filter"),
component.RunnerWithInstanceID(instanceID),
component.RunnerWithStorer("local", store),
component.RunnerWithStorer(store),
)
}

Expand Down
2 changes: 1 addition & 1 deletion sdk/component/internal/mocks/component_mock.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

49 changes: 0 additions & 49 deletions sdk/component/internal/storer/local/sqlite/enum_enum.go

This file was deleted.

5 changes: 0 additions & 5 deletions sdk/component/internal/storer/storer.go

This file was deleted.

Loading
Loading