From 4d74d5428b3319c7a06af441648aba8d85ce6102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?= <31509435+aleksasiriski@users.noreply.github.com> Date: Tue, 9 Jul 2024 02:31:36 +0200 Subject: [PATCH 1/9] feat(cache): dynamodb --- go.mod | 20 +++- go.sum | 36 +++++++ src/cache/db.go | 6 ++ src/cache/dynamodb/dynamodb.go | 134 ++++++++++++++++++++++++++ src/cache/dynamodb/dynamodb_test.go | 140 ++++++++++++++++++++++++++++ src/config/defaults.go | 3 + src/config/structs_server.go | 7 ++ 7 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 src/cache/dynamodb/dynamodb.go create mode 100644 src/cache/dynamodb/dynamodb_test.go diff --git a/go.mod b/go.mod index 44a4999d..5f2cb5db 100644 --- a/go.mod +++ b/go.mod @@ -25,13 +25,30 @@ require ( golang.org/x/tools v0.23.0 ) -require github.com/klauspost/compress v1.17.9 +require ( + github.com/aws/aws-sdk-go-v2 v1.30.1 + github.com/aws/aws-sdk-go-v2/config v1.27.24 + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.1 + github.com/klauspost/compress v1.17.9 +) require ( github.com/andybalholm/cascadia v1.3.2 // indirect github.com/antchfx/htmlquery v1.3.2 // indirect github.com/antchfx/xmlquery v1.4.1 // indirect github.com/antchfx/xpath v1.3.1 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.24 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 // indirect + github.com/aws/smithy-go v1.20.3 // indirect github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -43,6 +60,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/pprof v0.0.0-20240625030939-27f56978b8b0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/go.sum b/go.sum index f96bdeba..535cebb9 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,36 @@ github.com/antchfx/xpath v1.3.1 h1:PNbFuUqHwWl0xRjvUPjJ95Agbmdj2uzzIwmQKgu4oCk= github.com/antchfx/xpath v1.3.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI= github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= +github.com/aws/aws-sdk-go-v2 v1.30.1 h1:4y/5Dvfrhd1MxRDD77SrfsDaj8kUkkljU7XE83NPV+o= +github.com/aws/aws-sdk-go-v2 v1.30.1/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= +github.com/aws/aws-sdk-go-v2/config v1.27.24 h1:NM9XicZ5o1CBU/MZaHwFtimRpWx9ohAUAqkG6AqSqPo= +github.com/aws/aws-sdk-go-v2/config v1.27.24/go.mod h1:aXzi6QJTuQRVVusAO8/NxpdTeTyr/wRcybdDtfUwJSs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.24 h1:YclAsrnb1/GTQNt2nzv+756Iw4mF8AOzcDfweWwwm/M= +github.com/aws/aws-sdk-go-v2/credentials v1.17.24/go.mod h1:Hld7tmnAkoBQdTMNYZGzztzKRdA4fCdn9L83LOoigac= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 h1:Aznqksmd6Rfv2HQN9cpqIV/lQRMaIpJkLLaJ1ZI76no= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9/go.mod h1:WQr3MY7AxGNxaqAtsDWn+fBxmd4XvLkzeqQ8P1VM0/w= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 h1:5SAoZ4jYpGH4721ZNoS1znQrhOfZinOhc4XuTXx/nVc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13/go.mod h1:+rdA6ZLpaSeM7tSg/B0IEDinCIBJGmW8rKDFkYpP04g= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 h1:WIijqeaAO7TYFLbhsZmi2rgLEAtWOC1LhxCAVTJlSKw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13/go.mod h1:i+kbfa76PQbWw/ULoWnp51EYVWH4ENln76fLQE3lXT8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.1 h1:Szwz1vpZkvfhFMJ0X5uUECgHeUmPAxk1UGqAVs/pARw= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.1/go.mod h1:b4wouGyJlzkr2HAvPrDGgYNp1EtmlXOkzhEOvl0c0FQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.14 h1:X1J0Kd17n1PeXeoArNXlvnKewCyMvhVQh7iNMy6oi3s= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.14/go.mod h1:VYMN7l7dxp6xtQRjqIau6d7QAbmPG+yJ75GtCy70f18= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 h1:I9zMeF107l0rJrpnHpjEiiTSCKYAIw8mALiXcPsGBiA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15/go.mod h1:9xWJ3Q/S6Ojusz1UIkfycgD1mGirJfLLKqq3LPT7WN8= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 h1:p1GahKIjyMDZtiKoIn0/jAj/TkMzfzndDv5+zi2Mhgc= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.1/go.mod h1:/vWdhoIoYA5hYoPZ6fm7Sv4d8701PiG5VKe8/pPJL60= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 h1:ORnrOK0C4WmYV/uYt3koHEWBLYsRDwk2Np+eEoyV4Z0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2/go.mod h1:xyFHA4zGxgYkdD73VeezHt3vSKEG9EmFnGwoKlP00u4= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 h1:+woJ607dllHJQtsnJLi52ycuqHMwlW+Wqm2Ppsfp4nQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.1/go.mod h1:jiNR3JqT15Dm+QWq2SRgh0x0bCNSRP2L25+CqPNpJlQ= +github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= +github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/awslabs/aws-lambda-go-api-proxy v0.16.2 h1:CJyGEyO1CIwOnXTU40urf0mchf6t3voxpvUDikOU9LY= github.com/awslabs/aws-lambda-go-api-proxy v0.16.2/go.mod h1:vxxjwBHe/KbgFeNlAP/Tvp4SsVRL3WQamcWRxqVh0z0= github.com/bits-and-blooms/bitset v1.2.2-0.20220111210104-dfa3e347c392/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= @@ -113,6 +143,10 @@ github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSo github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= @@ -308,6 +342,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src/cache/db.go b/src/cache/db.go index a9663b9c..ed5352c0 100644 --- a/src/cache/db.go +++ b/src/cache/db.go @@ -6,6 +6,7 @@ import ( "github.com/rs/zerolog/log" + "github.com/hearchco/agent/src/cache/dynamodb" "github.com/hearchco/agent/src/cache/nocache" "github.com/hearchco/agent/src/cache/redis" "github.com/hearchco/agent/src/config" @@ -25,6 +26,11 @@ func New(ctx context.Context, cacheConf config.Cache) (DB, error) { if err != nil { err = fmt.Errorf("failed creating a redis cache: %w", err) } + case "dynamodb": + drv, err = dynamodb.New(ctx, cacheConf.KeyPrefix, cacheConf.DynamoDB) + if err != nil { + err = fmt.Errorf("failed creating a dynamodb cache: %w", err) + } default: drv, err = nocache.New() if err != nil { diff --git a/src/cache/dynamodb/dynamodb.go b/src/cache/dynamodb/dynamodb.go new file mode 100644 index 00000000..aca6f1d7 --- /dev/null +++ b/src/cache/dynamodb/dynamodb.go @@ -0,0 +1,134 @@ +package dynamodb + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/rs/zerolog/log" + + "github.com/hearchco/agent/src/config" + "github.com/hearchco/agent/src/utils/anonymize" +) + +type DRV struct { + ctx context.Context + keyPrefix string + client *dynamodb.Client + tableName string +} + +func New(ctx context.Context, keyPrefix string, conf config.DynamoDB) (DRV, error) { + cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(conf.Region)) + if err != nil { + log.Error().Err(err).Msg("Error loading AWS config") + return DRV{}, err + } + + client := dynamodb.NewFromConfig(cfg) + return DRV{ctx, keyPrefix, client, conf.Table}, nil +} + +func (drv DRV) Close() {} + +func (drv DRV) Set(k string, v any, ttl ...time.Duration) error { + log.Debug().Msg("Caching...") + cacheTimer := time.Now() + + key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k)) + item, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("dynamodb.Set(): error marshaling value: %w", err) + } + + attributes := map[string]types.AttributeValue{ + "Key": &types.AttributeValueMemberS{Value: key}, + "Value": &types.AttributeValueMemberS{Value: string(item)}, + } + + if len(ttl) > 0 { + expirationTime := time.Now().Add(ttl[0]).Unix() + attributes["TTL"] = &types.AttributeValueMemberN{Value: fmt.Sprintf("%d", expirationTime)} + } + + _, err = drv.client.PutItem(drv.ctx, &dynamodb.PutItemInput{ + TableName: aws.String(drv.tableName), + Item: attributes, + }) + if err != nil { + return fmt.Errorf("dynamodb.Set(): error setting KV in dynamodb: %w", err) + } + + log.Trace().Dur("duration", time.Since(cacheTimer)).Msg("Cached results") + return nil +} + +func (drv DRV) Get(k string, o any) error { + key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k)) + + result, err := drv.client.GetItem(drv.ctx, &dynamodb.GetItemInput{ + TableName: aws.String(drv.tableName), + Key: map[string]types.AttributeValue{ + "Key": &types.AttributeValueMemberS{Value: key}, + }, + }) + if err != nil { + return fmt.Errorf("dynamodb.Get(): error getting value from dynamodb for key %v: %w", key, err) + } + + if result.Item == nil { + log.Trace(). + Str("key", key). + Msg("Found no value in dynamodb") + return nil + } + + value := result.Item["Value"].(*types.AttributeValueMemberS).Value + if err := json.Unmarshal([]byte(value), o); err != nil { + return fmt.Errorf("dynamodb.Get(): error unmarshaling value from dynamodb for key %v: %w", key, err) + } + + return nil +} + +func (drv DRV) GetTTL(k string) (time.Duration, error) { + key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k)) + + result, err := drv.client.GetItem(drv.ctx, &dynamodb.GetItemInput{ + TableName: aws.String(drv.tableName), + Key: map[string]types.AttributeValue{ + "Key": &types.AttributeValueMemberS{Value: key}, + }, + }) + if err != nil { + return 0, fmt.Errorf("dynamodb.GetTTL(): error getting value from dynamodb for key %v: %w", key, err) + } + + if result.Item == nil { + log.Trace().Str("key", key).Msg("Found no value in dynamodb") + return 0, nil + } + + ttlAttribute, exists := result.Item["TTL"] + if !exists { + return 0, nil + } + + expirationTime, err := strconv.ParseInt(ttlAttribute.(*types.AttributeValueMemberN).Value, 10, 64) + if err != nil { + return 0, fmt.Errorf("dynamodb.GetTTL(): error parsing TTL value for key %v: %w", key, err) + } + + expiresIn := time.Until(time.Unix(expirationTime, 0)) + if expiresIn < 0 { + expiresIn = 0 + } + + return expiresIn, nil +} diff --git a/src/cache/dynamodb/dynamodb_test.go b/src/cache/dynamodb/dynamodb_test.go new file mode 100644 index 00000000..55679401 --- /dev/null +++ b/src/cache/dynamodb/dynamodb_test.go @@ -0,0 +1,140 @@ +package dynamodb + +import ( + "context" + "os" + "testing" + "time" + + "github.com/hearchco/agent/src/config" +) + +func newDynamoDBConf() config.DynamoDB { + awsRegion := os.Getenv("AWS_REGION") + if awsRegion == "" { + panic("AWS_REGION environment variable not set") + } + + tableName := os.Getenv("DYNAMODB_TABLE") + if tableName == "" { + panic("DYNAMODB_TABLE environment variable not set") + } + + return config.DynamoDB{Region: awsRegion, Table: tableName} +} + +var ( + conf = newDynamoDBConf() + keyPrefix = "TEST_" +) + +func TestNew(t *testing.T) { + ctx := context.Background() + _, err := New(ctx, keyPrefix, conf) + if err != nil { + t.Errorf("error creating dynamodb client: %v", err) + } +} + +func TestClose(t *testing.T) { + ctx := context.Background() + db, err := New(ctx, keyPrefix, conf) + if err != nil { + t.Errorf("error creating dynamodb client: %v", err) + } + + db.Close() +} + +func TestSet(t *testing.T) { + ctx := context.Background() + db, err := New(ctx, keyPrefix, conf) + if err != nil { + t.Errorf("error creating dynamodb client: %v", err) + } + + err = db.Set("testkeyset", "testvalue") + if err != nil { + t.Errorf("error setting key-value pair: %v", err) + } +} + +func TestSetTTL(t *testing.T) { + ctx := context.Background() + db, err := New(ctx, keyPrefix, conf) + if err != nil { + t.Errorf("error creating dynamodb client: %v", err) + } + + err = db.Set("testkeysetttl", "testvalue", 100*time.Second) + if err != nil { + t.Errorf("error setting key-value pair with TTL: %v", err) + } +} + +func TestGet(t *testing.T) { + ctx := context.Background() + db, err := New(ctx, keyPrefix, conf) + if err != nil { + t.Errorf("error creating dynamodb client: %v", err) + } + + err = db.Set("testkeyget", "testvalue") + if err != nil { + t.Errorf("error setting key-value pair: %v", err) + } + + var value string + err = db.Get("testkeyget", &value) + if err != nil { + t.Errorf("error getting value: %v", err) + } + + if value != "testvalue" { + t.Errorf("expected value: testvalue, got: %v", value) + } +} + +func TestGetTTL(t *testing.T) { + ctx := context.Background() + db, err := New(ctx, keyPrefix, conf) + if err != nil { + t.Errorf("error creating dynamodb client: %v", err) + } + + err = db.Set("testkeygetttl", "testvalue", 100*time.Second) + if err != nil { + t.Errorf("error setting key-value pair with TTL: %v", err) + } + + ttl, err := db.GetTTL("testkeygetttl") + if err != nil { + t.Errorf("error getting TTL: %v", err) + } + + // TTL is not exact, so we check for a range + if ttl > 100*time.Second || ttl < 99*time.Second { + t.Errorf("expected 100s >= ttl >= 99s, got: %v", ttl) + } +} + +func TestGetExpired(t *testing.T) { + ctx := context.Background() + db, err := New(ctx, keyPrefix, conf) + if err != nil { + t.Errorf("error creating dynamodb client: %v", err) + } + + err = db.Set("testkeygetexpired", "testvalue", 1*time.Second) + if err != nil { + t.Errorf("error setting key-value pair with TTL: %v", err) + } + + time.Sleep(1 * time.Second) + + var value string + err = db.Get("testkeygetexpired", &value) + if err == nil || value != "" { + t.Errorf("expected no value, got: %v, err: %v", value, err) + } +} diff --git a/src/config/defaults.go b/src/config/defaults.go index fb5ccd9a..b7342610 100644 --- a/src/config/defaults.go +++ b/src/config/defaults.go @@ -23,6 +23,9 @@ func New() Config { Host: "localhost", Port: 6379, }, + DynamoDB: DynamoDB{ + Table: "hearchco", + }, }, ImageProxy: ImageProxy{ Timeout: 3 * time.Second, diff --git a/src/config/structs_server.go b/src/config/structs_server.go index b0184eeb..932c34e2 100644 --- a/src/config/structs_server.go +++ b/src/config/structs_server.go @@ -51,6 +51,8 @@ type Cache struct { TTL TTL // Redis specific settings. Redis Redis + // DynamoDB specific settings. + DynamoDB DynamoDB } // ReaderTTL is format in which the config is read from the config file and environment variables. @@ -75,6 +77,11 @@ type Redis struct { Database uint8 `koanf:"database"` } +type DynamoDB struct { + Region string `koanf:"region"` + Table string `koanf:"table"` +} + // ReaderProxy is format in which the config is read from the config file and environment variables. // In format. // Example: 1s, 1m, 1h, 1d, 1w, 1M, 1y. From ca82348cc1e029d3e02bae1b0c971bb605424535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?= <31509435+aleksasiriski@users.noreply.github.com> Date: Wed, 10 Jul 2024 04:02:34 +0200 Subject: [PATCH 2/9] fix(cache): dynamodb with tests and CI --- .github/workflows/ci_test.yml | 47 ++++++++++++++++++++++++++++- Makefile | 12 ++++++-- scripts/test-dynamodb-docker.sh | 23 ++++++++++++++ scripts/test-dynamodb-podman.sh | 23 ++++++++++++++ scripts/test-dynamodb.sh | 19 ++++++++++++ src/cache/dynamodb/dynamodb.go | 30 +++++++++++++++++- src/cache/dynamodb/dynamodb_test.go | 13 ++++++-- src/config/structs_server.go | 2 ++ 8 files changed, 162 insertions(+), 7 deletions(-) create mode 100644 scripts/test-dynamodb-docker.sh create mode 100644 scripts/test-dynamodb-podman.sh create mode 100644 scripts/test-dynamodb.sh diff --git a/.github/workflows/ci_test.yml b/.github/workflows/ci_test.yml index bd0b71ff..e878de14 100644 --- a/.github/workflows/ci_test.yml +++ b/.github/workflows/ci_test.yml @@ -49,6 +49,15 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + dynamodb: + image: amazon/dynamodb-local + ports: + - 8000:8000 + options: >- + --health-cmd "curl -s http://localhost:8000/shell/ -o /dev/null" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - name: Checkout repository @@ -62,7 +71,7 @@ jobs: - name: Generate go code from go:generate comments run: make install - - name: Test units (w/o engines & redis) + - name: Test units (w/o engines & cache) run: make test - name: Test units (only redis) @@ -71,6 +80,42 @@ jobs: REDIS_HOST: localhost REDIS_PORT: "6379" + - name: Setup AWS CLI + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: "hearchco" + aws-secret-access-key: "hearchco" + aws-region: "hearchco-test-1" + + - name: Setup DynamoDB table KV + env: + DYNAMODB_TABLE: "hearchco" + DYNAMODB_ENDPOINT: "http://localhost:8000" + run: >- + aws dynamodb create-table + --table-name $DYNAMODB_TABLE + --attribute-definitions AttributeName=Key,AttributeType=S + --key-schema AttributeName=Key,KeyType=HASH + --billing-mode PAY_PER_REQUEST + --endpoint-url $DYNAMODB_ENDPOINT + + - name: Setup DynamoDB table TTL + env: + DYNAMODB_TABLE: "hearchco" + DYNAMODB_ENDPOINT: "http://localhost:8000" + run: >- + aws dynamodb update-time-to-live + --table-name $DYNAMODB_TABLE + --time-to-live-specification "Enabled=true, AttributeName=TTL" + --endpoint-url $DYNAMODB_ENDPOINT + + - name: Test units (only dynamodb) + run: make test-dynamodb + env: + AWS_REGION: "hearchco-test-1" + DYNAMODB_TABLE: "hearchco" + DYNAMODB_ENDPOINT: "http://localhost:8000" + test-engines: runs-on: ubuntu-latest permissions: diff --git a/Makefile b/Makefile index de1fc7d6..83d2e64c 100644 --- a/Makefile +++ b/Makefile @@ -32,9 +32,15 @@ test-redis-podman: sh ./scripts/test-redis-podman.sh test-redis-docker: sh ./scripts/test-redis-docker.sh -test-all: test test-redis test-engines -test-all-podman: test test-redis-podman test-engines -test-all-docker: test test-redis-docker test-engines +test-dynamodb: + sh ./scripts/test-dynamodb.sh +test-dynamodb-podman: + sh ./scripts/test-dynamodb-podman.sh +test-dynamodb-docker: + sh ./scripts/test-dynamodb-docker.sh +test-all: test test-redis test-dynamodb test-engines +test-all-podman: test test-redis-podman test-dynamodb-podman test-engines +test-all-docker: test test-redis-docker test-dynamodb-docker test-engines update: go get -u ./... diff --git a/scripts/test-dynamodb-docker.sh b/scripts/test-dynamodb-docker.sh new file mode 100644 index 00000000..c7addb04 --- /dev/null +++ b/scripts/test-dynamodb-docker.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +export AWS_REGION=hearchco-test-1 +export AWS_ACCESS_KEY_ID=hearchco +export AWS_SECRET_ACCESS_KEY=hearchco +export DYNAMODB_TABLE=hearchco_test +export DYNAMODB_ENDPOINT=http://localhost:8000 + +docker run --rm --name hearchco-dynamodb -d -p 8000:8000 docker.io/amazon/dynamodb-local && \ +sleep 5 && \ +aws dynamodb create-table \ + --table-name $DYNAMODB_TABLE \ + --attribute-definitions AttributeName=Key,AttributeType=S \ + --key-schema AttributeName=Key,KeyType=HASH \ + --billing-mode PAY_PER_REQUEST \ + --endpoint-url $DYNAMODB_ENDPOINT && \ +aws dynamodb update-time-to-live \ + --table-name $DYNAMODB_TABLE \ + --time-to-live-specification "Enabled=true, AttributeName=TTL" \ + --endpoint-url $DYNAMODB_ENDPOINT && \ +go test $(go list ./... | grep /dynamodb) -count=1 + +docker stop hearchco-dynamodb \ No newline at end of file diff --git a/scripts/test-dynamodb-podman.sh b/scripts/test-dynamodb-podman.sh new file mode 100644 index 00000000..3f235e7b --- /dev/null +++ b/scripts/test-dynamodb-podman.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +export AWS_REGION=hearchco-test-1 +export AWS_ACCESS_KEY_ID=hearchco +export AWS_SECRET_ACCESS_KEY=hearchco +export DYNAMODB_TABLE=hearchco_test +export DYNAMODB_ENDPOINT=http://localhost:8000 + +podman run --rm --name hearchco-dynamodb -d -p 8000:8000 docker.io/amazon/dynamodb-local && \ +sleep 5 && \ +aws dynamodb create-table \ + --table-name $DYNAMODB_TABLE \ + --attribute-definitions AttributeName=Key,AttributeType=S \ + --key-schema AttributeName=Key,KeyType=HASH \ + --billing-mode PAY_PER_REQUEST \ + --endpoint-url $DYNAMODB_ENDPOINT && \ +aws dynamodb update-time-to-live \ + --table-name $DYNAMODB_TABLE \ + --time-to-live-specification "Enabled=true, AttributeName=TTL" \ + --endpoint-url $DYNAMODB_ENDPOINT && \ +go test $(go list ./... | grep /dynamodb) -count=1 + +podman stop hearchco-dynamodb \ No newline at end of file diff --git a/scripts/test-dynamodb.sh b/scripts/test-dynamodb.sh new file mode 100644 index 00000000..5bfa4ba0 --- /dev/null +++ b/scripts/test-dynamodb.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +export AWS_REGION=hearchco-test-1 +export AWS_ACCESS_KEY_ID=hearchco +export AWS_SECRET_ACCESS_KEY=hearchco +export DYNAMODB_TABLE=hearchco_test +export DYNAMODB_ENDPOINT=http://localhost:8000 + +aws dynamodb create-table \ + --table-name $DYNAMODB_TABLE \ + --attribute-definitions AttributeName=Key,AttributeType=S \ + --key-schema AttributeName=Key,KeyType=HASH \ + --billing-mode PAY_PER_REQUEST \ + --endpoint-url $DYNAMODB_ENDPOINT && \ +aws dynamodb update-time-to-live \ + --table-name $DYNAMODB_TABLE \ + --time-to-live-specification "Enabled=true, AttributeName=TTL" \ + --endpoint-url $DYNAMODB_ENDPOINT && \ +go test $(go list ./... | grep /dynamodb) -count=1 diff --git a/src/cache/dynamodb/dynamodb.go b/src/cache/dynamodb/dynamodb.go index aca6f1d7..afbc37c3 100644 --- a/src/cache/dynamodb/dynamodb.go +++ b/src/cache/dynamodb/dynamodb.go @@ -31,7 +31,19 @@ func New(ctx context.Context, keyPrefix string, conf config.DynamoDB) (DRV, erro return DRV{}, err } - client := dynamodb.NewFromConfig(cfg) + var client *dynamodb.Client + if conf.Endpoint != "" { + log.Warn(). + Str("endpoint", conf.Endpoint). + Msg("Using custom endpoint") + + cfg.BaseEndpoint = &conf.Endpoint + resolver := dynamodb.NewDefaultEndpointResolverV2() + client = dynamodb.NewFromConfig(cfg, dynamodb.WithEndpointResolverV2(resolver)) + } else { + client = dynamodb.NewFromConfig(cfg) + } + return DRV{ctx, keyPrefix, client, conf.Table}, nil } @@ -89,6 +101,22 @@ func (drv DRV) Get(k string, o any) error { return nil } + // Required because TTL isn't guaranteed to remove the item immediately + if result.Item["TTL"] != nil { + expirationTime, err := strconv.ParseInt(result.Item["TTL"].(*types.AttributeValueMemberN).Value, 10, 64) + if err != nil { + return fmt.Errorf("dynamodb.Get(): error parsing TTL value for key %v: %w", key, err) + } + + expiresIn := time.Until(time.Unix(expirationTime, 0)) + if expiresIn < 0 { + log.Trace(). + Str("key", key). + Msg("Value has expired") + return nil + } + } + value := result.Item["Value"].(*types.AttributeValueMemberS).Value if err := json.Unmarshal([]byte(value), o); err != nil { return fmt.Errorf("dynamodb.Get(): error unmarshaling value from dynamodb for key %v: %w", key, err) diff --git a/src/cache/dynamodb/dynamodb_test.go b/src/cache/dynamodb/dynamodb_test.go index 55679401..db0f0343 100644 --- a/src/cache/dynamodb/dynamodb_test.go +++ b/src/cache/dynamodb/dynamodb_test.go @@ -20,7 +20,12 @@ func newDynamoDBConf() config.DynamoDB { panic("DYNAMODB_TABLE environment variable not set") } - return config.DynamoDB{Region: awsRegion, Table: tableName} + endpoint := os.Getenv("DYNAMODB_ENDPOINT") + if endpoint == "" { + panic("DYNAMODB_ENDPOINT environment variable not set") + } + + return config.DynamoDB{Region: awsRegion, Table: tableName, Endpoint: endpoint} } var ( @@ -134,7 +139,11 @@ func TestGetExpired(t *testing.T) { var value string err = db.Get("testkeygetexpired", &value) - if err == nil || value != "" { + if err != nil { + t.Errorf("error getting value: %v", err) + } + + if value != "" { t.Errorf("expected no value, got: %v, err: %v", value, err) } } diff --git a/src/config/structs_server.go b/src/config/structs_server.go index 932c34e2..1282197f 100644 --- a/src/config/structs_server.go +++ b/src/config/structs_server.go @@ -80,6 +80,8 @@ type Redis struct { type DynamoDB struct { Region string `koanf:"region"` Table string `koanf:"table"` + // Endpoint is used for local testing. + Endpoint string `koanf:"endpoint"` } // ReaderProxy is format in which the config is read from the config file and environment variables. From 7ed796fa0118676dd578bc4b6a852ad1f6f1252d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?= <31509435+aleksasiriski@users.noreply.github.com> Date: Wed, 10 Jul 2024 04:04:15 +0200 Subject: [PATCH 3/9] chore(Makefile): enters --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 83d2e64c..f83cec88 100644 --- a/Makefile +++ b/Makefile @@ -26,18 +26,21 @@ test: sh ./scripts/test.sh test-engines: sh ./scripts/test-engines.sh + test-redis: sh ./scripts/test-redis.sh test-redis-podman: sh ./scripts/test-redis-podman.sh test-redis-docker: sh ./scripts/test-redis-docker.sh + test-dynamodb: sh ./scripts/test-dynamodb.sh test-dynamodb-podman: sh ./scripts/test-dynamodb-podman.sh test-dynamodb-docker: sh ./scripts/test-dynamodb-docker.sh + test-all: test test-redis test-dynamodb test-engines test-all-podman: test test-redis-podman test-dynamodb-podman test-engines test-all-docker: test test-redis-docker test-dynamodb-docker test-engines From 164e0b95187302f68d0ccdbaee5e0fadf3310477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?= <31509435+aleksasiriski@users.noreply.github.com> Date: Wed, 10 Jul 2024 04:06:12 +0200 Subject: [PATCH 4/9] chore: tests w/o dynamodb --- scripts/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test.sh b/scripts/test.sh index 73cdf4f7..ac69c653 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -go test $(go list ./... | grep -v /engines/ | grep -v /redis) +go test $(go list ./... | grep -v /engines/ | grep -v /redis | grep -v /dynamodb) From 1f4be3103397a6c6180b483f89b77ed4ee131d1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?= <31509435+aleksasiriski@users.noreply.github.com> Date: Wed, 10 Jul 2024 04:15:22 +0200 Subject: [PATCH 5/9] ci(testing): dynamodb aws creds --- .github/workflows/ci_test.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_test.yml b/.github/workflows/ci_test.yml index e878de14..cd5f8a6d 100644 --- a/.github/workflows/ci_test.yml +++ b/.github/workflows/ci_test.yml @@ -82,13 +82,12 @@ jobs: - name: Setup AWS CLI uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: "hearchco" - aws-secret-access-key: "hearchco" - aws-region: "hearchco-test-1" - name: Setup DynamoDB table KV env: + AWS_REGION: "hearchco-test-1" + AWS_ACCESS_KEY_ID: "hearchco" + AWS_SECRET_ACCESS_KEY: "hearchco" DYNAMODB_TABLE: "hearchco" DYNAMODB_ENDPOINT: "http://localhost:8000" run: >- @@ -101,6 +100,9 @@ jobs: - name: Setup DynamoDB table TTL env: + AWS_REGION: "hearchco-test-1" + AWS_ACCESS_KEY_ID: "hearchco" + AWS_SECRET_ACCESS_KEY: "hearchco" DYNAMODB_TABLE: "hearchco" DYNAMODB_ENDPOINT: "http://localhost:8000" run: >- @@ -113,6 +115,8 @@ jobs: run: make test-dynamodb env: AWS_REGION: "hearchco-test-1" + AWS_ACCESS_KEY_ID: "hearchco" + AWS_SECRET_ACCESS_KEY: "hearchco" DYNAMODB_TABLE: "hearchco" DYNAMODB_ENDPOINT: "http://localhost:8000" From 821d0c7fecceefcbff5f2b4bf597815c60e9308a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?= <31509435+aleksasiriski@users.noreply.github.com> Date: Wed, 10 Jul 2024 04:18:31 +0200 Subject: [PATCH 6/9] ci(testing): aws region --- .github/workflows/ci_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci_test.yml b/.github/workflows/ci_test.yml index cd5f8a6d..bc9f5237 100644 --- a/.github/workflows/ci_test.yml +++ b/.github/workflows/ci_test.yml @@ -82,6 +82,8 @@ jobs: - name: Setup AWS CLI uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: "eu-central-1" - name: Setup DynamoDB table KV env: From 31e17d8e2855923344557a4dc3cac598369f4819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?= <31509435+aleksasiriski@users.noreply.github.com> Date: Wed, 10 Jul 2024 04:22:56 +0200 Subject: [PATCH 7/9] ci(testing): aws cli --- .github/workflows/ci_test.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci_test.yml b/.github/workflows/ci_test.yml index bc9f5237..b6a2301f 100644 --- a/.github/workflows/ci_test.yml +++ b/.github/workflows/ci_test.yml @@ -80,14 +80,20 @@ jobs: REDIS_HOST: localhost REDIS_PORT: "6379" - - name: Setup AWS CLI - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-region: "eu-central-1" + - name: Install AWS CLI + run: | + sudo apt-get update + sudo apt-get install -y awscli + + - name: Configure AWS CLI for Local DynamoDB + run: | + aws configure set aws_access_key_id dummy + aws configure set aws_secret_access_key dummy + aws configure set region eu-central-1 - name: Setup DynamoDB table KV env: - AWS_REGION: "hearchco-test-1" + AWS_REGION: "eu-central-1" AWS_ACCESS_KEY_ID: "hearchco" AWS_SECRET_ACCESS_KEY: "hearchco" DYNAMODB_TABLE: "hearchco" @@ -102,7 +108,7 @@ jobs: - name: Setup DynamoDB table TTL env: - AWS_REGION: "hearchco-test-1" + AWS_REGION: "eu-central-1" AWS_ACCESS_KEY_ID: "hearchco" AWS_SECRET_ACCESS_KEY: "hearchco" DYNAMODB_TABLE: "hearchco" @@ -116,7 +122,7 @@ jobs: - name: Test units (only dynamodb) run: make test-dynamodb env: - AWS_REGION: "hearchco-test-1" + AWS_REGION: "eu-central-1" AWS_ACCESS_KEY_ID: "hearchco" AWS_SECRET_ACCESS_KEY: "hearchco" DYNAMODB_TABLE: "hearchco" From 9ecbde8fa392b3b361deaf856b66f5126aaaea60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?= <31509435+aleksasiriski@users.noreply.github.com> Date: Wed, 10 Jul 2024 04:25:06 +0200 Subject: [PATCH 8/9] ci(testing): spacing --- .github/workflows/ci_test.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_test.yml b/.github/workflows/ci_test.yml index b6a2301f..f4aa124f 100644 --- a/.github/workflows/ci_test.yml +++ b/.github/workflows/ci_test.yml @@ -100,11 +100,11 @@ jobs: DYNAMODB_ENDPOINT: "http://localhost:8000" run: >- aws dynamodb create-table - --table-name $DYNAMODB_TABLE - --attribute-definitions AttributeName=Key,AttributeType=S - --key-schema AttributeName=Key,KeyType=HASH - --billing-mode PAY_PER_REQUEST - --endpoint-url $DYNAMODB_ENDPOINT + --table-name $DYNAMODB_TABLE + --attribute-definitions AttributeName=Key,AttributeType=S + --key-schema AttributeName=Key,KeyType=HASH + --billing-mode PAY_PER_REQUEST + --endpoint-url $DYNAMODB_ENDPOINT - name: Setup DynamoDB table TTL env: @@ -115,9 +115,9 @@ jobs: DYNAMODB_ENDPOINT: "http://localhost:8000" run: >- aws dynamodb update-time-to-live - --table-name $DYNAMODB_TABLE - --time-to-live-specification "Enabled=true, AttributeName=TTL" - --endpoint-url $DYNAMODB_ENDPOINT + --table-name $DYNAMODB_TABLE + --time-to-live-specification "Enabled=true, AttributeName=TTL" + --endpoint-url $DYNAMODB_ENDPOINT - name: Test units (only dynamodb) run: make test-dynamodb From da42afef06dc98584439342d67a09e93bfa8b84c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?= <31509435+aleksasiriski@users.noreply.github.com> Date: Wed, 10 Jul 2024 04:28:35 +0200 Subject: [PATCH 9/9] chore(scripts): remove aws tables from dynamodb.sh --- scripts/test-dynamodb.sh | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/scripts/test-dynamodb.sh b/scripts/test-dynamodb.sh index 5bfa4ba0..cd74f41e 100644 --- a/scripts/test-dynamodb.sh +++ b/scripts/test-dynamodb.sh @@ -1,19 +1,2 @@ #!/usr/bin/env bash - -export AWS_REGION=hearchco-test-1 -export AWS_ACCESS_KEY_ID=hearchco -export AWS_SECRET_ACCESS_KEY=hearchco -export DYNAMODB_TABLE=hearchco_test -export DYNAMODB_ENDPOINT=http://localhost:8000 - -aws dynamodb create-table \ - --table-name $DYNAMODB_TABLE \ - --attribute-definitions AttributeName=Key,AttributeType=S \ - --key-schema AttributeName=Key,KeyType=HASH \ - --billing-mode PAY_PER_REQUEST \ - --endpoint-url $DYNAMODB_ENDPOINT && \ -aws dynamodb update-time-to-live \ - --table-name $DYNAMODB_TABLE \ - --time-to-live-specification "Enabled=true, AttributeName=TTL" \ - --endpoint-url $DYNAMODB_ENDPOINT && \ go test $(go list ./... | grep /dynamodb) -count=1