From 8453698fe06bef92e0018050358795a966a40429 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Sun, 26 Nov 2023 10:39:06 -0800 Subject: [PATCH 1/8] Add RediStore for RateLimiting --- common/aws/elasticcache/client.go | 46 ++++++++++++++++++++ common/aws/elasticcache/client_test.go | 37 ++++++++++++++++ common/store/redis_store.go | 41 ++++++++++++++++++ common/store/redis_store_test.go | 59 ++++++++++++++++++++++++++ disperser/cmd/apiserver/config.go | 9 ++++ disperser/cmd/apiserver/main.go | 27 +++++++++--- go.mod | 2 + go.sum | 30 +++---------- 8 files changed, 221 insertions(+), 30 deletions(-) create mode 100644 common/aws/elasticcache/client.go create mode 100644 common/aws/elasticcache/client_test.go create mode 100644 common/store/redis_store.go create mode 100644 common/store/redis_store_test.go diff --git a/common/aws/elasticcache/client.go b/common/aws/elasticcache/client.go new file mode 100644 index 0000000000..ed59419c9d --- /dev/null +++ b/common/aws/elasticcache/client.go @@ -0,0 +1,46 @@ +package elasticcache + +import ( + "context" + "time" + + "github.com/Layr-Labs/eigenda/common" + "github.com/go-redis/redis/v8" +) + +type RedisClientConfig struct { + EndpointURL string + Port string +} + +type RedisClient struct { + redisClient *redis.Client + logger common.Logger // Ensure common.Logger is imported correctly +} + +func NewClient(cfg RedisClientConfig, logger common.Logger) (*RedisClient, error) { + redisClient := redis.NewClient(&redis.Options{ + Addr: cfg.EndpointURL + ":" + cfg.Port, + Password: "", // no password set + DB: 0, // use default DB + }) + + // Test the Redis connection + _, err := redisClient.Ping(context.Background()).Result() + if err != nil { + return nil, err // Return the error instead of logging and exiting + } + logger.Info("Redis connection successful") + + return &RedisClient{redisClient: redisClient, logger: logger}, nil +} + +// Get retrieves a value from Redis +func (c *RedisClient) Get(ctx context.Context, key string) *redis.StringCmd { + return c.redisClient.Get(ctx, key) +} + +// Set sets a value in Redis +func (c *RedisClient) Set(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.StatusCmd { + return c.redisClient.Set(ctx, key, value, expiration) +} diff --git a/common/aws/elasticcache/client_test.go b/common/aws/elasticcache/client_test.go new file mode 100644 index 0000000000..d6f1d47dd8 --- /dev/null +++ b/common/aws/elasticcache/client_test.go @@ -0,0 +1,37 @@ +package elasticcache_test + +import ( + "context" + "testing" + "time" + + "github.com/Layr-Labs/eigenda/common" + elasticCache "github.com/Layr-Labs/eigenda/common/aws/elasticcache" + "github.com/stretchr/testify/assert" +) + +func TestRedisClient(t *testing.T) { + // Set up the Redis client + cfg := elasticCache.RedisClientConfig{ + EndpointURL: "localhost", + Port: "6379", // Assuming Redis is running on the default port + } + logger := common.Logger{} // Replace with actual logger initialization + + client, err := elasticCache.NewClient(cfg, logger) + if err != nil { + t.Fatalf("Failed to create Redis client: %v", err) + } + + // Test setting a value + key := "testKey" + value := "testValue" + _, err = client.Set(context.Background(), key, value, 10*time.Second).Result() + assert.NoError(t, err, "Set should not return an error") + + // Test getting the value + stringCmd := client.Get(context.Background(), key) + result, err := stringCmd.Result() + assert.NoError(t, err, "Get should not return an error") + assert.Equal(t, value, result, "Get should return the value that was set") +} diff --git a/common/store/redis_store.go b/common/store/redis_store.go new file mode 100644 index 0000000000..f2feec335c --- /dev/null +++ b/common/store/redis_store.go @@ -0,0 +1,41 @@ +package store + +import ( + "context" + "encoding/json" + + "github.com/Layr-Labs/eigenda/common" + commoncache "github.com/Layr-Labs/eigenda/common/aws/elasticcache" +) + +type RedisStore[T any] struct { + client *commoncache.RedisClient +} + +func NewRedisStore[T any](client *commoncache.RedisClient) common.KVStore[T] { + return &RedisStore[T]{client: client} +} + +func (s *RedisStore[T]) GetItem(ctx context.Context, key string) (*T, error) { + val, err := s.client.Get(ctx, key).Result() + if err != nil { + return nil, err + } + + var item T + err = json.Unmarshal([]byte(val), &item) + if err != nil { + return nil, err + } + + return &item, nil +} + +func (s *RedisStore[T]) UpdateItem(ctx context.Context, key string, value *T) error { + jsonData, err := json.Marshal(value) + if err != nil { + return err + } + + return s.client.Set(ctx, key, jsonData, 0).Err() // 0 means no expiration +} diff --git a/common/store/redis_store_test.go b/common/store/redis_store_test.go new file mode 100644 index 0000000000..ab050d1fc9 --- /dev/null +++ b/common/store/redis_store_test.go @@ -0,0 +1,59 @@ +package store_test + +import ( + "context" + "os" + "testing" + "time" + + "github.com/Layr-Labs/eigenda/common" + "github.com/Layr-Labs/eigenda/common/aws/elasticcache" + "github.com/Layr-Labs/eigenda/common/store" + "github.com/Layr-Labs/eigenda/inabox/deploy" + "github.com/stretchr/testify/assert" +) + +func TestRedisStore(t *testing.T) { + + deployLocalStack = !(os.Getenv("DEPLOY_LOCALSTACK") == "false") + if !deployLocalStack { + localStackPort = os.Getenv("LOCALSTACK_PORT") + } + + if deployLocalStack { + var err error + dockertestPool, dockertestResource, err = deploy.StartDockertestWithLocalstackContainer(localStackPort) + if err != nil { + teardown() + panic("failed to start localstack container") + } + } + + // Set up the Redis client to point to your local Redis server + clientConfig := elasticcache.RedisClientConfig{ + EndpointURL: "localhost", + Port: "6379", + } + redisClient, err := elasticcache.NewClient(clientConfig) // Assuming logger can be nil + if err != nil { + t.Fatalf("Failed to create Redis client: %v", err) + } + + redisStore := store.NewRedisStore[common.RateBucketParams](redisClient) + + // Run your tests here + // Example: Test Set and Get + ctx := context.Background() + testKey := "testKey" + testValue := common.RateBucketParams{ + BucketLevels: []time.Duration{time.Second, time.Minute}, + LastRequestTime: time.Now().UTC(), + } + + err = redisStore.UpdateItem(ctx, testKey, &testValue) + assert.NoError(t, err, "UpdateItem should not return an error") + + result, err := redisStore.GetItem(ctx, testKey) + assert.NoError(t, err, "GetItem should not return an error") + assert.Equal(t, testValue, *result, "GetItem should return the value that was set") +} diff --git a/disperser/cmd/apiserver/config.go b/disperser/cmd/apiserver/config.go index b0af8244ae..b234065ad9 100644 --- a/disperser/cmd/apiserver/config.go +++ b/disperser/cmd/apiserver/config.go @@ -2,6 +2,7 @@ package main import ( "github.com/Layr-Labs/eigenda/common/aws" + "github.com/Layr-Labs/eigenda/common/aws/elasticcache" "github.com/Layr-Labs/eigenda/common/geth" "github.com/Layr-Labs/eigenda/common/logging" "github.com/Layr-Labs/eigenda/common/ratelimit" @@ -27,6 +28,8 @@ type Config struct { BLSOperatorStateRetrieverAddr string EigenDAServiceManagerAddr string + RateLimiterRedisClient bool + RedisClientConfig elasticcache.RedisClientConfig } func NewConfig(ctx *cli.Context) (Config, error) { @@ -59,6 +62,12 @@ func NewConfig(ctx *cli.Context) (Config, error) { BLSOperatorStateRetrieverAddr: ctx.GlobalString(flags.BlsOperatorStateRetrieverFlag.Name), EigenDAServiceManagerAddr: ctx.GlobalString(flags.EigenDAServiceManagerFlag.Name), + + // TODO Define a new config for RedisClientConfig + RedisClientConfig: elasticcache.RedisClientConfig{ + EndpointURL: ctx.GlobalString("redis-endpoint-url"), + Port: ctx.GlobalString("redis-port"), + }, } return config, nil } diff --git a/disperser/cmd/apiserver/main.go b/disperser/cmd/apiserver/main.go index 65484b3fb5..6c5039d06a 100644 --- a/disperser/cmd/apiserver/main.go +++ b/disperser/cmd/apiserver/main.go @@ -12,6 +12,7 @@ import ( "github.com/Layr-Labs/eigenda/disperser/common/blobstore" "github.com/Layr-Labs/eigenda/common/aws/dynamodb" + "github.com/Layr-Labs/eigenda/common/aws/elasticcache" "github.com/Layr-Labs/eigenda/common/aws/s3" "github.com/Layr-Labs/eigenda/common/geth" "github.com/Layr-Labs/eigenda/common/logging" @@ -97,16 +98,30 @@ func RunDisperserServer(ctx *cli.Context) error { globalParams := config.RatelimiterConfig.GlobalRateParams var bucketStore common.KVStore[common.RateBucketParams] - if config.BucketTableName != "" { - dynamoClient, err := dynamodb.NewClient(config.AwsClientConfig, logger) + + // Can be defined as a Factory of Stores used by RateLimiter + if config.RateLimiterRedisClient { + redisClient, err := elasticcache.NewClient(config.RedisClientConfig, logger) if err != nil { return err } - bucketStore = store.NewDynamoParamStore[common.RateBucketParams](dynamoClient, config.BucketTableName) + + bucketStore = store.NewRedisStore[common.RateBucketParams](redisClient) + } else { - bucketStore, err = store.NewLocalParamStore[common.RateBucketParams](config.BucketStoreSize) - if err != nil { - return err + if config.BucketTableName != "" { + dynamoClient, err := dynamodb.NewClient(config.AwsClientConfig, logger) + if err != nil { + return err + } + bucketStore = store.NewDynamoParamStore[common.RateBucketParams](dynamoClient, config.BucketTableName) + } + + if bucketStore == nil { + bucketStore, err = store.NewLocalParamStore[common.RateBucketParams](config.BucketStoreSize) + if err != nil { + return err + } } } ratelimiter = ratelimit.NewRateLimiter(globalParams, bucketStore, logger) diff --git a/go.mod b/go.mod index b9552cc540..bb22e6e833 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/fxamacker/cbor/v2 v2.5.0 github.com/gin-contrib/logger v0.2.6 github.com/gin-gonic/gin v1.9.1 + github.com/go-redis/redis/v8 v8.11.5 github.com/hashicorp/go-multierror v1.1.1 github.com/joho/godotenv v1.5.1 github.com/onsi/ginkgo/v2 v2.11.0 @@ -69,6 +70,7 @@ require ( github.com/containerd/continuity v0.3.0 // indirect github.com/crate-crypto/go-kzg-4844 v0.3.0 // indirect github.com/deckarep/golang-set/v2 v2.1.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/docker/cli v20.10.17+incompatible // indirect github.com/docker/docker v24.0.6+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect diff --git a/go.sum b/go.sum index 782781251f..8f404c9fc6 100644 --- a/go.sum +++ b/go.sum @@ -109,7 +109,6 @@ github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edY github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= -github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.2 h1:GDaNjuWSGu09guE9Oql0MSTNhNCLlWwO8y/xM5BzcbM= github.com/bytedance/sonic v1.9.2/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= @@ -117,7 +116,6 @@ github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqy github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= -github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= @@ -129,7 +127,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= -github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= @@ -162,7 +159,6 @@ github.com/crate-crypto/go-kzg-4844 v0.3.0 h1:UBlWE0CgyFqqzTI+IFyCzA7A3Zw4iip6uz github.com/crate-crypto/go-kzg-4844 v0.3.0/go.mod h1:SBP7ikXEgDnUPONgm33HtuDZEDtWa3L4QtN1ocJSEQ4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -175,6 +171,8 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docker/cli v20.10.17+incompatible h1:eO2KS7ZFeov5UJeaDmIs1NFEDRf32PaqRpvoEkKBy5M= github.com/docker/cli v20.10.17+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE= @@ -198,7 +196,6 @@ github.com/ethereum/go-ethereum v1.13.4/go.mod h1:I0U5VewuuTzvBtVzKo7b3hJzDhXOUt github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -221,7 +218,6 @@ github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnR github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= -github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk= github.com/gin-contrib/logger v0.2.6 h1:u+tvbiQhGEyuJgZSHNja3WD800ILduVyk5xKop160dw= github.com/gin-contrib/logger v0.2.6/go.mod h1:ZDkY/xiMqbZdz83enCHjMqxJUFRzB8bq0kjyMmjr3qU= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= @@ -234,7 +230,6 @@ github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SU github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= -github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= @@ -253,7 +248,6 @@ github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyr github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= -github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= @@ -263,8 +257,9 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k= github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= +github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= @@ -288,7 +283,6 @@ github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -336,7 +330,6 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= -github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -344,7 +337,6 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 h1:3JQNjnMRil1yD0IfZKHF9GxxWKDJGj8I0IqOUol//sw= -github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.3 h1:K8UWO1HUJpRMXBxbmaY1Y8IAMZC/RsKB+ArEnnK4l5o= @@ -405,16 +397,13 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= -github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2 h1:hRGSmZu7j271trc9sneMrpOW7GN5ngLm8YUZIPzf394= -github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -448,7 +437,6 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= -github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= @@ -471,15 +459,15 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -509,7 +497,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= -github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -528,7 +515,6 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= -github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= @@ -641,7 +627,6 @@ go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnw go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= -go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= @@ -778,7 +763,6 @@ golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -846,7 +830,6 @@ gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/R gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -862,7 +845,6 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= -gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= From 313ac825366560c4c018d9e7edaa81e48ed9169d Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Sun, 26 Nov 2023 10:48:31 -0800 Subject: [PATCH 2/8] Update Tests --- common/aws/elasticcache/client_test.go | 4 +--- common/store/redis_store_test.go | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/common/aws/elasticcache/client_test.go b/common/aws/elasticcache/client_test.go index d6f1d47dd8..a11710e420 100644 --- a/common/aws/elasticcache/client_test.go +++ b/common/aws/elasticcache/client_test.go @@ -5,7 +5,6 @@ import ( "testing" "time" - "github.com/Layr-Labs/eigenda/common" elasticCache "github.com/Layr-Labs/eigenda/common/aws/elasticcache" "github.com/stretchr/testify/assert" ) @@ -16,9 +15,8 @@ func TestRedisClient(t *testing.T) { EndpointURL: "localhost", Port: "6379", // Assuming Redis is running on the default port } - logger := common.Logger{} // Replace with actual logger initialization - client, err := elasticCache.NewClient(cfg, logger) + client, err := elasticCache.NewClient(cfg, nil) if err != nil { t.Fatalf("Failed to create Redis client: %v", err) } diff --git a/common/store/redis_store_test.go b/common/store/redis_store_test.go index ab050d1fc9..edd98669a3 100644 --- a/common/store/redis_store_test.go +++ b/common/store/redis_store_test.go @@ -34,7 +34,7 @@ func TestRedisStore(t *testing.T) { EndpointURL: "localhost", Port: "6379", } - redisClient, err := elasticcache.NewClient(clientConfig) // Assuming logger can be nil + redisClient, err := elasticcache.NewClient(clientConfig, nil) // Assuming logger can be nil if err != nil { t.Fatalf("Failed to create Redis client: %v", err) } From 54e1c831ebaa63c037b86c8ea559d828eb2a51ee Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Mon, 27 Nov 2023 22:03:52 -0800 Subject: [PATCH 3/8] update comment --- common/store/redis_store_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/store/redis_store_test.go b/common/store/redis_store_test.go index edd98669a3..95661b84fc 100644 --- a/common/store/redis_store_test.go +++ b/common/store/redis_store_test.go @@ -41,8 +41,7 @@ func TestRedisStore(t *testing.T) { redisStore := store.NewRedisStore[common.RateBucketParams](redisClient) - // Run your tests here - // Example: Test Set and Get + // Test Update and Get Item ctx := context.Background() testKey := "testKey" testValue := common.RateBucketParams{ From 760ed682cb00cdf51bf3ef8eb884b2bce63532b3 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Tue, 28 Nov 2023 09:18:44 -0800 Subject: [PATCH 4/8] Add shell script to start redis-cluster --- inabox/create-redis-cluster.sh | 31 +++++++++++++++++++++++++++++++ inabox/deploy/localstack.go | 10 +++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 inabox/create-redis-cluster.sh diff --git a/inabox/create-redis-cluster.sh b/inabox/create-redis-cluster.sh new file mode 100644 index 0000000000..b1eac939c7 --- /dev/null +++ b/inabox/create-redis-cluster.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -e + +REDIS_CLUSTER_ID="test-eigenda-redis-cluster" +REDIS_PORT="6379" +AWS_REGION="us-east-1" + +# Check if the Redis cluster already exists +function redis_cluster_exists() { + aws elasticache describe-cache-clusters --region $AWS_REGION | grep -q $REDIS_CLUSTER_ID + return $? +} + +# Start Redis service using LocalStack +function create_redis_cluster() { + aws elasticache create-cache-cluster \ + --cache-cluster-id $REDIS_CLUSTER_ID \ + --engine redis \ + --cache-node-type cache.t2.micro \ + --num-cache-nodes 1 \ + --port $REDIS_PORT \ + --region $AWS_REGION +} + +# Check if Redis cluster exists and create it if it does not +if redis_cluster_exists; then + echo "Redis cluster $REDIS_CLUSTER_ID already exists." +else + echo "Creating Redis cluster $REDIS_CLUSTER_ID." + create_redis_cluster +fi diff --git a/inabox/deploy/localstack.go b/inabox/deploy/localstack.go index 020f807b65..cf0f899312 100644 --- a/inabox/deploy/localstack.go +++ b/inabox/deploy/localstack.go @@ -124,8 +124,16 @@ func DeployResources(pool *dockertest.Pool, localStackPort, metadataTableName, b _, err = test_utils.CreateTable(context.Background(), cfg, bucketTableName, store.GenerateTableSchema(10, 10, bucketTableName)) - return err + // Create Redis Cluster + if err := pool.Retry(func() error { + fmt.Println("Creating Redis Cluster") + return execCmd("./create-redis-cluster.sh", []string{}, []string{fmt.Sprintf("AWS_URL=http://0.0.0.0:%s", localStackPort)}) + }); err != nil { + fmt.Println("Could not connect to docker to create Redis cluster:", err) + return err + } + return err } func PurgeDockertestResources(pool *dockertest.Pool, resource *dockertest.Resource) { From ce20a530ecaf0fbd9cec04840f3d1c486af20131 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Tue, 28 Nov 2023 13:41:23 -0800 Subject: [PATCH 5/8] update permissions on create redis cluster script --- inabox/create-redis-cluster.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 inabox/create-redis-cluster.sh diff --git a/inabox/create-redis-cluster.sh b/inabox/create-redis-cluster.sh old mode 100644 new mode 100755 From 1023b66e47958c2120cd0c6ba6faa6b0d3280b1c Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Tue, 28 Nov 2023 14:04:06 -0800 Subject: [PATCH 6/8] update create redis-cluster shell script --- inabox/create-redis-cluster.sh | 42 +++++++++++++++------------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/inabox/create-redis-cluster.sh b/inabox/create-redis-cluster.sh index b1eac939c7..560df673f5 100755 --- a/inabox/create-redis-cluster.sh +++ b/inabox/create-redis-cluster.sh @@ -1,31 +1,25 @@ #!/bin/bash -set -e -REDIS_CLUSTER_ID="test-eigenda-redis-cluster" -REDIS_PORT="6379" -AWS_REGION="us-east-1" +# Specify the Redis Docker image version +REDIS_IMAGE="redis:latest" -# Check if the Redis cluster already exists -function redis_cluster_exists() { - aws elasticache describe-cache-clusters --region $AWS_REGION | grep -q $REDIS_CLUSTER_ID - return $? -} +# Specify the Redis container name +REDIS_CONTAINER_NAME="test-eigenda-redis-cluster" -# Start Redis service using LocalStack -function create_redis_cluster() { - aws elasticache create-cache-cluster \ - --cache-cluster-id $REDIS_CLUSTER_ID \ - --engine redis \ - --cache-node-type cache.t2.micro \ - --num-cache-nodes 1 \ - --port $REDIS_PORT \ - --region $AWS_REGION -} +# Specify the port to be used for Redis +REDIS_PORT="6379" -# Check if Redis cluster exists and create it if it does not -if redis_cluster_exists; then - echo "Redis cluster $REDIS_CLUSTER_ID already exists." +# Check if the Redis container is already running +if [ $(docker ps -q -f name="$REDIS_CONTAINER_NAME") ]; then + echo "Redis container ($REDIS_CONTAINER_NAME) is already running." else - echo "Creating Redis cluster $REDIS_CLUSTER_ID." - create_redis_cluster + # Check if the Redis container exists but is stopped + if [ $(docker ps -aq -f name="$REDIS_CONTAINER_NAME") ]; then + # Start the existing Redis container + docker start "$REDIS_CONTAINER_NAME" + else + # Run a new Redis container + docker run --name "$REDIS_CONTAINER_NAME" -p "$REDIS_PORT:$REDIS_PORT" -d "$REDIS_IMAGE" + fi + echo "Redis server started and available on port $REDIS_PORT" fi From 74c264752190f244da67bb6a4275511f3eaee6c6 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Tue, 28 Nov 2023 21:44:29 -0800 Subject: [PATCH 7/8] add start docker ps for redis ut's --- common/aws/elasticcache/client_test.go | 48 +++++++++++++++++++++- common/store/redis_store_test.go | 56 ++++++++++++++++++++------ inabox/create-redis-cluster.sh | 33 +++++++-------- inabox/deploy/localstack.go | 2 +- 4 files changed, 108 insertions(+), 31 deletions(-) diff --git a/common/aws/elasticcache/client_test.go b/common/aws/elasticcache/client_test.go index a11710e420..4fed22e11c 100644 --- a/common/aws/elasticcache/client_test.go +++ b/common/aws/elasticcache/client_test.go @@ -2,13 +2,58 @@ package elasticcache_test import ( "context" + "log" + "os" "testing" "time" elasticCache "github.com/Layr-Labs/eigenda/common/aws/elasticcache" + cmock "github.com/Layr-Labs/eigenda/common/mock" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" "github.com/stretchr/testify/assert" ) +var pool *dockertest.Pool +var resource *dockertest.Resource + +func TestMain(m *testing.M) { + var err error + pool, err = dockertest.NewPool("") + if err != nil { + log.Fatalf("Could not connect to Docker: %v", err) + } + + resource, err = pool.RunWithOptions(&dockertest.RunOptions{ + Repository: "redis", + Tag: "latest", + PortBindings: map[docker.Port][]docker.PortBinding{ + "6379/tcp": {{HostIP: "", HostPort: "6379"}}, + }, + }) + if err != nil { + log.Fatalf("Could not start Redis container: %v", err) + } + + // Wait for Redis to be ready + if err := pool.Retry(func() error { + // Perform a health check... + return nil // return nil if healthy + }); err != nil { + log.Fatalf("Could not connect to Redis: %v", err) + } + + // Run tests + code := m.Run() + + // Teardown: Stop and remove the Redis container + if err := pool.Purge(resource); err != nil { + log.Fatalf("Could not purge Redis container: %v", err) + } + + os.Exit(code) +} + func TestRedisClient(t *testing.T) { // Set up the Redis client cfg := elasticCache.RedisClientConfig{ @@ -16,7 +61,8 @@ func TestRedisClient(t *testing.T) { Port: "6379", // Assuming Redis is running on the default port } - client, err := elasticCache.NewClient(cfg, nil) + logger := &cmock.Logger{} + client, err := elasticCache.NewClient(cfg, logger) if err != nil { t.Fatalf("Failed to create Redis client: %v", err) } diff --git a/common/store/redis_store_test.go b/common/store/redis_store_test.go index 95661b84fc..5d4b8f0c0c 100644 --- a/common/store/redis_store_test.go +++ b/common/store/redis_store_test.go @@ -2,39 +2,69 @@ package store_test import ( "context" + "log" "os" "testing" "time" "github.com/Layr-Labs/eigenda/common" "github.com/Layr-Labs/eigenda/common/aws/elasticcache" + cmock "github.com/Layr-Labs/eigenda/common/mock" "github.com/Layr-Labs/eigenda/common/store" - "github.com/Layr-Labs/eigenda/inabox/deploy" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" "github.com/stretchr/testify/assert" ) -func TestRedisStore(t *testing.T) { +var pool *dockertest.Pool +var resource *dockertest.Resource - deployLocalStack = !(os.Getenv("DEPLOY_LOCALSTACK") == "false") - if !deployLocalStack { - localStackPort = os.Getenv("LOCALSTACK_PORT") +func TestMain(m *testing.M) { + var err error + pool, err = dockertest.NewPool("") + if err != nil { + log.Fatalf("Could not connect to Docker: %v", err) } - if deployLocalStack { - var err error - dockertestPool, dockertestResource, err = deploy.StartDockertestWithLocalstackContainer(localStackPort) - if err != nil { - teardown() - panic("failed to start localstack container") - } + resource, err = pool.RunWithOptions(&dockertest.RunOptions{ + Repository: "redis", + Tag: "latest", + PortBindings: map[docker.Port][]docker.PortBinding{ + "6379/tcp": {{HostIP: "", HostPort: "6379"}}, + }, + }) + if err != nil { + log.Fatalf("Could not start Redis container: %v", err) } + // Wait for Redis to be ready + if err := pool.Retry(func() error { + // Perform a health check... + return nil // return nil if healthy + }); err != nil { + log.Fatalf("Could not connect to Redis: %v", err) + } + + // Run tests + code := m.Run() + + // Teardown: Stop and remove the Redis container + if err := pool.Purge(resource); err != nil { + log.Fatalf("Could not purge Redis container: %v", err) + } + + os.Exit(code) +} + +func TestRedisStore(t *testing.T) { + // Set up the Redis client to point to your local Redis server clientConfig := elasticcache.RedisClientConfig{ EndpointURL: "localhost", Port: "6379", } - redisClient, err := elasticcache.NewClient(clientConfig, nil) // Assuming logger can be nil + + redisClient, err := elasticcache.NewClient(clientConfig, &cmock.Logger{}) // Assuming logger can be nil if err != nil { t.Fatalf("Failed to create Redis client: %v", err) } diff --git a/inabox/create-redis-cluster.sh b/inabox/create-redis-cluster.sh index 560df673f5..f9fb5aeb81 100755 --- a/inabox/create-redis-cluster.sh +++ b/inabox/create-redis-cluster.sh @@ -1,25 +1,26 @@ #!/bin/bash -# Specify the Redis Docker image version -REDIS_IMAGE="redis:latest" +REDIS_CONTAINER_NAME="my-redis" +REDIS_PORT="6379" -# Specify the Redis container name -REDIS_CONTAINER_NAME="test-eigenda-redis-cluster" +echo "Pulling the latest Redis image from Docker Hub..." +docker pull redis:latest -# Specify the port to be used for Redis -REDIS_PORT="6379" +echo "Checking if Redis container ($REDIS_CONTAINER_NAME) already exists..." # Check if the Redis container is already running -if [ $(docker ps -q -f name="$REDIS_CONTAINER_NAME") ]; then +if [ "$(docker ps -q -f name=$REDIS_CONTAINER_NAME)" ]; then echo "Redis container ($REDIS_CONTAINER_NAME) is already running." + exit 0 +fi + +# Check if the Redis container exists but is stopped +if [ "$(docker ps -aq -f name=$REDIS_CONTAINER_NAME)" ]; then + echo "Starting existing Redis container ($REDIS_CONTAINER_NAME)..." + docker start "$REDIS_CONTAINER_NAME" else - # Check if the Redis container exists but is stopped - if [ $(docker ps -aq -f name="$REDIS_CONTAINER_NAME") ]; then - # Start the existing Redis container - docker start "$REDIS_CONTAINER_NAME" - else - # Run a new Redis container - docker run --name "$REDIS_CONTAINER_NAME" -p "$REDIS_PORT:$REDIS_PORT" -d "$REDIS_IMAGE" - fi - echo "Redis server started and available on port $REDIS_PORT" + echo "Starting a new Redis container ($REDIS_CONTAINER_NAME)..." + docker run --name "$REDIS_CONTAINER_NAME" -p "$REDIS_PORT:$REDIS_PORT" -d redis fi + +echo "Redis setup complete." \ No newline at end of file diff --git a/inabox/deploy/localstack.go b/inabox/deploy/localstack.go index cf0f899312..28559cad9d 100644 --- a/inabox/deploy/localstack.go +++ b/inabox/deploy/localstack.go @@ -127,7 +127,7 @@ func DeployResources(pool *dockertest.Pool, localStackPort, metadataTableName, b // Create Redis Cluster if err := pool.Retry(func() error { fmt.Println("Creating Redis Cluster") - return execCmd("./create-redis-cluster.sh", []string{}, []string{fmt.Sprintf("AWS_URL=http://0.0.0.0:%s", localStackPort)}) + return execCmd("./create-redis-cluster.sh", []string{}, nil) }); err != nil { fmt.Println("Could not connect to docker to create Redis cluster:", err) return err From 1fb6834bc15e4230fc18260de4560b7f258775c5 Mon Sep 17 00:00:00 2001 From: Siddharth More Date: Tue, 28 Nov 2023 22:08:16 -0800 Subject: [PATCH 8/8] remove create redis cluster shell script --- common/aws/elasticcache/client_test.go | 40 +++++++++++--------------- common/store/redis_store_test.go | 37 +++++++++--------------- inabox/create-redis-cluster.sh | 26 ----------------- inabox/deploy/localstack.go | 9 ------ 4 files changed, 30 insertions(+), 82 deletions(-) delete mode 100755 inabox/create-redis-cluster.sh diff --git a/common/aws/elasticcache/client_test.go b/common/aws/elasticcache/client_test.go index 4fed22e11c..284b4c5456 100644 --- a/common/aws/elasticcache/client_test.go +++ b/common/aws/elasticcache/client_test.go @@ -3,7 +3,6 @@ package elasticcache_test import ( "context" "log" - "os" "testing" "time" @@ -14,17 +13,15 @@ import ( "github.com/stretchr/testify/assert" ) -var pool *dockertest.Pool -var resource *dockertest.Resource - -func TestMain(m *testing.M) { - var err error - pool, err = dockertest.NewPool("") +func TestRedisClient(t *testing.T) { + // Start Docker pool + pool, err := dockertest.NewPool("") if err != nil { - log.Fatalf("Could not connect to Docker: %v", err) + t.Fatalf("Could not connect to Docker: %v", err) } - resource, err = pool.RunWithOptions(&dockertest.RunOptions{ + // Start Redis container + resource, err := pool.RunWithOptions(&dockertest.RunOptions{ Repository: "redis", Tag: "latest", PortBindings: map[docker.Port][]docker.PortBinding{ @@ -32,9 +29,16 @@ func TestMain(m *testing.M) { }, }) if err != nil { - log.Fatalf("Could not start Redis container: %v", err) + t.Fatalf("Could not start Redis container: %v", err) } + // Delay cleanup until after all tests have run + t.Cleanup(func() { + if err := pool.Purge(resource); err != nil { + t.Fatalf("Could not purge Redis container: %v", err) + } + }) + // Wait for Redis to be ready if err := pool.Retry(func() error { // Perform a health check... @@ -43,22 +47,10 @@ func TestMain(m *testing.M) { log.Fatalf("Could not connect to Redis: %v", err) } - // Run tests - code := m.Run() - - // Teardown: Stop and remove the Redis container - if err := pool.Purge(resource); err != nil { - log.Fatalf("Could not purge Redis container: %v", err) - } - - os.Exit(code) -} - -func TestRedisClient(t *testing.T) { - // Set up the Redis client + // Set up Redis client cfg := elasticCache.RedisClientConfig{ EndpointURL: "localhost", - Port: "6379", // Assuming Redis is running on the default port + Port: "6379", } logger := &cmock.Logger{} diff --git a/common/store/redis_store_test.go b/common/store/redis_store_test.go index 5d4b8f0c0c..3c035212f3 100644 --- a/common/store/redis_store_test.go +++ b/common/store/redis_store_test.go @@ -3,7 +3,6 @@ package store_test import ( "context" "log" - "os" "testing" "time" @@ -16,17 +15,15 @@ import ( "github.com/stretchr/testify/assert" ) -var pool *dockertest.Pool -var resource *dockertest.Resource - -func TestMain(m *testing.M) { - var err error - pool, err = dockertest.NewPool("") +func TestRedisStore(t *testing.T) { + // Start Docker pool + pool, err := dockertest.NewPool("") if err != nil { - log.Fatalf("Could not connect to Docker: %v", err) + t.Fatalf("Could not connect to Docker: %v", err) } - resource, err = pool.RunWithOptions(&dockertest.RunOptions{ + // Start Redis container + resource, err := pool.RunWithOptions(&dockertest.RunOptions{ Repository: "redis", Tag: "latest", PortBindings: map[docker.Port][]docker.PortBinding{ @@ -34,9 +31,16 @@ func TestMain(m *testing.M) { }, }) if err != nil { - log.Fatalf("Could not start Redis container: %v", err) + t.Fatalf("Could not start Redis container: %v", err) } + // Delay cleanup until after all tests have run + t.Cleanup(func() { + if err := pool.Purge(resource); err != nil { + t.Fatalf("Could not purge Redis container: %v", err) + } + }) + // Wait for Redis to be ready if err := pool.Retry(func() error { // Perform a health check... @@ -45,19 +49,6 @@ func TestMain(m *testing.M) { log.Fatalf("Could not connect to Redis: %v", err) } - // Run tests - code := m.Run() - - // Teardown: Stop and remove the Redis container - if err := pool.Purge(resource); err != nil { - log.Fatalf("Could not purge Redis container: %v", err) - } - - os.Exit(code) -} - -func TestRedisStore(t *testing.T) { - // Set up the Redis client to point to your local Redis server clientConfig := elasticcache.RedisClientConfig{ EndpointURL: "localhost", diff --git a/inabox/create-redis-cluster.sh b/inabox/create-redis-cluster.sh deleted file mode 100755 index f9fb5aeb81..0000000000 --- a/inabox/create-redis-cluster.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -REDIS_CONTAINER_NAME="my-redis" -REDIS_PORT="6379" - -echo "Pulling the latest Redis image from Docker Hub..." -docker pull redis:latest - -echo "Checking if Redis container ($REDIS_CONTAINER_NAME) already exists..." - -# Check if the Redis container is already running -if [ "$(docker ps -q -f name=$REDIS_CONTAINER_NAME)" ]; then - echo "Redis container ($REDIS_CONTAINER_NAME) is already running." - exit 0 -fi - -# Check if the Redis container exists but is stopped -if [ "$(docker ps -aq -f name=$REDIS_CONTAINER_NAME)" ]; then - echo "Starting existing Redis container ($REDIS_CONTAINER_NAME)..." - docker start "$REDIS_CONTAINER_NAME" -else - echo "Starting a new Redis container ($REDIS_CONTAINER_NAME)..." - docker run --name "$REDIS_CONTAINER_NAME" -p "$REDIS_PORT:$REDIS_PORT" -d redis -fi - -echo "Redis setup complete." \ No newline at end of file diff --git a/inabox/deploy/localstack.go b/inabox/deploy/localstack.go index 28559cad9d..51af406160 100644 --- a/inabox/deploy/localstack.go +++ b/inabox/deploy/localstack.go @@ -124,15 +124,6 @@ func DeployResources(pool *dockertest.Pool, localStackPort, metadataTableName, b _, err = test_utils.CreateTable(context.Background(), cfg, bucketTableName, store.GenerateTableSchema(10, 10, bucketTableName)) - // Create Redis Cluster - if err := pool.Retry(func() error { - fmt.Println("Creating Redis Cluster") - return execCmd("./create-redis-cluster.sh", []string{}, nil) - }); err != nil { - fmt.Println("Could not connect to docker to create Redis cluster:", err) - return err - } - return err }