feat(cache): dynamodb

.github/workflows/ci_test.yml

@@ -49,6 +49,15 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+      dynamodb:
+        image: amazon/dynamodb-local
+        ports:
+          - 8000:8000
+        options: >-
+          --health-cmd "curl -s http://localhost:8000/shell/ -o /dev/null"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
 
     steps:
       - name: Checkout repository
@@ -62,7 +71,7 @@ jobs:
       - name: Generate go code from go:generate comments
         run: make install
 
-      - name: Test units (w/o engines & redis)
+      - name: Test units (w/o engines & cache)
         run: make test
 
       - name: Test units (only redis)
@@ -71,6 +80,54 @@ jobs:
           REDIS_HOST: localhost
           REDIS_PORT: "6379"
 
+      - name: Install AWS CLI
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y awscli
+
+      - name: Configure AWS CLI for Local DynamoDB
+        run: |
+          aws configure set aws_access_key_id dummy
+          aws configure set aws_secret_access_key dummy
+          aws configure set region eu-central-1
+
+      - name: Setup DynamoDB table KV
+        env:
+          AWS_REGION: "eu-central-1"
+          AWS_ACCESS_KEY_ID: "hearchco"
+          AWS_SECRET_ACCESS_KEY: "hearchco"
+          DYNAMODB_TABLE: "hearchco"
+          DYNAMODB_ENDPOINT: "http://localhost:8000"
+        run: >-
+          aws dynamodb create-table
+          --table-name $DYNAMODB_TABLE
+          --attribute-definitions AttributeName=Key,AttributeType=S
+          --key-schema AttributeName=Key,KeyType=HASH
+          --billing-mode PAY_PER_REQUEST
+          --endpoint-url $DYNAMODB_ENDPOINT
+
+      - name: Setup DynamoDB table TTL
+        env:
+          AWS_REGION: "eu-central-1"
+          AWS_ACCESS_KEY_ID: "hearchco"
+          AWS_SECRET_ACCESS_KEY: "hearchco"
+          DYNAMODB_TABLE: "hearchco"
+          DYNAMODB_ENDPOINT: "http://localhost:8000"
+        run: >-
+          aws dynamodb update-time-to-live
+          --table-name $DYNAMODB_TABLE
+          --time-to-live-specification "Enabled=true, AttributeName=TTL"
+          --endpoint-url $DYNAMODB_ENDPOINT
+
+      - name: Test units (only dynamodb)
+        run: make test-dynamodb
+        env:
+          AWS_REGION: "eu-central-1"
+          AWS_ACCESS_KEY_ID: "hearchco"
+          AWS_SECRET_ACCESS_KEY: "hearchco"
+          DYNAMODB_TABLE: "hearchco"
+          DYNAMODB_ENDPOINT: "http://localhost:8000"
+
   test-engines:
     runs-on: ubuntu-latest
     permissions:

Makefile

@@ -26,15 +26,24 @@ test:
 	sh ./scripts/test.sh
 test-engines:
 	sh ./scripts/test-engines.sh
+
 test-redis:
 	sh ./scripts/test-redis.sh
 test-redis-podman:
 	sh ./scripts/test-redis-podman.sh
 test-redis-docker:
 	sh ./scripts/test-redis-docker.sh
-test-all: test test-redis test-engines
-test-all-podman: test test-redis-podman test-engines
-test-all-docker: test test-redis-docker test-engines
+
+test-dynamodb:
+	sh ./scripts/test-dynamodb.sh
+test-dynamodb-podman:
+	sh ./scripts/test-dynamodb-podman.sh
+test-dynamodb-docker:
+	sh ./scripts/test-dynamodb-docker.sh
+
+test-all: test test-redis test-dynamodb test-engines
+test-all-podman: test test-redis-podman test-dynamodb-podman test-engines
+test-all-docker: test test-redis-docker test-dynamodb-docker test-engines
 
 update:
 	go get -u ./...

go.mod

@@ -25,13 +25,30 @@ require (
 	golang.org/x/tools v0.23.0
 )
 
-require github.com/klauspost/compress v1.17.9
+require (
+	github.com/aws/aws-sdk-go-v2 v1.30.1
+	github.com/aws/aws-sdk-go-v2/config v1.27.24
+	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.1
+	github.com/klauspost/compress v1.17.9
+)
 
 require (
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/antchfx/htmlquery v1.3.2 // indirect
 	github.com/antchfx/xmlquery v1.4.1 // indirect
 	github.com/antchfx/xpath v1.3.1 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.24 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 // indirect
+	github.com/aws/smithy-go v1.20.3 // indirect
 	github.com/bits-and-blooms/bitset v1.13.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
@@ -43,6 +60,7 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/pprof v0.0.0-20240625030939-27f56978b8b0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect

go.sum

@@ -27,6 +27,36 @@ github.com/antchfx/xpath v1.3.1 h1:PNbFuUqHwWl0xRjvUPjJ95Agbmdj2uzzIwmQKgu4oCk=
 github.com/antchfx/xpath v1.3.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI=
 github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A=
+github.com/aws/aws-sdk-go-v2 v1.30.1 h1:4y/5Dvfrhd1MxRDD77SrfsDaj8kUkkljU7XE83NPV+o=
+github.com/aws/aws-sdk-go-v2 v1.30.1/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
+github.com/aws/aws-sdk-go-v2/config v1.27.24 h1:NM9XicZ5o1CBU/MZaHwFtimRpWx9ohAUAqkG6AqSqPo=
+github.com/aws/aws-sdk-go-v2/config v1.27.24/go.mod h1:aXzi6QJTuQRVVusAO8/NxpdTeTyr/wRcybdDtfUwJSs=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.24 h1:YclAsrnb1/GTQNt2nzv+756Iw4mF8AOzcDfweWwwm/M=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.24/go.mod h1:Hld7tmnAkoBQdTMNYZGzztzKRdA4fCdn9L83LOoigac=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 h1:Aznqksmd6Rfv2HQN9cpqIV/lQRMaIpJkLLaJ1ZI76no=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9/go.mod h1:WQr3MY7AxGNxaqAtsDWn+fBxmd4XvLkzeqQ8P1VM0/w=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 h1:5SAoZ4jYpGH4721ZNoS1znQrhOfZinOhc4XuTXx/nVc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13/go.mod h1:+rdA6ZLpaSeM7tSg/B0IEDinCIBJGmW8rKDFkYpP04g=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 h1:WIijqeaAO7TYFLbhsZmi2rgLEAtWOC1LhxCAVTJlSKw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13/go.mod h1:i+kbfa76PQbWw/ULoWnp51EYVWH4ENln76fLQE3lXT8=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
+github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.1 h1:Szwz1vpZkvfhFMJ0X5uUECgHeUmPAxk1UGqAVs/pARw=
+github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.1/go.mod h1:b4wouGyJlzkr2HAvPrDGgYNp1EtmlXOkzhEOvl0c0FQ=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI=
+github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.14 h1:X1J0Kd17n1PeXeoArNXlvnKewCyMvhVQh7iNMy6oi3s=
+github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.14/go.mod h1:VYMN7l7dxp6xtQRjqIau6d7QAbmPG+yJ75GtCy70f18=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 h1:I9zMeF107l0rJrpnHpjEiiTSCKYAIw8mALiXcPsGBiA=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15/go.mod h1:9xWJ3Q/S6Ojusz1UIkfycgD1mGirJfLLKqq3LPT7WN8=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 h1:p1GahKIjyMDZtiKoIn0/jAj/TkMzfzndDv5+zi2Mhgc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.1/go.mod h1:/vWdhoIoYA5hYoPZ6fm7Sv4d8701PiG5VKe8/pPJL60=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 h1:ORnrOK0C4WmYV/uYt3koHEWBLYsRDwk2Np+eEoyV4Z0=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2/go.mod h1:xyFHA4zGxgYkdD73VeezHt3vSKEG9EmFnGwoKlP00u4=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 h1:+woJ607dllHJQtsnJLi52ycuqHMwlW+Wqm2Ppsfp4nQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.1/go.mod h1:jiNR3JqT15Dm+QWq2SRgh0x0bCNSRP2L25+CqPNpJlQ=
+github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
+github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/awslabs/aws-lambda-go-api-proxy v0.16.2 h1:CJyGEyO1CIwOnXTU40urf0mchf6t3voxpvUDikOU9LY=
 github.com/awslabs/aws-lambda-go-api-proxy v0.16.2/go.mod h1:vxxjwBHe/KbgFeNlAP/Tvp4SsVRL3WQamcWRxqVh0z0=
 github.com/bits-and-blooms/bitset v1.2.2-0.20220111210104-dfa3e347c392/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
@@ -113,6 +143,10 @@ github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSo
 github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
 github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw=
 github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
 github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
@@ -308,6 +342,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

scripts/test-dynamodb-docker.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+export AWS_REGION=hearchco-test-1
+export AWS_ACCESS_KEY_ID=hearchco
+export AWS_SECRET_ACCESS_KEY=hearchco
+export DYNAMODB_TABLE=hearchco_test
+export DYNAMODB_ENDPOINT=http://localhost:8000
+
+docker run --rm --name hearchco-dynamodb -d -p 8000:8000 docker.io/amazon/dynamodb-local && \
+sleep 5 && \
+aws dynamodb create-table \
+    --table-name $DYNAMODB_TABLE \
+    --attribute-definitions AttributeName=Key,AttributeType=S \
+    --key-schema AttributeName=Key,KeyType=HASH \
+    --billing-mode PAY_PER_REQUEST \
+    --endpoint-url $DYNAMODB_ENDPOINT && \
+aws dynamodb update-time-to-live \
+    --table-name $DYNAMODB_TABLE \
+    --time-to-live-specification "Enabled=true, AttributeName=TTL" \
+    --endpoint-url $DYNAMODB_ENDPOINT && \
+go test $(go list ./... | grep /dynamodb) -count=1
+
+docker stop hearchco-dynamodb

scripts/test-dynamodb-podman.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+export AWS_REGION=hearchco-test-1
+export AWS_ACCESS_KEY_ID=hearchco
+export AWS_SECRET_ACCESS_KEY=hearchco
+export DYNAMODB_TABLE=hearchco_test
+export DYNAMODB_ENDPOINT=http://localhost:8000
+
+podman run --rm --name hearchco-dynamodb -d -p 8000:8000 docker.io/amazon/dynamodb-local && \
+sleep 5 && \
+aws dynamodb create-table \
+    --table-name $DYNAMODB_TABLE \
+    --attribute-definitions AttributeName=Key,AttributeType=S \
+    --key-schema AttributeName=Key,KeyType=HASH \
+    --billing-mode PAY_PER_REQUEST \
+    --endpoint-url $DYNAMODB_ENDPOINT && \
+aws dynamodb update-time-to-live \
+    --table-name $DYNAMODB_TABLE \
+    --time-to-live-specification "Enabled=true, AttributeName=TTL" \
+    --endpoint-url $DYNAMODB_ENDPOINT && \
+go test $(go list ./... | grep /dynamodb) -count=1
+
+podman stop hearchco-dynamodb

scripts/test-dynamodb.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+go test $(go list ./... | grep /dynamodb) -count=1

scripts/test.sh

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-go test $(go list ./... | grep -v /engines/ | grep -v /redis)
+go test $(go list ./... | grep -v /engines/ | grep -v /redis | grep -v /dynamodb)

src/cache/db.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/rs/zerolog/log"
 
+	"github.com/hearchco/agent/src/cache/dynamodb"
 	"github.com/hearchco/agent/src/cache/nocache"
 	"github.com/hearchco/agent/src/cache/redis"
 	"github.com/hearchco/agent/src/config"
@@ -25,6 +26,11 @@ func New(ctx context.Context, cacheConf config.Cache) (DB, error) {
 		if err != nil {
 			err = fmt.Errorf("failed creating a redis cache: %w", err)
 		}
+	case "dynamodb":
+		drv, err = dynamodb.New(ctx, cacheConf.KeyPrefix, cacheConf.DynamoDB)
+		if err != nil {
+			err = fmt.Errorf("failed creating a dynamodb cache: %w", err)
+		}
 	default:
 		drv, err = nocache.New()
 		if err != nil {