Release/0.7.2

.github/workflows/docker-publish-release.yaml

@@ -0,0 +1,78 @@
+name: docker-publish-release
+
+on:
+  push:
+    tags:
+      - v*
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      force:
+        description: "Force untagged release (expert mode)"
+        required: false
+        default: false
+        type: boolean
+
+env:
+  REGISTRY: ghcr.io
+  CACHE-FROM: /tmp/.buildx-cache
+  CACHE-TO: /tmp/.buildx-cache-new
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Install GitVersion
+      uses: gittools/actions/gitversion/[email protected]
+      with:
+        versionSpec: '5.x'
+
+    - name: Determine SemVer
+      uses: gittools/actions/gitversion/[email protected]
+      with:
+        useConfigFile: true
+
+    - run: |
+        echo "SemVer ${{ env.fullSemVer }} Forced ${{ github.event.inputs.force }}"
+      name: Display SemVer
+
+    - name: Setup Buildx
+      uses: docker/setup-buildx-action@v1
+      with:
+        install: true
+        driver-opts: image=moby/buildkit:master
+
+    - name: Cache docker layers
+      uses: actions/cache@v2
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-
+      if: ${{ success() }}
+
+    - name: Log into registry ${{ env.REGISTRY }}
+      uses: docker/login-action@v2
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+      if: ${{ success() }}
+
+    # Build And Push Image
+    - name: Build docker image release
+      run: make docker-release-build
+      if: ${{ success() }}
+
+    # Publish if release is tagged or force == true
+    - name: Push docker image release
+      run: make docker-release-push
+      if: startsWith(github.ref, 'refs/tags/v') || github.event.inputs.force == 'true'

GitVersion.yml

@@ -0,0 +1,29 @@
+increment: None
+branches:
+  main:
+    mode: ContinuousDelivery
+    tag: pre
+    increment: Patch
+    prevent-increment-of-merged-branch-version: true
+    track-merge-target: false
+    regex: ^master$|^main$
+    source-branches:
+    - release
+    tracks-release-branches: true
+    is-release-branch: false
+    is-mainline: true
+    pre-release-weight: 55000
+  release:
+    mode: ContinuousDelivery
+    tag: rc
+    increment: None
+    prevent-increment-of-merged-branch-version: true
+    track-merge-target: false
+    regex: ^v*|^releases?[/-]
+    source-branches:
+    - main
+    - release
+    tracks-release-branches: false
+    is-release-branch: true
+    is-mainline: false
+    pre-release-weight: 30000

Makefile

@@ -1,5 +1,22 @@
 .PHONY: compile-el compile-dl clean protoc lint build unit-tests integration-tests-churner integration-tests-indexer integration-tests-inabox integration-tests-inabox-nochurner integration-tests-graph-indexer
 
+ifeq ($(wildcard .git/*),)
+$(warning semver disabled - building from release zip)
+GITCOMMIT := ""
+GITDATE := ""
+SEMVER := $(shell basename $(CURDIR))
+else
+GITCOMMIT := $(shell git rev-parse --short HEAD)
+GITDATE := $(shell git log -1 --format=%cd --date=unix)
+SEMVER := $(shell docker run --rm --volume "$(PWD):/repo" gittools/gitversion:5.12.0 /repo -output json -showvariable SemVer)
+ifeq ($(SEMVER), )
+$(warning semver disabled - docker not installed)
+SEMVER := "0.0.0"
+endif
+endif
+
+RELEASE_TAG := $(or $(RELEASE_TAG),latest)
+
 PROTOS := ./api/proto
 PROTOS_DISPERSER := ./disperser/api/proto
 PROTO_GEN := ./api/grpc
@@ -75,3 +92,12 @@ integration-tests-graph-indexer:
 integration-tests-dataapi:
 	make dataapi-build
 	go test -v ./disperser/dataapi
+
+docker-release-build:
+	RELEASE_TAG=${SEMVER} docker compose -f docker-compose-release.yaml build --build-arg SEMVER=${SEMVER} --build-arg GITCOMMIT=${GITCOMMIT} --build-arg GITDATE=${GITDATE} 
+
+docker-release-push:
+	RELEASE_TAG=${SEMVER} docker compose -f docker-compose-release.yaml push
+
+semver:
+	echo "${SEMVER}"

common/ratelimit.go

@@ -15,17 +15,22 @@ import (
 // ID is the authenticated Account ID. For retrieval requests, the requester ID will be the requester's IP address.
 type RequesterID = string
 
+// RequesterName is the friendly name of the party making the request. In the case
+// of a rollup making a dispersal request, the RequesterName is the name of the rollup.
+type RequesterName = string
+
 type RequestParams struct {
-	RequesterID RequesterID
-	BlobSize    uint
-	Rate        RateParam
-	Info        interface{}
+	RequesterID   RequesterID
+	RequesterName RequesterName
+	BlobSize      uint
+	Rate          RateParam
+	Info          interface{}
 }
 
 type RateLimiter interface {
 	// AllowRequest checks whether the request should be allowed. If the request is allowed, the function returns true.
 	// If the request is not allowed, the function returns false and the RequestParams of the request that was not allowed.
-	// In order to for the request to be allowed, all of the requests represented by the RequestParams slice must be allowed.
+	// In order for the request to be allowed, all of the requests represented by the RequestParams slice must be allowed.
 	// Each RequestParams object represents a single request. Each request is subjected to the same GlobalRateParams, but the
 	// individual parameters of the request can differ.
 	//
@@ -37,7 +42,7 @@ type RateLimiter interface {
 
 type GlobalRateParams struct {
 	// BucketSizes are the time scales at which the rate limit is enforced.
-	// For each time scale, the rate limiter will make sure that the give rate (possibly subject to a relaxation given
+	// For each time scale, the rate limiter will make sure that the given rate (possibly subject to a relaxation given
 	// by one of the Multipliers) is observed when the request bandwidth is averaged at this time scale.
 	// In terms of implementation, the rate limiter uses a set of "time buckets". A time bucket, i, is filled to a maximum of
 	// `BucketSizes[i]` at a rate of 1, and emptied by an amount equal to `(size of request)/RateParam` each time a

common/ratelimit/limiter.go

@@ -2,10 +2,13 @@ package ratelimit
 
 import (
 	"context"
+	"strconv"
 	"time"
 
 	"github.com/Layr-Labs/eigenda/common"
 	"github.com/Layr-Labs/eigensdk-go/logging"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 type BucketStore = common.KVStore[common.RateBucketParams]
@@ -15,13 +18,20 @@ type rateLimiter struct {
 	bucketStore      BucketStore
 
 	logger logging.Logger
+
+	// Prometheus metrics
+	bucketLevels *prometheus.GaugeVec
 }
 
-func NewRateLimiter(rateParams common.GlobalRateParams, bucketStore BucketStore, logger logging.Logger) common.RateLimiter {
+func NewRateLimiter(reg prometheus.Registerer, rateParams common.GlobalRateParams, bucketStore BucketStore, logger logging.Logger) common.RateLimiter {
 	return &rateLimiter{
 		globalRateParams: rateParams,
 		bucketStore:      bucketStore,
 		logger:           logger.With("component", "RateLimiter"),
+		bucketLevels: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
+			Name: "rate_limiter_bucket_levels",
+			Help: "Current level of each bucket for rate limiting",
+		}, []string{"requester_id", "requester_name", "bucket_index"}),
 	}
 }
 
@@ -109,7 +119,18 @@ func (d *rateLimiter) checkAllowed(ctx context.Context, params common.RequestPar
 		bucketParams.BucketLevels[i] = getBucketLevel(bucketParams.BucketLevels[i], size, interval, deduction)
 		allowed = allowed && bucketParams.BucketLevels[i] > 0
 
-		d.logger.Debug("Bucket level", "key", params.RequesterID, "prevLevel", prevLevel, "level", bucketParams.BucketLevels[i], "size", size, "interval", interval, "deduction", deduction, "allowed", allowed)
+		d.logger.Debug("Bucket level updated", "key", params.RequesterID, "name", params.RequesterName, "prevLevel", prevLevel, "level", bucketParams.BucketLevels[i], "size", size, "interval", interval, "deduction", deduction, "allowed", allowed)
+
+		// Update metrics only if the requester name is provided. We're making
+		// an assumption that the requester name is only provided for authenticated
+		// requests so it should limit the cardinality of the requester_id label.
+		if params.RequesterName != "" {
+			d.bucketLevels.With(prometheus.Labels{
+				"requester_id":   params.RequesterID,
+				"requester_name": params.RequesterName,
+				"bucket_index":   strconv.Itoa(i),
+			}).Set(float64(bucketParams.BucketLevels[i]))
+		}
 	}
 
 	return allowed, bucketParams

common/ratelimit/ratelimit_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/Layr-Labs/eigenda/common/ratelimit"
 	"github.com/Layr-Labs/eigenda/common/store"
 	"github.com/Layr-Labs/eigensdk-go/logging"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -25,7 +26,7 @@ func makeTestRatelimiter() (common.RateLimiter, error) {
 		return nil, err
 	}
 
-	ratelimiter := ratelimit.NewRateLimiter(globalParams, bucketStore, logging.NewNoopLogger())
+	ratelimiter := ratelimit.NewRateLimiter(prometheus.NewRegistry(), globalParams, bucketStore, logging.NewNoopLogger())
 
 	return ratelimiter, nil