Merge pull request #69 from Layr-Labs/sm-postgresOverhaul

Replace sqlite with PostgreSQL
Layr-Labs · Oct 30, 2024 · f5a5e67 · f5a5e67
2 parents e69d5f2 + a0c0dd0
commit f5a5e67
Show file tree

Hide file tree

Showing 119 changed files with 3,500 additions and 5,332 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -3,12 +3,32 @@ on: push
 jobs:
   test:
     runs-on: ubuntu-24.04
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_DB: sidecar
+          POSTGRES_USER: sidecar
+          POSTGRES_PASSWORD: sidecar
+        ports:
+          - 5432:5432
+        # Add health check to ensure postgres is ready before running tests
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Install dependencies
         run: make deps
       - name: Run tests
+        env: # Pass the database connection details to the tests
+          SIDECAR_DATABASE_HOST: localhost
+          SIDECAR_DATABASE_PORT: 5432
+          SIDECAR_DATABASE_USER: sidecar
+          SIDECAR_DATABASE_PASSWORD: sidecar
         run: make ci-test
   lint:
     runs-on: ubuntu-24.04

diff --git a/.gitignore b/.gitignore
@@ -23,3 +23,5 @@ test-db
 *.dylib
 *.dylib.dSYM
 /sqlite-extensions/build
+.DS_Store
+node_modules
diff --git a/.testdataVersion b/.testdataVersion
@@ -1 +1 @@
-070a7fa07a31795caaf6e2423d5c60f429d1d7a4
+6f661f5aa11fcb055349004032dbfd9f778428f7
diff --git a/Makefile b/Makefile
@@ -1,12 +1,8 @@
 .PHONY: deps
 
-PROJECT_ROOT = $(shell pwd)
-CGO_CFLAGS = "-I$(PROJECT_ROOT)/sqlite-extensions"
-CGO_LDFLAGS = "-L$(PROJECT_ROOT)/sqlite-extensions/build/lib -lcalculations -Wl,-rpath,$(PROJECT_ROOT)/sqlite-extensions/build/lib"
-PYTHONPATH = $(PROJECT_ROOT)/sqlite-extensions
 CGO_ENABLED = 1
 GO=$(shell which go)
-ALL_FLAGS=CGO_CFLAGS=$(CGO_CFLAGS) CGO_LDFLAGS=$(CGO_LDFLAGS) PYTHONPATH=$(PYTHONPATH) CGO_ENABLED=$(CGO_ENABLED)
+ALL_FLAGS=
 
 deps/dev:
 	${GO} install github.com/golangci/golangci-lint/cmd/[email protected]
@@ -29,7 +25,6 @@ clean:
 
 .PHONY: build/cmd/sidecar
 build/cmd/sidecar:
-	cd sqlite-extensions && make all && cd -
 	$(ALL_FLAGS) $(GO) build -o bin/sidecar main.go
 
 .PHONY: build

diff --git a/README.md b/README.md
@@ -3,9 +3,7 @@
 ## Dependencies
 
 * Go 1.22
-* Sqlite3 (version 9.x.x)
-* Python3 (version 3.12)
-* GCC (for building sqlite3 extensions)
+* PostgreSQL >= 15.x
 * Homebrew (if on MacOS)
 
 ## Supported build environments

diff --git a/cmd/debugger/main.go b/cmd/debugger/main.go
@@ -3,14 +3,18 @@ package main
 import (
 	"context"
 	"fmt"
+	"github.com/Layr-Labs/go-sidecar/internal/contractStore/postgresContractStore"
+	"github.com/Layr-Labs/go-sidecar/internal/eigenState/rewardSubmissions"
+	"github.com/Layr-Labs/go-sidecar/internal/eigenState/submittedDistributionRoots"
+	"github.com/Layr-Labs/go-sidecar/internal/postgres"
+	pgStorage "github.com/Layr-Labs/go-sidecar/internal/storage/postgres"
 	"log"
 
 	"github.com/Layr-Labs/go-sidecar/internal/clients/ethereum"
 	"github.com/Layr-Labs/go-sidecar/internal/clients/etherscan"
 	"github.com/Layr-Labs/go-sidecar/internal/config"
 	"github.com/Layr-Labs/go-sidecar/internal/contractCaller"
 	"github.com/Layr-Labs/go-sidecar/internal/contractManager"
-	"github.com/Layr-Labs/go-sidecar/internal/contractStore/sqliteContractStore"
 	"github.com/Layr-Labs/go-sidecar/internal/eigenState/avsOperators"
 	"github.com/Layr-Labs/go-sidecar/internal/eigenState/operatorShares"
 	"github.com/Layr-Labs/go-sidecar/internal/eigenState/stakerDelegations"
@@ -21,10 +25,8 @@ import (
 	"github.com/Layr-Labs/go-sidecar/internal/logger"
 	"github.com/Layr-Labs/go-sidecar/internal/metrics"
 	"github.com/Layr-Labs/go-sidecar/internal/pipeline"
+	"github.com/Layr-Labs/go-sidecar/internal/postgres/migrations"
 	"github.com/Layr-Labs/go-sidecar/internal/sidecar"
-	"github.com/Layr-Labs/go-sidecar/internal/sqlite"
-	"github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations"
-	sqliteBlockStore "github.com/Layr-Labs/go-sidecar/internal/storage/sqlite"
 	"go.uber.org/zap"
 )
 
@@ -42,30 +44,32 @@ func main() {
 	etherscanClient := etherscan.NewEtherscanClient(cfg, l)
 	client := ethereum.NewClient(cfg.EthereumRpcConfig.BaseUrl, l)
 
-	db := sqlite.NewSqlite(&sqlite.SqliteConfig{
-		Path:           cfg.GetSqlitePath(),
-		ExtensionsPath: cfg.SqliteConfig.ExtensionsPath,
-	}, l)
+	pgConfig := postgres.PostgresConfigFromDbConfig(&cfg.DatabaseConfig)
+	pgConfig.CreateDbIfNotExists = true
 
-	grm, err := sqlite.NewGormSqliteFromSqlite(db)
+	pg, err := postgres.NewPostgres(pgConfig)
 	if err != nil {
-		l.Error("Failed to create gorm instance", zap.Error(err))
-		panic(err)
+		l.Fatal("Failed to setup postgres connection", zap.Error(err))
 	}
 
-	migrator := migrations.NewSqliteMigrator(grm, l)
+	grm, err := postgres.NewGormFromPostgresConnection(pg.Db)
+	if err != nil {
+		l.Fatal("Failed to create gorm instance", zap.Error(err))
+	}
+
+	migrator := migrations.NewMigrator(pg.Db, grm, l)
 	if err = migrator.MigrateAll(); err != nil {
-		log.Fatalf("Failed to migrate: %v", err)
+		l.Fatal("Failed to migrate", zap.Error(err))
 	}
 
-	contractStore := sqliteContractStore.NewSqliteContractStore(grm, l, cfg)
+	contractStore := postgresContractStore.NewPostgresContractStore(grm, l, cfg)
 	if err := contractStore.InitializeCoreContracts(); err != nil {
 		log.Fatalf("Failed to initialize core contracts: %v", err)
 	}
 
 	cm := contractManager.NewContractManager(contractStore, etherscanClient, client, sdc, l)
 
-	mds := sqliteBlockStore.NewSqliteBlockStore(grm, l, cfg)
+	mds := pgStorage.NewPostgresBlockStore(grm, l, cfg)
 	if err != nil {
 		log.Fatalln(err)
 	}
@@ -84,6 +88,12 @@ func main() {
 	if _, err := stakerShares.NewStakerSharesModel(sm, grm, l, cfg); err != nil {
 		l.Sugar().Fatalw("Failed to create StakerSharesModel", zap.Error(err))
 	}
+	if _, err := submittedDistributionRoots.NewSubmittedDistributionRootsModel(sm, grm, l, cfg); err != nil {
+		l.Sugar().Fatalw("Failed to create SubmittedDistributionRootsModel", zap.Error(err))
+	}
+	if _, err := rewardSubmissions.NewRewardSubmissionsModel(sm, grm, l, cfg); err != nil {
+		l.Sugar().Fatalw("Failed to create RewardSubmissionsModel", zap.Error(err))
+	}
 
 	fetchr := fetcher.NewFetcher(client, cfg, l)
 

diff --git a/cmd/root.go b/cmd/root.go
@@ -34,9 +34,11 @@ func init() {
 
 	rootCmd.PersistentFlags().String("etherscan.api-keys", "", `Comma-separated string of keys. e.g. "key1,key2,key3"`)
 
-	rootCmd.PersistentFlags().Bool("sqlite.in-memory", false, `"true" or "false"`)
-	rootCmd.PersistentFlags().String("sqlite.db-file-path", "", `e.g. "/tmp/sidecar.db"`)
-	rootCmd.PersistentFlags().StringArray("sqlite.extensions-path", []string{}, `e.g. "./sqlite-extensions"`)
+	rootCmd.PersistentFlags().String(config.DatabaseHost, "localhost", `Defaults to 'localhost'. Set to something else if you are running PostgreSQL on your own`)
+	rootCmd.PersistentFlags().Int(config.DatabasePort, 5432, `Defaults to '5432'`)
+	rootCmd.PersistentFlags().String(config.DatabaseUser, "sidecar", `Defaults to 'sidecar'`)
+	rootCmd.PersistentFlags().String(config.DatabasePassword, "", ``)
+	rootCmd.PersistentFlags().String(config.DatabaseDbName, "sidecar", `Defaults to 'sidecar'`)
 
 	rootCmd.PersistentFlags().Int("rpc.grpc-port", 7100, `e.g. 7100`)
 	rootCmd.PersistentFlags().Int("rpc.http-port", 7101, `e.g. 7101`)
@@ -51,7 +53,7 @@ func init() {
 }
 
 func initConfig(cmd *cobra.Command) {
-	viper.SetEnvPrefix("sidecar")
+	viper.SetEnvPrefix(config.ENV_PREFIX)
 
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_", ".", "_"))
 

diff --git a/cmd/run.go b/cmd/run.go
@@ -3,6 +3,9 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"github.com/Layr-Labs/go-sidecar/internal/contractStore/postgresContractStore"
+	"github.com/Layr-Labs/go-sidecar/internal/postgres"
+	pgStorage "github.com/Layr-Labs/go-sidecar/internal/storage/postgres"
 	"log"
 	"time"
 
@@ -11,7 +14,6 @@ import (
 	"github.com/Layr-Labs/go-sidecar/internal/config"
 	"github.com/Layr-Labs/go-sidecar/internal/contractCaller"
 	"github.com/Layr-Labs/go-sidecar/internal/contractManager"
-	"github.com/Layr-Labs/go-sidecar/internal/contractStore/sqliteContractStore"
 	"github.com/Layr-Labs/go-sidecar/internal/eigenState/avsOperators"
 	"github.com/Layr-Labs/go-sidecar/internal/eigenState/operatorShares"
 	"github.com/Layr-Labs/go-sidecar/internal/eigenState/rewardSubmissions"
@@ -24,11 +26,9 @@ import (
 	"github.com/Layr-Labs/go-sidecar/internal/logger"
 	"github.com/Layr-Labs/go-sidecar/internal/metrics"
 	"github.com/Layr-Labs/go-sidecar/internal/pipeline"
+	"github.com/Layr-Labs/go-sidecar/internal/postgres/migrations"
 	"github.com/Layr-Labs/go-sidecar/internal/shutdown"
 	"github.com/Layr-Labs/go-sidecar/internal/sidecar"
-	"github.com/Layr-Labs/go-sidecar/internal/sqlite"
-	"github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations"
-	sqliteBlockStore "github.com/Layr-Labs/go-sidecar/internal/storage/sqlite"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
@@ -53,37 +53,32 @@ var runCmd = &cobra.Command{
 		etherscanClient := etherscan.NewEtherscanClient(cfg, l)
 		client := ethereum.NewClient(cfg.EthereumRpcConfig.BaseUrl, l)
 
-		if !cfg.SqliteConfig.InMemory {
-			if err := sqlite.InitSqliteDir(cfg.GetSqlitePath()); err != nil {
-				l.Error("Failed to initialize sqlite directory", zap.Error(err))
-				panic(err)
-			}
-		}
+		pgConfig := postgres.PostgresConfigFromDbConfig(&cfg.DatabaseConfig)
+		pgConfig.CreateDbIfNotExists = true
 
-		db := sqlite.NewSqlite(&sqlite.SqliteConfig{
-			Path:           cfg.GetSqlitePath(),
-			ExtensionsPath: cfg.SqliteConfig.ExtensionsPath,
-		}, l)
+		pg, err := postgres.NewPostgres(pgConfig)
+		if err != nil {
+			l.Fatal("Failed to setup postgres connection", zap.Error(err))
+		}
 
-		grm, err := sqlite.NewGormSqliteFromSqlite(db)
+		grm, err := postgres.NewGormFromPostgresConnection(pg.Db)
 		if err != nil {
-			l.Error("Failed to create gorm instance", zap.Error(err))
-			panic(err)
+			l.Fatal("Failed to create gorm instance", zap.Error(err))
 		}
 
-		migrator := migrations.NewSqliteMigrator(grm, l)
+		migrator := migrations.NewMigrator(pg.Db, grm, l)
 		if err = migrator.MigrateAll(); err != nil {
-			log.Fatalf("Failed to migrate: %v", err)
+			l.Fatal("Failed to migrate", zap.Error(err))
 		}
 
-		contractStore := sqliteContractStore.NewSqliteContractStore(grm, l, cfg)
+		contractStore := postgresContractStore.NewPostgresContractStore(grm, l, cfg)
 		if err := contractStore.InitializeCoreContracts(); err != nil {
 			log.Fatalf("Failed to initialize core contracts: %v", err)
 		}
 
 		cm := contractManager.NewContractManager(contractStore, etherscanClient, client, sdc, l)
 
-		mds := sqliteBlockStore.NewSqliteBlockStore(grm, l, cfg)
+		mds := pgStorage.NewPostgresBlockStore(grm, l, cfg)
 		if err != nil {
 			log.Fatalln(err)
 		}
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		070a7fa07a31795caaf6e2423d5c60f429d1d7a4
		6f661f5aa11fcb055349004032dbfd9f778428f7