From 3912b401c10c6f5cc5517cda3e700cb32fde0cc3 Mon Sep 17 00:00:00 2001 From: Sean McGary Date: Thu, 12 Sep 2024 16:19:17 -0500 Subject: [PATCH] All numeric columns need to be text to avoid potential truncation In some cases, sqlite is turning numbers into scientific notation which results in lost precision. --- .../operatorShares/operatorShares.go | 13 +- .../rewardSubmissions/rewardSubmissions.go | 2 +- .../eigenState/stakerShares/stakerShares.go | 9 +- .../202409061720_operatorShareChanges/up.go | 2 +- .../migrations/202409082234_stakerShare/up.go | 2 +- .../202409101540_rewardSubmissions/up.go | 4 +- internal/storage/tables.go | 182 ------------------ 7 files changed, 22 insertions(+), 192 deletions(-) delete mode 100644 internal/storage/tables.go diff --git a/internal/eigenState/operatorShares/operatorShares.go b/internal/eigenState/operatorShares/operatorShares.go index ebc4037e..8f923daf 100644 --- a/internal/eigenState/operatorShares/operatorShares.go +++ b/internal/eigenState/operatorShares/operatorShares.go @@ -30,7 +30,7 @@ import ( type OperatorShares struct { Operator string Strategy string - Shares string `gorm:"type:numeric"` + Shares string BlockNumber uint64 CreatedAt time.Time } @@ -167,6 +167,7 @@ func (osm *OperatorSharesModel) GetStateTransitions() (types.StateTransitions[Ac } else { record.Shares = record.Shares.Add(record.Shares, shares) } + fmt.Printf("OperatorShares %+v\n", record) return record, nil } @@ -268,7 +269,7 @@ func (osm *OperatorSharesModel) prepareState(blockNumber uint64) ([]OperatorShar select operator, strategy, - shares + cast(shares as text) as shares from operator_shares where block_number = @previousBlock @@ -290,6 +291,7 @@ func (osm *OperatorSharesModel) prepareState(blockNumber uint64) ([]OperatorShar // Map the existing records to a map for easier lookup mappedRecords := make(map[SlotId]OperatorShares) for _, record := range existingRecords { + fmt.Printf("Existing OperatorShares %+v\n", record) slotId := NewSlotId(record.Operator, record.Strategy) mappedRecords[slotId] = record } @@ -308,7 +310,12 @@ func (osm *OperatorSharesModel) prepareState(blockNumber uint64) ([]OperatorShar if existingRecord, ok := mappedRecords[slotId]; ok { existingShares, success := numbers.NewBig257().SetString(existingRecord.Shares, 10) if !success { - osm.logger.Sugar().Errorw("Failed to convert existing shares to big.Int") + osm.logger.Sugar().Errorw("Failed to convert existing shares to big.Int", + zap.String("shares", existingRecord.Shares), + zap.String("operator", existingRecord.Operator), + zap.String("strategy", existingRecord.Strategy), + zap.Uint64("blockNumber", blockNumber), + ) continue } prepared.Shares = existingShares.Add(existingShares, newState.Shares) diff --git a/internal/eigenState/rewardSubmissions/rewardSubmissions.go b/internal/eigenState/rewardSubmissions/rewardSubmissions.go index e565f84b..119d4408 100644 --- a/internal/eigenState/rewardSubmissions/rewardSubmissions.go +++ b/internal/eigenState/rewardSubmissions/rewardSubmissions.go @@ -32,7 +32,7 @@ type RewardSubmission struct { Amount string Strategy string StrategyIndex uint64 - Multiplier string `gorm:"type:numeric"` + Multiplier string StartTimestamp *time.Time `gorm:"type:DATETIME"` EndTimestamp *time.Time `gorm:"type:DATETIME"` Duration uint64 diff --git a/internal/eigenState/stakerShares/stakerShares.go b/internal/eigenState/stakerShares/stakerShares.go index cf20200b..588895be 100644 --- a/internal/eigenState/stakerShares/stakerShares.go +++ b/internal/eigenState/stakerShares/stakerShares.go @@ -30,7 +30,7 @@ import ( type StakerShares struct { Staker string Strategy string - Shares string `gorm:"type:numeric"` + Shares string BlockNumber uint64 CreatedAt time.Time } @@ -579,7 +579,12 @@ func (ss *StakerSharesModel) prepareState(blockNumber uint64) ([]StakerSharesDif if existingRecord, ok := mappedRecords[slotId]; ok { existingShares, success := numbers.NewBig257().SetString(existingRecord.Shares, 10) if !success { - ss.logger.Sugar().Errorw("Failed to convert existing shares to big.Int") + ss.logger.Sugar().Errorw("Failed to convert existing shares to big.Int", + zap.String("shares", existingRecord.Shares), + zap.String("staker", existingRecord.Staker), + zap.String("strategy", existingRecord.Strategy), + zap.Uint64("blockNumber", blockNumber), + ) continue } prepared.Shares = existingShares.Add(existingShares, newState.Shares) diff --git a/internal/sqlite/migrations/202409061720_operatorShareChanges/up.go b/internal/sqlite/migrations/202409061720_operatorShareChanges/up.go index c95d18ed..2e7de198 100644 --- a/internal/sqlite/migrations/202409061720_operatorShareChanges/up.go +++ b/internal/sqlite/migrations/202409061720_operatorShareChanges/up.go @@ -12,7 +12,7 @@ func (m *SqliteMigration) Up(grm *gorm.DB) error { `create table if not exists operator_shares ( operator TEXT NOT NULL, strategy TEXT NOT NULL, - shares NUMERIC NOT NULL, + shares TEXT NOT NULL, block_number INTEGER NOT NULL, created_at DATETIME default current_timestamp, unique (operator, strategy, block_number) diff --git a/internal/sqlite/migrations/202409082234_stakerShare/up.go b/internal/sqlite/migrations/202409082234_stakerShare/up.go index 2d3d5751..b162b84b 100644 --- a/internal/sqlite/migrations/202409082234_stakerShare/up.go +++ b/internal/sqlite/migrations/202409082234_stakerShare/up.go @@ -12,7 +12,7 @@ func (m *SqliteMigration) Up(grm *gorm.DB) error { `create table if not exists staker_shares ( staker TEXT NOT NULL, strategy TEXT NOT NULL, - shares NUMERIC NOT NULL, + shares TEXT NOT NULL, block_number INTEGER NOT NULL, created_at DATETIME default current_timestamp, unique (staker, strategy, block_number) diff --git a/internal/sqlite/migrations/202409101540_rewardSubmissions/up.go b/internal/sqlite/migrations/202409101540_rewardSubmissions/up.go index 222c0ccb..4275e8e4 100644 --- a/internal/sqlite/migrations/202409101540_rewardSubmissions/up.go +++ b/internal/sqlite/migrations/202409101540_rewardSubmissions/up.go @@ -13,10 +13,10 @@ func (m *SqliteMigration) Up(grm *gorm.DB) error { avs TEXT NOT NULL, reward_hash TEST NOT NULL, token TEXT NOT NULL, - amount NUMERIC NOT NULL, + amount TEXT NOT NULL, strategy TEXT NOT NULL, strategy_index INTEGER NOT NULL, - multiplier NUMERIC NOT NULL, + multiplier TEXT NOT NULL, start_timestamp DATETIME NOT NULL, end_timestamp DATETIME NOT NULL, duration INTEGER NOT NULL, diff --git a/internal/storage/tables.go b/internal/storage/tables.go deleted file mode 100644 index 277c73e4..00000000 --- a/internal/storage/tables.go +++ /dev/null @@ -1,182 +0,0 @@ -package storage - -import ( - "math/big" - "time" -) - -// ---------------------------------------------------------------------------- -// Append only tables of state -// ---------------------------------------------------------------------------- - -/* -create table if not exists staker_share_changes ( - - id serial primary key, - staker varchar, - strategy varchar, - shares numeric, - transaction_hash varchar, - log_index bigint, - block_number bigint, - created_at timestamp with time zone - -);. -*/ -type StakerShareChanges struct { - Id uint64 `gorm:"type:serial"` - Staker string - Strategy string - Shares big.Int `gorm:"type:numeric"` - TransactionHash string - LogIndex uint64 - BlockNumber uint64 - CreatedAt time.Time -} - -/* -create table if not exists active_reward_submissions ( - - id serial primary key, - avs varchar, - reward_hash varchar, - token varchar, - amount numeric, - strategy varchar, - multiplier numeric, - strategy_index bigint, - transaction_hash varchar, - log_index bigint, - block_number bigint, - start_timestamp timestamp, - end_timestamp timestamp, - duration bigint - created_at timestamp with time zone - -);. -*/ -type ActiveRewardSubmissions struct { - Id uint64 `gorm:"type:serial"` - Avs string - RewardHash string - Token string - Amount big.Int `gorm:"type:numeric"` - Strategy string - Multiplier big.Int `gorm:"type:numeric"` - StrategyIndex uint64 - TransactionHash string - LogIndex uint64 - BlockNumber uint64 - StartTimestamp time.Time - EndTimestamp time.Time - Duration uint64 - CreatedAt time.Time -} - -/* -create table if not exists active_reward_for_all_submissions ( - - id serial primary key, - avs varchar, - reward_hash varchar, - token varchar, - amount numeric, - strategy varchar, - multiplier numeric, - strategy_index bigint, - transaction_hash varchar, - log_index bigint, - block_number bigint, - start_timestamp timestamp, - end_timestamp timestamp, - duration bigint - created_at timestamp with time zone - -);. -*/ -type RewardForAllSubmissions struct { - Id uint64 `gorm:"type:serial"` - Avs string - RewardHash string - Token string - Amount big.Int `gorm:"type:numeric"` - Strategy string - Multiplier big.Int `gorm:"type:numeric"` - StrategyIndex uint64 - TransactionHash string - LogIndex uint64 - BlockNumber uint64 - StartTimestamp time.Time - EndTimestamp time.Time - Duration uint64 - CreatedAt time.Time -} - -// ---------------------------------------------------------------------------- -// Block-based "summary" tables -// ---------------------------------------------------------------------------- -/* -create table if not exists active_rewards ( - - avs varchar, - reward_hash varchar, - token varchar, - amount numeric, - strategy varchar, - multiplier numeric, - strategy_index bigint, - block_number bigint, - start_timestamp timestamp, - end_timestamp timestamp, - duration bigint, - created_at timestamp with time zone - -). -*/ -type ActiveRewards struct { - Avs string - RewardHash string - Token string - Amount big.Int `gorm:"type:numeric"` - Strategy string - Multiplier big.Int `gorm:"type:numeric"` - StrategyIndex uint64 - BlockNumber uint64 - StartTimestamp time.Time - EndTimestamp time.Time - Duration uint64 - CreatedAt time.Time -} - -/* -create table if not exists active_reward_for_all ( - - avs varchar, - reward_hash varchar, - token varchar, - amount numeric, - strategy varchar, - multiplier numeric, - strategy_index bigint, - block_number bigint, - start_timestamp timestamp, - end_timestamp timestamp, - duration bigint, - created_at timestamp with time zone - -). -*/ -type ActiveRewardForAll struct { - Avs string - RewardHash string - Token string - Amount big.Int `gorm:"type:numeric"` - Strategy string - Multiplier big.Int `gorm:"type:numeric"` - StrategyIndex uint64 - BlockNumber uint64 - StartTimestamp time.Time - EndTimestamp time.Time - Duration uint64 - CreatedAt time.Time -}