Skip to content

Commit

Permalink
Subtract big
Browse files Browse the repository at this point in the history
  • Loading branch information
@seanmcgary
seanmcgary committed Oct 10, 2024
1 parent 3ecffcb commit a982f3a
Show file tree
Hide file tree
Showing 6 changed files with 62 additions and 132 deletions.
6 changes: 6 additions & 0 deletions internal/sqlite/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,4 +78,10 @@ limit 100
## Post-nile staker final totals query

```sql
select
total_staker_operator_payout,
operator_tokens,
staker_tokens
from dbt_mainnet_ethereum_rewards."staker_rewards__2024-09-01 00:00:00_s_2024-09-02 16:00:02"
limit 100
```
4 changes: 0 additions & 4 deletions internal/sqlite/sqlite.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,10 +105,6 @@ func NewSqlite(cfg *SqliteConfig, l *zap.Logger) gorm.Dialector {
l.Sugar().Errorw("Failed to register aggregator sum_big_windowed", "error", err)
return err
}
if err := conn.RegisterFunc("subtract_big", numbers.SubtractBig, true); err != nil {
l.Sugar().Errorw("Failed to register function subtract_big", "error", err)
return err
}
if err := conn.RegisterFunc("numeric_multiply", numbers.NumericMultiply, true); err != nil {
l.Sugar().Errorw("Failed to register function NumericMultiply", "error", err)
return err
Expand Down
141 changes: 13 additions & 128 deletions internal/sqlite/sqlite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,13 @@ import (
func Test_Sqlite(t *testing.T) {
l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: true})

s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

t.Run("Should use the bytesToHex function", func(t *testing.T) {
query := `
with json_values as (
Expand All @@ -28,12 +35,6 @@ func Test_Sqlite(t *testing.T) {
from json_values
limit 1
`
s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

type results struct {
WithdrawalHex string
Expand Down Expand Up @@ -64,13 +65,6 @@ func Test_Sqlite(t *testing.T) {
},
}

s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

createQuery := `
create table shares (
shares TEXT NOT NULL
Expand Down Expand Up @@ -102,13 +96,6 @@ func Test_Sqlite(t *testing.T) {
t.Run("Should call calc_raw_tokens_per_day", func(t *testing.T) {
query := `select calc_raw_tokens_per_day('100', 100) as amt`

s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

var result string
res := grm.Raw(query).Scan(&result)
assert.Nil(t, res.Error)
Expand All @@ -120,18 +107,10 @@ func Test_Sqlite(t *testing.T) {
t.Run("Custom C functions", func(t *testing.T) {
t.Run("Should call pre_nile_tokens_per_day", func(t *testing.T) {
expectedRoundedValue := "1428571428571427000000000000000000000"

amount := "1428571428571428571428571428571428571.4142857142857143"

query := `select pre_nile_tokens_per_day(@amount) as amt`

s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

var result string
res := grm.Raw(query, sql.Named("amount", amount)).Scan(&result)
assert.Nil(t, res.Error)
Expand Down Expand Up @@ -171,13 +150,6 @@ func Test_Sqlite(t *testing.T) {
[]string{"0.418129833809252", "142857142857142700000000", "59732833401321600000000"},
}

s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

for _, v := range values {
stakerProportion := v[0]
tokensPerDay := v[1]
Expand Down Expand Up @@ -299,13 +271,6 @@ func Test_Sqlite(t *testing.T) {
[]string{"0.00000054801092", "71428571428571350000000", "39143637142857100"},
}

s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

for _, v := range values {
stakerProportion := v[0]
tokensPerDay := v[1]
Expand Down Expand Up @@ -427,13 +392,6 @@ func Test_Sqlite(t *testing.T) {
[]string{"0.000001560297627", "178571428571428571", "278624576249"},
}

s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

for _, v := range values {
stakerProportion := v[0]
tokensPerDay := v[1]
Expand All @@ -453,7 +411,6 @@ func Test_Sqlite(t *testing.T) {
}
})
t.Run("Should call amazon_operator_token_rewards", func(t *testing.T) {
// t.Skip("what")
values := [][]string{
[]string{"13167642495403300000000", "1316764249540330000000"},
[]string{"48660996099751700000000", "4866099609975170000000"},
Expand Down Expand Up @@ -485,13 +442,6 @@ func Test_Sqlite(t *testing.T) {
[]string{"59732833401321600000000", "5973283340132160000000"},
}

s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

for _, v := range values {
totalStakerOperatorPayout := v[0]
expectedResult := v[1]
Expand Down Expand Up @@ -607,13 +557,6 @@ func Test_Sqlite(t *testing.T) {
[]string{"39143637142857100", "3914363714285710"},
}

s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

for _, v := range values {
totalStakerOperatorPayout := v[0]
expectedResult := v[1]
Expand Down Expand Up @@ -730,14 +673,7 @@ func Test_Sqlite(t *testing.T) {
[]string{"278624576249", "27862457624"},
}

s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

for _, v := range values {
for _, v := range values[:20] {
totalStakerOperatorPayout := v[0]
expectedResult := v[1]

Expand All @@ -749,8 +685,7 @@ func Test_Sqlite(t *testing.T) {
assert.Equal(t, expectedResult, result, "totalStakerOperatorPayout: %s", totalStakerOperatorPayout)
}
})
t.Run("Should call big_subtract to determine amazon staker token total", func(t *testing.T) {
t.Skip("Implement me")
t.Run("Should call subtract_big to determine amazon staker token total", func(t *testing.T) {
values := [][]string{
[]string{"13167642495403300000000", "1316764249540330000000", "11850878245862970000000"},
[]string{"48660996099751700000000", "4866099609975170000000", "43794896489776530000000"},
Expand Down Expand Up @@ -782,14 +717,7 @@ func Test_Sqlite(t *testing.T) {
[]string{"59732833401321600000000", "5973283340132160000000", "53759550061189440000000"},
}

s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

for _, v := range values {
for _, v := range values[:20] {
totalStakerOperatorPayout := v[0]
operatorTokens := v[1]
expectedResult := v[2]
Expand All @@ -802,8 +730,7 @@ func Test_Sqlite(t *testing.T) {
assert.Equal(t, expectedResult, result, "totalStakerOperatorPayout: %s", totalStakerOperatorPayout)
}
})
t.Run("Should call big_subtract to determine nile staker token total", func(t *testing.T) {
t.Skip("Implement me")
t.Run("Should call subtract_big to determine nile staker token total", func(t *testing.T) {
values := [][]string{
[]string{"17169644785714300", "1716964478571430", "15452680307142870"},
[]string{"13047198500000000", "1304719850000000", "11742478650000000"},
Expand Down Expand Up @@ -907,14 +834,7 @@ func Test_Sqlite(t *testing.T) {
[]string{"39143637142857100", "3914363714285710", "35229273428571390"},
}

s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

for _, v := range values {
for _, v := range values[:20] {
totalStakerOperatorPayout := v[0]
operatorTokens := v[1]
expectedResult := v[2]
Expand All @@ -927,8 +847,7 @@ func Test_Sqlite(t *testing.T) {
assert.Equal(t, expectedResult, result, "totalStakerOperatorPayout: %s", totalStakerOperatorPayout)
}
})
t.Run("Should call big_subtract to determine post-nile staker token total", func(t *testing.T) {
t.Skip("Implement me")
t.Run("Should call subtract_big to determine post-nile staker token total", func(t *testing.T) {
values := [][]string{
[]string{"584881405357", "58488140535", "526393264822"},
[]string{"25329506785", "2532950678", "22796556107"},
Expand Down Expand Up @@ -1032,13 +951,6 @@ func Test_Sqlite(t *testing.T) {
[]string{"278624576249", "27862457624", "250762118625"},
}

s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

for _, v := range values {
totalStakerOperatorPayout := v[0]
operatorTokens := v[1]
Expand All @@ -1055,13 +967,6 @@ func Test_Sqlite(t *testing.T) {
t.Run("Should call big_gt", func(t *testing.T) {
query := `select big_gt('100', '1') as gt`

s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

var result string
res := grm.Raw(query).Scan(&result)
assert.Nil(t, res.Error)
Expand All @@ -1071,13 +976,6 @@ func Test_Sqlite(t *testing.T) {
t.Run("Should call numeric_multiply_c", func(t *testing.T) {
query := `select numeric_multiply_c('100', '.1')`

s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

var result string
res := grm.Raw(query).Scan(&result)
assert.Nil(t, res.Error)
Expand All @@ -1088,13 +986,6 @@ func Test_Sqlite(t *testing.T) {
t.Skip("This function is actually an aggregate function")
query := `select sum_big_c('100', '1')`

s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

var result string
res := grm.Raw(query).Scan(&result)
assert.Nil(t, res.Error)
Expand All @@ -1114,12 +1005,6 @@ func Test_Sqlite(t *testing.T) {
[]string{"2000000000000000000000000000000000000000", "2000000000000000000000000000000000000000", "1.00000000000000000000"},
[]string{"2000000000000000000000000000000000000000", "2000000000000000000000000000000000000000", "1.00000000000000000000"},
}
s := NewSqlite(&SqliteConfig{
Path: SqliteInMemoryPath,
ExtensionsPath: []string{tests.GetSqliteExtensionsPath()},
}, l)
grm, err := NewGormSqliteFromSqlite(s)
assert.Nil(t, err)

for _, v := range values {
stakerWeight := v[0]
Expand Down
36 changes: 36 additions & 0 deletions sqlite-extensions/calculations.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -460,6 +460,36 @@ void calculate_staker_proportion(sqlite3_context *context, int argc, sqlite3_val
sqlite3_result_text(context, tokens, -1, SQLITE_TRANSIENT);
}

char* _subtract_big(const char* a, const char* b) {
return call_python_func("subtractBig", a, b);
}

void subtract_big(sqlite3_context *context, int argc, sqlite3_value **argv) {
if (argc != 2) {
sqlite3_result_error(context, "subtract_big() requires two arguments", -1);
return;
}
const char* a = (const char*)sqlite3_value_text(argv[0]);
if (!a) {
sqlite3_result_null(context);
return;
}

const char* b = (const char*)sqlite3_value_text(argv[1]);
if (!b) {
sqlite3_result_null(context);
return;
}

char* tokens = _subtract_big(a, b);
if (!tokens) {
sqlite3_result_null(context);
return;
}

sqlite3_result_text(context, tokens, -1, SQLITE_TRANSIENT);
}

int sqlite3_calculations_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) {
SQLITE_EXTENSION_INIT2(pApi);

Expand Down Expand Up @@ -530,6 +560,12 @@ int sqlite3_calculations_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_ro
return rc;
}

rc = sqlite3_create_function(db, "subtract_big", 2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, subtract_big, 0, 0);
if (rc != SQLITE_OK) {
fprintf(stderr, "Failed to create function: %s\n", sqlite3_errmsg(db));
return rc;
}

return SQLITE_OK;
}

Expand Down
3 changes: 3 additions & 0 deletions sqlite-extensions/calculations.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ static void sum_big_finalize(sqlite3_context* context);
char* _calculate_staker_proportion(const char* stakerWeight, const char* totalWeight);
void calculate_staker_proportion(sqlite3_context *context, int argc, sqlite3_value **argv);

char* _subtract_big(const char* a, const char* b);
void subtract_big(sqlite3_context *context, int argc, sqlite3_value **argv);

void sqlite3_calculations_shutdown(void);

#endif // CALCULATIONS_H
4 changes: 4 additions & 0 deletions sqlite-extensions/calculations.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,3 +117,7 @@ def calculateStakerProportion(stakerWeightStr: str, totalWeightStr: str) -> str:
preProportion2 = preProportion1 / Decimal('1000000000000000')

return "{}".format(preProportion2, 'f')

def subtractBig(a:str, b:str) -> str:
diff = Decimal(a) - Decimal(b)
return format(diff, 'f')

0 comments on commit a982f3a

Please sign in to comment.