From d9ff88727149598a80e9c7767610343962de6650 Mon Sep 17 00:00:00 2001 From: Nozim Mehrubonov Date: Sun, 17 Dec 2023 20:08:16 +0000 Subject: [PATCH 01/14] Add metamorph records clearing --- .../jobs/clear_block_transactions_map.go | 3 +- .../jobs/clear_block_transactions_map_test.go | 4 +- background_worker/jobs/clear_blocks.go | 4 +- background_worker/jobs/clear_blocks_test.go | 4 +- background_worker/jobs/clear_metamorph.go | 42 +++++++++++ .../jobs/clear_metamorph_test.go | 70 +++++++++++++++++++ background_worker/jobs/clear_transactions.go | 2 +- .../jobs/clear_transactions_test.go | 4 +- background_worker/scheduler.go | 4 +- .../000001_create_transactions.down.sql | 1 + .../000001_create_transactions.up.sql | 20 ++++++ .../postgres/000002_create_blocks.down.sql | 1 + .../postgres/000002_create_blocks.up.sql | 8 +++ 13 files changed, 155 insertions(+), 12 deletions(-) create mode 100644 background_worker/jobs/clear_metamorph.go create mode 100644 background_worker/jobs/clear_metamorph_test.go create mode 100644 database/migrations/metamorph/postgres/000001_create_transactions.down.sql create mode 100644 database/migrations/metamorph/postgres/000001_create_transactions.up.sql create mode 100644 database/migrations/metamorph/postgres/000002_create_blocks.down.sql create mode 100644 database/migrations/metamorph/postgres/000002_create_blocks.up.sql diff --git a/background_worker/jobs/clear_block_transactions_map.go b/background_worker/jobs/clear_block_transactions_map.go index 078201cf0..5c9c4160d 100644 --- a/background_worker/jobs/clear_block_transactions_map.go +++ b/background_worker/jobs/clear_block_transactions_map.go @@ -9,9 +9,10 @@ import ( _ "github.com/lib/pq" ) -func (c ClearJob) ClearBlockTransactionsMap(params ClearRecrodsParams) error { +func (c ClearJob) ClearBlockTransactionsMap(params ClearRecordsParams) error { Log(INFO, "Connecting to database ...") conn, err := sqlx.Open(params.Scheme(), params.String()) + if err != nil { Log(ERROR, fmt.Sprintf("unable to create connection %s", err)) return err diff --git a/background_worker/jobs/clear_block_transactions_map_test.go b/background_worker/jobs/clear_block_transactions_map_test.go index 258699760..cfbe1fd2f 100644 --- a/background_worker/jobs/clear_block_transactions_map_test.go +++ b/background_worker/jobs/clear_block_transactions_map_test.go @@ -16,11 +16,11 @@ import ( ) type ClearBlockTransactionsMapSuite struct { - DatabaseTestSuite + BlockTXDBTestSuite } func (s *ClearBlockTransactionsMapSuite) Test() { - params := ClearRecrodsParams{ + params := ClearRecordsParams{ DBConnectionParams: DefaultParams, RecordRetentionDays: 10, } diff --git a/background_worker/jobs/clear_blocks.go b/background_worker/jobs/clear_blocks.go index 4353eda52..61790e645 100644 --- a/background_worker/jobs/clear_blocks.go +++ b/background_worker/jobs/clear_blocks.go @@ -14,7 +14,7 @@ const ( numericalDateHourLayout = "2006010215" ) -type ClearRecrodsParams struct { +type ClearRecordsParams struct { dbconn.DBConnectionParams RecordRetentionDays int } @@ -41,7 +41,7 @@ func NewClearJob(opts ...func(job *ClearJob)) *ClearJob { return c } -func (c ClearJob) ClearBlocks(params ClearRecrodsParams) error { +func (c ClearJob) ClearBlocks(params ClearRecordsParams) error { Log(INFO, "Connecting to database ...") conn, err := sqlx.Open(params.Scheme(), params.String()) diff --git a/background_worker/jobs/clear_blocks_test.go b/background_worker/jobs/clear_blocks_test.go index 0e3921f53..1fcd07812 100644 --- a/background_worker/jobs/clear_blocks_test.go +++ b/background_worker/jobs/clear_blocks_test.go @@ -16,11 +16,11 @@ import ( ) type ClearJobSuite struct { - DatabaseTestSuite + BlockTXDBTestSuite } func (s *ClearJobSuite) Test() { - params := ClearRecrodsParams{ + params := ClearRecordsParams{ DBConnectionParams: DefaultParams, RecordRetentionDays: 10, } diff --git a/background_worker/jobs/clear_metamorph.go b/background_worker/jobs/clear_metamorph.go new file mode 100644 index 000000000..c134233d3 --- /dev/null +++ b/background_worker/jobs/clear_metamorph.go @@ -0,0 +1,42 @@ +package jobs + +import ( + "fmt" + "github.com/jmoiron/sqlx" +) + +func runDelete(table string, params ClearRecordsParams) error { + Log(INFO, "Connecting to database ...") + conn, err := sqlx.Open(params.Scheme(), params.String()) + if err != nil { + Log(ERROR, fmt.Sprintf("unable to create connection %s", err)) + return err + } + interval := fmt.Sprintf("%d days", params.RecordRetentionDays) + + stmt, err := conn.Preparex(fmt.Sprintf("DELETE FROM %s WHERE inserted_at <= (CURRENT_DATE - $1::interval)", table)) + if err != nil { + Log(ERROR, fmt.Sprintf("unable to prepare statement %s", err)) + return err + } + + res, err := stmt.Exec(interval) + if err != nil { + Log(ERROR, "unable to delete block rows") + return err + } + rows, _ := res.RowsAffected() + Log(INFO, fmt.Sprintf("Successfully deleted %d rows", rows)) + return nil +} + +func ClearMetamorph(params ClearRecordsParams) error { + if err := runDelete("blocks", params); err != nil { + return err + } + if err := runDelete("transactions", params); err != nil { + return err + } + + return nil +} diff --git a/background_worker/jobs/clear_metamorph_test.go b/background_worker/jobs/clear_metamorph_test.go new file mode 100644 index 000000000..47d150c78 --- /dev/null +++ b/background_worker/jobs/clear_metamorph_test.go @@ -0,0 +1,70 @@ +package jobs + +import ( + . "github.com/bitcoin-sv/arc/database_testing" + "github.com/bitcoin-sv/arc/metamorph/store" + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "testing" + "time" +) + +type ClearMetamorphSuite struct { + MetamorphDBTestSuite +} + +func (s *ClearMetamorphSuite) Test() { + + for i := 0; i < 5; i++ { + blk := GetTestMMBlock() + blk.InsertedAt = time.Now().Add(-20 * 24 * time.Hour) + s.InsertBlock(blk) + } + + for i := 0; i < 5; i++ { + blk := GetTestMMBlock() + blk.InsertedAt = time.Now().Add(-1 * 24 * time.Hour) + s.InsertBlock(blk) + } + + for i := 0; i < 5; i++ { + tx := GetTestMMTransaction() + tx.InsertedAt = time.Now().Add(-20 * 24 * time.Hour) + + s.InsertTransaction(tx) + } + + for i := 0; i < 5; i++ { + tx := GetTestMMTransaction() + tx.InsertedAt = time.Now().Add(-1 * 24 * time.Hour) + + s.InsertTransaction(tx) + } + + err := ClearMetamorph(ClearRecordsParams{ + DBConnectionParams: DefaultMMParams, + RecordRetentionDays: 14, + }) + + require.NoError(s.T(), err) + + db, err := sqlx.Open("postgres", DefaultMMParams.String()) + require.NoError(s.T(), err) + + var blks []store.Block + require.NoError(s.T(), db.Select(&blks, "SELECT * from blocks")) + + assert.Len(s.T(), blks, 5) + + var stx []store.Transaction + require.NoError(s.T(), db.Select(&stx, "SELECT * from transactions")) + assert.Len(s.T(), stx, 5) + +} + +func TestRunClearMM(t *testing.T) { + s := new(ClearMetamorphSuite) + suite.Run(t, s) +} diff --git a/background_worker/jobs/clear_transactions.go b/background_worker/jobs/clear_transactions.go index 022144952..03fa7d475 100644 --- a/background_worker/jobs/clear_transactions.go +++ b/background_worker/jobs/clear_transactions.go @@ -8,7 +8,7 @@ import ( _ "github.com/lib/pq" ) -func (c ClearJob) ClearTransactions(params ClearRecrodsParams) error { +func (c ClearJob) ClearTransactions(params ClearRecordsParams) error { Log(INFO, "Connecting to database ...") conn, err := sqlx.Open(params.Scheme(), params.String()) diff --git a/background_worker/jobs/clear_transactions_test.go b/background_worker/jobs/clear_transactions_test.go index 5c41ca681..6a2706104 100644 --- a/background_worker/jobs/clear_transactions_test.go +++ b/background_worker/jobs/clear_transactions_test.go @@ -16,11 +16,11 @@ import ( ) type ClearTransactionsSuite struct { - DatabaseTestSuite + BlockTXDBTestSuite } func (s *ClearTransactionsSuite) Test() { - params := ClearRecrodsParams{ + params := ClearRecordsParams{ DBConnectionParams: DefaultParams, RecordRetentionDays: 10, } diff --git a/background_worker/scheduler.go b/background_worker/scheduler.go index bdc76fa36..9d874031d 100644 --- a/background_worker/scheduler.go +++ b/background_worker/scheduler.go @@ -9,10 +9,10 @@ import ( type ARCScheduler struct { Scheduler *gocron.Scheduler IntervalInHours int - Params jobs.ClearRecrodsParams + Params jobs.ClearRecordsParams } -func (sched *ARCScheduler) RunJob(table string, job func(params jobs.ClearRecrodsParams) error) { +func (sched *ARCScheduler) RunJob(table string, job func(params jobs.ClearRecordsParams) error) { _, err := sched.Scheduler.Every(sched.IntervalInHours).Hours().Do(func() { jobs.Log(jobs.INFO, fmt.Sprintf("Clearing expired %s...", table)) err := job(sched.Params) diff --git a/database/migrations/metamorph/postgres/000001_create_transactions.down.sql b/database/migrations/metamorph/postgres/000001_create_transactions.down.sql new file mode 100644 index 000000000..91345dc77 --- /dev/null +++ b/database/migrations/metamorph/postgres/000001_create_transactions.down.sql @@ -0,0 +1 @@ +DROP TABLE transactions; diff --git a/database/migrations/metamorph/postgres/000001_create_transactions.up.sql b/database/migrations/metamorph/postgres/000001_create_transactions.up.sql new file mode 100644 index 000000000..292e36b34 --- /dev/null +++ b/database/migrations/metamorph/postgres/000001_create_transactions.up.sql @@ -0,0 +1,20 @@ +CREATE TABLE transactions +( + hash BYTEA PRIMARY KEY, + stored_at TIMESTAMPTZ, + announced_at TIMESTAMPTZ, + mined_at TIMESTAMPTZ, + status INTEGER, + block_height BIGINT, + block_hash BYTEA, + callback_url TEXT, + callback_token TEXT, + merkle_proof TEXT, + reject_reason TEXT, + raw_tx BYTEA, + locked_by TEXT, + inserted_at TIMESTAMPTZ +); + +CREATE INDEX ix_metamorph_transactions_locked_by ON transactions (locked_by); +CREATE INDEX ix_metamorph_transactions_inserted_at_num ON transactions (inserted_at); diff --git a/database/migrations/metamorph/postgres/000002_create_blocks.down.sql b/database/migrations/metamorph/postgres/000002_create_blocks.down.sql new file mode 100644 index 000000000..7a4fcb3dc --- /dev/null +++ b/database/migrations/metamorph/postgres/000002_create_blocks.down.sql @@ -0,0 +1 @@ +DROP TABLE blocks; diff --git a/database/migrations/metamorph/postgres/000002_create_blocks.up.sql b/database/migrations/metamorph/postgres/000002_create_blocks.up.sql new file mode 100644 index 000000000..753529483 --- /dev/null +++ b/database/migrations/metamorph/postgres/000002_create_blocks.up.sql @@ -0,0 +1,8 @@ +CREATE TABLE blocks +( + hash BYTEA PRIMARY KEY, + processed_at TIMESTAMPTZ, + inserted_at TIMESTAMPTZ +); + +CREATE INDEX ix_metamorph_blocks_inserted_at ON blocks (inserted_at); From c000368db8d9e78def1091ba1f6f216258338cf7 Mon Sep 17 00:00:00 2001 From: Nozim Mehrubonov Date: Sun, 17 Dec 2023 20:10:12 +0000 Subject: [PATCH 02/14] Add metamorph database testing utility code --- database_testing/blocktx_db_test_suite.go | 184 ++++++++++++++++++++ database_testing/metamorph_db_test_suite.go | 155 +++++++++++++++++ 2 files changed, 339 insertions(+) create mode 100644 database_testing/blocktx_db_test_suite.go create mode 100644 database_testing/metamorph_db_test_suite.go diff --git a/database_testing/blocktx_db_test_suite.go b/database_testing/blocktx_db_test_suite.go new file mode 100644 index 000000000..09cdc2270 --- /dev/null +++ b/database_testing/blocktx_db_test_suite.go @@ -0,0 +1,184 @@ +package database_testing + +import ( + "errors" + "fmt" + mrand "math/rand" + "path/filepath" + "runtime" + "time" + + "github.com/bitcoin-sv/arc/blocktx/store" + "github.com/bitcoin-sv/arc/dbconn" + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/postgres" + _ "github.com/golang-migrate/migrate/v4/source/file" + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "golang.org/x/exp/rand" +) + +func GetRandomBytes() string { + return fmt.Sprintf("%d %d", mrand.Int63(), mrand.Int63())[:32] +} +func GetTestBlock() *store.Block { + now := time.Now() + return &store.Block{ + ID: int64(mrand.Intn(10000)), + Hash: GetRandomBytes(), + PreviousHash: fmt.Sprintf("%d", rand.Int63()), + MerkleRoot: fmt.Sprintf("%d", rand.Int63()), + Height: int64(mrand.Intn(100)), + Orphaned: false, + ProcessedAt: now, + } +} + +func GetTestTransaction() *store.Transaction { + return &store.Transaction{ + ID: int64(mrand.Intn(10000)), + Hash: GetRandomBytes(), + Source: fmt.Sprintf("testtx %d", mrand.Int63()), + MerklePath: fmt.Sprintf("testtx %d", mrand.Int63()), + } +} + +type DBConnectionParams struct { + Host string + Port int + Username string + Password string + DBName string +} + +func (p DBConnectionParams) String() string { + return fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=disable", p.Username, p.Password, p.Host, p.Port, p.DBName) +} + +var DefaultParams = dbconn.New( + "localhost", + 5432, + "arcuser", + "arcpass", + "blocktx_test", + "postgres", + "disable", +) + +// BlockTXDBTestSuite test helper suite to +// 1. create database +// 2. run database/postgres +// 3. use in test scenario +// 4. tear down when tests are finished +type BlockTXDBTestSuite struct { + suite.Suite + Connection *sqlx.Conn +} + +func (s *BlockTXDBTestSuite) SetupSuite() { + _, callerFilePath, _, _ := runtime.Caller(0) + + testDir := filepath.Dir(callerFilePath) + + path := "file://" + testDir + "/../database/migrations/blocktx/postgres" + m, err := migrate.New(path, DefaultParams.String()) + + require.NoError(s.T(), err) + + if err := m.Up(); err != nil { + if !errors.Is(err, migrate.ErrNoChange) { + require.NoError(s.T(), err) + } + } +} + +func (s *BlockTXDBTestSuite) SetupTest() { + s.truncateTables() +} + + +func (s *BlockTXDBTestSuite) Conn() *sqlx.Conn { + return s.Connection +} + +func (s *BlockTXDBTestSuite) InsertBlock(block *store.Block) { + db, err := sqlx.Open("postgres", DefaultParams.String()) + require.NoError(s.T(), err) + + q := `INSERT INTO blocks( + id, + hash, + prevhash, + merkleroot, + orphanedyn, + height, + processed_at, + inserted_at) + VALUES( + :id, + :hash, + :prevhash, + :merkleroot, + :orphanedyn, + :height, + :processed_at, + :inserted_at + );` + + _, err = db.NamedExec(q, + block) + require.NoError(s.T(), err) + +} + +func (s *BlockTXDBTestSuite) InsertTransaction(tx *store.Transaction) { + db, err := sqlx.Open("postgres", DefaultParams.String()) + require.NoError(s.T(), err) + q := `INSERT INTO transactions( + id, + hash, + source, + merkle_path) + VALUES( + :id, + :hash, + :source, + :merkle_path);` + + _, err = db.NamedExec(q, tx) + + require.NoError(s.T(), err, fmt.Sprintf("tx %+v", tx)) +} + +func (s *BlockTXDBTestSuite) InsertBlockTransactionMap(btx *store.BlockTransactionMap) { + db, err := sqlx.Open("postgres", DefaultParams.String()) + require.NoError(s.T(), err) + + q := `INSERT INTO block_transactions_map( + blockid, + txid, + pos) + VALUES( + :blockid, + :txid, + :pos);` + + _, err = db.NamedExec(q, btx) + require.NoError(s.T(), err) +} + +// TearDownTest clear all the tables +func (s *BlockTXDBTestSuite) TearDownTest() { + s.truncateTables() +} + + +func (s *BlockTXDBTestSuite) truncateTables() { + db, err := sqlx.Open("postgres", DefaultParams.String()) + require.NoError(s.T(), err) + + db.MustExec("truncate table blocks;") + db.MustExec("truncate table transactions;") + db.MustExec("truncate table block_transactions_map;") +} diff --git a/database_testing/metamorph_db_test_suite.go b/database_testing/metamorph_db_test_suite.go new file mode 100644 index 000000000..e25ef6f28 --- /dev/null +++ b/database_testing/metamorph_db_test_suite.go @@ -0,0 +1,155 @@ +package database_testing + +import ( + "errors" + "fmt" + "path/filepath" + "runtime" + "time" + + "github.com/bitcoin-sv/arc/dbconn" + "github.com/bitcoin-sv/arc/metamorph/store" + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/postgres" + _ "github.com/golang-migrate/migrate/v4/source/file" + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +func GetTestMMBlock() *store.Block { + return &store.Block{ + Hash: GetRandomBytes(), + ProcessedAt: time.Now().UTC(), + InsertedAt: time.Now().UTC(), + } +} + +func GetTestMMTransaction() *store.Transaction { + return &store.Transaction{ + Hash: GetRandomBytes(), + StoredAt: time.Date(2023, 10, 4, 22, 0, 0, 0, time.UTC), + AnnouncedAt: time.Date(2023, 10, 5, 12, 0, 0, 0, time.UTC), + MinedAt: time.Date(2023, 10, 7, 10, 0, 0, 0, time.UTC), + Status: 1, + BlockHeight: int64(12345), + BlockHash: []byte{0x11, 0x22, 0x33, 0x44}, + CallbackURL: "https://example.com/callback", + CallbackToken: "1234567890abcdef", + MerkleProof: "4d1f934bd7a5223b95656220d39c64b3a66b0f770137f6611052381806e90275", + RawTX: []byte("01020304"), // example raw transaction data + LockedBy: "0x1234567890abcdef", + InsertedAt: time.Now().UTC(), + } +} + +var DefaultMMParams = dbconn.New( + "localhost", + 5432, + "arcuser", + "arcpass", + "metamorph_test", + "postgres", + "disable", +) + +type MetamorphDBTestSuite struct { + suite.Suite + Connection *sqlx.Conn +} + +func (s *MetamorphDBTestSuite) SetupSuite() { + _, callerFilePath, _, _ := runtime.Caller(0) + + testDir := filepath.Dir(callerFilePath) + + path := "file://" + testDir + "/../database/migrations/metamorph/postgres" + m, err := migrate.New(path, DefaultMMParams.String()) + + require.NoError(s.T(), err) + + if err := m.Up(); err != nil { + if !errors.Is(err, migrate.ErrNoChange) { + require.NoError(s.T(), err) + } + } +} + +func (s *MetamorphDBTestSuite) SetupTest() { + s.truncateTables() +} + +func (s *MetamorphDBTestSuite) truncateTables() { + s.T().Log("truncating tables") + db, err := sqlx.Open("postgres", DefaultMMParams.String()) + require.NoError(s.T(), err) + + db.MustExec("truncate table blocks;") + db.MustExec("truncate table transactions;") +} + +func (s *MetamorphDBTestSuite) Conn() *sqlx.Conn { + return s.Connection +} + +func (s *MetamorphDBTestSuite) InsertBlock(block *store.Block) { + db, err := sqlx.Open("postgres", DefaultMMParams.String()) + require.NoError(s.T(), err) + + q := `INSERT INTO blocks( + hash, + processed_at, + inserted_at) + VALUES( + :hash, + :processed_at, + :inserted_at + );` + + _, err = db.NamedExec(q, + block) + require.NoError(s.T(), err) + +} + +func (s *MetamorphDBTestSuite) InsertTransaction(tx *store.Transaction) { + db, err := sqlx.Open("postgres", DefaultMMParams.String()) + require.NoError(s.T(), err) + q := `INSERT INTO transactions (hash, + stored_at, + announced_at, + mined_at, + status, + block_height, + block_hash, + callback_url, + callback_token, + merkle_proof, + reject_reason, + raw_tx, + locked_by, + inserted_at) VALUES ( + :hash, + :stored_at, + :announced_at, + :mined_at, + :status, + :block_height, + :block_hash, + :callback_url, + :callback_token, + :merkle_proof, + :reject_reason, + :raw_tx, + :locked_by, + :inserted_at);` + + _, err = db.NamedExec(q, tx) + + require.NoError(s.T(), err, fmt.Sprintf("tx %+v", tx)) +} + +// TearDownTest clear all the tables +func (s *MetamorphDBTestSuite) TearDownTest() { + s.truncateTables() +} From 8996f57b55e19f92629e94732a99514e2c7f82b9 Mon Sep 17 00:00:00 2001 From: Nozim Mehrubonov Date: Sun, 17 Dec 2023 20:14:45 +0000 Subject: [PATCH 03/14] Fix compilation errors --- database_testing/database_test_suite.go | 182 ------------------------ metamorph/store/model.go | 26 ++++ 2 files changed, 26 insertions(+), 182 deletions(-) delete mode 100644 database_testing/database_test_suite.go create mode 100644 metamorph/store/model.go diff --git a/database_testing/database_test_suite.go b/database_testing/database_test_suite.go deleted file mode 100644 index 956f3a7fc..000000000 --- a/database_testing/database_test_suite.go +++ /dev/null @@ -1,182 +0,0 @@ -package database_testing - -import ( - "errors" - "fmt" - mrand "math/rand" - "path/filepath" - "runtime" - "time" - - "github.com/bitcoin-sv/arc/blocktx/store" - "github.com/bitcoin-sv/arc/dbconn" - "github.com/golang-migrate/migrate/v4" - _ "github.com/golang-migrate/migrate/v4/database/postgres" - _ "github.com/golang-migrate/migrate/v4/source/file" - "github.com/jmoiron/sqlx" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - "golang.org/x/exp/rand" -) - -func GetRandomBytes() string { - return fmt.Sprintf("%d %d", mrand.Int63(), mrand.Int63())[:32] -} -func GetTestBlock() *store.Block { - now := time.Now() - return &store.Block{ - ID: int64(mrand.Intn(10000)), - Hash: GetRandomBytes(), - PreviousHash: fmt.Sprintf("%d", rand.Int63()), - MerkleRoot: fmt.Sprintf("%d", rand.Int63()), - Height: int64(mrand.Intn(100)), - Orphaned: false, - ProcessedAt: now, - } -} - -func GetTestTransaction() *store.Transaction { - return &store.Transaction{ - ID: int64(mrand.Intn(10000)), - Hash: GetRandomBytes(), - Source: fmt.Sprintf("testtx %d", mrand.Int63()), - MerklePath: fmt.Sprintf("testtx %d", mrand.Int63()), - } -} - -type DBConnectionParams struct { - Host string - Port int - Username string - Password string - DBName string -} - -func (p DBConnectionParams) String() string { - return fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=disable", p.Username, p.Password, p.Host, p.Port, p.DBName) -} - -var DefaultParams = dbconn.New( - "localhost", - 5432, - "arcuser", - "arcpass", - "blocktx_test", - "postgres", - "disable", -) - -// DatabaseTestSuite test helper suite to -// 1. create database -// 2. run database/migrations -// 3. use in test scenario -// 4. tear down when tests are finished -type DatabaseTestSuite struct { - suite.Suite - Connection *sqlx.Conn -} - -func (s *DatabaseTestSuite) SetupSuite() { - _, callerFilePath, _, _ := runtime.Caller(0) - - testDir := filepath.Dir(callerFilePath) - - path := "file://" + testDir + "/../database/migrations/blocktx/postgres" - m, err := migrate.New(path, DefaultParams.String()) - - require.NoError(s.T(), err) - - if err := m.Up(); err != nil { - if !errors.Is(err, migrate.ErrNoChange) { - require.NoError(s.T(), err) - } - } -} - -func (s *DatabaseTestSuite) SetupTest() { - s.truncateTables() -} - -func (s *DatabaseTestSuite) truncateTables() { - db, err := sqlx.Open("postgres", DefaultParams.String()) - require.NoError(s.T(), err) - - db.MustExec("truncate table blocks;") - db.MustExec("truncate table transactions;") - db.MustExec("truncate table block_transactions_map;") -} - -func (s *DatabaseTestSuite) Conn() *sqlx.Conn { - return s.Connection -} - -func (s *DatabaseTestSuite) InsertBlock(block *store.Block) { - db, err := sqlx.Open("postgres", DefaultParams.String()) - require.NoError(s.T(), err) - - q := `INSERT INTO blocks( - id, - hash, - prevhash, - merkleroot, - orphanedyn, - height, - processed_at, - inserted_at) - VALUES( - :id, - :hash, - :prevhash, - :merkleroot, - :orphanedyn, - :height, - :processed_at, - :inserted_at - );` - - _, err = db.NamedExec(q, - block) - require.NoError(s.T(), err) - -} - -func (s *DatabaseTestSuite) InsertTransaction(tx *store.Transaction) { - db, err := sqlx.Open("postgres", DefaultParams.String()) - require.NoError(s.T(), err) - q := `INSERT INTO transactions( - id, - hash, - source, - merkle_path) - VALUES( - :id, - :hash, - :source, - :merkle_path);` - - _, err = db.NamedExec(q, tx) - - require.NoError(s.T(), err, fmt.Sprintf("tx %+v", tx)) -} - -func (s *DatabaseTestSuite) InsertBlockTransactionMap(btx *store.BlockTransactionMap) { - db, err := sqlx.Open("postgres", DefaultParams.String()) - require.NoError(s.T(), err) - - q := `INSERT INTO block_transactions_map( - blockid, - txid, - pos) - VALUES( - :blockid, - :txid, - :pos);` - - _, err = db.NamedExec(q, btx) - require.NoError(s.T(), err) -} - -// TearDownTest clear all the tables -func (s *DatabaseTestSuite) TearDownTest() { - s.truncateTables() -} diff --git a/metamorph/store/model.go b/metamorph/store/model.go new file mode 100644 index 000000000..9a282a362 --- /dev/null +++ b/metamorph/store/model.go @@ -0,0 +1,26 @@ +package store + +import "time" + +type Block struct { + Hash string `db:"hash"` + ProcessedAt time.Time `db:"processed_at"` + InsertedAt time.Time `db:"inserted_at"` +} + +type Transaction struct { + Hash string `db:"hash"` + StoredAt time.Time `db:"stored_at"` + AnnouncedAt time.Time `db:"announced_at"` + MinedAt time.Time `db:"mined_at"` + Status int `db:"status"` + BlockHeight int64 `db:"block_height"` + BlockHash []byte `db:"block_hash"` + CallbackURL string `db:"callback_url"` + CallbackToken string `db:"callback_token"` + MerkleProof string `db:"merkle_proof"` + RejectReason string `db:"reject_reason"` + RawTX []byte `db:"raw_tx"` + LockedBy string `db:"locked_by"` + InsertedAt time.Time `db:"inserted_at"` +} From ad90d798645f25d559e6094240740c109bb3b441 Mon Sep 17 00:00:00 2001 From: Nozim Mehrubonov Date: Sun, 17 Dec 2023 20:16:27 +0000 Subject: [PATCH 04/14] Fix typo --- cmd/background_worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/background_worker.go b/cmd/background_worker.go index 8416df189..7cf78d197 100644 --- a/cmd/background_worker.go +++ b/cmd/background_worker.go @@ -57,7 +57,7 @@ func StartBackGroundWorker(logger *slog.Logger) (func(), error) { return nil, err } - params := jobs.ClearRecrodsParams{ + params := jobs.ClearRecordsParams{ DBConnectionParams: dbconn.New( dbHost, dbPort, From 882f5f9e368ed2a75019b9461b2ca7fc4b385fa7 Mon Sep 17 00:00:00 2001 From: Nozim Mehrubonov Date: Sun, 17 Dec 2023 20:18:09 +0000 Subject: [PATCH 05/14] Amend blocktx tests --- blocktx/store/sql/get_block_for_height_test.go | 2 +- blocktx/store/sql/get_block_test.go | 2 +- blocktx/store/sql/get_block_transactions_test.go | 2 +- blocktx/store/sql/get_last_processed_block_test.go | 2 +- blocktx/store/sql/get_mined_transaction_for_block_test.go | 2 +- blocktx/store/sql/get_transaction_block_test.go | 2 +- blocktx/store/sql/get_transaction_merkle_path_test.go | 2 +- blocktx/store/sql/get_transaction_source_test.go | 2 +- blocktx/store/sql/insert_block_test.go | 2 +- blocktx/store/sql/insert_block_transactions_test.go | 2 +- blocktx/store/sql/orphan_height_test.go | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/blocktx/store/sql/get_block_for_height_test.go b/blocktx/store/sql/get_block_for_height_test.go index 101c65b49..ad92ea2c8 100644 --- a/blocktx/store/sql/get_block_for_height_test.go +++ b/blocktx/store/sql/get_block_for_height_test.go @@ -13,7 +13,7 @@ import ( ) type GetBlockByHeightTestSuite struct { - DatabaseTestSuite + BlockTXDBTestSuite } func (s *GetBlockByHeightTestSuite) Test() { diff --git a/blocktx/store/sql/get_block_test.go b/blocktx/store/sql/get_block_test.go index 085dd88dd..bb4282175 100644 --- a/blocktx/store/sql/get_block_test.go +++ b/blocktx/store/sql/get_block_test.go @@ -14,7 +14,7 @@ import ( ) type GetBlockTestSuite struct { - DatabaseTestSuite + BlockTXDBTestSuite } func (s *GetBlockTestSuite) Test() { diff --git a/blocktx/store/sql/get_block_transactions_test.go b/blocktx/store/sql/get_block_transactions_test.go index 9b048e5c1..68469d878 100644 --- a/blocktx/store/sql/get_block_transactions_test.go +++ b/blocktx/store/sql/get_block_transactions_test.go @@ -15,7 +15,7 @@ import ( ) type GetBlockTransactionsSuite struct { - DatabaseTestSuite + BlockTXDBTestSuite } func (s *GetBlockTransactionsSuite) Test() { diff --git a/blocktx/store/sql/get_last_processed_block_test.go b/blocktx/store/sql/get_last_processed_block_test.go index 29056d6ee..24a8ef1c9 100644 --- a/blocktx/store/sql/get_last_processed_block_test.go +++ b/blocktx/store/sql/get_last_processed_block_test.go @@ -14,7 +14,7 @@ import ( ) type GetLastProcessedBlockSuite struct { - DatabaseTestSuite + BlockTXDBTestSuite } func (s *GetLastProcessedBlockSuite) Test() { diff --git a/blocktx/store/sql/get_mined_transaction_for_block_test.go b/blocktx/store/sql/get_mined_transaction_for_block_test.go index df0bfd94b..ee7f00ba2 100644 --- a/blocktx/store/sql/get_mined_transaction_for_block_test.go +++ b/blocktx/store/sql/get_mined_transaction_for_block_test.go @@ -13,7 +13,7 @@ import ( ) type MinedTransactionForBlockSuite struct { - DatabaseTestSuite + BlockTXDBTestSuite } func (s *MinedTransactionForBlockSuite) Test() { diff --git a/blocktx/store/sql/get_transaction_block_test.go b/blocktx/store/sql/get_transaction_block_test.go index 7bb7df224..1704205b4 100644 --- a/blocktx/store/sql/get_transaction_block_test.go +++ b/blocktx/store/sql/get_transaction_block_test.go @@ -13,7 +13,7 @@ import ( ) type GetTransactionBlockSuite struct { - DatabaseTestSuite + BlockTXDBTestSuite } func (s *GetTransactionBlockSuite) Test() { diff --git a/blocktx/store/sql/get_transaction_merkle_path_test.go b/blocktx/store/sql/get_transaction_merkle_path_test.go index 35af8a205..140985c39 100644 --- a/blocktx/store/sql/get_transaction_merkle_path_test.go +++ b/blocktx/store/sql/get_transaction_merkle_path_test.go @@ -13,7 +13,7 @@ import ( ) type GetTransactionMerklePathSuite struct { - DatabaseTestSuite + BlockTXDBTestSuite } func (s *GetTransactionMerklePathSuite) Test() { diff --git a/blocktx/store/sql/get_transaction_source_test.go b/blocktx/store/sql/get_transaction_source_test.go index 2416391b5..bf7067c4d 100644 --- a/blocktx/store/sql/get_transaction_source_test.go +++ b/blocktx/store/sql/get_transaction_source_test.go @@ -13,7 +13,7 @@ import ( ) type GetTransactionSourceSuite struct { - DatabaseTestSuite + BlockTXDBTestSuite } func (s *GetTransactionSourceSuite) Test() { diff --git a/blocktx/store/sql/insert_block_test.go b/blocktx/store/sql/insert_block_test.go index 38fc27596..6cf776d6b 100644 --- a/blocktx/store/sql/insert_block_test.go +++ b/blocktx/store/sql/insert_block_test.go @@ -14,7 +14,7 @@ import ( ) type InsertBlockSuite struct { - DatabaseTestSuite + BlockTXDBTestSuite } func (s *InsertBlockSuite) Test() { diff --git a/blocktx/store/sql/insert_block_transactions_test.go b/blocktx/store/sql/insert_block_transactions_test.go index bd6933e64..9bd61b09d 100644 --- a/blocktx/store/sql/insert_block_transactions_test.go +++ b/blocktx/store/sql/insert_block_transactions_test.go @@ -14,7 +14,7 @@ import ( ) type InsertBlockTransactionsSuite struct { - DatabaseTestSuite + BlockTXDBTestSuite } type Tx struct { diff --git a/blocktx/store/sql/orphan_height_test.go b/blocktx/store/sql/orphan_height_test.go index 054087d78..ccba73238 100644 --- a/blocktx/store/sql/orphan_height_test.go +++ b/blocktx/store/sql/orphan_height_test.go @@ -13,7 +13,7 @@ import ( ) type OrphanHeightSutie struct { - DatabaseTestSuite + BlockTXDBTestSuite } func (s *OrphanHeightSutie) Test() { From c5ebd6e9dba7ffaa5421e108f2b62c0eb80b4da0 Mon Sep 17 00:00:00 2001 From: Nozim Mehrubonov Date: Sun, 17 Dec 2023 20:20:27 +0000 Subject: [PATCH 06/14] Fix compilation errors --- cmd/background_worker/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/background_worker/main.go b/cmd/background_worker/main.go index eb0be1c8f..16e639199 100644 --- a/cmd/background_worker/main.go +++ b/cmd/background_worker/main.go @@ -27,7 +27,7 @@ func main() { } } - params := jobs.ClearRecrodsParams{ + params := jobs.ClearRecordsParams{ DBConnectionParams: dbconn.New( viper.GetString("cleanBlocks.host"), viper.GetInt("cleanBlocks.port"), From 37f507ded5bc503ad196107259008ce817ce1ab2 Mon Sep 17 00:00:00 2001 From: Nozim Mehrubonov Date: Tue, 19 Dec 2023 16:27:21 +0000 Subject: [PATCH 07/14] Fix test failure --- blocktx/store/sql/get_transaction_blocks_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blocktx/store/sql/get_transaction_blocks_test.go b/blocktx/store/sql/get_transaction_blocks_test.go index 83132ac17..d81c52c71 100644 --- a/blocktx/store/sql/get_transaction_blocks_test.go +++ b/blocktx/store/sql/get_transaction_blocks_test.go @@ -15,7 +15,7 @@ import ( ) type GetTransactionBlocksSuite struct { - DatabaseTestSuite + BlockTXDBTestSuite } func (s *GetTransactionBlocksSuite) Test() { From 788bc83a4c33572bf74a9c909d4a60d313e74ae9 Mon Sep 17 00:00:00 2001 From: Nozim Mehrubonov Date: Tue, 19 Dec 2023 16:33:14 +0000 Subject: [PATCH 08/14] Fix compilation error --- blocktx/store/sql/get_block_gaps_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blocktx/store/sql/get_block_gaps_test.go b/blocktx/store/sql/get_block_gaps_test.go index 5f854c23e..d289a2ec4 100644 --- a/blocktx/store/sql/get_block_gaps_test.go +++ b/blocktx/store/sql/get_block_gaps_test.go @@ -14,7 +14,7 @@ import ( ) type GetBlockGapTestSuite struct { - DatabaseTestSuite + BlockTXDBTestSuite } func (s *GetBlockGapTestSuite) Test() { From 92868e3145adf7422ce26f6a323d54009173a69e Mon Sep 17 00:00:00 2001 From: Nozim Mehrubonov Date: Tue, 19 Dec 2023 16:39:25 +0000 Subject: [PATCH 09/14] Fix formatting in blocktx db utility --- database_testing/blocktx_db_test_suite.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/database_testing/blocktx_db_test_suite.go b/database_testing/blocktx_db_test_suite.go index 09cdc2270..c8966a49e 100644 --- a/database_testing/blocktx_db_test_suite.go +++ b/database_testing/blocktx_db_test_suite.go @@ -97,7 +97,6 @@ func (s *BlockTXDBTestSuite) SetupTest() { s.truncateTables() } - func (s *BlockTXDBTestSuite) Conn() *sqlx.Conn { return s.Connection } @@ -173,7 +172,6 @@ func (s *BlockTXDBTestSuite) TearDownTest() { s.truncateTables() } - func (s *BlockTXDBTestSuite) truncateTables() { db, err := sqlx.Open("postgres", DefaultParams.String()) require.NoError(s.T(), err) From 475b2a09b7ad6c09f6f1e62f7e024c537061c67b Mon Sep 17 00:00:00 2001 From: Nozim Mehrubonov Date: Tue, 26 Dec 2023 09:07:46 +0000 Subject: [PATCH 10/14] Add timestamp columnes for metamorph tables --- .../metamorph/postgres/000002_create_timestamps.down.sql | 2 ++ .../metamorph/postgres/000002_create_timestamps.up.sql | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 database/migrations/metamorph/postgres/000002_create_timestamps.down.sql create mode 100644 database/migrations/metamorph/postgres/000002_create_timestamps.up.sql diff --git a/database/migrations/metamorph/postgres/000002_create_timestamps.down.sql b/database/migrations/metamorph/postgres/000002_create_timestamps.down.sql new file mode 100644 index 000000000..f6d41b7b6 --- /dev/null +++ b/database/migrations/metamorph/postgres/000002_create_timestamps.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE transactions ADD COLUMN inserted_at TIMESTAMPTZ; +ALTER TABLE blocks ADD COLUMN inserted_at TIMESTAMPTZ; \ No newline at end of file diff --git a/database/migrations/metamorph/postgres/000002_create_timestamps.up.sql b/database/migrations/metamorph/postgres/000002_create_timestamps.up.sql new file mode 100644 index 000000000..a766d98bf --- /dev/null +++ b/database/migrations/metamorph/postgres/000002_create_timestamps.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE blocks DROP COLUMN inserted_at; +ALTER TABLE transactions DROP COLUMN inserted_at; \ No newline at end of file From 0bcd8cbc4e34445cdca9fc605d9fd79b96bd7bb2 Mon Sep 17 00:00:00 2001 From: Nozim Mehrubonov Date: Tue, 26 Dec 2023 13:24:20 +0000 Subject: [PATCH 11/14] Rename migrations --- ...eate_timestamps.down.sql => 000003_create_timestamps.down.sql} | 0 ...2_create_timestamps.up.sql => 000003_create_timestamps.up.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename database/migrations/metamorph/postgres/{000002_create_timestamps.down.sql => 000003_create_timestamps.down.sql} (100%) rename database/migrations/metamorph/postgres/{000002_create_timestamps.up.sql => 000003_create_timestamps.up.sql} (100%) diff --git a/database/migrations/metamorph/postgres/000002_create_timestamps.down.sql b/database/migrations/metamorph/postgres/000003_create_timestamps.down.sql similarity index 100% rename from database/migrations/metamorph/postgres/000002_create_timestamps.down.sql rename to database/migrations/metamorph/postgres/000003_create_timestamps.down.sql diff --git a/database/migrations/metamorph/postgres/000002_create_timestamps.up.sql b/database/migrations/metamorph/postgres/000003_create_timestamps.up.sql similarity index 100% rename from database/migrations/metamorph/postgres/000002_create_timestamps.up.sql rename to database/migrations/metamorph/postgres/000003_create_timestamps.up.sql From 53ebaaebb608ede43b25cc900baaed5031580108 Mon Sep 17 00:00:00 2001 From: Nozim Mehrubonov Date: Thu, 4 Jan 2024 08:41:18 +0000 Subject: [PATCH 12/14] Fix test errors, amend migrations --- .../postgres/000001_create_transactions.up.sql | 5 +++-- .../metamorph/postgres/000002_create_blocks.up.sql | 10 ++++------ .../postgres/000003_create_timestamps.down.sql | 4 ++-- .../metamorph/postgres/000003_create_timestamps.up.sql | 4 ++-- database_testing/metamorph_db_test_suite.go | 3 +-- metamorph/store/model.go | 8 +++++--- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/database/migrations/metamorph/postgres/000001_create_transactions.up.sql b/database/migrations/metamorph/postgres/000001_create_transactions.up.sql index 292e36b34..61c337214 100644 --- a/database/migrations/metamorph/postgres/000001_create_transactions.up.sql +++ b/database/migrations/metamorph/postgres/000001_create_transactions.up.sql @@ -13,8 +13,9 @@ CREATE TABLE transactions reject_reason TEXT, raw_tx BYTEA, locked_by TEXT, - inserted_at TIMESTAMPTZ + inserted_at_num INTEGER DEFAULT TO_NUMBER(TO_CHAR((NOW()) AT TIME ZONE 'UTC', 'yyyymmddhh24'), '9999999999') NOT NULL + ); CREATE INDEX ix_metamorph_transactions_locked_by ON transactions (locked_by); -CREATE INDEX ix_metamorph_transactions_inserted_at_num ON transactions (inserted_at); +CREATE INDEX ix_metamorph_transactions_inserted_at_num ON transactions (inserted_at_num); diff --git a/database/migrations/metamorph/postgres/000002_create_blocks.up.sql b/database/migrations/metamorph/postgres/000002_create_blocks.up.sql index 753529483..4789db9a7 100644 --- a/database/migrations/metamorph/postgres/000002_create_blocks.up.sql +++ b/database/migrations/metamorph/postgres/000002_create_blocks.up.sql @@ -1,8 +1,6 @@ -CREATE TABLE blocks -( - hash BYTEA PRIMARY KEY, - processed_at TIMESTAMPTZ, - inserted_at TIMESTAMPTZ +CREATE TABLE blocks ( + hash BYTEA PRIMARY KEY, + processed_at TIMESTAMPTZ, + inserted_at_num INTEGER DEFAULT TO_NUMBER(TO_CHAR((NOW()) AT TIME ZONE 'UTC', 'yyyymmddhh24'), '9999999999') NOT NULL ); -CREATE INDEX ix_metamorph_blocks_inserted_at ON blocks (inserted_at); diff --git a/database/migrations/metamorph/postgres/000003_create_timestamps.down.sql b/database/migrations/metamorph/postgres/000003_create_timestamps.down.sql index f6d41b7b6..426c824ac 100644 --- a/database/migrations/metamorph/postgres/000003_create_timestamps.down.sql +++ b/database/migrations/metamorph/postgres/000003_create_timestamps.down.sql @@ -1,2 +1,2 @@ -ALTER TABLE transactions ADD COLUMN inserted_at TIMESTAMPTZ; -ALTER TABLE blocks ADD COLUMN inserted_at TIMESTAMPTZ; \ No newline at end of file +ALTER TABLE blocks DROP COLUMN inserted_at; +ALTER TABLE transactions DROP COLUMN inserted_at; diff --git a/database/migrations/metamorph/postgres/000003_create_timestamps.up.sql b/database/migrations/metamorph/postgres/000003_create_timestamps.up.sql index a766d98bf..f6d41b7b6 100644 --- a/database/migrations/metamorph/postgres/000003_create_timestamps.up.sql +++ b/database/migrations/metamorph/postgres/000003_create_timestamps.up.sql @@ -1,2 +1,2 @@ -ALTER TABLE blocks DROP COLUMN inserted_at; -ALTER TABLE transactions DROP COLUMN inserted_at; \ No newline at end of file +ALTER TABLE transactions ADD COLUMN inserted_at TIMESTAMPTZ; +ALTER TABLE blocks ADD COLUMN inserted_at TIMESTAMPTZ; \ No newline at end of file diff --git a/database_testing/metamorph_db_test_suite.go b/database_testing/metamorph_db_test_suite.go index e25ef6f28..9ad5023e7 100644 --- a/database_testing/metamorph_db_test_suite.go +++ b/database_testing/metamorph_db_test_suite.go @@ -65,9 +65,8 @@ func (s *MetamorphDBTestSuite) SetupSuite() { path := "file://" + testDir + "/../database/migrations/metamorph/postgres" m, err := migrate.New(path, DefaultMMParams.String()) - require.NoError(s.T(), err) - + s.T().Log("applied migrations") if err := m.Up(); err != nil { if !errors.Is(err, migrate.ErrNoChange) { require.NoError(s.T(), err) diff --git a/metamorph/store/model.go b/metamorph/store/model.go index 9a282a362..354f4907b 100644 --- a/metamorph/store/model.go +++ b/metamorph/store/model.go @@ -3,9 +3,10 @@ package store import "time" type Block struct { - Hash string `db:"hash"` - ProcessedAt time.Time `db:"processed_at"` - InsertedAt time.Time `db:"inserted_at"` + Hash string `db:"hash"` + ProcessedAt time.Time `db:"processed_at"` + InsertedAt time.Time `db:"inserted_at"` + InsertedAtNum int64 `db:"inserted_at_num"` } type Transaction struct { @@ -23,4 +24,5 @@ type Transaction struct { RawTX []byte `db:"raw_tx"` LockedBy string `db:"locked_by"` InsertedAt time.Time `db:"inserted_at"` + InsertedAtNum int64 `db:"inserted_at_num"` } From cdebed28294a1b8f6ecf967b2e52145fb739eabe Mon Sep 17 00:00:00 2001 From: Nozim Mehrubonov Date: Fri, 5 Jan 2024 12:17:09 +0000 Subject: [PATCH 13/14] Amend schema name migrations --- .../000001_create_transactions.down.sql | 2 +- .../000001_create_transactions.up.sql | 33 +++++++++---------- .../postgres/000002_create_blocks.down.sql | 2 +- .../postgres/000002_create_blocks.up.sql | 9 ++--- .../postgres/000003_create_functions.down.sql | 2 ++ .../postgres/000003_create_functions.up.sql | 16 +++++++++ .../000003_create_timestamps.down.sql | 2 -- .../postgres/000003_create_timestamps.up.sql | 2 -- .../000004_create_timestamps.down.sql | 2 ++ .../postgres/000004_create_timestamps.up.sql | 2 ++ 10 files changed, 45 insertions(+), 27 deletions(-) create mode 100644 database/migrations/metamorph/postgres/000003_create_functions.down.sql create mode 100644 database/migrations/metamorph/postgres/000003_create_functions.up.sql delete mode 100644 database/migrations/metamorph/postgres/000003_create_timestamps.down.sql delete mode 100644 database/migrations/metamorph/postgres/000003_create_timestamps.up.sql create mode 100644 database/migrations/metamorph/postgres/000004_create_timestamps.down.sql create mode 100644 database/migrations/metamorph/postgres/000004_create_timestamps.up.sql diff --git a/database/migrations/metamorph/postgres/000001_create_transactions.down.sql b/database/migrations/metamorph/postgres/000001_create_transactions.down.sql index 91345dc77..693c84add 100644 --- a/database/migrations/metamorph/postgres/000001_create_transactions.down.sql +++ b/database/migrations/metamorph/postgres/000001_create_transactions.down.sql @@ -1 +1 @@ -DROP TABLE transactions; +DROP TABLE metamorph.transactions; diff --git a/database/migrations/metamorph/postgres/000001_create_transactions.up.sql b/database/migrations/metamorph/postgres/000001_create_transactions.up.sql index 61c337214..29108f7c5 100644 --- a/database/migrations/metamorph/postgres/000001_create_transactions.up.sql +++ b/database/migrations/metamorph/postgres/000001_create_transactions.up.sql @@ -1,21 +1,20 @@ -CREATE TABLE transactions -( - hash BYTEA PRIMARY KEY, - stored_at TIMESTAMPTZ, - announced_at TIMESTAMPTZ, - mined_at TIMESTAMPTZ, - status INTEGER, - block_height BIGINT, - block_hash BYTEA, - callback_url TEXT, +CREATE SCHEMA metamorph; +CREATE TABLE metamorph.transactions ( + hash BYTEA PRIMARY KEY, + stored_at TIMESTAMPTZ, + announced_at TIMESTAMPTZ, + mined_at TIMESTAMPTZ, + status INTEGER, + block_height BIGINT, + block_hash BYTEA, + callback_url TEXT, callback_token TEXT, - merkle_proof TEXT, - reject_reason TEXT, - raw_tx BYTEA, - locked_by TEXT, + merkle_proof TEXT, + reject_reason TEXT, + raw_tx BYTEA, + locked_by TEXT, inserted_at_num INTEGER DEFAULT TO_NUMBER(TO_CHAR((NOW()) AT TIME ZONE 'UTC', 'yyyymmddhh24'), '9999999999') NOT NULL - ); -CREATE INDEX ix_metamorph_transactions_locked_by ON transactions (locked_by); -CREATE INDEX ix_metamorph_transactions_inserted_at_num ON transactions (inserted_at_num); +CREATE INDEX ix_metamorph_transactions_locked_by ON metamorph.transactions (locked_by); +CREATE INDEX ix_metamorph_transactions_inserted_at_num ON metamorph.transactions (inserted_at_num); diff --git a/database/migrations/metamorph/postgres/000002_create_blocks.down.sql b/database/migrations/metamorph/postgres/000002_create_blocks.down.sql index 7a4fcb3dc..06d8b13d6 100644 --- a/database/migrations/metamorph/postgres/000002_create_blocks.down.sql +++ b/database/migrations/metamorph/postgres/000002_create_blocks.down.sql @@ -1 +1 @@ -DROP TABLE blocks; +DROP TABLE metamorph.blocks; diff --git a/database/migrations/metamorph/postgres/000002_create_blocks.up.sql b/database/migrations/metamorph/postgres/000002_create_blocks.up.sql index 4789db9a7..4cc1cc0eb 100644 --- a/database/migrations/metamorph/postgres/000002_create_blocks.up.sql +++ b/database/migrations/metamorph/postgres/000002_create_blocks.up.sql @@ -1,6 +1,7 @@ -CREATE TABLE blocks ( - hash BYTEA PRIMARY KEY, - processed_at TIMESTAMPTZ, - inserted_at_num INTEGER DEFAULT TO_NUMBER(TO_CHAR((NOW()) AT TIME ZONE 'UTC', 'yyyymmddhh24'), '9999999999') NOT NULL +CREATE TABLE metamorph.blocks ( + hash BYTEA PRIMARY KEY, + processed_at TIMESTAMPTZ, + inserted_at_num INTEGER DEFAULT TO_NUMBER(TO_CHAR((NOW()) AT TIME ZONE 'UTC', 'yyyymmddhh24'), '9999999999') NOT NULL ); +CREATE INDEX ix_metamorph_blocks_inserted_at_num ON metamorph.blocks (inserted_at_num); diff --git a/database/migrations/metamorph/postgres/000003_create_functions.down.sql b/database/migrations/metamorph/postgres/000003_create_functions.down.sql new file mode 100644 index 000000000..22f8397b5 --- /dev/null +++ b/database/migrations/metamorph/postgres/000003_create_functions.down.sql @@ -0,0 +1,2 @@ +DROP FUNCTION reverse_bytes_iter(bytes bytea, length int, midpoint int, index int); +DROP FUNCTION reverse_bytes(bytes bytea); diff --git a/database/migrations/metamorph/postgres/000003_create_functions.up.sql b/database/migrations/metamorph/postgres/000003_create_functions.up.sql new file mode 100644 index 000000000..a49eec2fb --- /dev/null +++ b/database/migrations/metamorph/postgres/000003_create_functions.up.sql @@ -0,0 +1,16 @@ +CREATE OR REPLACE FUNCTION reverse_bytes_iter(bytes bytea, length int, midpoint int, index int) + RETURNS bytea AS + $$ +SELECT CASE WHEN index >= midpoint THEN bytes ELSE + reverse_bytes_iter( + set_byte( + set_byte(bytes, index, get_byte(bytes, length-index)), + length-index, get_byte(bytes, index) + ), + length, midpoint, index + 1 + ) + END; +$$ LANGUAGE SQL IMMUTABLE; + +CREATE +OR REPLACE FUNCTION reverse_bytes(bytes bytea) RETURNS bytea AS 'SELECT reverse_bytes_iter(bytes, octet_length(bytes)-1, octet_length(bytes)/2, 0)' LANGUAGE SQL IMMUTABLE; diff --git a/database/migrations/metamorph/postgres/000003_create_timestamps.down.sql b/database/migrations/metamorph/postgres/000003_create_timestamps.down.sql deleted file mode 100644 index 426c824ac..000000000 --- a/database/migrations/metamorph/postgres/000003_create_timestamps.down.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE blocks DROP COLUMN inserted_at; -ALTER TABLE transactions DROP COLUMN inserted_at; diff --git a/database/migrations/metamorph/postgres/000003_create_timestamps.up.sql b/database/migrations/metamorph/postgres/000003_create_timestamps.up.sql deleted file mode 100644 index f6d41b7b6..000000000 --- a/database/migrations/metamorph/postgres/000003_create_timestamps.up.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE transactions ADD COLUMN inserted_at TIMESTAMPTZ; -ALTER TABLE blocks ADD COLUMN inserted_at TIMESTAMPTZ; \ No newline at end of file diff --git a/database/migrations/metamorph/postgres/000004_create_timestamps.down.sql b/database/migrations/metamorph/postgres/000004_create_timestamps.down.sql new file mode 100644 index 000000000..d3725bd2d --- /dev/null +++ b/database/migrations/metamorph/postgres/000004_create_timestamps.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE metamorph.blocks DROP COLUMN inserted_at; +ALTER TABLE metamorph.transactions DROP COLUMN inserted_at; diff --git a/database/migrations/metamorph/postgres/000004_create_timestamps.up.sql b/database/migrations/metamorph/postgres/000004_create_timestamps.up.sql new file mode 100644 index 000000000..4af3ecf85 --- /dev/null +++ b/database/migrations/metamorph/postgres/000004_create_timestamps.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE metamorph.transactions ADD COLUMN inserted_at TIMESTAMPTZ; +ALTER TABLE metamorph.blocks ADD COLUMN inserted_at TIMESTAMPTZ; \ No newline at end of file From aab927abb3019dc9d9feb91807eafbc36c996cfe Mon Sep 17 00:00:00 2001 From: Nozim Mehrubonov Date: Fri, 5 Jan 2024 12:59:45 +0000 Subject: [PATCH 14/14] Amend metamorph tests --- background_worker/jobs/clear_metamorph.go | 5 +++-- background_worker/jobs/clear_metamorph_test.go | 9 +++++---- database_testing/metamorph_db_test_suite.go | 8 ++++---- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/background_worker/jobs/clear_metamorph.go b/background_worker/jobs/clear_metamorph.go index c134233d3..7f3bfedaf 100644 --- a/background_worker/jobs/clear_metamorph.go +++ b/background_worker/jobs/clear_metamorph.go @@ -2,6 +2,7 @@ package jobs import ( "fmt" + "github.com/jmoiron/sqlx" ) @@ -31,10 +32,10 @@ func runDelete(table string, params ClearRecordsParams) error { } func ClearMetamorph(params ClearRecordsParams) error { - if err := runDelete("blocks", params); err != nil { + if err := runDelete("metamorph.blocks", params); err != nil { return err } - if err := runDelete("transactions", params); err != nil { + if err := runDelete("metamorph.transactions", params); err != nil { return err } diff --git a/background_worker/jobs/clear_metamorph_test.go b/background_worker/jobs/clear_metamorph_test.go index 47d150c78..bfe40dbb0 100644 --- a/background_worker/jobs/clear_metamorph_test.go +++ b/background_worker/jobs/clear_metamorph_test.go @@ -1,14 +1,15 @@ package jobs import ( + "testing" + "time" + . "github.com/bitcoin-sv/arc/database_testing" "github.com/bitcoin-sv/arc/metamorph/store" "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "testing" - "time" ) type ClearMetamorphSuite struct { @@ -54,12 +55,12 @@ func (s *ClearMetamorphSuite) Test() { require.NoError(s.T(), err) var blks []store.Block - require.NoError(s.T(), db.Select(&blks, "SELECT * from blocks")) + require.NoError(s.T(), db.Select(&blks, "SELECT * from metamorph.blocks")) assert.Len(s.T(), blks, 5) var stx []store.Transaction - require.NoError(s.T(), db.Select(&stx, "SELECT * from transactions")) + require.NoError(s.T(), db.Select(&stx, "SELECT * from metamorph.transactions")) assert.Len(s.T(), stx, 5) } diff --git a/database_testing/metamorph_db_test_suite.go b/database_testing/metamorph_db_test_suite.go index 9ad5023e7..13cd09a0c 100644 --- a/database_testing/metamorph_db_test_suite.go +++ b/database_testing/metamorph_db_test_suite.go @@ -83,8 +83,8 @@ func (s *MetamorphDBTestSuite) truncateTables() { db, err := sqlx.Open("postgres", DefaultMMParams.String()) require.NoError(s.T(), err) - db.MustExec("truncate table blocks;") - db.MustExec("truncate table transactions;") + db.MustExec("truncate table metamorph.blocks;") + db.MustExec("truncate table metamorph.transactions;") } func (s *MetamorphDBTestSuite) Conn() *sqlx.Conn { @@ -95,7 +95,7 @@ func (s *MetamorphDBTestSuite) InsertBlock(block *store.Block) { db, err := sqlx.Open("postgres", DefaultMMParams.String()) require.NoError(s.T(), err) - q := `INSERT INTO blocks( + q := `INSERT INTO metamorph.blocks( hash, processed_at, inserted_at) @@ -114,7 +114,7 @@ func (s *MetamorphDBTestSuite) InsertBlock(block *store.Block) { func (s *MetamorphDBTestSuite) InsertTransaction(tx *store.Transaction) { db, err := sqlx.Open("postgres", DefaultMMParams.String()) require.NoError(s.T(), err) - q := `INSERT INTO transactions (hash, + q := `INSERT INTO metamorph.transactions (hash, stored_at, announced_at, mined_at,