From a5c648e5268026b47a999285c663134fef069538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Wed, 15 Jan 2025 10:53:16 +0100 Subject: [PATCH 01/10] refactor(ARCO-319): Add setting which denotes whether new block should be considered longest --- config/config.go | 1 + config/defaults.go | 1 + config/example_config.yaml | 1 + test/config/config.yaml | 1 + 4 files changed, 4 insertions(+) diff --git a/config/config.go b/config/config.go index 04426964d..54ee8954e 100644 --- a/config/config.go +++ b/config/config.go @@ -127,6 +127,7 @@ type BlocktxConfig struct { MaxAllowedBlockHeightMismatch int `mapstructure:"maxAllowedBlockHeightMismatch"` MessageQueue *MessageQueueConfig `mapstructure:"mq"` P2pReadBufferSize int `mapstructure:"p2pReadBufferSize"` + IncomingIsLongest bool `mapstructure:"incomingIsLongest"` } type DbConfig struct { diff --git a/config/defaults.go b/config/defaults.go index e5047a70d..a6aea5696 100644 --- a/config/defaults.go +++ b/config/defaults.go @@ -128,6 +128,7 @@ func getBlocktxConfig() *BlocktxConfig { MaxBlockProcessingDuration: 5 * time.Minute, MessageQueue: &MessageQueueConfig{}, P2pReadBufferSize: 8 * 1024 * 1024, + IncomingIsLongest: false, } } diff --git a/config/example_config.yaml b/config/example_config.yaml index 27ff1c80d..e900d6e0b 100644 --- a/config/example_config.yaml +++ b/config/example_config.yaml @@ -105,6 +105,7 @@ blocktx: registerTxsInterval: 10s # time interval to read from the channel registered transactions maxBlockProcessingDuration: 5m # maximum time a blocktx can spend on processing a block before unlocking it to be requested again monitorPeers: false # if enabled, peers which do not receive alive signal from nodes will be restarted + incomingIsLongest: false # whether each new block received is considered to be from the longest blockchain. If there are a lot of block gaps in blocktx database it is advisable to set this to true fillGaps: enabled: true interval: 15m # time interval to check and fill gaps in processed blocks diff --git a/test/config/config.yaml b/test/config/config.yaml index f65539a5e..184d68cdc 100644 --- a/test/config/config.yaml +++ b/test/config/config.yaml @@ -68,6 +68,7 @@ metamorph: checkSeenOnNetworkOlderThan: 3h checkSeenOnNetworkPeriod: 4h monitorPeers: true + incomingIsLongest: false profilerAddr: localhost:9992 health: serverDialAddr: localhost:8005 From faecb48cee4e7a8f0af5527a246a1ab73f9caeae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Wed, 15 Jan 2025 11:29:00 +0100 Subject: [PATCH 02/10] refactor(ARCO-319): Implement setting --- cmd/arc/services/blocktx.go | 1 + internal/blocktx/processor.go | 19 ++++++++++--------- internal/blocktx/processor_opts.go | 6 ++++++ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/cmd/arc/services/blocktx.go b/cmd/arc/services/blocktx.go index c3183d95e..568426e76 100644 --- a/cmd/arc/services/blocktx.go +++ b/cmd/arc/services/blocktx.go @@ -126,6 +126,7 @@ func StartBlockTx(logger *slog.Logger, arcConfig *config.ArcConfig) (func(), err blocktx.WithRegisterTxsInterval(btxConfig.RegisterTxsInterval), blocktx.WithMessageQueueClient(mqClient), blocktx.WithMaxBlockProcessingDuration(btxConfig.MaxBlockProcessingDuration), + blocktx.WithIncomingIsLongest(btxConfig.IncomingIsLongest), ) blockRequestCh := make(chan blocktx.BlockRequest, blockProcessingBuffer) diff --git a/internal/blocktx/processor.go b/internal/blocktx/processor.go index f463c5db9..0257b5b4a 100644 --- a/internal/blocktx/processor.go +++ b/internal/blocktx/processor.go @@ -71,6 +71,7 @@ type Processor struct { tracingAttributes []attribute.KeyValue stats *processorStats statCollectionInterval time.Duration + incomingIsLongest bool now func() time.Time maxBlockProcessingDuration time.Duration @@ -456,15 +457,17 @@ func (p *Processor) verifyAndInsertBlock(ctx context.Context, blockMsg *p2p.Bloc MerkleRoot: merkleRoot[:], Height: blockMsg.Height, Chainwork: calculateChainwork(blockMsg.Header.Bits).String(), - Status: blocktx_api.Status_LONGEST, // temporary fix (!), TODO: remove this when gaps are filling quickly again } - // TODO: uncomment when gaps are filling quickly again - // err = p.assignBlockStatus(ctx, incomingBlock, previousBlockHash) - // if err != nil { - // p.logger.Error("unable to assign block status", slog.String("hash", blockHash.String()), slog.Uint64("height", incomingBlock.Height), slog.String("err", err.Error())) - // return nil, err - // } + if p.incomingIsLongest { + incomingBlock.Status = blocktx_api.Status_LONGEST + } else { + err = p.assignBlockStatus(ctx, incomingBlock, previousBlockHash) + if err != nil { + p.logger.Error("unable to assign block status", slog.String("hash", blockHash.String()), slog.Uint64("height", incomingBlock.Height), slog.String("err", err.Error())) + return nil, err + } + } p.logger.Info("Inserting block", slog.String("hash", blockHash.String()), slog.Uint64("height", incomingBlock.Height), slog.String("status", incomingBlock.Status.String())) @@ -477,7 +480,6 @@ func (p *Processor) verifyAndInsertBlock(ctx context.Context, blockMsg *p2p.Bloc return incomingBlock, nil } -//lint:ignore U1000 Ignored until gaps are filling quickly again TODO: remove this ignore func (p *Processor) assignBlockStatus(ctx context.Context, block *blocktx_api.Block, prevBlockHash chainhash.Hash) (err error) { ctx, span := tracing.StartTracing(ctx, "assignBlockStatus", p.tracingEnabled, p.tracingAttributes...) defer func() { @@ -538,7 +540,6 @@ func (p *Processor) assignBlockStatus(ctx context.Context, block *blocktx_api.Bl return nil } -//lint:ignore U1000 Ignored until gaps are filling quickly again TODO: remove this ignore func (p *Processor) longestTipExists(ctx context.Context) (bool, error) { _, err := p.store.GetChainTip(ctx) if err != nil && !errors.Is(err, store.ErrBlockNotFound) { diff --git a/internal/blocktx/processor_opts.go b/internal/blocktx/processor_opts.go index 29b9291db..3cb5b9401 100644 --- a/internal/blocktx/processor_opts.go +++ b/internal/blocktx/processor_opts.go @@ -79,3 +79,9 @@ func WithMaxBlockProcessingDuration(d time.Duration) func(*Processor) { processor.maxBlockProcessingDuration = d } } + +func WithIncomingIsLongest(enabled bool) func(*Processor) { + return func(processor *Processor) { + processor.incomingIsLongest = enabled + } +} From 0d47f0acaf5332befd28bb0f15e606c8a3f539bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Wed, 15 Jan 2025 14:21:55 +0100 Subject: [PATCH 03/10] refactor(ARCO-319): Enable tests --- .../reorg/blocktx.block_transactions.yaml | 8 ++--- .../fixtures/reorg/blocktx.blocks.yaml | 31 +++++++++++-------- .../blocktx.registered_transactions.yaml | 8 +++++ .../blocktx.block_transactions.yaml | 10 +++--- .../blocktx.registered_transactions.yaml | 8 +++++ .../blocktx.block_transactions_map.yaml | 8 ----- .../reorg_integration_test.go | 5 +-- internal/blocktx/processor_test.go | 3 -- .../blocktx.registered_transactions.yaml | 28 ++++++++--------- test/submit_05_reorg_test.go | 3 -- 10 files changed, 58 insertions(+), 54 deletions(-) create mode 100644 internal/blocktx/integration_test/fixtures/reorg/blocktx.registered_transactions.yaml create mode 100644 internal/blocktx/integration_test/fixtures/reorg_orphans/blocktx.registered_transactions.yaml delete mode 100644 internal/blocktx/integration_test/fixtures/stale_orphans/blocktx.block_transactions_map.yaml diff --git a/internal/blocktx/integration_test/fixtures/reorg/blocktx.block_transactions.yaml b/internal/blocktx/integration_test/fixtures/reorg/blocktx.block_transactions.yaml index 61a6a8bb3..cd25464fb 100644 --- a/internal/blocktx/integration_test/fixtures/reorg/blocktx.block_transactions.yaml +++ b/internal/blocktx/integration_test/fixtures/reorg/blocktx.block_transactions.yaml @@ -1,12 +1,12 @@ - block_id: 1002 hash: 0xb16cea53fc823e146fbb9ae4ad3124f7c273f30562585ad6e4831495d609f430 - merkle_tree_index: 3 + merkle_tree_index: 0 - block_id: 1999 # the same tx also in stale block hash: 0xb16cea53fc823e146fbb9ae4ad3124f7c273f30562585ad6e4831495d609f430 - merkle_tree_index: 999 + merkle_tree_index: 0 - block_id: 1999 # the same tx also in stale block hash: 0xcd3d2f97dfc0cdb6a07ec4b72df5e1794c9553ff2f62d90ed4add047e8088853 - merkle_tree_index: 999 + merkle_tree_index: 1 - block_id: 1004 hash: 0xece2b7e40d98749c03c551b783420d6e3fdc3c958244bbf275437839585829a6 - merkle_tree_index: 5 + merkle_tree_index: 0 diff --git a/internal/blocktx/integration_test/fixtures/reorg/blocktx.blocks.yaml b/internal/blocktx/integration_test/fixtures/reorg/blocktx.blocks.yaml index 58b3fddeb..af089a49b 100644 --- a/internal/blocktx/integration_test/fixtures/reorg/blocktx.blocks.yaml +++ b/internal/blocktx/integration_test/fixtures/reorg/blocktx.blocks.yaml @@ -1,6 +1,6 @@ - inserted_at: 2023-12-15 14:00:00 id: 1001 - hash: 0xf97e20396f02ab990ed31b9aec70c240f48b7e5ea239aa050000000000000000 + hash: 0x67708796ef57464ed9eaf2a663d3da32372e4c2fb65558020000000000000000 prevhash: 0xb71ab063c5f96cad71cdc59dcc94182a20a69cbd7eed2d070000000000000000 merkleroot: 0x7f4019eb006f5333cce752df387fa8443035c22291eb771ee5b16a02b81c8483 height: 822014 @@ -22,18 +22,6 @@ status: 10 is_longest: true chainwork: '62209952899966' -- inserted_at: 2023-12-15 14:30:00 - id: 1999 - hash: 0x82471bbf045ab13825a245b37de71d77ec12513b37e2524ec11551d18c19f7c3 - prevhash: 0x67708796ef57464ed9eaf2a663d3da32372e4c2fb65558020000000000000000 - merkleroot: 0x7382df1b717287ab87e5e3e25759697c4c45eea428f701cdd0c77ad3fc707257 - height: 822015 - processed_at: 2023-12-15 14:30:00 - size: 20160000 - tx_count: 6523 - status: 20 # STALE - competing block - is_longest: false - chainwork: '62209952899966' - inserted_at: 2023-12-15 14:40:00 id: 1003 hash: 0xe1df1273e6e7270f96b508545d7aa80aebda7d758dc82e080000000000000000 @@ -58,6 +46,23 @@ status: 10 is_longest: true chainwork: '62209952899966' + +# Stale +- inserted_at: 2023-12-15 14:30:00 + id: 1999 + hash: 0x82471bbf045ab13825a245b37de71d77ec12513b37e2524ec11551d18c19f7c3 + prevhash: 0x67708796ef57464ed9eaf2a663d3da32372e4c2fb65558020000000000000000 + merkleroot: 0x7382df1b717287ab87e5e3e25759697c4c45eea428f701cdd0c77ad3fc707257 + height: 822015 + processed_at: 2023-12-15 14:30:00 + size: 20160000 + tx_count: 6523 + status: 20 # STALE - competing block + is_longest: false + chainwork: '62209952899966' + + + - inserted_at: 2023-12-15 14:50:00 id: 10052 hash: 0x000000000000000003b15d668b54c4b91ae81a86298ee209d9f39fd7a769bcde diff --git a/internal/blocktx/integration_test/fixtures/reorg/blocktx.registered_transactions.yaml b/internal/blocktx/integration_test/fixtures/reorg/blocktx.registered_transactions.yaml new file mode 100644 index 000000000..2dab66118 --- /dev/null +++ b/internal/blocktx/integration_test/fixtures/reorg/blocktx.registered_transactions.yaml @@ -0,0 +1,8 @@ +- hash: 0xcd3d2f97dfc0cdb6a07ec4b72df5e1794c9553ff2f62d90ed4add047e8088853 + inserted_at: 2023-12-15 14:00:00 +- hash: 0xb16cea53fc823e146fbb9ae4ad3124f7c273f30562585ad6e4831495d609f430 + inserted_at: 2023-12-15 14:00:00 +- hash: 0x2ff4430eb883c6f6c0640a5d716b2d107bbc0efa5aeaa237aec796d4686b0a8f + inserted_at: 2023-12-15 14:00:00 +- hash: 0xece2b7e40d98749c03c551b783420d6e3fdc3c958244bbf275437839585829a6 + inserted_at: 2023-12-15 14:00:00 diff --git a/internal/blocktx/integration_test/fixtures/reorg_orphans/blocktx.block_transactions.yaml b/internal/blocktx/integration_test/fixtures/reorg_orphans/blocktx.block_transactions.yaml index f54d8d8c5..0d010ab7f 100644 --- a/internal/blocktx/integration_test/fixtures/reorg_orphans/blocktx.block_transactions.yaml +++ b/internal/blocktx/integration_test/fixtures/reorg_orphans/blocktx.block_transactions.yaml @@ -1,15 +1,15 @@ - block_id: 1002 hash: 0xcd3d2f97dfc0cdb6a07ec4b72df5e1794c9553ff2f62d90ed4add047e8088853 - merkle_tree_index: 1 + merkle_tree_index: 0 - block_id: 1002 hash: 0xb16cea53fc823e146fbb9ae4ad3124f7c273f30562585ad6e4831495d609f430 - merkle_tree_index: 3 + merkle_tree_index: 1 - block_id: 1004 hash: 0xb16cea53fc823e146fbb9ae4ad3124f7c273f30562585ad6e4831495d609f430 - merkle_tree_index: 3 + merkle_tree_index: 0 - block_id: 1003 hash: 0x2ff4430eb883c6f6c0640a5d716b2d107bbc0efa5aeaa237aec796d4686b0a8f - merkle_tree_index: 4 + merkle_tree_index: 0 - block_id: 1006 hash: 0xece2b7e40d98749c03c551b783420d6e3fdc3c958244bbf275437839585829a6 - merkle_tree_index: 5 + merkle_tree_index: 0 diff --git a/internal/blocktx/integration_test/fixtures/reorg_orphans/blocktx.registered_transactions.yaml b/internal/blocktx/integration_test/fixtures/reorg_orphans/blocktx.registered_transactions.yaml new file mode 100644 index 000000000..2dab66118 --- /dev/null +++ b/internal/blocktx/integration_test/fixtures/reorg_orphans/blocktx.registered_transactions.yaml @@ -0,0 +1,8 @@ +- hash: 0xcd3d2f97dfc0cdb6a07ec4b72df5e1794c9553ff2f62d90ed4add047e8088853 + inserted_at: 2023-12-15 14:00:00 +- hash: 0xb16cea53fc823e146fbb9ae4ad3124f7c273f30562585ad6e4831495d609f430 + inserted_at: 2023-12-15 14:00:00 +- hash: 0x2ff4430eb883c6f6c0640a5d716b2d107bbc0efa5aeaa237aec796d4686b0a8f + inserted_at: 2023-12-15 14:00:00 +- hash: 0xece2b7e40d98749c03c551b783420d6e3fdc3c958244bbf275437839585829a6 + inserted_at: 2023-12-15 14:00:00 diff --git a/internal/blocktx/integration_test/fixtures/stale_orphans/blocktx.block_transactions_map.yaml b/internal/blocktx/integration_test/fixtures/stale_orphans/blocktx.block_transactions_map.yaml deleted file mode 100644 index 4a64b5c89..000000000 --- a/internal/blocktx/integration_test/fixtures/stale_orphans/blocktx.block_transactions_map.yaml +++ /dev/null @@ -1,8 +0,0 @@ -- block_id: 1002 - merkle_tree_index: 3 - inserted_at: 2023-12-10 14:00:00 - hash: 0xb16cea53fc823e146fbb9ae4ad3124f7c273f30562585ad6e4831495d609f430 -- block_id: 1005 - merkle_tree_index: 5 - inserted_at: 2023-12-10 14:00:00 - hash: 0xece2b7e40d98749c03c551b783420d6e3fdc3c958244bbf275437839585829a6 diff --git a/internal/blocktx/integration_test/reorg_integration_test.go b/internal/blocktx/integration_test/reorg_integration_test.go index ed754d4b2..31f9698ee 100644 --- a/internal/blocktx/integration_test/reorg_integration_test.go +++ b/internal/blocktx/integration_test/reorg_integration_test.go @@ -46,9 +46,6 @@ import ( ) func TestReorg(t *testing.T) { - // TODO: remove the skip when gaps are filling quickly again - t.Skip("Skipping until gaps are being processed quickly again") - if testing.Short() { t.Skip("skipping integration test") } @@ -150,7 +147,7 @@ func TestReorg(t *testing.T) { blockHash822015Fork = "82471bbf045ab13825a245b37de71d77ec12513b37e2524ec11551d18c19f7c3" blockHash822016Fork = "032c3688bc7536b2d787f3a196b1145a09bf33183cd1448ff6b1a9dfbb022db8" - blockHash822014StartOfChain = "f97e20396f02ab990ed31b9aec70c240f48b7e5ea239aa050000000000000000" + blockHash822014StartOfChain = "67708796ef57464ed9eaf2a663d3da32372e4c2fb65558020000000000000000" blockHash822015 = "c9b4e1e4dcf9188416027511671b9346be8ef93c0ddf59060000000000000000" blockHash822016 = "e1df1273e6e7270f96b508545d7aa80aebda7d758dc82e080000000000000000" blockHash822017 = "76404890880cb36ce68100abb05b3a958e17c0ed274d5c0a0000000000000000" diff --git a/internal/blocktx/processor_test.go b/internal/blocktx/processor_test.go index c1330b5d9..6c05a0295 100644 --- a/internal/blocktx/processor_test.go +++ b/internal/blocktx/processor_test.go @@ -247,9 +247,6 @@ func TestHandleBlock(t *testing.T) { } func TestHandleBlockReorgAndOrphans(t *testing.T) { - // TODO: remove the skip when gaps are filling quickly again - t.Skip("Skipping until gaps are being processed quickly again") - testCases := []struct { name string blockAlreadyExists bool diff --git a/internal/blocktx/store/postgresql/fixtures/insert_block_transactions/blocktx.registered_transactions.yaml b/internal/blocktx/store/postgresql/fixtures/insert_block_transactions/blocktx.registered_transactions.yaml index 1faee19e9..a1fb0d6fc 100644 --- a/internal/blocktx/store/postgresql/fixtures/insert_block_transactions/blocktx.registered_transactions.yaml +++ b/internal/blocktx/store/postgresql/fixtures/insert_block_transactions/blocktx.registered_transactions.yaml @@ -1,16 +1,16 @@ -#- hash: 0x76732b80598326a18d3bf0a86518adbdf95d0ddc6ff6693004440f4776168c3b -# inserted_at: 2023-12-15 14:00:00 -#- hash: 0x164e85a5d5bc2b2372e8feaa266e5e4b7d0808f8d2b784fb1f7349c4726392b0 -# inserted_at: 2023-12-15 14:00:00 -#- hash: 0xb4201cc6fc5768abff14adf75042ace6061da9176ee5bb943291b9ba7d7f5743 -# inserted_at: 2023-12-15 14:00:00 -#- hash: 0x37bd6c87927e75faeb3b3c939f64721cda48e1bb98742676eebe83aceee1a669 -# inserted_at: 2023-12-15 14:00:00 -#- hash: 0x952f80e20a0330f3b9c2dfd1586960064e797218b5c5df665cada221452c17eb -# inserted_at: 2023-12-15 14:00:00 -#- hash: 0x861a281b27de016e50887288de87eab5ca56a1bb172cdff6dba965474ce0f608 -# inserted_at: 2023-12-15 14:00:00 -#- hash: 0x9421cc760c5405af950a76dc3e4345eaefd4e7322f172a3aee5e0ddc7b4f8313 -# inserted_at: 2023-12-15 14:00:00 +- hash: 0x76732b80598326a18d3bf0a86518adbdf95d0ddc6ff6693004440f4776168c3b + inserted_at: 2023-12-15 14:00:00 +- hash: 0x164e85a5d5bc2b2372e8feaa266e5e4b7d0808f8d2b784fb1f7349c4726392b0 + inserted_at: 2023-12-15 14:00:00 +- hash: 0xb4201cc6fc5768abff14adf75042ace6061da9176ee5bb943291b9ba7d7f5743 + inserted_at: 2023-12-15 14:00:00 +- hash: 0x37bd6c87927e75faeb3b3c939f64721cda48e1bb98742676eebe83aceee1a669 + inserted_at: 2023-12-15 14:00:00 +- hash: 0x952f80e20a0330f3b9c2dfd1586960064e797218b5c5df665cada221452c17eb + inserted_at: 2023-12-15 14:00:00 +- hash: 0x861a281b27de016e50887288de87eab5ca56a1bb172cdff6dba965474ce0f608 + inserted_at: 2023-12-15 14:00:00 +- hash: 0x9421cc760c5405af950a76dc3e4345eaefd4e7322f172a3aee5e0ddc7b4f8313 + inserted_at: 2023-12-15 14:00:00 - hash: 0x8b7d038db4518ac4c665abfc5aeaacbd2124ad8ca70daa8465ed2c4427c41b9b inserted_at: 2023-12-15 14:00:00 diff --git a/test/submit_05_reorg_test.go b/test/submit_05_reorg_test.go index 3badd0672..971e59b80 100644 --- a/test/submit_05_reorg_test.go +++ b/test/submit_05_reorg_test.go @@ -15,9 +15,6 @@ import ( ) func TestReorg(t *testing.T) { - // TODO: remove the skip when gaps are filling quickly again - t.Skip("Skipping until gaps are being processed quickly again") - address, privateKey := node_client.FundNewWallet(t, bitcoind) utxos := node_client.GetUtxos(t, bitcoind, address) From a8fa8e83e79188a01415f622aead7d48049f2872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Wed, 15 Jan 2025 13:31:14 +0100 Subject: [PATCH 04/10] refactor(ARCO-319): Fix log --- test/init_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/init_test.go b/test/init_test.go index 061bbab8c..1b4f87378 100644 --- a/test/init_test.go +++ b/test/init_test.go @@ -25,7 +25,7 @@ func TestMain(m *testing.M) { } func setupSut() { - log.Printf("init tests") + log.Println("init tests") if os.Getenv("TEST_LOCAL") != "" { nodeHost = "localhost" From a0ba815884b4f569cb5235e7bf3ddb18028e2911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Thu, 16 Jan 2025 12:03:26 +0100 Subject: [PATCH 05/10] refactor(ARCO-319): e2e tests fill gaps in 1s intervals --- test/config/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/config/config.yaml b/test/config/config.yaml index 184d68cdc..397566ff4 100644 --- a/test/config/config.yaml +++ b/test/config/config.yaml @@ -98,7 +98,7 @@ blocktx: registerTxsInterval: 200ms fillGaps: enabled: true - interval: 30s + interval: 1s # This allows e2e tests to pass as there will be no gaps when initial blocks are created maxAllowedBlockHeightMismatch: 3 api: From cf8a95cbcade8c54d4cbb7d64fd1b85682f3007e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Thu, 16 Jan 2025 14:39:25 +0100 Subject: [PATCH 06/10] refactor(ARCO-319): Add missing settings to test config --- test/config/config.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/test/config/config.yaml b/test/config/config.yaml index 397566ff4..0df6866e8 100644 --- a/test/config/config.yaml +++ b/test/config/config.yaml @@ -63,13 +63,13 @@ metamorph: maxOpenConns: 80 sslMode: disable processorCacheExpiryTime: 24h + unseenTransactionRebroadcastingInterval: 60s maxRetries: 1000 processStatusUpdateInterval: 50ms - checkSeenOnNetworkOlderThan: 3h - checkSeenOnNetworkPeriod: 4h + recheckSeen: + fromAgo: 24h + untilAgo: 1h monitorPeers: true - incomingIsLongest: false - profilerAddr: localhost:9992 health: serverDialAddr: localhost:8005 minimumHealthyConnections: 2 @@ -94,12 +94,15 @@ blocktx: maxOpenConns: 80 sslMode: disable recordRetentionDays: 28 - profilerAddr: localhost:9993 registerTxsInterval: 200ms + maxBlockProcessingDuration: 5m + monitorPeers: true + incomingIsLongest: false fillGaps: enabled: true interval: 1s # This allows e2e tests to pass as there will be no gaps when initial blocks are created maxAllowedBlockHeightMismatch: 3 + p2pReadBufferSize: 8388608 api: address: 0.0.0.0:9090 @@ -143,6 +146,9 @@ callbacker: dialAddr: callbacker:8021 health: serverDialAddr: localhost:8022 + delay: 0s + pause: 0s + batchSendInterval: 5s db: mode: postgres postgres: @@ -154,6 +160,8 @@ callbacker: maxIdleConns: 10 maxOpenConns: 80 sslMode: disable + pruneInterval: 24h + pruneOlderThan: 336h failedCallbackCheckInterval: 1m delayDuration: 5s expiration: 24h From 69a06c0be50b9915231d6189847d24512df90ce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Thu, 16 Jan 2025 14:40:04 +0100 Subject: [PATCH 07/10] refactor(ARCO-319): Set maxBlockProcessingDuration in e2e test to 2s so that all gaps get closed before e2e tests start --- internal/blocktx/processor.go | 3 +-- test/config/config.yaml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/blocktx/processor.go b/internal/blocktx/processor.go index 0257b5b4a..2bb19d400 100644 --- a/internal/blocktx/processor.go +++ b/internal/blocktx/processor.go @@ -48,7 +48,6 @@ const ( registerTxsBatchSizeDefault = 100 registerRequestTxBatchSizeDefault = 100 waitForBlockProcessing = 5 * time.Minute - lockTime = 5 * time.Minute parallellism = 5 ) @@ -167,7 +166,7 @@ func (p *Processor) StartBlockRequesting() { peer := req.Peer // lock block for the current instance to process - processedBy, err := p.store.SetBlockProcessing(p.ctx, hash, p.hostname, lockTime, maxBlocksInProgress) + processedBy, err := p.store.SetBlockProcessing(p.ctx, hash, p.hostname, p.maxBlockProcessingDuration, maxBlocksInProgress) if err != nil { if errors.Is(err, store.ErrBlockProcessingMaximumReached) { p.logger.Debug("block processing maximum reached", slog.String("hash", hash.String()), slog.String("processed_by", processedBy)) diff --git a/test/config/config.yaml b/test/config/config.yaml index 0df6866e8..d25eea628 100644 --- a/test/config/config.yaml +++ b/test/config/config.yaml @@ -95,7 +95,7 @@ blocktx: sslMode: disable recordRetentionDays: 28 registerTxsInterval: 200ms - maxBlockProcessingDuration: 5m + maxBlockProcessingDuration: 2s monitorPeers: true incomingIsLongest: false fillGaps: From 830570a6af11dce77dca201313adc7ebc55ff00c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Thu, 16 Jan 2025 15:03:32 +0100 Subject: [PATCH 08/10] refactor(ARCO-319): Fix comment --- internal/blocktx/store/postgresql/get_orphaned_chain.go | 3 ++- internal/blocktx/store/postgresql/upsert_block.go | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/blocktx/store/postgresql/get_orphaned_chain.go b/internal/blocktx/store/postgresql/get_orphaned_chain.go index 79f28b59b..0d85f83d9 100644 --- a/internal/blocktx/store/postgresql/get_orphaned_chain.go +++ b/internal/blocktx/store/postgresql/get_orphaned_chain.go @@ -3,8 +3,9 @@ package postgresql import ( "context" - "github.com/bitcoin-sv/arc/internal/blocktx/blocktx_api" "github.com/libsv/go-p2p/chaincfg/chainhash" + + "github.com/bitcoin-sv/arc/internal/blocktx/blocktx_api" ) // GetOrphansBackToNonOrphanAncestor recursively searches for blocks marked diff --git a/internal/blocktx/store/postgresql/upsert_block.go b/internal/blocktx/store/postgresql/upsert_block.go index cab0188fb..1b4b855ec 100644 --- a/internal/blocktx/store/postgresql/upsert_block.go +++ b/internal/blocktx/store/postgresql/upsert_block.go @@ -16,11 +16,11 @@ func (p *PostgreSQL) UpsertBlock(ctx context.Context, block *blocktx_api.Block) }() // This query will insert a block ONLY if one of the 3 conditions is met: - // 1. Block being inserted is `ORPHANED` and there's no previous block in the database + // 1. Block being inserted is `ORPHANED` or `LONGEST` and there's no previous block in the database // 2. The block being inserted has the same status as its previous block // 3. The block being inserted has status `STALE` but the previous block was `LONGEST` // Any other situation would mean an error in block processing - // (probably because of another block being inserted by other blocktx instance at the same time) + // (probably because of another block which is being inserted by another blocktx instance at the same time) // and requires the block to be received and processed again. qInsert := ` INSERT INTO blocktx.blocks (hash, prevhash, merkleroot, height, status, chainwork, is_longest) @@ -55,3 +55,8 @@ func (p *PostgreSQL) UpsertBlock(ctx context.Context, block *blocktx_api.Block) return blockID, nil } + +//func (p *PostgreSQL) UpdateLongest(ctx context.Context, blockHash *chainhash.Hash) (err error) { +// +// return nil +//} From 548d0b8502356a377f934a6a1f22b9d3da059568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Thu, 16 Jan 2025 15:16:50 +0100 Subject: [PATCH 09/10] refactor(ARCO-319): Re-add unique index for block height that is longest --- .../000023_unique_height_is_longest.down.sql | 1 + .../000023_unique_height_is_longest.up.sql | 4 ++ .../blocktx/store/postgresql/postgres_test.go | 37 ++++++++----------- 3 files changed, 21 insertions(+), 21 deletions(-) create mode 100644 internal/blocktx/store/postgresql/migrations/000023_unique_height_is_longest.down.sql create mode 100644 internal/blocktx/store/postgresql/migrations/000023_unique_height_is_longest.up.sql diff --git a/internal/blocktx/store/postgresql/migrations/000023_unique_height_is_longest.down.sql b/internal/blocktx/store/postgresql/migrations/000023_unique_height_is_longest.down.sql new file mode 100644 index 000000000..dc31e0fe9 --- /dev/null +++ b/internal/blocktx/store/postgresql/migrations/000023_unique_height_is_longest.down.sql @@ -0,0 +1 @@ +DROP INDEX blocktx.pux_height_is_longest; diff --git a/internal/blocktx/store/postgresql/migrations/000023_unique_height_is_longest.up.sql b/internal/blocktx/store/postgresql/migrations/000023_unique_height_is_longest.up.sql new file mode 100644 index 000000000..e499d5093 --- /dev/null +++ b/internal/blocktx/store/postgresql/migrations/000023_unique_height_is_longest.up.sql @@ -0,0 +1,4 @@ +-- This will make sure that there can only be ONE block at any +-- given height that is considered part of the LONGEST chain. +CREATE UNIQUE INDEX pux_height_is_longest ON blocktx.blocks(height) +WHERE is_longest; diff --git a/internal/blocktx/store/postgresql/postgres_test.go b/internal/blocktx/store/postgresql/postgres_test.go index 4ff579784..f5f36295d 100644 --- a/internal/blocktx/store/postgresql/postgres_test.go +++ b/internal/blocktx/store/postgresql/postgres_test.go @@ -137,8 +137,7 @@ func TestPostgresDB(t *testing.T) { blockHash1 := testutils.RevChainhash(t, "000000000000000001b8adefc1eb98896c80e30e517b9e2655f1f929d9958a48") blockHash2 := testutils.RevChainhash(t, "00000000000000000a081a539601645abe977946f8f6466a3c9e0c34d50be4a8") - // TODO: uncomment when all block gaps are filled - // blockHashViolating := testutils.RevChainhash(t, "00000000b69bd8e4dc60580117617a466d5c76ada85fb7b87e9baea01f9d9984") + blockHashViolating := testutils.RevChainhash(t, "00000000b69bd8e4dc60580117617a466d5c76ada85fb7b87e9baea01f9d9984") merkleRoot := testutils.RevChainhash(t, "31e25c5ac7c143687f55fc49caf0f552ba6a16d4f785e4c9a9a842179a085f0c") expectedBlock := &blocktx_api.Block{ Hash: blockHash2[:], @@ -148,14 +147,13 @@ func TestPostgresDB(t *testing.T) { Status: blocktx_api.Status_LONGEST, Processed: true, } - // TODO: uncomment when all block gaps are filled - // expectedBlockViolatingUniqueIndex := &blocktx_api.Block{ - // Hash: blockHashViolating[:], - // PreviousHash: blockHash1[:], - // MerkleRoot: merkleRoot[:], - // Height: 100, - // Status: blocktx_api.Status_LONGEST, - // } + expectedBlockViolatingUniqueIndex := &blocktx_api.Block{ + Hash: blockHashViolating[:], + PreviousHash: blockHash1[:], + MerkleRoot: merkleRoot[:], + Height: 100, + Status: blocktx_api.Status_LONGEST, + } expectedBlockOverrideStatus := &blocktx_api.Block{ Hash: blockHash2[:], PreviousHash: blockHash1[:], @@ -178,11 +176,10 @@ func TestPostgresDB(t *testing.T) { require.NoError(t, err) require.Equal(t, expectedBlock, actualBlockResp) - // TODO: uncomment when all block gaps are filled // when - // _, err = postgresDB.UpsertBlock(ctx, expectedBlockViolatingUniqueIndex) + _, err = postgresDB.UpsertBlock(ctx, expectedBlockViolatingUniqueIndex) // then - // require.ErrorIs(t, err, store.ErrFailedToInsertBlock) + require.ErrorIs(t, err, store.ErrFailedToInsertBlock) // when id, err = postgresDB.UpsertBlock(ctx, expectedBlockOverrideStatus) @@ -394,11 +391,10 @@ func TestPostgresDB(t *testing.T) { {Hash: hash4Stale[:], Status: blocktx_api.Status_LONGEST}, } - // TODO: uncomment when all block gaps are filled - // blockStatusUpdatesViolating := []store.BlockStatusUpdate{ - // // there is already a LONGEST block at that height - // {Hash: hash1Longest[:], Status: blocktx_api.Status_LONGEST}, - // } + blockStatusUpdatesViolating := []store.BlockStatusUpdate{ + // there is already a LONGEST block at that height + {Hash: hash1Longest[:], Status: blocktx_api.Status_LONGEST}, + } // when err := postgresDB.UpdateBlocksStatuses(ctx, blockStatusUpdates) @@ -421,10 +417,9 @@ func TestPostgresDB(t *testing.T) { require.NoError(t, err) require.Equal(t, blocktx_api.Status_LONGEST, stale4.Status) - // TODO: uncomment when all block gaps are filled // when - // err = postgresDB.UpdateBlocksStatuses(ctx, blockStatusUpdatesViolating) - // require.ErrorIs(t, err, store.ErrFailedToUpdateBlockStatuses) + err = postgresDB.UpdateBlocksStatuses(ctx, blockStatusUpdatesViolating) + require.ErrorIs(t, err, store.ErrFailedToUpdateBlockStatuses) }) t.Run("get mined txs", func(t *testing.T) { From c6a08593fc4fc54df096bfac63defdb32f5fa689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=B6ckli?= Date: Thu, 16 Jan 2025 15:23:56 +0100 Subject: [PATCH 10/10] refactor(ARCO-319): Rm comment --- internal/blocktx/store/postgresql/upsert_block.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/internal/blocktx/store/postgresql/upsert_block.go b/internal/blocktx/store/postgresql/upsert_block.go index 1b4b855ec..1d9a361cf 100644 --- a/internal/blocktx/store/postgresql/upsert_block.go +++ b/internal/blocktx/store/postgresql/upsert_block.go @@ -55,8 +55,3 @@ func (p *PostgreSQL) UpsertBlock(ctx context.Context, block *blocktx_api.Block) return blockID, nil } - -//func (p *PostgreSQL) UpdateLongest(ctx context.Context, blockHash *chainhash.Hash) (err error) { -// -// return nil -//}