Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: re-activate reorg support #737

Open
wants to merge 28 commits into
base: main
Choose a base branch
from
Open

feat: re-activate reorg support #737

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
8d31625
refactor: Order by height descending
boecklim Jan 10, 2025
e26ff91
refactor: Use query statement for unique processing of block instead …
boecklim Jan 10, 2025
4f7ef42
refactor: Use time as parameter outside of query
boecklim Jan 10, 2025
a88033a
refactor: Get block gaps function does not join with block processing…
boecklim Jan 10, 2025
f872493
refactor: Set block processing with parameter to restrict maximum num…
boecklim Jan 10, 2025
db67814
refactor: No need to delete block processing entries
boecklim Jan 10, 2025
ea35343
refactor: Fix processor test
boecklim Jan 10, 2025
f80bfb9
refactor: Fix get block gaps test
boecklim Jan 10, 2025
0385a33
refactor: Rm commented code
boecklim Jan 10, 2025
525d5a3
refactor: Rm & generate mocks
boecklim Jan 10, 2025
8eb6db0
refactor: Make gen
boecklim Jan 10, 2025
a19179c
refactor: Rm blocktx watcher
boecklim Jan 10, 2025
1b971bd
refactor: Fix test
boecklim Jan 10, 2025
c8da42a
refactor: Upgrade golang.org/x/net & github.com/jackc/pgx/v5
boecklim Jan 14, 2025
c096396
refactor: TestBatchCallback
boecklim Jan 14, 2025
5b68ac8
refactor: Go mod tidy
boecklim Jan 14, 2025
d6f1195
refactor: Use advisory lock
boecklim Jan 14, 2025
34a441b
refactor: Fix migration
boecklim Jan 14, 2025
89e35f6
refactor(ARCO-312): Fix comments
boecklim Jan 15, 2025
968840d
refactor(ARCO-312): Refactor processor opts
boecklim Jan 15, 2025
f59adab
refactor(ARCO-312): Calculcate Merkle path
boecklim Jan 15, 2025
69bfd58
refactor(ARCO-312): Run workflow on push to PR into any branch
boecklim Jan 15, 2025
de9f969
refactor(ARCO-312): Don't run workflow on push into main branch
boecklim Jan 15, 2025
2072131
refactor(ARCO-312): Add setting which denotes whether new block shoul…
boecklim Jan 15, 2025
78fc981
refactor(ARCO-312): Implement setting
boecklim Jan 15, 2025
17b3576
refactor(ARCO-312): temp logs
boecklim Jan 15, 2025
3b6c60d
refactor(ARCO-312): temp logs
boecklim Jan 15, 2025
670ca36
refactor(ARCO-312): Ensure ordered processing of initial blocks
boecklim Jan 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions .github/workflows/go.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,8 @@
name: Go

on:
push:
branches: ["main"]
pull_request:
branches: ["main", "reorg-support"] # TODO: to be removed before merge to main
branches: ["**"]

jobs:
build:
Expand Down
12 changes: 6 additions & 6 deletions cmd/arc/services/blocktx.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,17 @@ import (

"go.opentelemetry.io/otel/attribute"

"github.com/bitcoin-sv/arc/internal/grpc_opts"
"github.com/bitcoin-sv/arc/internal/message_queue/nats/client/nats_core"
"github.com/bitcoin-sv/arc/internal/message_queue/nats/client/nats_jetstream"
"github.com/bitcoin-sv/arc/internal/message_queue/nats/nats_connection"
"github.com/bitcoin-sv/arc/internal/tracing"
"github.com/libsv/go-p2p"

"github.com/bitcoin-sv/arc/config"
"github.com/bitcoin-sv/arc/internal/blocktx"
"github.com/bitcoin-sv/arc/internal/blocktx/store"
"github.com/bitcoin-sv/arc/internal/blocktx/store/postgresql"

"github.com/bitcoin-sv/arc/internal/grpc_opts"
"github.com/bitcoin-sv/arc/internal/message_queue/nats/client/nats_core"
"github.com/bitcoin-sv/arc/internal/message_queue/nats/client/nats_jetstream"
"github.com/bitcoin-sv/arc/internal/message_queue/nats/nats_connection"
"github.com/bitcoin-sv/arc/internal/tracing"
"github.com/bitcoin-sv/arc/internal/version"
)

Expand Down Expand Up @@ -127,6 +126,7 @@ func StartBlockTx(logger *slog.Logger, arcConfig *config.ArcConfig) (func(), err
blocktx.WithRegisterTxsInterval(btxConfig.RegisterTxsInterval),
blocktx.WithMessageQueueClient(mqClient),
blocktx.WithMaxBlockProcessingDuration(btxConfig.MaxBlockProcessingDuration),
blocktx.WithIncomingIsLongest(btxConfig.IncomingIsLongest),
)

blockRequestCh := make(chan blocktx.BlockRequest, blockProcessingBuffer)
Expand Down
11 changes: 1 addition & 10 deletions cmd/arc/services/k8s_watcher.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@ import (
"log/slog"

"github.com/bitcoin-sv/arc/config"
"github.com/bitcoin-sv/arc/internal/blocktx"
"github.com/bitcoin-sv/arc/internal/blocktx/blocktx_api"
"github.com/bitcoin-sv/arc/internal/k8s_watcher"
"github.com/bitcoin-sv/arc/internal/k8s_watcher/k8s_client"
"github.com/bitcoin-sv/arc/internal/metamorph"
Expand All @@ -28,14 +26,7 @@ func StartK8sWatcher(logger *slog.Logger, arcConfig *config.ArcConfig) (func(),
return nil, fmt.Errorf("failed to get k8s-client: %v", err)
}

blocktxConn, err := blocktx.DialGRPC(arcConfig.Blocktx.DialAddr, arcConfig.Prometheus.Endpoint, arcConfig.GrpcMessageSize, nil)
if err != nil {
return nil, fmt.Errorf("failed to connect to block-tx server: %v", err)
}

blocktxClient := blocktx.NewClient(blocktx_api.NewBlockTxAPIClient(blocktxConn))

k8sWatcher := k8s_watcher.New(metamorphClient, blocktxClient, k8sClient, arcConfig.K8sWatcher.Namespace, k8s_watcher.WithLogger(logger))
k8sWatcher := k8s_watcher.New(metamorphClient, k8sClient, arcConfig.K8sWatcher.Namespace, k8s_watcher.WithLogger(logger))
err = k8sWatcher.Start()
if err != nil {
return nil, fmt.Errorf("faile to start k8s-watcher: %v", err)
Expand Down
1 change: 1 addition & 0 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ type BlocktxConfig struct {
MaxAllowedBlockHeightMismatch int `mapstructure:"maxAllowedBlockHeightMismatch"`
MessageQueue *MessageQueueConfig `mapstructure:"mq"`
P2pReadBufferSize int `mapstructure:"p2pReadBufferSize"`
IncomingIsLongest bool `mapstructure:"incomingIsLongest"`
}

type DbConfig struct {
Expand Down
1 change: 1 addition & 0 deletions config/defaults.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ func getBlocktxConfig() *BlocktxConfig {
MaxBlockProcessingDuration: 5 * time.Minute,
MessageQueue: &MessageQueueConfig{},
P2pReadBufferSize: 8 * 1024 * 1024,
IncomingIsLongest: false,
}
}

Expand Down
1 change: 1 addition & 0 deletions config/example_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ blocktx:
registerTxsInterval: 10s # time interval to read from the channel registered transactions
maxBlockProcessingDuration: 5m # maximum time a blocktx can spend on processing a block before unlocking it to be requested again
monitorPeers: false # if enabled, peers which do not receive alive signal from nodes will be restarted
incomingIsLongest: false # whether each new block received is considered to be from the longest blockchain. If there are a lot of block gaps in blocktx database it is advisable to set this to true
fillGaps:
enabled: true
interval: 15m # time interval to check and fill gaps in processed blocks
Expand Down
9 changes: 5 additions & 4 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ require (
github.com/google/uuid v1.6.0
github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0
github.com/jackc/pgx/v5 v5.3.1
github.com/jackc/pgx/v5 v5.7.2
github.com/jedib0t/go-pretty/v6 v6.5.9
github.com/jmoiron/sqlx v1.3.5
github.com/labstack/echo-contrib v0.17.1
Expand Down Expand Up @@ -46,7 +46,6 @@ require (
go.opentelemetry.io/otel/sdk v1.28.0
go.opentelemetry.io/otel/trace v1.31.0
golang.org/x/sync v0.10.0
google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4
google.golang.org/grpc v1.65.0
google.golang.org/protobuf v1.34.2
gopkg.in/yaml.v3 v3.0.1
Expand Down Expand Up @@ -101,7 +100,8 @@ require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/invopop/yaml v0.3.1 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.9 // indirect
Expand Down Expand Up @@ -158,14 +158,15 @@ require (
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.32.0 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/net v0.30.0 // indirect
golang.org/x/net v0.34.0 // indirect
golang.org/x/oauth2 v0.22.0 // indirect
golang.org/x/sys v0.29.0 // indirect
golang.org/x/term v0.28.0 // indirect
golang.org/x/text v0.21.0 // indirect
golang.org/x/time v0.7.0 // indirect
golang.org/x/tools v0.24.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240805194559-2c9e96a0b5d4 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
Expand Down
14 changes: 8 additions & 6 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,14 +165,16 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgproto3/v2 v2.3.2 h1:7eY55bdBeCz1F2fTzSz69QC+pG46jYq9/jtSPiJ5nn0=
github.com/jackc/pgproto3/v2 v2.3.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw=
github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
github.com/jackc/pgx/v4 v4.18.1 h1:YP7G1KABtKpB5IHrO9vYwSrCOhs7p3uqhvhhQBptya0=
github.com/jackc/pgx/v4 v4.18.1/go.mod h1:FydWkUyadDmdNH/mHnGob881GawxeEm7TcMCzkb+qQE=
github.com/jackc/pgx/v5 v5.3.1 h1:Fcr8QJ1ZeLi5zsPZqQeUZhNhxfkkKBOgJuYkJHoBOtU=
github.com/jackc/pgx/v5 v5.3.1/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8=
github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI=
github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jedib0t/go-pretty/v6 v6.5.9 h1:ACteMBRrrmm1gMsXe9PSTOClQ63IXDUt03H5U+UV8OU=
github.com/jedib0t/go-pretty/v6 v6.5.9/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E=
github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
Expand Down Expand Up @@ -417,8 +419,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA=
golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down
45 changes: 18 additions & 27 deletions internal/blocktx/blocktx_api/blocktx_api.pb.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 0 additions & 3 deletions internal/blocktx/blocktx_api/blocktx_api.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,6 @@ service BlockTxAPI {
// ClearRegisteredTransactions clears registered transactions
rpc ClearRegisteredTransactions(ClearData) returns (RowsAffectedResponse) {}

// DelUnfinishedBlockProcessing deletes unfinished block processing
rpc DelUnfinishedBlockProcessing(DelUnfinishedBlockProcessingRequest) returns (RowsAffectedResponse) {}

// VerifyMerkleRoots verifies the merkle roots existance in blocktx db and returns unverified block heights
rpc VerifyMerkleRoots(MerkleRootsVerificationRequest) returns (MerkleRootVerificationResponse) {}
}
Expand Down
Loading
Loading