From 7d27b1119092e3195238a1897d6012c3372e6c53 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Fri, 2 Aug 2024 19:03:05 -0500 Subject: [PATCH] Remove shards (#5066) --- Builds/levelization/results/loops.txt | 10 +- Builds/levelization/results/ordering.txt | 11 +- cfg/rippled-example.cfg | 47 +- cfg/rippled-reporting.cfg | 47 +- cmake/RippledCore.cmake | 1 - include/xrpl/basics/ThreadSafetyAnalysis.h | 63 - include/xrpl/proto/ripple.proto | 65 - include/xrpl/protocol/HashPrefix.h | 3 - include/xrpl/protocol/SystemParameters.h | 3 - include/xrpl/protocol/jss.h | 12 +- src/test/net/DatabaseDownloader_test.cpp | 315 --- src/test/nodestore/DatabaseShard_test.cpp | 1894 -------------- src/test/nodestore/Database_test.cpp | 31 - src/test/rpc/NodeToShardRPC_test.cpp | 414 --- src/test/rpc/RPCCall_test.cpp | 294 --- src/test/rpc/ShardArchiveHandler_test.cpp | 705 ------ src/test/shamap/common.h | 13 +- src/xrpld/app/consensus/RCLConsensus.cpp | 1 - src/xrpld/app/ledger/InboundLedger.h | 1 - src/xrpld/app/ledger/LedgerMaster.h | 3 - src/xrpld/app/ledger/detail/InboundLedger.cpp | 52 +- .../app/ledger/detail/InboundLedgers.cpp | 25 +- src/xrpld/app/ledger/detail/LedgerMaster.cpp | 101 +- src/xrpld/app/main/Application.cpp | 194 +- src/xrpld/app/main/Application.h | 10 - src/xrpld/app/main/DBInit.h | 120 - src/xrpld/app/main/Main.cpp | 6 - src/xrpld/app/misc/NetworkOPs.cpp | 6 +- src/xrpld/app/misc/SHAMapStoreImp.cpp | 25 +- src/xrpld/app/rdb/Download.h | 79 - src/xrpld/app/rdb/README.md | 33 +- src/xrpld/app/rdb/ShardArchive.h | 78 - src/xrpld/app/rdb/State.h | 1 - src/xrpld/app/rdb/UnitaryShard.h | 155 -- .../rdb/backend/detail/{detail => }/Node.cpp | 97 +- src/xrpld/app/rdb/backend/detail/Node.h | 26 +- .../rdb/backend/detail/PostgresDatabase.cpp | 1 - .../app/rdb/backend/detail/SQLiteDatabase.cpp | 893 +------ src/xrpld/app/rdb/backend/detail/Shard.h | 90 - .../app/rdb/backend/detail/detail/Shard.cpp | 147 -- src/xrpld/app/rdb/detail/Download.cpp | 152 -- .../app/rdb/detail/RelationalDatabase.cpp | 1 - src/xrpld/app/rdb/detail/ShardArchive.cpp | 68 - src/xrpld/app/rdb/detail/UnitaryShard.cpp | 320 --- src/xrpld/core/Config.h | 1 - src/xrpld/core/ConfigSections.h | 6 - src/xrpld/core/Job.h | 1 - src/xrpld/core/JobTypes.h | 1 - src/xrpld/net/DatabaseBody.h | 179 -- src/xrpld/net/DatabaseDownloader.h | 76 - src/xrpld/net/HTTPDownloader.h | 130 - src/xrpld/net/HTTPStream.h | 165 -- src/xrpld/net/ShardDownloader.md | 311 --- src/xrpld/net/detail/DatabaseBody.ipp | 231 -- src/xrpld/net/detail/DatabaseDownloader.cpp | 92 - src/xrpld/net/detail/HTTPDownloader.cpp | 340 --- src/xrpld/net/detail/HTTPStream.cpp | 203 -- src/xrpld/net/detail/RPCCall.cpp | 42 - src/xrpld/net/uml/interrupt_sequence.pu | 233 -- src/xrpld/net/uml/states.pu | 69 - src/xrpld/nodestore/Database.h | 96 +- src/xrpld/nodestore/DatabaseShard.h | 298 --- src/xrpld/nodestore/DeterministicShard.md | 162 -- src/xrpld/nodestore/Manager.h | 1 - src/xrpld/nodestore/README.md | 194 -- src/xrpld/nodestore/ShardInfo.h | 122 - src/xrpld/nodestore/ShardPool.md | 43 - src/xrpld/nodestore/ShardSizeTuning.md | 213 -- src/xrpld/nodestore/Types.h | 9 - src/xrpld/nodestore/backend/NuDBFactory.cpp | 23 +- src/xrpld/nodestore/detail/Database.cpp | 121 - .../nodestore/detail/DatabaseNodeImp.cpp | 1 - src/xrpld/nodestore/detail/DatabaseNodeImp.h | 6 - .../nodestore/detail/DatabaseRotatingImp.cpp | 12 - .../nodestore/detail/DatabaseRotatingImp.h | 3 - .../nodestore/detail/DatabaseShardImp.cpp | 2253 ----------------- src/xrpld/nodestore/detail/DatabaseShardImp.h | 429 ---- .../nodestore/detail/DeterministicShard.cpp | 216 -- .../nodestore/detail/DeterministicShard.h | 174 -- src/xrpld/nodestore/detail/Shard.cpp | 1272 ---------- src/xrpld/nodestore/detail/Shard.h | 432 ---- src/xrpld/nodestore/detail/ShardInfo.cpp | 136 - src/xrpld/nodestore/detail/TaskQueue.cpp | 76 - src/xrpld/nodestore/detail/TaskQueue.h | 64 - src/xrpld/overlay/Overlay.h | 9 - src/xrpld/overlay/Peer.h | 3 - src/xrpld/overlay/detail/Message.cpp | 4 - src/xrpld/overlay/detail/OverlayImpl.cpp | 109 - src/xrpld/overlay/detail/OverlayImpl.h | 16 - src/xrpld/overlay/detail/PeerImp.cpp | 385 +-- src/xrpld/overlay/detail/PeerImp.h | 17 - src/xrpld/overlay/detail/ProtocolMessage.h | 24 - src/xrpld/overlay/detail/TrafficCount.cpp | 6 - src/xrpld/overlay/detail/TrafficCount.h | 2 - src/xrpld/perflog/detail/PerfLogImp.cpp | 6 +- src/xrpld/rpc/ShardArchiveHandler.h | 176 -- src/xrpld/rpc/ShardVerificationScheduler.h | 84 - src/xrpld/rpc/detail/Handler.cpp | 3 - src/xrpld/rpc/detail/ShardArchiveHandler.cpp | 585 ----- .../rpc/detail/ShardVerificationScheduler.cpp | 68 - src/xrpld/rpc/handlers/CrawlShards.cpp | 73 - src/xrpld/rpc/handlers/DownloadShard.cpp | 176 -- src/xrpld/rpc/handlers/GetCounts.cpp | 30 +- src/xrpld/rpc/handlers/Handlers.h | 6 - src/xrpld/rpc/handlers/NodeToShard.cpp | 86 - src/xrpld/rpc/handlers/Tx.cpp | 1 - src/xrpld/shamap/Family.h | 19 +- src/xrpld/shamap/NodeFamily.h | 12 +- src/xrpld/shamap/ShardFamily.h | 125 - src/xrpld/shamap/detail/SHAMap.cpp | 5 +- src/xrpld/shamap/detail/SHAMapSync.cpp | 13 +- src/xrpld/shamap/detail/ShardFamily.cpp | 198 -- 112 files changed, 115 insertions(+), 17175 deletions(-) delete mode 100644 include/xrpl/basics/ThreadSafetyAnalysis.h delete mode 100644 src/test/net/DatabaseDownloader_test.cpp delete mode 100644 src/test/nodestore/DatabaseShard_test.cpp delete mode 100644 src/test/rpc/NodeToShardRPC_test.cpp delete mode 100644 src/test/rpc/ShardArchiveHandler_test.cpp delete mode 100644 src/xrpld/app/rdb/Download.h delete mode 100644 src/xrpld/app/rdb/ShardArchive.h delete mode 100644 src/xrpld/app/rdb/UnitaryShard.h rename src/xrpld/app/rdb/backend/detail/{detail => }/Node.cpp (93%) delete mode 100644 src/xrpld/app/rdb/backend/detail/Shard.h delete mode 100644 src/xrpld/app/rdb/backend/detail/detail/Shard.cpp delete mode 100644 src/xrpld/app/rdb/detail/Download.cpp delete mode 100644 src/xrpld/app/rdb/detail/ShardArchive.cpp delete mode 100644 src/xrpld/app/rdb/detail/UnitaryShard.cpp delete mode 100644 src/xrpld/net/DatabaseBody.h delete mode 100644 src/xrpld/net/DatabaseDownloader.h delete mode 100644 src/xrpld/net/HTTPDownloader.h delete mode 100644 src/xrpld/net/HTTPStream.h delete mode 100644 src/xrpld/net/ShardDownloader.md delete mode 100644 src/xrpld/net/detail/DatabaseBody.ipp delete mode 100644 src/xrpld/net/detail/DatabaseDownloader.cpp delete mode 100644 src/xrpld/net/detail/HTTPDownloader.cpp delete mode 100644 src/xrpld/net/detail/HTTPStream.cpp delete mode 100644 src/xrpld/net/uml/interrupt_sequence.pu delete mode 100644 src/xrpld/net/uml/states.pu delete mode 100644 src/xrpld/nodestore/DatabaseShard.h delete mode 100644 src/xrpld/nodestore/DeterministicShard.md delete mode 100644 src/xrpld/nodestore/ShardInfo.h delete mode 100644 src/xrpld/nodestore/ShardPool.md delete mode 100644 src/xrpld/nodestore/ShardSizeTuning.md delete mode 100644 src/xrpld/nodestore/detail/DatabaseShardImp.cpp delete mode 100644 src/xrpld/nodestore/detail/DatabaseShardImp.h delete mode 100644 src/xrpld/nodestore/detail/DeterministicShard.cpp delete mode 100644 src/xrpld/nodestore/detail/DeterministicShard.h delete mode 100644 src/xrpld/nodestore/detail/Shard.cpp delete mode 100644 src/xrpld/nodestore/detail/Shard.h delete mode 100644 src/xrpld/nodestore/detail/ShardInfo.cpp delete mode 100644 src/xrpld/nodestore/detail/TaskQueue.cpp delete mode 100644 src/xrpld/nodestore/detail/TaskQueue.h delete mode 100644 src/xrpld/rpc/ShardArchiveHandler.h delete mode 100644 src/xrpld/rpc/ShardVerificationScheduler.h delete mode 100644 src/xrpld/rpc/detail/ShardArchiveHandler.cpp delete mode 100644 src/xrpld/rpc/detail/ShardVerificationScheduler.cpp delete mode 100644 src/xrpld/rpc/handlers/CrawlShards.cpp delete mode 100644 src/xrpld/rpc/handlers/DownloadShard.cpp delete mode 100644 src/xrpld/rpc/handlers/NodeToShard.cpp delete mode 100644 src/xrpld/shamap/ShardFamily.h delete mode 100644 src/xrpld/shamap/detail/ShardFamily.cpp diff --git a/Builds/levelization/results/loops.txt b/Builds/levelization/results/loops.txt index ee7e6fd3bc6..f703a3a9d5d 100644 --- a/Builds/levelization/results/loops.txt +++ b/Builds/levelization/results/loops.txt @@ -16,11 +16,8 @@ Loop: xrpld.app xrpld.ledger Loop: xrpld.app xrpld.net xrpld.app > xrpld.net -Loop: xrpld.app xrpld.nodestore - xrpld.app > xrpld.nodestore - Loop: xrpld.app xrpld.overlay - xrpld.overlay ~= xrpld.app + xrpld.overlay == xrpld.app Loop: xrpld.app xrpld.peerfinder xrpld.app > xrpld.peerfinder @@ -38,10 +35,7 @@ Loop: xrpld.core xrpld.perflog xrpld.perflog ~= xrpld.core Loop: xrpld.net xrpld.rpc - xrpld.rpc > xrpld.net - -Loop: xrpld.nodestore xrpld.overlay - xrpld.overlay ~= xrpld.nodestore + xrpld.rpc ~= xrpld.net Loop: xrpld.overlay xrpld.rpc xrpld.rpc ~= xrpld.overlay diff --git a/Builds/levelization/results/ordering.txt b/Builds/levelization/results/ordering.txt index 87f9b03a54e..2856d783a1f 100644 --- a/Builds/levelization/results/ordering.txt +++ b/Builds/levelization/results/ordering.txt @@ -74,19 +74,13 @@ test.ledger > xrpld.app test.ledger > xrpld.core test.ledger > xrpld.ledger test.ledger > xrpl.protocol -test.net > test.jtx -test.net > test.toplevel -test.net > test.unit_test -test.net > xrpld.net test.nodestore > test.jtx test.nodestore > test.toplevel test.nodestore > test.unit_test test.nodestore > xrpl.basics -test.nodestore > xrpld.app test.nodestore > xrpld.core test.nodestore > xrpld.nodestore test.nodestore > xrpld.unity -test.nodestore > xrpl.protocol test.overlay > test.jtx test.overlay > test.unit_test test.overlay > xrpl.basics @@ -109,13 +103,11 @@ test.resource > test.unit_test test.resource > xrpl.basics test.resource > xrpl.resource test.rpc > test.jtx -test.rpc > test.nodestore test.rpc > test.toplevel test.rpc > xrpl.basics test.rpc > xrpld.app test.rpc > xrpld.core test.rpc > xrpld.net -test.rpc > xrpld.nodestore test.rpc > xrpld.overlay test.rpc > xrpld.rpc test.rpc > xrpl.json @@ -150,6 +142,7 @@ xrpld.app > test.unit_test xrpld.app > xrpl.basics xrpld.app > xrpld.conditions xrpld.app > xrpld.consensus +xrpld.app > xrpld.nodestore xrpld.app > xrpld.perflog xrpld.app > xrpl.json xrpld.app > xrpl.protocol @@ -186,14 +179,12 @@ xrpld.peerfinder > xrpl.basics xrpld.peerfinder > xrpld.core xrpld.peerfinder > xrpl.protocol xrpld.perflog > xrpl.basics -xrpld.perflog > xrpld.nodestore xrpld.perflog > xrpl.json xrpld.perflog > xrpl.protocol xrpld.rpc > xrpl.basics xrpld.rpc > xrpld.core xrpld.rpc > xrpld.ledger xrpld.rpc > xrpld.nodestore -xrpld.rpc > xrpld.shamap xrpld.rpc > xrpl.json xrpld.rpc > xrpl.protocol xrpld.rpc > xrpl.resource diff --git a/cfg/rippled-example.cfg b/cfg/rippled-example.cfg index 2ba2afa727d..b283900d013 100644 --- a/cfg/rippled-example.cfg +++ b/cfg/rippled-example.cfg @@ -1094,7 +1094,7 @@ # default value for the unspecified parameter. # # Note: the cache will not be created if online_delete -# is specified, or if shards are used. +# is specified. # # fast_load Boolean. If set, load the last persisted ledger # from disk upon process start before syncing to @@ -1107,10 +1107,6 @@ # earliest_seq The default is 32570 to match the XRP ledger # network's earliest allowed sequence. Alternate # networks may set this value. Minimum value of 1. -# If a [shard_db] section is defined, and this -# value is present either [node_db] or [shard_db], -# it must be defined with the same value in both -# sections. # # online_delete Minimum value of 256. Enable automatic purging # of older ledger information. Maintain at least this @@ -1192,32 +1188,6 @@ # your rippled.cfg file. # Partial pathnames are relative to the location of the rippled executable. # -# [shard_db] Settings for the Shard Database (optional) -# -# Format (without spaces): -# One or more lines of case-insensitive key / value pairs: -# '=' -# ... -# -# Example: -# path=db/shards/nudb -# -# Required keys: -# path Location to store the database -# -# Optional keys: -# max_historical_shards -# The maximum number of historical shards -# to store. -# -# [historical_shard_paths] Additional storage paths for the Shard Database (optional) -# -# Format (without spaces): -# One or more lines, each expressing a full path for storing historical shards: -# /mnt/disk1 -# /mnt/disk2 -# ... -# # [sqlite] Tuning settings for the SQLite databases (optional) # # Format (without spaces): @@ -1674,21 +1644,6 @@ path=/var/lib/rippled/db/nudb online_delete=512 advisory_delete=0 -# This is the persistent datastore for shards. It is important for the health -# of the ripple network that rippled operators shard as much as practical. -# NuDB requires SSD storage. Helpful information can be found at -# https://xrpl.org/history-sharding.html -#[shard_db] -#path=/var/lib/rippled/db/shards/nudb -#max_historical_shards=50 -# -# This optional section can be configured with a list -# of paths to use for storing historical shards. Each -# path must correspond to a unique filesystem. -#[historical_shard_paths] -#/path/1 -#/path/2 - [database_path] /var/lib/rippled/db diff --git a/cfg/rippled-reporting.cfg b/cfg/rippled-reporting.cfg index 290bcc5418a..9776ef5ee45 100644 --- a/cfg/rippled-reporting.cfg +++ b/cfg/rippled-reporting.cfg @@ -1039,17 +1039,13 @@ # default value for the unspecified parameter. # # Note: the cache will not be created if online_delete -# is specified, or if shards are used. +# is specified. # # Optional keys for NuDB or RocksDB: # # earliest_seq The default is 32570 to match the XRP ledger # network's earliest allowed sequence. Alternate # networks may set this value. Minimum value of 1. -# If a [shard_db] section is defined, and this -# value is present either [node_db] or [shard_db], -# it must be defined with the same value in both -# sections. # # online_delete Minimum value of 256. Enable automatic purging # of older ledger information. Maintain at least this @@ -1135,32 +1131,6 @@ # your rippled.cfg file. # Partial pathnames are relative to the location of the rippled executable. # -# [shard_db] Settings for the Shard Database (optional) -# -# Format (without spaces): -# One or more lines of case-insensitive key / value pairs: -# '=' -# ... -# -# Example: -# path=db/shards/nudb -# -# Required keys: -# path Location to store the database -# -# Optional keys: -# max_historical_shards -# The maximum number of historical shards -# to store. -# -# [historical_shard_paths] Additional storage paths for the Shard Database (optional) -# -# Format (without spaces): -# One or more lines, each expressing a full path for storing historical shards: -# /mnt/disk1 -# /mnt/disk2 -# ... -# # [sqlite] Tuning settings for the SQLite databases (optional) # # Format (without spaces): @@ -1616,21 +1586,6 @@ path=/var/lib/rippled-reporting/db/nudb # online_delete=512 # advisory_delete=0 -# This is the persistent datastore for shards. It is important for the health -# of the ripple network that rippled operators shard as much as practical. -# NuDB requires SSD storage. Helpful information can be found at -# https://xrpl.org/history-sharding.html -#[shard_db] -#path=/var/lib/rippled/db/shards/nudb -#max_historical_shards=50 -# -# This optional section can be configured with a list -# of paths to use for storing historical shards. Each -# path must correspond to a unique filesystem. -#[historical_shard_paths] -#/path/1 -#/path/2 - [database_path] /var/lib/rippled-reporting/db diff --git a/cmake/RippledCore.cmake b/cmake/RippledCore.cmake index 6a0060f7b32..18a424c484b 100644 --- a/cmake/RippledCore.cmake +++ b/cmake/RippledCore.cmake @@ -142,7 +142,6 @@ if(xrpld) set_source_files_properties( # these two seem to produce conflicts in beast teardown template methods src/test/rpc/ValidatorRPC_test.cpp - src/test/rpc/ShardArchiveHandler_test.cpp src/test/ledger/Invariants_test.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION TRUE) endif() diff --git a/include/xrpl/basics/ThreadSafetyAnalysis.h b/include/xrpl/basics/ThreadSafetyAnalysis.h deleted file mode 100644 index b1889d5b4c6..00000000000 --- a/include/xrpl/basics/ThreadSafetyAnalysis.h +++ /dev/null @@ -1,63 +0,0 @@ -#ifndef RIPPLE_BASICS_THREAD_SAFTY_ANALYSIS_H_INCLUDED -#define RIPPLE_BASICS_THREAD_SAFTY_ANALYSIS_H_INCLUDED - -#ifdef RIPPLE_ENABLE_THREAD_SAFETY_ANNOTATIONS -#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) -#else -#define THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op -#endif - -#define CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(capability(x)) - -#define SCOPED_CAPABILITY THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) - -#define GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x)) - -#define PT_GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x)) - -#define ACQUIRED_BEFORE(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__)) - -#define ACQUIRED_AFTER(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__)) - -#define REQUIRES(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(requires_capability(__VA_ARGS__)) - -#define REQUIRES_SHARED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(requires_shared_capability(__VA_ARGS__)) - -#define ACQUIRE(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(acquire_capability(__VA_ARGS__)) - -#define ACQUIRE_SHARED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(acquire_shared_capability(__VA_ARGS__)) - -#define RELEASE(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(release_capability(__VA_ARGS__)) - -#define RELEASE_SHARED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(release_shared_capability(__VA_ARGS__)) - -#define RELEASE_GENERIC(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(release_generic_capability(__VA_ARGS__)) - -#define TRY_ACQUIRE(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_capability(__VA_ARGS__)) - -#define TRY_ACQUIRE_SHARED(...) \ - THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_shared_capability(__VA_ARGS__)) - -#define EXCLUDES(...) THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__)) - -#define ASSERT_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(assert_capability(x)) - -#define ASSERT_SHARED_CAPABILITY(x) \ - THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_capability(x)) - -#define RETURN_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) - -#define NO_THREAD_SAFETY_ANALYSIS \ - THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) - -#endif diff --git a/include/xrpl/proto/ripple.proto b/include/xrpl/proto/ripple.proto index 74cbfe8f6cb..a06bbd9a311 100644 --- a/include/xrpl/proto/ripple.proto +++ b/include/xrpl/proto/ripple.proto @@ -18,10 +18,6 @@ enum MessageType mtHAVE_SET = 35; mtVALIDATION = 41; mtGET_OBJECTS = 42; - mtGET_SHARD_INFO = 50; - mtSHARD_INFO = 51; - mtGET_PEER_SHARD_INFO = 52; - mtPEER_SHARD_INFO = 53; mtVALIDATORLIST = 54; mtSQUELCH = 55; mtVALIDATORLISTCOLLECTION = 56; @@ -29,8 +25,6 @@ enum MessageType mtPROOF_PATH_RESPONSE = 58; mtREPLAY_DELTA_REQ = 59; mtREPLAY_DELTA_RESPONSE = 60; - mtGET_PEER_SHARD_INFO_V2 = 61; - mtPEER_SHARD_INFO_V2 = 62; mtHAVE_TRANSACTIONS = 63; mtTRANSACTIONS = 64; } @@ -89,71 +83,12 @@ message TMLink required bytes nodePubKey = 1 [deprecated=true]; // node public key } -// Request info on shards held -message TMGetPeerShardInfo -{ - required uint32 hops = 1 [deprecated=true]; // number of hops to travel - optional bool lastLink = 2 [deprecated=true]; // true if last link in the peer chain - repeated TMLink peerChain = 3 [deprecated=true]; // public keys used to route messages -} - -// Info about shards held -message TMPeerShardInfo -{ - required string shardIndexes = 1 [deprecated=true]; // rangeSet of shard indexes - optional bytes nodePubKey = 2 [deprecated=true]; // node public key - optional string endpoint = 3 [deprecated=true]; // ipv6 or ipv4 address - optional bool lastLink = 4 [deprecated=true]; // true if last link in the peer chain - repeated TMLink peerChain = 5 [deprecated=true]; // public keys used to route messages -} - // Peer public key message TMPublicKey { required bytes publicKey = 1; } -// Request peer shard information -message TMGetPeerShardInfoV2 -{ - // Peer public keys used to route messages - repeated TMPublicKey peerChain = 1; - - // Remaining times to relay - required uint32 relays = 2; -} - -// Peer shard information -message TMPeerShardInfoV2 -{ - message TMIncomplete - { - required uint32 shardIndex = 1; - required uint32 state = 2; - - // State completion percent, 1 - 100 - optional uint32 progress = 3; - } - - // Message creation time - required uint32 timestamp = 1; - - // Incomplete shards being acquired or verified - repeated TMIncomplete incomplete = 2; - - // Verified immutable shards (RangeSet) - optional string finalized = 3; - - // Public key of node that authored the shard info - required bytes publicKey = 4; - - // Digital signature of node that authored the shard info - required bytes signature = 5; - - // Peer public keys used to route messages - repeated TMPublicKey peerChain = 6; -} - // A transaction can have only one input and one output. // If you want to send an amount that is greater than any single address of yours // you must first combine coins from one address to another. diff --git a/include/xrpl/protocol/HashPrefix.h b/include/xrpl/protocol/HashPrefix.h index 0979756b6e1..bc9c23d1910 100644 --- a/include/xrpl/protocol/HashPrefix.h +++ b/include/xrpl/protocol/HashPrefix.h @@ -84,9 +84,6 @@ enum class HashPrefix : std::uint32_t { /** Payment Channel Claim */ paymentChannelClaim = detail::make_hash_prefix('C', 'L', 'M'), - - /** shard info for signing */ - shardInfo = detail::make_hash_prefix('S', 'H', 'D'), }; template diff --git a/include/xrpl/protocol/SystemParameters.h b/include/xrpl/protocol/SystemParameters.h index c99944193ae..7531a0d5fb9 100644 --- a/include/xrpl/protocol/SystemParameters.h +++ b/include/xrpl/protocol/SystemParameters.h @@ -72,9 +72,6 @@ static constexpr std::uint32_t XRP_LEDGER_EARLIEST_SEQ{32570u}; * used in asserts and tests. */ static constexpr std::uint32_t XRP_LEDGER_EARLIEST_FEES{562177u}; -/** The number of ledgers in a shard */ -static constexpr std::uint32_t DEFAULT_LEDGERS_PER_SHARD{16384u}; - /** The minimum amount of support an amendment should have. @note This value is used by legacy code and will become obsolete diff --git a/include/xrpl/protocol/jss.h b/include/xrpl/protocol/jss.h index a46e15f39ef..84628da286f 100644 --- a/include/xrpl/protocol/jss.h +++ b/include/xrpl/protocol/jss.h @@ -89,7 +89,6 @@ JSS(EscrowFinish); // transaction type. JSS(Fee); // in/out: TransactionSign; field. JSS(FeeSettings); // ledger type. JSS(Flags); // in/out: TransactionSign; field. -JSS(incomplete_shards); // out: OverlayImpl, PeerImp JSS(Invalid); // JSS(LastLedgerSequence); // in: TransactionSign; field JSS(LastUpdateTime); // field. @@ -260,7 +259,6 @@ JSS(code); // out: errors JSS(command); // in: RPCHandler JSS(complete); // out: NetworkOPs, InboundLedger JSS(complete_ledgers); // out: NetworkOPs, PeerImp -JSS(complete_shards); // out: OverlayImpl, PeerImp JSS(consensus); // out: NetworkOPs, LedgerConsensus JSS(converge_time); // out: NetworkOPs JSS(converge_time_s); // out: NetworkOPs @@ -270,8 +268,6 @@ JSS(counters); // in/out: retrieve counters JSS(ctid); // in/out: Tx RPC JSS(currency_a); // out: BookChanges JSS(currency_b); // out: BookChanges -JSS(currentShard); // out: NodeToShardStatus -JSS(currentShardIndex); // out: NodeToShardStatus JSS(currency); // in: paths/PathRequest, STAmount // out: STPathSet, STAmount, // AccountLines @@ -344,8 +340,6 @@ JSS(fetch_pack); // out: NetworkOPs JSS(FIELDS); // out: RPC server_definitions // matches definitions.json format JSS(first); // out: rpc/Version -JSS(firstSequence); // out: NodeToShardStatus -JSS(firstShardIndex); // out: NodeToShardStatus JSS(finished); JSS(fix_txns); // in: LedgerCleaner JSS(flags); // out: AccountOffers, @@ -376,7 +370,7 @@ JSS(ident); // in: AccountCurrencies, AccountInfo, JSS(ignore_default); // in: AccountLines JSS(inLedger); // out: tx/Transaction JSS(inbound); // out: PeerImp -JSS(index); // in: LedgerEntry, DownloadShard +JSS(index); // in: LedgerEntry // out: STLedgerEntry, // LedgerEntry, TxHistory, LedgerData JSS(info); // out: ServerInfo, ConsensusInfo, FetchInfo @@ -406,8 +400,6 @@ JSS(key); // out JSS(key_type); // in/out: WalletPropose, TransactionSign JSS(latency); // out: PeerImp JSS(last); // out: RPCVersion -JSS(lastSequence); // out: NodeToShardStatus -JSS(lastShardIndex); // out: NodeToShardStatus JSS(last_close); // out: NetworkOPs JSS(last_refresh_time); // out: ValidatorSite JSS(last_refresh_status); // out: ValidatorSite @@ -631,7 +623,6 @@ JSS(server_status); // out: NetworkOPs JSS(server_version); // out: NetworkOPs JSS(settle_delay); // out: AccountChannels JSS(severity); // in: LogLevel -JSS(shards); // in/out: GetCounts, DownloadShard JSS(signature); // out: NetworkOPs, ChannelAuthorize JSS(signature_verified); // out: ChannelVerify JSS(signing_key); // out: NetworkOPs @@ -655,7 +646,6 @@ JSS(state_now); // in: Subscribe JSS(status); // error JSS(stop); // in: LedgerCleaner JSS(stop_history_tx_only); // in: Unsubscribe, stop history tx stream -JSS(storedSeqs); // out: NodeToShardStatus JSS(streams); // in: Subscribe, Unsubscribe JSS(strict); // in: AccountCurrencies, AccountInfo JSS(sub_index); // in: LedgerEntry diff --git a/src/test/net/DatabaseDownloader_test.cpp b/src/test/net/DatabaseDownloader_test.cpp deleted file mode 100644 index 99e98dd3d01..00000000000 --- a/src/test/net/DatabaseDownloader_test.cpp +++ /dev/null @@ -1,315 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright 2019 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { -namespace test { - -#define REPORT_FAILURE(D) reportFailure(D, __FILE__, __LINE__) - -class DatabaseDownloader_test : public beast::unit_test::suite -{ - std::shared_ptr - createServer(jtx::Env& env, bool ssl = true) - { - std::vector list; - list.push_back(TrustedPublisherServer::randomValidator()); - return make_TrustedPublisherServer( - env.app().getIOService(), - list, - env.timeKeeper().now() + std::chrono::seconds{3600}, - // No future VLs - {}, - ssl); - } - - struct DownloadCompleter - { - std::mutex m; - std::condition_variable cv; - bool called = false; - boost::filesystem::path dest; - - void - operator()(boost::filesystem::path dst) - { - std::unique_lock lk(m); - called = true; - dest = std::move(dst); - cv.notify_one(); - }; - - bool - waitComplete() - { - std::unique_lock lk(m); - - auto stat = cv.wait_for( - lk, std::chrono::seconds(10), [this] { return called; }); - - called = false; - return stat; - }; - }; - DownloadCompleter cb; - - struct Downloader - { - test::StreamSink sink_; - beast::Journal journal_; - std::shared_ptr ptr_; - - Downloader(jtx::Env& env) - : journal_{sink_} - , ptr_{make_DatabaseDownloader( - env.app().getIOService(), - env.app().config(), - journal_)} - { - } - - ~Downloader() - { - ptr_->stop(); - } - - DatabaseDownloader* - operator->() - { - return ptr_.get(); - } - - DatabaseDownloader const* - operator->() const - { - return ptr_.get(); - } - }; - - void - reportFailure(Downloader const& dl, char const* file, int line) - { - std::stringstream ss; - ss << "Failed. LOGS:\n" - << dl.sink_.messages().str() - << "\nDownloadCompleter failure." - "\nDatabaseDownloader session active? " - << std::boolalpha << dl->sessionIsActive() - << "\nDatabaseDownloader is stopping? " << std::boolalpha - << dl->isStopping(); - - fail(ss.str(), file, line); - } - - void - testDownload(bool verify) - { - testcase << std::string("Basic download - SSL ") + - (verify ? "Verify" : "No Verify"); - - using namespace jtx; - - ripple::test::detail::FileDirGuard cert{ - *this, "_cert", "ca.pem", TrustedPublisherServer::ca_cert()}; - - Env env{*this, envconfig([&cert, &verify](std::unique_ptr cfg) { - if ((cfg->SSL_VERIFY = verify)) // yes, this is assignment - cfg->SSL_VERIFY_FILE = cert.file().string(); - return cfg; - })}; - - Downloader dl{env}; - - // create a TrustedPublisherServer as a simple HTTP - // server to request from. Use the /textfile endpoint - // to get a simple text file sent as response. - auto server = createServer(env); - log << "Downloading DB from " << server->local_endpoint() << std::endl; - - ripple::test::detail::FileDirGuard const data{ - *this, "downloads", "data", "", false, false}; - // initiate the download and wait for the callback - // to be invoked - auto stat = dl->download( - server->local_endpoint().address().to_string(), - std::to_string(server->local_endpoint().port()), - "/textfile", - 11, - data.file(), - std::function{std::ref(cb)}); - if (!BEAST_EXPECT(stat)) - { - REPORT_FAILURE(dl); - return; - } - if (!BEAST_EXPECT(cb.waitComplete())) - { - REPORT_FAILURE(dl); - return; - } - BEAST_EXPECT(cb.dest == data.file()); - if (!BEAST_EXPECT(boost::filesystem::exists(data.file()))) - return; - BEAST_EXPECT(boost::filesystem::file_size(data.file()) > 0); - } - - void - testFailures() - { - testcase("Error conditions"); - - using namespace jtx; - - Env env{*this}; - - { - // bad hostname - boost::system::error_code ec; - boost::asio::ip::tcp::resolver resolver{env.app().getIOService()}; - auto const results = resolver.resolve("badhostname", "443", ec); - // we require an error in resolving this name in order - // for this test to pass. Some networks might have DNS hijacking - // that prevent NXDOMAIN, in which case the failure is not - // possible, so we skip the test. - if (ec) - { - Downloader dl{env}; - ripple::test::detail::FileDirGuard const datafile{ - *this, "downloads", "data", "", false, false}; - BEAST_EXPECT(dl->download( - "badhostname", - "443", - "", - 11, - datafile.file(), - std::function{ - std::ref(cb)})); - if (!BEAST_EXPECT(cb.waitComplete())) - { - REPORT_FAILURE(dl); - } - BEAST_EXPECT(!boost::filesystem::exists(datafile.file())); - BEAST_EXPECTS( - dl.sink_.messages().str().find("async_resolve") != - std::string::npos, - dl.sink_.messages().str()); - } - } - { - // can't connect - Downloader dl{env}; - ripple::test::detail::FileDirGuard const datafile{ - *this, "downloads", "data", "", false, false}; - auto server = createServer(env); - auto host = server->local_endpoint().address().to_string(); - auto port = std::to_string(server->local_endpoint().port()); - log << "Downloading DB from " << server->local_endpoint() - << std::endl; - server->stop(); - BEAST_EXPECT(dl->download( - host, - port, - "", - 11, - datafile.file(), - std::function{std::ref(cb)})); - if (!BEAST_EXPECT(cb.waitComplete())) - { - REPORT_FAILURE(dl); - } - BEAST_EXPECT(!boost::filesystem::exists(datafile.file())); - BEAST_EXPECTS( - dl.sink_.messages().str().find("async_connect") != - std::string::npos, - dl.sink_.messages().str()); - } - { - // not ssl (failed handlshake) - Downloader dl{env}; - ripple::test::detail::FileDirGuard const datafile{ - *this, "downloads", "data", "", false, false}; - auto server = createServer(env, false); - log << "Downloading DB from " << server->local_endpoint() - << std::endl; - BEAST_EXPECT(dl->download( - server->local_endpoint().address().to_string(), - std::to_string(server->local_endpoint().port()), - "", - 11, - datafile.file(), - std::function{std::ref(cb)})); - if (!BEAST_EXPECT(cb.waitComplete())) - { - REPORT_FAILURE(dl); - } - BEAST_EXPECT(!boost::filesystem::exists(datafile.file())); - BEAST_EXPECTS( - dl.sink_.messages().str().find("async_handshake") != - std::string::npos, - dl.sink_.messages().str()); - } - { - // huge file (content length) - Downloader dl{env}; - ripple::test::detail::FileDirGuard const datafile{ - *this, "downloads", "data", "", false, false}; - auto server = createServer(env); - log << "Downloading DB from " << server->local_endpoint() - << std::endl; - BEAST_EXPECT(dl->download( - server->local_endpoint().address().to_string(), - std::to_string(server->local_endpoint().port()), - "/textfile/huge", - 11, - datafile.file(), - std::function{std::ref(cb)})); - if (!BEAST_EXPECT(cb.waitComplete())) - { - REPORT_FAILURE(dl); - } - BEAST_EXPECT(!boost::filesystem::exists(datafile.file())); - BEAST_EXPECTS( - dl.sink_.messages().str().find("Insufficient disk space") != - std::string::npos, - dl.sink_.messages().str()); - } - } - -public: - void - run() override - { - testDownload(true); - testDownload(false); - testFailures(); - } -}; - -#undef REPORT_FAILURE - -BEAST_DEFINE_TESTSUITE(DatabaseDownloader, net, ripple); -} // namespace test -} // namespace ripple diff --git a/src/test/nodestore/DatabaseShard_test.cpp b/src/test/nodestore/DatabaseShard_test.cpp deleted file mode 100644 index e185c43d157..00000000000 --- a/src/test/nodestore/DatabaseShard_test.cpp +++ /dev/null @@ -1,1894 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace ripple { -namespace NodeStore { - -/** std::uniform_int_distribution is platform dependent. - * Unit test for deterministic shards is the following: it generates - * predictable accounts and transactions, packs them into ledgers - * and makes the shard. The hash of this shard should be equal to the - * given value. On different platforms (precisely, Linux and Mac) - * hashes of the resulting shard was different. It was unvestigated - * that the problem is in the class std::uniform_int_distribution - * which generates different pseudorandom sequences on different - * platforms, but we need predictable sequence. - */ -template -struct uniformIntDistribution -{ - using resultType = IntType; - - const resultType A, B; - - struct paramType - { - const resultType A, B; - - paramType(resultType aa, resultType bb) : A(aa), B(bb) - { - } - }; - - explicit uniformIntDistribution( - const resultType a = 0, - const resultType b = std::numeric_limits::max()) - : A(a), B(b) - { - } - - explicit uniformIntDistribution(const paramType& params) - : A(params.A), B(params.B) - { - } - - template - resultType - operator()(Generator& g) const - { - return rnd(g, A, B); - } - - template - resultType - operator()(Generator& g, const paramType& params) const - { - return rnd(g, params.A, params.B); - } - - resultType - a() const - { - return A; - } - - resultType - b() const - { - return B; - } - - resultType - min() const - { - return A; - } - - resultType - max() const - { - return B; - } - -private: - template - resultType - rnd(Generator& g, const resultType a, const resultType b) const - { - static_assert( - std::is_convertible:: - value, - "Ups..."); - static_assert( - Generator::min() == 0, "If non-zero we have handle the offset"); - const resultType range = b - a + 1; - assert(Generator::max() >= range); // Just for safety - const resultType rejectLim = g.max() % range; - resultType n; - do - n = g(); - while (n <= rejectLim); - return (n % range) + a; - } -}; - -template -Integral -randInt(Engine& engine, Integral min, Integral max) -{ - assert(max > min); - - // This should have no state and constructing it should - // be very cheap. If that turns out not to be the case - // it could be hand-optimized. - return uniformIntDistribution(min, max)(engine); -} - -template -Integral -randInt(Engine& engine, Integral max) -{ - return randInt(engine, Integral(0), max); -} - -// Tests DatabaseShard class -// -class DatabaseShard_test : public TestBase -{ - static constexpr std::uint32_t maxSizeGb = 10; - static constexpr std::uint32_t maxHistoricalShards = 100; - static constexpr std::uint32_t ledgersPerShard = 256; - static constexpr std::uint32_t earliestSeq = ledgersPerShard + 1; - static constexpr std::uint32_t dataSizeMax = 4; - static constexpr std::uint32_t iniAmount = 1000000; - static constexpr std::uint32_t nTestShards = 4; - static constexpr std::chrono::seconds shardStoreTimeout = - std::chrono::seconds(60); - test::SuiteJournal journal_; - beast::temp_dir defNodeDir; - - struct TestData - { - /* ring used to generate pseudo-random sequence */ - beast::xor_shift_engine rng_; - /* number of shards to generate */ - int numShards_; - /* vector of accounts used to send test transactions */ - std::vector accounts_; - /* nAccounts_[i] is the number of these accounts existed before i-th - * ledger */ - std::vector nAccounts_; - /* payAccounts_[i][j] = {from, to} is the pair which consists of two - * number of accounts: source and destinations, which participate in - * j-th payment on i-th ledger */ - std::vector>> payAccounts_; - /* xrpAmount_[i] is the amount for all payments on i-th ledger */ - std::vector xrpAmount_; - /* ledgers_[i] is the i-th ledger which contains the above described - * accounts and payments */ - std::vector> ledgers_; - - TestData( - std::uint64_t const seedValue, - int dataSize = dataSizeMax, - int numShards = 1) - : rng_(seedValue), numShards_(numShards) - { - std::uint32_t n = 0; - std::uint32_t nLedgers = ledgersPerShard * numShards; - - nAccounts_.reserve(nLedgers); - payAccounts_.reserve(nLedgers); - xrpAmount_.reserve(nLedgers); - - for (std::uint32_t i = 0; i < nLedgers; ++i) - { - int p; - if (n >= 2) - p = randInt(rng_, 2 * dataSize); - else - p = 0; - - std::vector> pay; - pay.reserve(p); - - for (int j = 0; j < p; ++j) - { - int from, to; - do - { - from = randInt(rng_, n - 1); - to = randInt(rng_, n - 1); - } while (from == to); - - pay.push_back(std::make_pair(from, to)); - } - - n += !randInt(rng_, nLedgers / dataSize); - - if (n > accounts_.size()) - { - char str[9]; - for (int j = 0; j < 8; ++j) - str[j] = 'a' + randInt(rng_, 'z' - 'a'); - str[8] = 0; - accounts_.emplace_back(str); - } - - nAccounts_.push_back(n); - payAccounts_.push_back(std::move(pay)); - xrpAmount_.push_back(randInt(rng_, 90) + 10); - } - } - - bool - isNewAccounts(int seq) - { - return nAccounts_[seq] > (seq ? nAccounts_[seq - 1] : 0); - } - - void - makeLedgerData(test::jtx::Env& env_, std::uint32_t seq) - { - using namespace test::jtx; - - // The local fee may go up, especially in the online delete tests - while (env_.app().getFeeTrack().lowerLocalFee()) - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - - if (isNewAccounts(seq)) - env_.fund(XRP(iniAmount), accounts_[nAccounts_[seq] - 1]); - - for (std::uint32_t i = 0; i < payAccounts_[seq].size(); ++i) - { - env_( - pay(accounts_[payAccounts_[seq][i].first], - accounts_[payAccounts_[seq][i].second], - XRP(xrpAmount_[seq]))); - } - } - - bool - makeLedgers(test::jtx::Env& env_, std::uint32_t startIndex = 0) - { - if (startIndex == 0) - { - for (std::uint32_t i = 3; i <= ledgersPerShard; ++i) - { - if (!env_.close()) - return false; - std::shared_ptr ledger = - env_.app().getLedgerMaster().getClosedLedger(); - if (ledger->info().seq != i) - return false; - } - } - - for (std::uint32_t i = 0; i < ledgersPerShard * numShards_; ++i) - { - auto const index = i + (startIndex * ledgersPerShard); - - makeLedgerData(env_, i); - if (!env_.close()) - return false; - std::shared_ptr ledger = - env_.app().getLedgerMaster().getClosedLedger(); - if (ledger->info().seq != index + ledgersPerShard + 1) - return false; - ledgers_.push_back(ledger); - } - - return true; - } - }; - - void - testLedgerData( - TestData& data, - std::shared_ptr ledger, - std::uint32_t seq) - { - using namespace test::jtx; - - auto rootCount{0}; - auto accCount{0}; - auto sothCount{0}; - for (auto const& sles : ledger->sles) - { - if (sles->getType() == ltACCOUNT_ROOT) - { - int sq = sles->getFieldU32(sfSequence); - int reqsq = -1; - const auto id = sles->getAccountID(sfAccount); - - for (int i = 0; i < data.accounts_.size(); ++i) - { - if (id == data.accounts_[i].id()) - { - reqsq = ledgersPerShard + 1; - for (int j = 0; j <= seq; ++j) - if (data.nAccounts_[j] > i + 1 || - (data.nAccounts_[j] == i + 1 && - !data.isNewAccounts(j))) - { - for (int k = 0; k < data.payAccounts_[j].size(); - ++k) - if (data.payAccounts_[j][k].first == i) - reqsq++; - } - else - reqsq++; - ++accCount; - break; - } - } - if (reqsq == -1) - { - reqsq = data.nAccounts_[seq] + 1; - ++rootCount; - } - BEAST_EXPECT(sq == reqsq); - } - else - ++sothCount; - } - BEAST_EXPECT(rootCount == 1); - BEAST_EXPECT(accCount == data.nAccounts_[seq]); - BEAST_EXPECT(sothCount == 3); - - auto iniCount{0}; - auto setCount{0}; - auto payCount{0}; - auto tothCount{0}; - for (auto const& tx : ledger->txs) - { - if (tx.first->getTxnType() == ttPAYMENT) - { - std::int64_t xrpAmount = - tx.first->getFieldAmount(sfAmount).xrp().decimalXRP(); - if (xrpAmount == iniAmount) - ++iniCount; - else - { - ++payCount; - BEAST_EXPECT(xrpAmount == data.xrpAmount_[seq]); - } - } - else if (tx.first->getTxnType() == ttACCOUNT_SET) - ++setCount; - else - ++tothCount; - } - int newacc = data.isNewAccounts(seq) ? 1 : 0; - BEAST_EXPECT(iniCount == newacc); - BEAST_EXPECT(setCount == newacc); - BEAST_EXPECT(payCount == data.payAccounts_[seq].size()); - BEAST_EXPECT(tothCount == !seq); - } - - bool - saveLedger( - Database& db, - Ledger const& ledger, - std::shared_ptr const& next = {}) - { - // Store header - { - Serializer s(sizeof(std::uint32_t) + sizeof(LedgerInfo)); - s.add32(HashPrefix::ledgerMaster); - addRaw(ledger.info(), s); - db.store( - hotLEDGER, - std::move(s.modData()), - ledger.info().hash, - ledger.info().seq); - } - - // Store the state map - auto visitAcc = [&](SHAMapTreeNode const& node) { - Serializer s; - node.serializeWithPrefix(s); - db.store( - node.getType() == SHAMapNodeType::tnINNER ? hotUNKNOWN - : hotACCOUNT_NODE, - std::move(s.modData()), - node.getHash().as_uint256(), - ledger.info().seq); - return true; - }; - - if (ledger.stateMap().getHash().isNonZero()) - { - if (!ledger.stateMap().isValid()) - return false; - if (next && next->info().parentHash == ledger.info().hash) - { - auto have = next->stateMap().snapShot(false); - ledger.stateMap().snapShot(false)->visitDifferences( - &(*have), visitAcc); - } - else - ledger.stateMap().snapShot(false)->visitNodes(visitAcc); - } - - // Store the transaction map - auto visitTx = [&](SHAMapTreeNode& node) { - Serializer s; - node.serializeWithPrefix(s); - db.store( - node.getType() == SHAMapNodeType::tnINNER ? hotUNKNOWN - : hotTRANSACTION_NODE, - std::move(s.modData()), - node.getHash().as_uint256(), - ledger.info().seq); - return true; - }; - - if (ledger.info().txHash.isNonZero()) - { - if (!ledger.txMap().isValid()) - return false; - ledger.txMap().snapShot(false)->visitNodes(visitTx); - } - - return true; - } - - void - checkLedger(TestData& data, DatabaseShard& db, Ledger const& ledger) - { - auto fetched = db.fetchLedger(ledger.info().hash, ledger.info().seq); - if (!BEAST_EXPECT(fetched)) - return; - - testLedgerData(data, fetched, ledger.info().seq - ledgersPerShard - 1); - - // verify the metadata/header info by serializing to json - BEAST_EXPECT( - getJson(LedgerFill{ - ledger, nullptr, LedgerFill::full | LedgerFill::expand}) == - getJson(LedgerFill{ - *fetched, nullptr, LedgerFill::full | LedgerFill::expand})); - - BEAST_EXPECT( - getJson(LedgerFill{ - ledger, nullptr, LedgerFill::full | LedgerFill::binary}) == - getJson(LedgerFill{ - *fetched, nullptr, LedgerFill::full | LedgerFill::binary})); - - // walk shamap and validate each node - auto fcompAcc = [&](SHAMapTreeNode& node) -> bool { - Serializer s; - node.serializeWithPrefix(s); - auto nSrc{NodeObject::createObject( - node.getType() == SHAMapNodeType::tnINNER ? hotUNKNOWN - : hotACCOUNT_NODE, - std::move(s.modData()), - node.getHash().as_uint256())}; - if (!BEAST_EXPECT(nSrc)) - return false; - - auto nDst = db.fetchNodeObject( - node.getHash().as_uint256(), ledger.info().seq); - if (!BEAST_EXPECT(nDst)) - return false; - - BEAST_EXPECT(isSame(nSrc, nDst)); - - return true; - }; - if (ledger.stateMap().getHash().isNonZero()) - ledger.stateMap().snapShot(false)->visitNodes(fcompAcc); - - auto fcompTx = [&](SHAMapTreeNode& node) -> bool { - Serializer s; - node.serializeWithPrefix(s); - auto nSrc{NodeObject::createObject( - node.getType() == SHAMapNodeType::tnINNER ? hotUNKNOWN - : hotTRANSACTION_NODE, - std::move(s.modData()), - node.getHash().as_uint256())}; - if (!BEAST_EXPECT(nSrc)) - return false; - - auto nDst = db.fetchNodeObject( - node.getHash().as_uint256(), ledger.info().seq); - if (!BEAST_EXPECT(nDst)) - return false; - - BEAST_EXPECT(isSame(nSrc, nDst)); - - return true; - }; - if (ledger.info().txHash.isNonZero()) - ledger.txMap().snapShot(false)->visitNodes(fcompTx); - } - - std::string - bitmask2Rangeset(std::uint64_t bitmask) - { - std::string set; - if (!bitmask) - return set; - bool empty = true; - - for (std::uint32_t i = 0; i < 64 && bitmask; i++) - { - if (bitmask & (1ll << i)) - { - if (!empty) - set += ","; - set += std::to_string(i); - empty = false; - } - } - - RangeSet rs; - BEAST_EXPECT(from_string(rs, set)); - return ripple::to_string(rs); - } - - std::unique_ptr - testConfig( - std::string const& shardDir, - std::string const& nodeDir = std::string()) - { - using namespace test::jtx; - - return envconfig([&](std::unique_ptr cfg) { - // Shard store configuration - cfg->overwrite(ConfigSection::shardDatabase(), "path", shardDir); - cfg->overwrite( - ConfigSection::shardDatabase(), - "max_historical_shards", - std::to_string(maxHistoricalShards)); - cfg->overwrite( - ConfigSection::shardDatabase(), - "ledgers_per_shard", - std::to_string(ledgersPerShard)); - cfg->overwrite( - ConfigSection::shardDatabase(), - "earliest_seq", - std::to_string(earliestSeq)); - - // Node store configuration - cfg->overwrite( - ConfigSection::nodeDatabase(), - "path", - nodeDir.empty() ? defNodeDir.path() : nodeDir); - cfg->overwrite( - ConfigSection::nodeDatabase(), - "ledgers_per_shard", - std::to_string(ledgersPerShard)); - cfg->overwrite( - ConfigSection::nodeDatabase(), - "earliest_seq", - std::to_string(earliestSeq)); - return cfg; - }); - } - - std::optional - waitShard( - DatabaseShard& shardStore, - std::uint32_t shardIndex, - std::chrono::seconds timeout = shardStoreTimeout) - { - auto const end{std::chrono::system_clock::now() + timeout}; - while (shardStore.getNumTasks() || - !boost::icl::contains( - shardStore.getShardInfo()->finalized(), shardIndex)) - { - if (!BEAST_EXPECT(std::chrono::system_clock::now() < end)) - return std::nullopt; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - return shardIndex; - } - - std::optional - createShard( - TestData& data, - DatabaseShard& shardStore, - int maxShardIndex = 1, - int shardOffset = 0) - { - int shardIndex{-1}; - - for (std::uint32_t i = 0; i < ledgersPerShard; ++i) - { - auto const ledgerSeq{shardStore.prepareLedger( - (maxShardIndex + 1) * ledgersPerShard)}; - if (!BEAST_EXPECT(ledgerSeq != std::nullopt)) - return std::nullopt; - - shardIndex = shardStore.seqToShardIndex(*ledgerSeq); - - int const arrInd = *ledgerSeq - (ledgersPerShard * shardOffset) - - ledgersPerShard - 1; - BEAST_EXPECT( - arrInd >= 0 && arrInd < maxShardIndex * ledgersPerShard); - BEAST_EXPECT(saveLedger(shardStore, *data.ledgers_[arrInd])); - if (arrInd % ledgersPerShard == (ledgersPerShard - 1)) - { - uint256 const finalKey_{0}; - Serializer s; - s.add32(Shard::version); - s.add32(shardStore.firstLedgerSeq(shardIndex)); - s.add32(shardStore.lastLedgerSeq(shardIndex)); - s.addRaw(data.ledgers_[arrInd]->info().hash.data(), 256 / 8); - shardStore.store( - hotUNKNOWN, std::move(s.modData()), finalKey_, *ledgerSeq); - } - shardStore.setStored(data.ledgers_[arrInd]); - } - - return waitShard(shardStore, shardIndex); - } - - void - testStandalone() - { - testcase("Standalone"); - - using namespace test::jtx; - - beast::temp_dir shardDir; - DummyScheduler scheduler; - { - Env env{*this, testConfig(shardDir.path())}; - std::unique_ptr shardStore{ - make_ShardStore(env.app(), scheduler, 2, journal_)}; - - BEAST_EXPECT(shardStore); - BEAST_EXPECT(shardStore->init()); - BEAST_EXPECT(shardStore->ledgersPerShard() == ledgersPerShard); - BEAST_EXPECT(shardStore->seqToShardIndex(ledgersPerShard + 1) == 1); - BEAST_EXPECT(shardStore->seqToShardIndex(2 * ledgersPerShard) == 1); - BEAST_EXPECT( - shardStore->seqToShardIndex(2 * ledgersPerShard + 1) == 2); - BEAST_EXPECT( - shardStore->earliestShardIndex() == - (earliestSeq - 1) / ledgersPerShard); - BEAST_EXPECT(shardStore->firstLedgerSeq(1) == ledgersPerShard + 1); - BEAST_EXPECT(shardStore->lastLedgerSeq(1) == 2 * ledgersPerShard); - BEAST_EXPECT(shardStore->getRootDir().string() == shardDir.path()); - } - - { - Env env{*this, testConfig(shardDir.path())}; - std::unique_ptr shardStore{ - make_ShardStore(env.app(), scheduler, 2, journal_)}; - - env.app().config().overwrite( - ConfigSection::shardDatabase(), "ledgers_per_shard", "512"); - BEAST_EXPECT(!shardStore->init()); - } - - Env env{*this, testConfig(shardDir.path())}; - std::unique_ptr shardStore{ - make_ShardStore(env.app(), scheduler, 2, journal_)}; - - env.app().config().overwrite( - ConfigSection::shardDatabase(), - "earliest_seq", - std::to_string(std::numeric_limits::max())); - BEAST_EXPECT(!shardStore->init()); - } - - void - testCreateShard(std::uint64_t const seedValue) - { - testcase("Create shard"); - - using namespace test::jtx; - - beast::temp_dir shardDir; - Env env{*this, testConfig(shardDir.path())}; - DatabaseShard* db = env.app().getShardStore(); - BEAST_EXPECT(db); - - TestData data(seedValue); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - if (!createShard(data, *db, 1)) - return; - - for (std::uint32_t i = 0; i < ledgersPerShard; ++i) - checkLedger(data, *db, *data.ledgers_[i]); - } - - void - testReopenDatabase(std::uint64_t const seedValue) - { - testcase("Reopen shard store"); - - using namespace test::jtx; - - beast::temp_dir shardDir; - { - Env env{*this, testConfig(shardDir.path())}; - DatabaseShard* db = env.app().getShardStore(); - BEAST_EXPECT(db); - - TestData data(seedValue, 4, 2); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - for (auto i = 0; i < 2; ++i) - { - if (!createShard(data, *db, 2)) - return; - } - } - { - Env env{*this, testConfig(shardDir.path())}; - DatabaseShard* db = env.app().getShardStore(); - BEAST_EXPECT(db); - - TestData data(seedValue, 4, 2); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - for (std::uint32_t i = 1; i <= 2; ++i) - waitShard(*db, i); - - for (std::uint32_t i = 0; i < 2 * ledgersPerShard; ++i) - checkLedger(data, *db, *data.ledgers_[i]); - } - } - - void - testGetFinalShards(std::uint64_t const seedValue) - { - testcase("Get final shards"); - - using namespace test::jtx; - - beast::temp_dir shardDir; - Env env{*this, testConfig(shardDir.path())}; - DatabaseShard* db = env.app().getShardStore(); - BEAST_EXPECT(db); - - TestData data(seedValue, 2, nTestShards); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - BEAST_EXPECT(db->getShardInfo()->finalized().empty()); - - for (auto i = 0; i < nTestShards; ++i) - { - auto const shardIndex{createShard(data, *db, nTestShards)}; - if (!BEAST_EXPECT( - shardIndex && *shardIndex >= 1 && - *shardIndex <= nTestShards)) - { - return; - } - - BEAST_EXPECT(boost::icl::contains( - db->getShardInfo()->finalized(), *shardIndex)); - } - } - - void - testPrepareShards(std::uint64_t const seedValue) - { - testcase("Prepare shards"); - - using namespace test::jtx; - - beast::temp_dir shardDir; - Env env{*this, testConfig(shardDir.path())}; - DatabaseShard* db = env.app().getShardStore(); - BEAST_EXPECT(db); - - TestData data(seedValue, 1, nTestShards); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - BEAST_EXPECT(db->getPreShards() == ""); - BEAST_EXPECT(!db->prepareShards({})); - - std::uint64_t bitMask = 0; - for (std::uint32_t i = 0; i < nTestShards * 2; ++i) - { - std::uint32_t const shardIndex{ - randInt(data.rng_, nTestShards - 1) + 1}; - if (bitMask & (1ll << shardIndex)) - { - db->removePreShard(shardIndex); - bitMask &= ~(1ll << shardIndex); - } - else - { - BEAST_EXPECT(db->prepareShards({shardIndex})); - bitMask |= 1ll << shardIndex; - } - BEAST_EXPECT(db->getPreShards() == bitmask2Rangeset(bitMask)); - } - - // test illegal cases - // adding shards with too large number - BEAST_EXPECT(!db->prepareShards({0})); - BEAST_EXPECT(db->getPreShards() == bitmask2Rangeset(bitMask)); - BEAST_EXPECT(!db->prepareShards({nTestShards + 1})); - BEAST_EXPECT(db->getPreShards() == bitmask2Rangeset(bitMask)); - BEAST_EXPECT(!db->prepareShards({nTestShards + 2})); - BEAST_EXPECT(db->getPreShards() == bitmask2Rangeset(bitMask)); - - // create shards which are not prepared for import - BEAST_EXPECT(db->getShardInfo()->finalized().empty()); - - std::uint64_t bitMask2 = 0; - for (auto i = 0; i < nTestShards; ++i) - { - auto const shardIndex{createShard(data, *db, nTestShards)}; - if (!BEAST_EXPECT( - shardIndex && *shardIndex >= 1 && - *shardIndex <= nTestShards)) - { - return; - } - - BEAST_EXPECT(boost::icl::contains( - db->getShardInfo()->finalized(), *shardIndex)); - - bitMask2 |= 1ll << *shardIndex; - BEAST_EXPECT((bitMask & bitMask2) == 0); - if ((bitMask | bitMask2) == ((1ll << nTestShards) - 1) << 1) - break; - } - - // try to create another shard - BEAST_EXPECT( - db->prepareLedger((nTestShards + 1) * ledgersPerShard) == - std::nullopt); - } - - void - testImportShard(std::uint64_t const seedValue) - { - testcase("Import shard"); - - using namespace test::jtx; - - beast::temp_dir importDir; - TestData data(seedValue, 2); - - { - Env env{*this, testConfig(importDir.path())}; - DatabaseShard* db = env.app().getShardStore(); - BEAST_EXPECT(db); - - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - if (!createShard(data, *db, 1)) - return; - - for (std::uint32_t i = 0; i < ledgersPerShard; ++i) - checkLedger(data, *db, *data.ledgers_[i]); - - data.ledgers_.clear(); - } - - boost::filesystem::path importPath(importDir.path()); - importPath /= "1"; - - { - beast::temp_dir shardDir; - Env env{*this, testConfig(shardDir.path())}; - DatabaseShard* db = env.app().getShardStore(); - BEAST_EXPECT(db); - - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - BEAST_EXPECT(!db->importShard(1, importPath / "not_exist")); - BEAST_EXPECT(db->prepareShards({1})); - BEAST_EXPECT(db->getPreShards() == "1"); - - using namespace boost::filesystem; - remove_all(importPath / LgrDBName); - remove_all(importPath / TxDBName); - - if (!BEAST_EXPECT(db->importShard(1, importPath))) - return; - - BEAST_EXPECT(db->getPreShards() == ""); - - auto n = waitShard(*db, 1); - if (!BEAST_EXPECT(n && *n == 1)) - return; - - for (std::uint32_t i = 0; i < ledgersPerShard; ++i) - checkLedger(data, *db, *data.ledgers_[i]); - } - } - - void - testCorruptedDatabase(std::uint64_t const seedValue) - { - testcase("Corrupted shard store"); - - using namespace test::jtx; - - beast::temp_dir shardDir; - { - TestData data(seedValue, 4, 2); - { - Env env{*this, testConfig(shardDir.path())}; - DatabaseShard* db = env.app().getShardStore(); - BEAST_EXPECT(db); - - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - for (auto i = 0; i < 2; ++i) - { - if (!BEAST_EXPECT(createShard(data, *db, 2))) - return; - } - } - - boost::filesystem::path path = shardDir.path(); - path /= std::string("2"); - path /= "nudb.dat"; - - FILE* f = fopen(path.string().c_str(), "r+b"); - if (!BEAST_EXPECT(f)) - return; - char buf[256]; - beast::rngfill(buf, sizeof(buf), data.rng_); - BEAST_EXPECT(fwrite(buf, 1, 256, f) == 256); - fclose(f); - } - - Env env{*this, testConfig(shardDir.path())}; - DatabaseShard* db = env.app().getShardStore(); - BEAST_EXPECT(db); - - TestData data(seedValue, 4, 2); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - for (std::uint32_t shardIndex = 1; shardIndex <= 1; ++shardIndex) - waitShard(*db, shardIndex); - - BEAST_EXPECT(boost::icl::contains(db->getShardInfo()->finalized(), 1)); - - for (std::uint32_t i = 0; i < 1 * ledgersPerShard; ++i) - checkLedger(data, *db, *data.ledgers_[i]); - } - - void - testIllegalFinalKey(std::uint64_t const seedValue) - { - testcase("Illegal finalKey"); - - using namespace test::jtx; - - for (int i = 0; i < 5; ++i) - { - beast::temp_dir shardDir; - { - Env env{*this, testConfig(shardDir.path())}; - DatabaseShard* db = env.app().getShardStore(); - BEAST_EXPECT(db); - - TestData data(seedValue + i, 2); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - int shardIndex{-1}; - for (std::uint32_t j = 0; j < ledgersPerShard; ++j) - { - auto const ledgerSeq{ - db->prepareLedger(2 * ledgersPerShard)}; - if (!BEAST_EXPECT(ledgerSeq != std::nullopt)) - return; - - shardIndex = db->seqToShardIndex(*ledgerSeq); - int arrInd = *ledgerSeq - ledgersPerShard - 1; - BEAST_EXPECT(arrInd >= 0 && arrInd < ledgersPerShard); - BEAST_EXPECT(saveLedger(*db, *data.ledgers_[arrInd])); - if (arrInd % ledgersPerShard == (ledgersPerShard - 1)) - { - uint256 const finalKey_{0}; - Serializer s; - s.add32(Shard::version + (i == 0)); - s.add32(db->firstLedgerSeq(shardIndex) + (i == 1)); - s.add32(db->lastLedgerSeq(shardIndex) - (i == 3)); - s.addRaw( - data.ledgers_[arrInd - (i == 4)] - ->info() - .hash.data(), - 256 / 8); - db->store( - hotUNKNOWN, - std::move(s.modData()), - finalKey_, - *ledgerSeq); - } - db->setStored(data.ledgers_[arrInd]); - } - - if (i == 2) - { - waitShard(*db, shardIndex); - BEAST_EXPECT(boost::icl::contains( - db->getShardInfo()->finalized(), 1)); - } - else - { - boost::filesystem::path path(shardDir.path()); - path /= "1"; - boost::system::error_code ec; - auto start = std::chrono::system_clock::now(); - auto end = start + shardStoreTimeout; - while (std::chrono::system_clock::now() < end && - boost::filesystem::exists(path, ec)) - { - std::this_thread::yield(); - } - - BEAST_EXPECT(db->getShardInfo()->finalized().empty()); - } - } - - { - Env env{*this, testConfig(shardDir.path())}; - DatabaseShard* db = env.app().getShardStore(); - BEAST_EXPECT(db); - - TestData data(seedValue + i, 2); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - if (i == 2) - { - waitShard(*db, 1); - BEAST_EXPECT(boost::icl::contains( - db->getShardInfo()->finalized(), 1)); - - for (std::uint32_t j = 0; j < ledgersPerShard; ++j) - checkLedger(data, *db, *data.ledgers_[j]); - } - else - BEAST_EXPECT(db->getShardInfo()->finalized().empty()); - } - } - } - - std::string - ripemd160File(std::string filename) - { - using beast::hash_append; - std::ifstream input(filename, std::ios::in | std::ios::binary); - char buf[4096]; - ripemd160_hasher h; - - while (input.read(buf, 4096), input.gcount() > 0) - hash_append(h, buf, input.gcount()); - - auto const binResult = static_cast(h); - const auto charDigest = binResult.data(); - std::string result; - boost::algorithm::hex( - charDigest, - charDigest + sizeof(binResult), - std::back_inserter(result)); - - return result; - } - - void - testDeterministicShard(std::uint64_t const seedValue) - { - testcase("Deterministic shards"); - - using namespace test::jtx; - - for (int i = 0; i < 2; i++) - { - beast::temp_dir shardDir; - { - Env env{*this, testConfig(shardDir.path())}; - DatabaseShard* db = env.app().getShardStore(); - BEAST_EXPECT(db); - - TestData data(seedValue, 4); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - if (!BEAST_EXPECT(createShard(data, *db) != std::nullopt)) - return; - } - - boost::filesystem::path path(shardDir.path()); - path /= "1"; - - auto static const ripemd160Key = - ripemd160File((path / "nudb.key").string()); - auto static const ripemd160Dat = - ripemd160File((path / "nudb.dat").string()); - - { - Env env{*this, testConfig(shardDir.path())}; - DatabaseShard* db = env.app().getShardStore(); - BEAST_EXPECT(db); - - TestData data(seedValue, 4); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - if (!BEAST_EXPECT(waitShard(*db, 1) != std::nullopt)) - return; - - for (std::uint32_t j = 0; j < ledgersPerShard; ++j) - checkLedger(data, *db, *data.ledgers_[j]); - } - - BEAST_EXPECT( - ripemd160File((path / "nudb.key").string()) == ripemd160Key); - BEAST_EXPECT( - ripemd160File((path / "nudb.dat").string()) == ripemd160Dat); - } - } - - void - testImportNodeStore(std::uint64_t const seedValue) - { - testcase("Import node store"); - - using namespace test::jtx; - - beast::temp_dir shardDir; - { - beast::temp_dir nodeDir; - Env env{*this, testConfig(shardDir.path(), nodeDir.path())}; - DatabaseShard* db = env.app().getShardStore(); - Database& ndb = env.app().getNodeStore(); - BEAST_EXPECT(db); - - TestData data(seedValue, 4, 2); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - for (std::uint32_t i = 0; i < 2 * ledgersPerShard; ++i) - BEAST_EXPECT(saveLedger(ndb, *data.ledgers_[i])); - - BEAST_EXPECT(db->getShardInfo()->finalized().empty()); - db->importDatabase(ndb); - for (std::uint32_t i = 1; i <= 2; ++i) - waitShard(*db, i); - - auto const finalShards{db->getShardInfo()->finalized()}; - for (std::uint32_t shardIndex : {1, 2}) - BEAST_EXPECT(boost::icl::contains(finalShards, shardIndex)); - } - { - Env env{*this, testConfig(shardDir.path())}; - DatabaseShard* db = env.app().getShardStore(); - BEAST_EXPECT(db); - - TestData data(seedValue, 4, 2); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - for (std::uint32_t i = 1; i <= 2; ++i) - waitShard(*db, i); - - auto const finalShards{db->getShardInfo()->finalized()}; - for (std::uint32_t shardIndex : {1, 2}) - BEAST_EXPECT(boost::icl::contains(finalShards, shardIndex)); - - for (std::uint32_t i = 0; i < 2 * ledgersPerShard; ++i) - checkLedger(data, *db, *data.ledgers_[i]); - } - } - - void - testImportWithOnlineDelete(std::uint64_t const seedValue) - { - testcase("Import node store with online delete"); - - using namespace test::jtx; - using test::CaptureLogs; - - beast::temp_dir shardDir; - beast::temp_dir nodeDir; - std::string capturedLogs; - - { - auto c = testConfig(shardDir.path(), nodeDir.path()); - auto& section = c->section(ConfigSection::nodeDatabase()); - section.set("online_delete", "550"); - section.set("advisory_delete", "1"); - - // Adjust the log level to capture relevant output - c->section(SECTION_RPC_STARTUP) - .append( - "{ \"command\": \"log_level\", \"severity\": \"trace\" " - "}"); - - std::unique_ptr logs(new CaptureLogs(&capturedLogs)); - Env env{*this, std::move(c), std::move(logs)}; - - DatabaseShard* db = env.app().getShardStore(); - Database& ndb = env.app().getNodeStore(); - BEAST_EXPECT(db); - - auto& store = env.app().getSHAMapStore(); - - // Allow online delete to delete the startup ledgers - // so that it will take some time for the import to - // catch up to the point of the next rotation - store.setCanDelete(10); - - // Create some ledgers for the shard store to import - auto const shardCount = 5; - TestData data(seedValue, 4, shardCount); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - store.rendezvous(); - auto const lastRotated = store.getLastRotated(); - BEAST_EXPECT(lastRotated >= 553 && lastRotated < 1103); - - // Start the import - db->importDatabase(ndb); - - while (!db->getDatabaseImportSequence()) - { - // Wait until the import starts - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - - // Enable unimpeded online deletion now that the import has started - store.setCanDelete(std::numeric_limits::max()); - - auto pauseVerifier = std::thread([lastRotated, &store, db, this] { - // The import should still be running when this thread starts - BEAST_EXPECT(db->getDatabaseImportSequence()); - auto rotationProgress = lastRotated; - while (auto const ledgerSeq = db->getDatabaseImportSequence()) - { - // Make sure database rotations dont interfere - // with the import - - auto const last = store.getLastRotated(); - if (last != rotationProgress) - { - // A rotation occurred during shard import. Not - // necessarily an error - - BEAST_EXPECT( - !ledgerSeq || ledgerSeq >= rotationProgress); - rotationProgress = last; - } - } - }); - - auto join = [&pauseVerifier]() { - if (pauseVerifier.joinable()) - pauseVerifier.join(); - }; - - // Create more ledgers to trigger online deletion - data = TestData(seedValue * 2); - if (!BEAST_EXPECT(data.makeLedgers(env, shardCount))) - { - join(); - return; - } - - join(); - BEAST_EXPECT(store.getLastRotated() != lastRotated); - } - - // Database rotation should have been postponed at some - // point during the import - auto const expectedLogMessage = - "rotation would interfere with ShardStore import"; - BEAST_EXPECT( - capturedLogs.find(expectedLogMessage) != std::string::npos); - } - - void - testImportWithHistoricalPaths(std::uint64_t const seedValue) - { - testcase("Import with historical paths"); - - using namespace test::jtx; - - // Test importing with multiple historical paths - { - beast::temp_dir shardDir; - std::array historicalDirs; - std::array historicalPaths; - - std::transform( - historicalDirs.begin(), - historicalDirs.end(), - historicalPaths.begin(), - [](const beast::temp_dir& dir) { return dir.path(); }); - - beast::temp_dir nodeDir; - auto c = testConfig(shardDir.path(), nodeDir.path()); - - auto& historyPaths = c->section(SECTION_HISTORICAL_SHARD_PATHS); - historyPaths.append( - {historicalPaths[0].string(), - historicalPaths[1].string(), - historicalPaths[2].string(), - historicalPaths[3].string()}); - - Env env{*this, std::move(c)}; - DatabaseShard* db = env.app().getShardStore(); - Database& ndb = env.app().getNodeStore(); - BEAST_EXPECT(db); - - auto const shardCount = 4; - - TestData data(seedValue, 4, shardCount); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - for (std::uint32_t i = 0; i < shardCount * ledgersPerShard; ++i) - BEAST_EXPECT(saveLedger(ndb, *data.ledgers_[i])); - - BEAST_EXPECT(db->getShardInfo()->finalized().empty()); - - db->importDatabase(ndb); - for (std::uint32_t i = 1; i <= shardCount; ++i) - waitShard(*db, i); - - auto const final{db->getShardInfo()->finalized()}; - for (std::uint32_t shardIndex : {1, 2, 3, 4}) - BEAST_EXPECT(boost::icl::contains(final, shardIndex)); - - auto const mainPathCount = std::distance( - boost::filesystem::directory_iterator(shardDir.path()), - boost::filesystem::directory_iterator()); - - // Only the two most recent shards - // should be stored at the main path - BEAST_EXPECT(mainPathCount == 2); - - auto const historicalPathCount = std::accumulate( - historicalPaths.begin(), - historicalPaths.end(), - 0, - [](int const sum, boost::filesystem::path const& path) { - return sum + - std::distance( - boost::filesystem::directory_iterator(path), - boost::filesystem::directory_iterator()); - }); - - // All historical shards should be stored - // at historical paths - BEAST_EXPECT(historicalPathCount == shardCount - 2); - } - - // Test importing with a single historical path - { - beast::temp_dir shardDir; - beast::temp_dir historicalDir; - beast::temp_dir nodeDir; - - auto c = testConfig(shardDir.path(), nodeDir.path()); - - auto& historyPaths = c->section(SECTION_HISTORICAL_SHARD_PATHS); - historyPaths.append({historicalDir.path()}); - - Env env{*this, std::move(c)}; - DatabaseShard* db = env.app().getShardStore(); - Database& ndb = env.app().getNodeStore(); - BEAST_EXPECT(db); - - auto const shardCount = 4; - - TestData data(seedValue * 2, 4, shardCount); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - for (std::uint32_t i = 0; i < shardCount * ledgersPerShard; ++i) - BEAST_EXPECT(saveLedger(ndb, *data.ledgers_[i])); - - BEAST_EXPECT(db->getShardInfo()->finalized().empty()); - - db->importDatabase(ndb); - for (std::uint32_t i = 1; i <= shardCount; ++i) - waitShard(*db, i); - - auto const finalShards{db->getShardInfo()->finalized()}; - for (std::uint32_t shardIndex : {1, 2, 3, 4}) - BEAST_EXPECT(boost::icl::contains(finalShards, shardIndex)); - - auto const mainPathCount = std::distance( - boost::filesystem::directory_iterator(shardDir.path()), - boost::filesystem::directory_iterator()); - - // Only the two most recent shards - // should be stored at the main path - BEAST_EXPECT(mainPathCount == 2); - - auto const historicalPathCount = std::distance( - boost::filesystem::directory_iterator(historicalDir.path()), - boost::filesystem::directory_iterator()); - - // All historical shards should be stored - // at historical paths - BEAST_EXPECT(historicalPathCount == shardCount - 2); - } - } - - void - testPrepareWithHistoricalPaths(std::uint64_t const seedValue) - { - testcase("Prepare with historical paths"); - - using namespace test::jtx; - - // Create the primary shard directory - beast::temp_dir primaryDir; - auto config{testConfig(primaryDir.path())}; - - // Create four historical directories - std::array historicalDirs; - { - auto& paths{config->section(SECTION_HISTORICAL_SHARD_PATHS)}; - for (auto const& dir : historicalDirs) - paths.append(dir.path()); - } - - Env env{*this, std::move(config)}; - - // Create some shards - std::uint32_t constexpr numShards{4}; - TestData data(seedValue, 4, numShards); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - auto shardStore{env.app().getShardStore()}; - BEAST_EXPECT(shardStore); - - for (auto i = 0; i < numShards; ++i) - { - auto const shardIndex{createShard(data, *shardStore, numShards)}; - if (!BEAST_EXPECT( - shardIndex && *shardIndex >= 1 && *shardIndex <= numShards)) - { - return; - } - } - - { - // Confirm finalized shards are in the shard store - auto const finalized{shardStore->getShardInfo()->finalized()}; - BEAST_EXPECT(boost::icl::length(finalized) == numShards); - BEAST_EXPECT(boost::icl::first(finalized) == 1); - BEAST_EXPECT(boost::icl::last(finalized) == numShards); - } - - using namespace boost::filesystem; - auto const dirContains = [](beast::temp_dir const& dir, - std::uint32_t shardIndex) { - boost::filesystem::path const path(std::to_string(shardIndex)); - for (auto const& it : directory_iterator(dir.path())) - if (boost::filesystem::path(it).stem() == path) - return true; - return false; - }; - auto const historicalDirsContains = [&](std::uint32_t shardIndex) { - for (auto const& dir : historicalDirs) - if (dirContains(dir, shardIndex)) - return true; - return false; - }; - - // Confirm two most recent shards are in the primary shard directory - for (auto const shardIndex : {numShards - 1, numShards}) - { - BEAST_EXPECT(dirContains(primaryDir, shardIndex)); - BEAST_EXPECT(!historicalDirsContains(shardIndex)); - } - - // Confirm remaining shards are in the historical shard directories - for (auto shardIndex = 1; shardIndex < numShards - 1; ++shardIndex) - { - BEAST_EXPECT(!dirContains(primaryDir, shardIndex)); - BEAST_EXPECT(historicalDirsContains(shardIndex)); - } - - // Create some more shards to exercise recent shard rotation - data = TestData(seedValue * 2, 4, numShards); - if (!BEAST_EXPECT(data.makeLedgers(env, numShards))) - return; - - for (auto i = 0; i < numShards; ++i) - { - auto const shardIndex{ - createShard(data, *shardStore, numShards * 2, numShards)}; - if (!BEAST_EXPECT( - shardIndex && *shardIndex >= numShards + 1 && - *shardIndex <= numShards * 2)) - { - return; - } - } - - { - // Confirm finalized shards are in the shard store - auto const finalized{shardStore->getShardInfo()->finalized()}; - BEAST_EXPECT(boost::icl::length(finalized) == numShards * 2); - BEAST_EXPECT(boost::icl::first(finalized) == 1); - BEAST_EXPECT(boost::icl::last(finalized) == numShards * 2); - } - - // Confirm two most recent shards are in the primary shard directory - for (auto const shardIndex : {numShards * 2 - 1, numShards * 2}) - { - BEAST_EXPECT(dirContains(primaryDir, shardIndex)); - BEAST_EXPECT(!historicalDirsContains(shardIndex)); - } - - // Confirm remaining shards are in the historical shard directories - for (auto shardIndex = 1; shardIndex < numShards * 2 - 1; ++shardIndex) - { - BEAST_EXPECT(!dirContains(primaryDir, shardIndex)); - BEAST_EXPECT(historicalDirsContains(shardIndex)); - } - } - - void - testOpenShardManagement(std::uint64_t const seedValue) - { - testcase("Open shard management"); - - using namespace test::jtx; - - beast::temp_dir shardDir; - Env env{*this, testConfig(shardDir.path())}; - - auto shardStore{env.app().getShardStore()}; - BEAST_EXPECT(shardStore); - - // Create one shard more than the open final limit - auto const openFinalLimit{env.app().config().getValueFor( - SizedItem::openFinalLimit, std::nullopt)}; - auto const numShards{openFinalLimit + 1}; - - TestData data(seedValue, 2, numShards); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - BEAST_EXPECT(shardStore->getShardInfo()->finalized().empty()); - - int oldestShardIndex{-1}; - for (auto i = 0; i < numShards; ++i) - { - auto shardIndex{createShard(data, *shardStore, numShards)}; - if (!BEAST_EXPECT( - shardIndex && *shardIndex >= 1 && *shardIndex <= numShards)) - { - return; - } - - BEAST_EXPECT(boost::icl::contains( - shardStore->getShardInfo()->finalized(), *shardIndex)); - - if (oldestShardIndex == -1) - oldestShardIndex = *shardIndex; - } - - // The number of open shards exceeds the open limit by one. - // A sweep will close enough shards to be within the limit. - shardStore->sweep(); - - // Read from the closed shard and automatically open it - auto const ledgerSeq{shardStore->lastLedgerSeq(oldestShardIndex)}; - auto const index{ledgerSeq - ledgersPerShard - 1}; - BEAST_EXPECT(shardStore->fetchNodeObject( - data.ledgers_[index]->info().hash, ledgerSeq)); - } - - void - testShardInfo(std::uint64_t const seedValue) - { - testcase("Shard info"); - - using namespace test::jtx; - beast::temp_dir shardDir; - Env env{*this, testConfig(shardDir.path())}; - - auto shardStore{env.app().getShardStore()}; - BEAST_EXPECT(shardStore); - - // Check shard store is empty - { - auto const shardInfo{shardStore->getShardInfo()}; - BEAST_EXPECT( - shardInfo->msgTimestamp().time_since_epoch().count() == 0); - BEAST_EXPECT(shardInfo->finalizedToString().empty()); - BEAST_EXPECT(shardInfo->finalized().empty()); - BEAST_EXPECT(shardInfo->incompleteToString().empty()); - BEAST_EXPECT(shardInfo->incomplete().empty()); - } - - // Create an incomplete shard with index 1 - TestData data(seedValue, dataSizeMax, 2); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - if (!BEAST_EXPECT(shardStore->prepareLedger(2 * ledgersPerShard))) - return; - - // Check shard is incomplete - { - auto const shardInfo{shardStore->getShardInfo()}; - BEAST_EXPECT(shardInfo->finalizedToString().empty()); - BEAST_EXPECT(shardInfo->finalized().empty()); - BEAST_EXPECT(shardInfo->incompleteToString() == "1:0"); - BEAST_EXPECT( - shardInfo->incomplete().find(1) != - shardInfo->incomplete().end()); - } - - // Finalize the shard - { - auto shardIndex{createShard(data, *shardStore)}; - if (!BEAST_EXPECT(shardIndex && *shardIndex == 1)) - return; - } - - // Check shard is finalized - { - auto const shardInfo{shardStore->getShardInfo()}; - BEAST_EXPECT(shardInfo->finalizedToString() == "1"); - BEAST_EXPECT(boost::icl::contains(shardInfo->finalized(), 1)); - BEAST_EXPECT(shardInfo->incompleteToString().empty()); - BEAST_EXPECT(shardInfo->incomplete().empty()); - BEAST_EXPECT(!shardInfo->update(1, ShardState::finalized, 0)); - BEAST_EXPECT(shardInfo->setFinalizedFromString("2")); - BEAST_EXPECT(shardInfo->finalizedToString() == "2"); - BEAST_EXPECT(boost::icl::contains(shardInfo->finalized(), 2)); - } - - // Create an incomplete shard with index 2 - if (!BEAST_EXPECT(shardStore->prepareLedger(3 * ledgersPerShard))) - return; - - // Store 10 percent of the ledgers - for (std::uint32_t i = 0; i < (ledgersPerShard / 10); ++i) - { - auto const ledgerSeq{ - shardStore->prepareLedger(3 * ledgersPerShard)}; - if (!BEAST_EXPECT(ledgerSeq != std::nullopt)) - return; - - auto const arrInd{*ledgerSeq - ledgersPerShard - 1}; - if (!BEAST_EXPECT(saveLedger(*shardStore, *data.ledgers_[arrInd]))) - return; - - shardStore->setStored(data.ledgers_[arrInd]); - } - - auto const shardInfo{shardStore->getShardInfo()}; - BEAST_EXPECT(shardInfo->incompleteToString() == "2:10"); - BEAST_EXPECT( - shardInfo->incomplete().find(2) != shardInfo->incomplete().end()); - - auto const timeStamp{env.app().timeKeeper().now()}; - shardInfo->setMsgTimestamp(timeStamp); - BEAST_EXPECT(timeStamp == shardInfo->msgTimestamp()); - - // Check message - auto const msg{shardInfo->makeMessage(env.app())}; - Serializer s; - s.add32(HashPrefix::shardInfo); - - BEAST_EXPECT(msg.timestamp() != 0); - s.add32(msg.timestamp()); - - // Verify incomplete shard - { - BEAST_EXPECT(msg.incomplete_size() == 1); - - auto const& incomplete{msg.incomplete(0)}; - BEAST_EXPECT(incomplete.shardindex() == 2); - s.add32(incomplete.shardindex()); - - BEAST_EXPECT( - static_cast(incomplete.state()) == - ShardState::acquire); - s.add32(incomplete.state()); - - BEAST_EXPECT(incomplete.has_progress()); - BEAST_EXPECT(incomplete.progress() == 10); - s.add32(incomplete.progress()); - } - - // Verify finalized shard - BEAST_EXPECT(msg.has_finalized()); - BEAST_EXPECT(msg.finalized() == "1"); - s.addRaw(msg.finalized().data(), msg.finalized().size()); - - // Verify public key - auto slice{makeSlice(msg.publickey())}; - BEAST_EXPECT(publicKeyType(slice)); - - // Verify signature - BEAST_EXPECT(verify( - PublicKey(slice), s.slice(), makeSlice(msg.signature()), false)); - - BEAST_EXPECT(msg.peerchain_size() == 0); - } - - void - testSQLiteDatabase(std::uint64_t const seedValue) - { - testcase("SQLite Database"); - - using namespace test::jtx; - - beast::temp_dir shardDir; - Env env{*this, testConfig(shardDir.path())}; - - auto shardStore{env.app().getShardStore()}; - BEAST_EXPECT(shardStore); - - auto const shardCount = 3; - TestData data(seedValue, 3, shardCount); - if (!BEAST_EXPECT(data.makeLedgers(env))) - return; - - BEAST_EXPECT(shardStore->getShardInfo()->finalized().empty()); - BEAST_EXPECT(shardStore->getShardInfo()->incompleteToString().empty()); - - auto rdb = - dynamic_cast(&env.app().getRelationalDatabase()); - - BEAST_EXPECT(rdb); - - for (std::uint32_t i = 0; i < shardCount; ++i) - { - // Populate the shard store - - auto n = createShard(data, *shardStore, shardCount); - if (!BEAST_EXPECT(n && *n >= 1 && *n <= shardCount)) - return; - } - - // Close these databases to force the SQLiteDatabase - // to use the shard databases and lookup tables. - rdb->closeLedgerDB(); - rdb->closeTransactionDB(); - - // Lambda for comparing Ledger objects - auto infoCmp = [](auto const& a, auto const& b) { - return a.hash == b.hash && a.txHash == b.txHash && - a.accountHash == b.accountHash && - a.parentHash == b.parentHash && a.drops == b.drops && - a.accepted == b.accepted && a.closeFlags == b.closeFlags && - a.closeTimeResolution == b.closeTimeResolution && - a.closeTime == b.closeTime; - }; - - for (auto const& ledger : data.ledgers_) - { - // Compare each test ledger to the data retrieved - // from the SQLiteDatabase class - - if (shardStore->seqToShardIndex(ledger->seq()) < - shardStore->earliestShardIndex() || - ledger->info().seq < shardStore->earliestLedgerSeq()) - continue; - - auto info = rdb->getLedgerInfoByHash(ledger->info().hash); - - BEAST_EXPECT(info); - BEAST_EXPECT(infoCmp(*info, ledger->info())); - - for (auto const& transaction : ledger->txs) - { - // Compare each test transaction to the data - // retrieved from the SQLiteDatabase class - - error_code_i error{rpcSUCCESS}; - - auto reference = rdb->getTransaction( - transaction.first->getTransactionID(), {}, error); - - BEAST_EXPECT(error == rpcSUCCESS); - if (!BEAST_EXPECT(reference.index() == 0)) - continue; - - auto txn = std::get<0>(reference).first->getSTransaction(); - - BEAST_EXPECT( - transaction.first->getFullText() == txn->getFullText()); - } - } - - // Create additional ledgers to test a pathway in - // 'ripple::saveLedgerMeta' wherein fetching the - // accepted ledger fails - data = TestData(seedValue * 2, 4, 1); - if (!BEAST_EXPECT(data.makeLedgers(env, shardCount))) - return; - } - -public: - DatabaseShard_test() : journal_("DatabaseShard_test", *this) - { - } - - void - run() override - { - auto seedValue = [] { - static std::uint64_t seedValue = 41; - seedValue += 10; - return seedValue; - }; - - testStandalone(); - testCreateShard(seedValue()); - testReopenDatabase(seedValue()); - testGetFinalShards(seedValue()); - testPrepareShards(seedValue()); - testImportShard(seedValue()); - testCorruptedDatabase(seedValue()); - testIllegalFinalKey(seedValue()); - testDeterministicShard(seedValue()); - testImportNodeStore(seedValue()); - testImportWithOnlineDelete(seedValue()); - testImportWithHistoricalPaths(seedValue()); - testPrepareWithHistoricalPaths(seedValue()); - testOpenShardManagement(seedValue()); - testShardInfo(seedValue()); - testSQLiteDatabase(seedValue()); - } -}; - -BEAST_DEFINE_TESTSUITE_MANUAL(DatabaseShard, NodeStore, ripple); - -} // namespace NodeStore -} // namespace ripple diff --git a/src/test/nodestore/Database_test.cpp b/src/test/nodestore/Database_test.cpp index 6774857dfc6..d866247da89 100644 --- a/src/test/nodestore/Database_test.cpp +++ b/src/test/nodestore/Database_test.cpp @@ -616,37 +616,6 @@ class Database_test : public TestBase std::strcmp(e.what(), "earliest_seq set more than once") == 0); } - - // Verify default ledgers per shard - { - std::unique_ptr db = - Manager::instance().make_Database( - megabytes(4), scheduler, 2, nodeParams, journal_); - BEAST_EXPECT( - db->ledgersPerShard() == DEFAULT_LEDGERS_PER_SHARD); - } - - // Set an invalid ledgers per shard - try - { - nodeParams.set("ledgers_per_shard", "100"); - std::unique_ptr db = - Manager::instance().make_Database( - megabytes(4), scheduler, 2, nodeParams, journal_); - } - catch (std::runtime_error const& e) - { - BEAST_EXPECT( - std::strcmp(e.what(), "Invalid ledgers_per_shard") == 0); - } - - // Set a valid ledgers per shard - nodeParams.set("ledgers_per_shard", "256"); - std::unique_ptr db = Manager::instance().make_Database( - megabytes(4), scheduler, 2, nodeParams, journal_); - - // Verify database uses the ledgers per shard - BEAST_EXPECT(db->ledgersPerShard() == 256); } } diff --git a/src/test/rpc/NodeToShardRPC_test.cpp b/src/test/rpc/NodeToShardRPC_test.cpp deleted file mode 100644 index ec1ff367c1e..00000000000 --- a/src/test/rpc/NodeToShardRPC_test.cpp +++ /dev/null @@ -1,414 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2021 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { -namespace test { - -class NodeToShardRPC_test : public beast::unit_test::suite -{ - bool - importCompleted( - NodeStore::DatabaseShard* shardStore, - std::uint8_t const numberOfShards, - Json::Value const& result) - { - auto const info = shardStore->getShardInfo(); - - // Assume completed if the import isn't running - auto const completed = - result[jss::error_message] == "Database import not running"; - - if (completed) - { - BEAST_EXPECT( - info->incomplete().size() + info->finalized().size() == - numberOfShards); - } - - return completed; - } - -public: - void - testDisabled() - { - testcase("Disabled"); - - beast::temp_dir tempDir; - - jtx::Env env = [&] { - auto c = jtx::envconfig(); - auto& sectionNode = c->section(ConfigSection::nodeDatabase()); - sectionNode.set("earliest_seq", "257"); - sectionNode.set("ledgers_per_shard", "256"); - c->setupControl(true, true, true); - - return jtx::Env(*this, std::move(c)); - }(); - - std::uint8_t const numberOfShards = 10; - - // Create some ledgers so that we can initiate a - // shard store database import. - for (int i = 0; i < 256 * (numberOfShards + 1); ++i) - { - env.close(); - } - - { - auto shardStore = env.app().getShardStore(); - if (!BEAST_EXPECT(!shardStore)) - return; - } - - { - // Try the node_to_shard status RPC command. Should fail. - - Json::Value jvParams; - jvParams[jss::action] = "status"; - - auto const result = env.rpc( - "json", "node_to_shard", to_string(jvParams))[jss::result]; - - BEAST_EXPECT(result[jss::error_code] == rpcNOT_ENABLED); - } - - { - // Try to start a shard store import via the RPC - // interface. Should fail. - - Json::Value jvParams; - jvParams[jss::action] = "start"; - - auto const result = env.rpc( - "json", "node_to_shard", to_string(jvParams))[jss::result]; - - BEAST_EXPECT(result[jss::error_code] == rpcNOT_ENABLED); - } - - { - // Try the node_to_shard status RPC command. Should fail. - - Json::Value jvParams; - jvParams[jss::action] = "status"; - - auto const result = env.rpc( - "json", "node_to_shard", to_string(jvParams))[jss::result]; - - BEAST_EXPECT(result[jss::error_code] == rpcNOT_ENABLED); - } - } - - void - testStart() - { - testcase("Start"); - - beast::temp_dir tempDir; - - jtx::Env env = [&] { - auto c = jtx::envconfig(); - auto& section = c->section(ConfigSection::shardDatabase()); - section.set("path", tempDir.path()); - section.set("max_historical_shards", "20"); - section.set("ledgers_per_shard", "256"); - section.set("earliest_seq", "257"); - auto& sectionNode = c->section(ConfigSection::nodeDatabase()); - sectionNode.set("earliest_seq", "257"); - sectionNode.set("ledgers_per_shard", "256"); - c->setupControl(true, true, true); - - return jtx::Env(*this, std::move(c)); - }(); - - std::uint8_t const numberOfShards = 10; - - // Create some ledgers so that we can initiate a - // shard store database import. - for (int i = 0; i < env.app().getShardStore()->ledgersPerShard() * - (numberOfShards + 1); - ++i) - { - env.close(); - } - - auto shardStore = env.app().getShardStore(); - if (!BEAST_EXPECT(shardStore)) - return; - - { - // Initiate a shard store import via the RPC - // interface. - - Json::Value jvParams; - jvParams[jss::action] = "start"; - - auto const result = env.rpc( - "json", "node_to_shard", to_string(jvParams))[jss::result]; - - BEAST_EXPECT( - result[jss::message] == "Database import initiated..."); - } - - while (!shardStore->getDatabaseImportSequence()) - { - // Wait until the import starts - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - - { - // Verify that the import is in progress with - // the node_to_shard status RPC command - - Json::Value jvParams; - jvParams[jss::action] = "status"; - - auto const result = env.rpc( - "json", "node_to_shard", to_string(jvParams))[jss::result]; - - BEAST_EXPECT( - result[jss::status] == "success" || - importCompleted(shardStore, numberOfShards, result)); - - std::chrono::seconds const maxWait{180}; - - { - auto const start = std::chrono::system_clock::now(); - while (true) - { - // Verify that the status object accurately - // reflects import progress. - - auto const completeShards = - shardStore->getShardInfo()->finalized(); - - if (!completeShards.empty()) - { - auto const result = env.rpc( - "json", - "node_to_shard", - to_string(jvParams))[jss::result]; - - if (!importCompleted( - shardStore, numberOfShards, result)) - { - BEAST_EXPECT(result[jss::firstShardIndex] == 1); - BEAST_EXPECT(result[jss::lastShardIndex] == 10); - } - } - - if (boost::icl::contains(completeShards, 1)) - { - auto const result = env.rpc( - "json", - "node_to_shard", - to_string(jvParams))[jss::result]; - - BEAST_EXPECT( - result[jss::currentShardIndex] >= 1 || - importCompleted( - shardStore, numberOfShards, result)); - - break; - } - - if (std::this_thread::sleep_for( - std::chrono::milliseconds{100}); - std::chrono::system_clock::now() - start > maxWait) - { - BEAST_EXPECTS( - false, - "Import timeout: could just be a slow machine."); - break; - } - } - } - - { - // Wait for the import to complete - auto const start = std::chrono::system_clock::now(); - while (!boost::icl::contains( - shardStore->getShardInfo()->finalized(), 10)) - { - if (std::this_thread::sleep_for( - std::chrono::milliseconds{100}); - std::chrono::system_clock::now() - start > maxWait) - { - BEAST_EXPECT(importCompleted( - shardStore, numberOfShards, result)); - break; - } - } - } - } - } - - void - testStop() - { - testcase("Stop"); - - beast::temp_dir tempDir; - - jtx::Env env = [&] { - auto c = jtx::envconfig(); - auto& section = c->section(ConfigSection::shardDatabase()); - section.set("path", tempDir.path()); - section.set("max_historical_shards", "20"); - section.set("ledgers_per_shard", "256"); - section.set("earliest_seq", "257"); - auto& sectionNode = c->section(ConfigSection::nodeDatabase()); - sectionNode.set("earliest_seq", "257"); - sectionNode.set("ledgers_per_shard", "256"); - c->setupControl(true, true, true); - - return jtx::Env( - *this, std::move(c), nullptr, beast::severities::kDisabled); - }(); - - std::uint8_t const numberOfShards = 10; - - // Create some ledgers so that we can initiate a - // shard store database import. - for (int i = 0; i < env.app().getShardStore()->ledgersPerShard() * - (numberOfShards + 1); - ++i) - { - env.close(); - } - - auto shardStore = env.app().getShardStore(); - if (!BEAST_EXPECT(shardStore)) - return; - - { - // Initiate a shard store import via the RPC - // interface. - - Json::Value jvParams; - jvParams[jss::action] = "start"; - - auto const result = env.rpc( - "json", "node_to_shard", to_string(jvParams))[jss::result]; - - BEAST_EXPECT( - result[jss::message] == "Database import initiated..."); - } - - { - // Verify that the import is in progress with - // the node_to_shard status RPC command - - Json::Value jvParams; - jvParams[jss::action] = "status"; - - auto const result = env.rpc( - "json", "node_to_shard", to_string(jvParams))[jss::result]; - - BEAST_EXPECT( - result[jss::status] == "success" || - importCompleted(shardStore, numberOfShards, result)); - - std::chrono::seconds const maxWait{30}; - auto const start = std::chrono::system_clock::now(); - - while (shardStore->getShardInfo()->finalized().empty()) - { - // Wait for at least one shard to complete - - if (std::this_thread::sleep_for(std::chrono::milliseconds{100}); - std::chrono::system_clock::now() - start > maxWait) - { - BEAST_EXPECTS( - false, "Import timeout: could just be a slow machine."); - break; - } - } - } - - { - Json::Value jvParams; - jvParams[jss::action] = "stop"; - - auto const result = env.rpc( - "json", "node_to_shard", to_string(jvParams))[jss::result]; - - BEAST_EXPECT( - result[jss::message] == "Database import halt initiated..." || - importCompleted(shardStore, numberOfShards, result)); - } - - std::chrono::seconds const maxWait{30}; - auto const start = std::chrono::system_clock::now(); - - while (true) - { - // Wait until we can verify that the import has - // stopped - - Json::Value jvParams; - jvParams[jss::action] = "status"; - - auto const result = env.rpc( - "json", "node_to_shard", to_string(jvParams))[jss::result]; - - // When the import has stopped, polling the - // status returns an error - if (result.isMember(jss::error)) - { - if (BEAST_EXPECT(result.isMember(jss::error_message))) - { - BEAST_EXPECT( - result[jss::error_message] == - "Database import not running"); - } - - break; - } - - if (std::this_thread::sleep_for(std::chrono::milliseconds{100}); - std::chrono::system_clock::now() - start > maxWait) - { - BEAST_EXPECTS( - false, "Import timeout: could just be a slow machine."); - break; - } - } - } - - void - run() override - { - testDisabled(); - testStart(); - testStop(); - } -}; - -BEAST_DEFINE_TESTSUITE_MANUAL(NodeToShardRPC, rpc, ripple); -} // namespace test -} // namespace ripple diff --git a/src/test/rpc/RPCCall_test.cpp b/src/test/rpc/RPCCall_test.cpp index f3aaf468a9e..5f13c9799a1 100644 --- a/src/test/rpc/RPCCall_test.cpp +++ b/src/test/rpc/RPCCall_test.cpp @@ -2545,231 +2545,6 @@ static RPCCallTestData const rpcCallTestArray[] = { ] })"}, - // download_shard - // -------------------------------------------------------------- - {"download_shard: minimal.", - __LINE__, - { - "download_shard", - "20", - "url_NotValidated", - }, - RPCCallTestData::no_exception, - R"({ - "method" : "download_shard", - "params" : [ - { - "api_version" : %API_VER%, - "shards" : [ - { - "index" : 20, - "url" : "url_NotValidated" - } - ] - } - ] - })"}, - {"download_shard:", - __LINE__, - { - "download_shard", - "20", - "url_NotValidated", - }, - RPCCallTestData::no_exception, - R"({ - "method" : "download_shard", - "params" : [ - { - "api_version" : %API_VER%, - "shards" : [ - { - "index" : 20, - "url" : "url_NotValidated" - } - ] - } - ] - })"}, - {"download_shard: many shards.", - __LINE__, - { - "download_shard", - "200000000", - "url_NotValidated0", - "199999999", - "url_NotValidated1", - "199999998", - "url_NotValidated2", - "199999997", - "url_NotValidated3", - }, - RPCCallTestData::no_exception, - R"({ - "method" : "download_shard", - "params" : [ - { - "api_version" : %API_VER%, - "shards" : [ - { - "index" : 200000000, - "url" : "url_NotValidated0" - }, - { - "index" : 199999999, - "url" : "url_NotValidated1" - }, - { - "index" : 199999998, - "url" : "url_NotValidated2" - }, - { - "index" : 199999997, - "url" : "url_NotValidated3" - } - ] - } - ] - })"}, - {"download_shard: many shards.", - __LINE__, - { - "download_shard", - "2000000", - "url_NotValidated0", - "2000001", - "url_NotValidated1", - "2000002", - "url_NotValidated2", - "2000003", - "url_NotValidated3", - "2000004", - "url_NotValidated4", - }, - RPCCallTestData::no_exception, - R"({ - "method" : "download_shard", - "params" : [ - { - "api_version" : %API_VER%, - "shards" : [ - { - "index" : 2000000, - "url" : "url_NotValidated0" - }, - { - "index" : 2000001, - "url" : "url_NotValidated1" - }, - { - "index" : 2000002, - "url" : "url_NotValidated2" - }, - { - "index" : 2000003, - "url" : "url_NotValidated3" - }, - { - "index" : 2000004, - "url" : "url_NotValidated4" - } - ] - } - ] - })"}, - {"download_shard: too few arguments.", - __LINE__, - {"download_shard", "20"}, - RPCCallTestData::no_exception, - R"({ - "method" : "download_shard", - "params" : [ - { - "error" : "badSyntax", - "error_code" : 1, - "error_message" : "Syntax error." - } - ] - })"}, - {// Note: this should return an error but not throw. - "download_shard: novalidate too few arguments.", - __LINE__, - {"download_shard", "novalidate", "20"}, - RPCCallTestData::bad_cast, - R"()"}, - {"download_shard: novalidate at end.", - __LINE__, - { - "download_shard", - "20", - "url_NotValidated", - "novalidate", - }, - RPCCallTestData::no_exception, - R"({ - "method" : "download_shard", - "params" : [ - { - "api_version" : %API_VER%, - "shards" : [ - { - "index" : 20, - "url" : "url_NotValidated" - } - ] - } - ] - })"}, - {"download_shard: novalidate in middle.", - __LINE__, - { - "download_shard", - "20", - "url_NotValidated20", - "novalidate", - "200", - "url_NotValidated200", - }, - RPCCallTestData::no_exception, - R"({ - "method" : "download_shard", - "params" : [ - { - "error" : "invalidParams", - "error_code" : 31, - "error_message" : "Invalid parameters." - } - ] - })"}, - {// Note: this should return an error but not throw. - "download_shard: arguments swapped.", - __LINE__, - { - "download_shard", - "url_NotValidated", - "20", - }, - RPCCallTestData::bad_cast, - R"()"}, - {"download_shard: index too small.", - __LINE__, - { - "download_shard", - "-1", - "url_NotValidated", - }, - RPCCallTestData::bad_cast, - R"()"}, - {"download_shard: index too big.", - __LINE__, - { - "download_shard", - "4294967296", - "url_NotValidated", - }, - RPCCallTestData::bad_cast, - R"()"}, - // feature // --------------------------------------------------------------------- {"feature: minimal.", @@ -4232,75 +4007,6 @@ static RPCCallTestData const rpcCallTestArray[] = { ] })"}, - // node_to_shard - // ------------------------------------------------------------------- - {"node_to_shard: status.", - __LINE__, - {"node_to_shard", "status"}, - RPCCallTestData::no_exception, - R"({ - "method" : "node_to_shard", - "params" : [ - { - "api_version" : %API_VER%, - "action" : "status" - } - ] - })"}, - {"node_to_shard: start.", - __LINE__, - {"node_to_shard", "start"}, - RPCCallTestData::no_exception, - R"({ - "method" : "node_to_shard", - "params" : [ - { - "api_version" : %API_VER%, - "action" : "start" - } - ] - })"}, - {"node_to_shard: stop.", - __LINE__, - {"node_to_shard", "stop"}, - RPCCallTestData::no_exception, - R"({ - "method" : "node_to_shard", - "params" : [ - { - "api_version" : %API_VER%, - "action" : "stop" - } - ] - })"}, - {"node_to_shard: too many arguments.", - __LINE__, - {"node_to_shard", "start", "stop"}, - RPCCallTestData::no_exception, - R"({ - "method" : "node_to_shard", - "params" : [ - { - "error" : "badSyntax", - "error_code" : 1, - "error_message" : "Syntax error." - } - ] - })"}, - {"node_to_shard: invalid argument.", - __LINE__, - {"node_to_shard", "invalid"}, - RPCCallTestData::no_exception, - R"({ - "method" : "node_to_shard", - "params" : [ - { - "api_version" : %API_VER%, - "action" : "invalid" - } - ] - })"}, - // owner_info // ------------------------------------------------------------------ {"owner_info: minimal.", diff --git a/src/test/rpc/ShardArchiveHandler_test.cpp b/src/test/rpc/ShardArchiveHandler_test.cpp deleted file mode 100644 index 82f12fe49a9..00000000000 --- a/src/test/rpc/ShardArchiveHandler_test.cpp +++ /dev/null @@ -1,705 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { -namespace test { - -class ShardArchiveHandler_test : public beast::unit_test::suite -{ - using Downloads = std::vector>; - - std::shared_ptr - createServer(jtx::Env& env, bool ssl = true) - { - std::vector list; - list.push_back(TrustedPublisherServer::randomValidator()); - return make_TrustedPublisherServer( - env.app().getIOService(), - list, - env.timeKeeper().now() + std::chrono::seconds{3600}, - // No future VLs - {}, - ssl); - } - -public: - // Test the shard downloading module by queueing - // a download and verifying the contents of the - // state database. - void - testSingleDownloadAndStateDB() - { - testcase("testSingleDownloadAndStateDB"); - - beast::temp_dir tempDir; - - auto c = jtx::envconfig(); - auto& section = c->section(ConfigSection::shardDatabase()); - section.set("path", tempDir.path()); - section.set("max_historical_shards", "20"); - c->setupControl(true, true, true); - - jtx::Env env(*this, std::move(c)); - auto handler = env.app().getShardArchiveHandler(); - BEAST_EXPECT(handler); - BEAST_EXPECT(dynamic_cast(handler) == nullptr); - - std::string const rawUrl = "https://foo:443/1.tar.lz4"; - parsedURL url; - - parseUrl(url, rawUrl); - handler->add(1, {url, rawUrl}); - - { - std::lock_guard lock(handler->m_); - std::uint64_t rowCount = 0; - - readArchiveDB( - *handler->sqlDB_, [&](std::string const& url, int state) { - BEAST_EXPECT(state == 1); - BEAST_EXPECT(url == rawUrl); - ++rowCount; - }); - - BEAST_EXPECT(rowCount == 1); - } - - handler->release(); - } - - // Test the shard downloading module by queueing - // three downloads and verifying the contents of - // the state database. - void - testDownloadsAndStateDB() - { - testcase("testDownloadsAndStateDB"); - - beast::temp_dir tempDir; - - auto c = jtx::envconfig(); - auto& section = c->section(ConfigSection::shardDatabase()); - section.set("path", tempDir.path()); - section.set("max_historical_shards", "20"); - c->setupControl(true, true, true); - - jtx::Env env(*this, std::move(c)); - auto handler = env.app().getShardArchiveHandler(); - BEAST_EXPECT(handler); - BEAST_EXPECT(dynamic_cast(handler) == nullptr); - - Downloads const dl = { - {1, "https://foo:443/1.tar.lz4"}, - {2, "https://foo:443/2.tar.lz4"}, - {3, "https://foo:443/3.tar.lz4"}}; - - for (auto const& entry : dl) - { - parsedURL url; - parseUrl(url, entry.second); - handler->add(entry.first, {url, entry.second}); - } - - { - std::lock_guard lock(handler->m_); - std::uint64_t pos = 0; - - readArchiveDB( - *handler->sqlDB_, [&](std::string const& url, int state) { - BEAST_EXPECT(state == dl[pos].first); - BEAST_EXPECT(url == dl[pos].second); - ++pos; - }); - - BEAST_EXPECT(pos == dl.size()); - } - - handler->release(); - } - - // Test the shard downloading module by initiating - // and completing ten downloads and verifying the - // contents of the filesystem and the handler's - // archives. - void - testDownloadsAndFileSystem() - { - testcase("testDownloadsAndFileSystem"); - - beast::temp_dir tempDir; - - auto c = jtx::envconfig(); - { - auto& section{c->section(ConfigSection::shardDatabase())}; - section.set("path", tempDir.path()); - section.set("max_historical_shards", "20"); - section.set("ledgers_per_shard", "256"); - section.set("earliest_seq", "257"); - } - { - auto& section{c->section(ConfigSection::nodeDatabase())}; - section.set("ledgers_per_shard", "256"); - section.set("earliest_seq", "257"); - } - c->setupControl(true, true, true); - - jtx::Env env( - *this, std::move(c), nullptr, beast::severities::kDisabled); - - std::uint8_t const numberOfDownloads = 10; - - // Create some ledgers so that the ShardArchiveHandler - // can verify the last ledger hash for the shard - // downloads. - for (int i = 0; i < env.app().getShardStore()->ledgersPerShard() * - (numberOfDownloads + 1); - ++i) - { - env.close(); - } - - auto handler = env.app().getShardArchiveHandler(); - BEAST_EXPECT(handler); - BEAST_EXPECT(dynamic_cast(handler) == nullptr); - - auto server = createServer(env); - auto host = server->local_endpoint().address().to_string(); - auto port = std::to_string(server->local_endpoint().port()); - server->stop(); - - Downloads const dl = [count = numberOfDownloads, &host, &port] { - Downloads ret; - - for (int i = 1; i <= count; ++i) - { - ret.push_back( - {i, - (boost::format("https://%s:%d/%d.tar.lz4") % host % port % - i) - .str()}); - } - - return ret; - }(); - - for (auto const& entry : dl) - { - parsedURL url; - parseUrl(url, entry.second); - handler->add(entry.first, {url, entry.second}); - } - - BEAST_EXPECT(handler->start()); - - auto stateDir = - RPC::ShardArchiveHandler::getDownloadDirectory(env.app().config()); - - std::unique_lock lock(handler->m_); - - BEAST_EXPECT( - boost::filesystem::exists(stateDir) || handler->archives_.empty()); - - using namespace std::chrono_literals; - auto waitMax = 60s; - - while (!handler->archives_.empty()) - { - lock.unlock(); - std::this_thread::sleep_for(1s); - - if (waitMax -= 1s; waitMax <= 0s) - { - BEAST_EXPECT(false); - break; - } - - lock.lock(); - } - - BEAST_EXPECT(!boost::filesystem::exists(stateDir)); - } - - // Test the shard downloading module by initiating - // and completing ten downloads and verifying the - // contents of the filesystem and the handler's - // archives. Then restart the application and ensure - // that the handler is created and started automatically. - void - testDownloadsAndRestart() - { - testcase("testDownloadsAndRestart"); - - beast::temp_dir tempDir; - - { - auto c = jtx::envconfig(); - { - auto& section{c->section(ConfigSection::shardDatabase())}; - section.set("path", tempDir.path()); - section.set("max_historical_shards", "20"); - section.set("ledgers_per_shard", "256"); - section.set("earliest_seq", "257"); - } - { - auto& section{c->section(ConfigSection::nodeDatabase())}; - section.set("ledgers_per_shard", "256"); - section.set("earliest_seq", "257"); - } - c->setupControl(true, true, true); - - jtx::Env env( - *this, std::move(c), nullptr, beast::severities::kDisabled); - - std::uint8_t const numberOfDownloads = 10; - - // Create some ledgers so that the ShardArchiveHandler - // can verify the last ledger hash for the shard - // downloads. - for (int i = 0; i < env.app().getShardStore()->ledgersPerShard() * - (numberOfDownloads + 1); - ++i) - { - env.close(); - } - - auto handler = env.app().getShardArchiveHandler(); - BEAST_EXPECT(handler); - BEAST_EXPECT( - dynamic_cast(handler) == nullptr); - - auto server = createServer(env); - auto host = server->local_endpoint().address().to_string(); - auto port = std::to_string(server->local_endpoint().port()); - server->stop(); - - Downloads const dl = [count = numberOfDownloads, &host, &port] { - Downloads ret; - - for (int i = 1; i <= count; ++i) - { - ret.push_back( - {i, - (boost::format("https://%s:%d/%d.tar.lz4") % host % - port % i) - .str()}); - } - - return ret; - }(); - - for (auto const& entry : dl) - { - parsedURL url; - parseUrl(url, entry.second); - handler->add(entry.first, {url, entry.second}); - } - - auto stateDir = RPC::ShardArchiveHandler::getDownloadDirectory( - env.app().config()); - - boost::filesystem::copy_file( - stateDir / stateDBName, - boost::filesystem::path(tempDir.path()) / stateDBName); - - BEAST_EXPECT(handler->start()); - - std::unique_lock lock(handler->m_); - - BEAST_EXPECT( - boost::filesystem::exists(stateDir) || - handler->archives_.empty()); - - using namespace std::chrono_literals; - auto waitMax = 60s; - - while (!handler->archives_.empty()) - { - lock.unlock(); - std::this_thread::sleep_for(1s); - - if (waitMax -= 1s; waitMax <= 0s) - { - BEAST_EXPECT(false); - break; - } - - lock.lock(); - } - - BEAST_EXPECT(!boost::filesystem::exists(stateDir)); - - boost::filesystem::create_directory(stateDir); - - boost::filesystem::copy_file( - boost::filesystem::path(tempDir.path()) / stateDBName, - stateDir / stateDBName); - } - - auto c = jtx::envconfig(); - { - auto& section{c->section(ConfigSection::shardDatabase())}; - section.set("path", tempDir.path()); - section.set("max_historical_shards", "20"); - section.set("shard_verification_retry_interval", "1"); - section.set("shard_verification_max_attempts", "10000"); - section.set("ledgers_per_shard", "256"); - section.set("earliest_seq", "257"); - } - { - auto& section{c->section(ConfigSection::nodeDatabase())}; - section.set("ledgers_per_shard", "256"); - section.set("earliest_seq", "257"); - } - c->setupControl(true, true, true); - - jtx::Env env( - *this, std::move(c), nullptr, beast::severities::kDisabled); - std::uint8_t const numberOfDownloads = 10; - - // Create some ledgers so that the ShardArchiveHandler - // can verify the last ledger hash for the shard - // downloads. - for (int i = 0; i < env.app().getShardStore()->ledgersPerShard() * - (numberOfDownloads + 1); - ++i) - { - env.close(); - } - - auto handler = env.app().getShardArchiveHandler(); - BEAST_EXPECT(dynamic_cast(handler) != nullptr); - - auto stateDir = - RPC::ShardArchiveHandler::getDownloadDirectory(env.app().config()); - - std::unique_lock lock(handler->m_); - - BEAST_EXPECT( - boost::filesystem::exists(stateDir) || handler->archives_.empty()); - - using namespace std::chrono_literals; - auto waitMax = 60s; - - while (!handler->archives_.empty()) - { - lock.unlock(); - std::this_thread::sleep_for(1s); - - if (waitMax -= 1s; waitMax <= 0s) - { - BEAST_EXPECT(false); - break; - } - - lock.lock(); - } - - BEAST_EXPECT(!boost::filesystem::exists(stateDir)); - } - - // Ensure that downloads fail when the shard - // database cannot store any more shards - void - testShardCountFailure() - { - testcase("testShardCountFailure"); - std::string capturedLogs; - - { - beast::temp_dir tempDir; - - auto c = jtx::envconfig(); - { - auto& section{c->section(ConfigSection::shardDatabase())}; - section.set("path", tempDir.path()); - section.set("max_historical_shards", "1"); - section.set("ledgers_per_shard", "256"); - section.set("earliest_seq", "257"); - } - { - auto& section{c->section(ConfigSection::nodeDatabase())}; - section.set("ledgers_per_shard", "256"); - section.set("earliest_seq", "257"); - } - c->setupControl(true, true, true); - - std::unique_ptr logs(new CaptureLogs(&capturedLogs)); - jtx::Env env(*this, std::move(c), std::move(logs)); - - std::uint8_t const numberOfDownloads = 10; - - // Create some ledgers so that the ShardArchiveHandler - // can verify the last ledger hash for the shard - // downloads. - for (int i = 0; i < env.app().getShardStore()->ledgersPerShard() * - (numberOfDownloads + 1); - ++i) - { - env.close(); - } - - auto handler = env.app().getShardArchiveHandler(); - BEAST_EXPECT(handler); - BEAST_EXPECT( - dynamic_cast(handler) == nullptr); - - auto server = createServer(env); - auto host = server->local_endpoint().address().to_string(); - auto port = std::to_string(server->local_endpoint().port()); - server->stop(); - - Downloads const dl = [count = numberOfDownloads, &host, &port] { - Downloads ret; - - for (int i = 1; i <= count; ++i) - { - ret.push_back( - {i, - (boost::format("https://%s:%d/%d.tar.lz4") % host % - port % i) - .str()}); - } - - return ret; - }(); - - for (auto const& entry : dl) - { - parsedURL url; - parseUrl(url, entry.second); - handler->add(entry.first, {url, entry.second}); - } - - BEAST_EXPECT(!handler->start()); - auto stateDir = RPC::ShardArchiveHandler::getDownloadDirectory( - env.app().config()); - - handler->release(); - BEAST_EXPECT(!boost::filesystem::exists(stateDir)); - } - - auto const expectedErrorMessage = - "shards 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 maximum number of historical " - "shards reached"; - BEAST_EXPECT( - capturedLogs.find(expectedErrorMessage) != std::string::npos); - - { - beast::temp_dir tempDir; - - auto c = jtx::envconfig(); - { - auto& section{c->section(ConfigSection::shardDatabase())}; - section.set("path", tempDir.path()); - section.set("max_historical_shards", "0"); - section.set("ledgers_per_shard", "256"); - section.set("earliest_seq", "257"); - } - { - auto& section{c->section(ConfigSection::nodeDatabase())}; - section.set("ledgers_per_shard", "256"); - section.set("earliest_seq", "257"); - } - c->setupControl(true, true, true); - - std::unique_ptr logs(new CaptureLogs(&capturedLogs)); - jtx::Env env(*this, std::move(c), std::move(logs)); - - std::uint8_t const numberOfDownloads = 1; - - // Create some ledgers so that the ShardArchiveHandler - // can verify the last ledger hash for the shard - // downloads. - for (int i = 0; i < env.app().getShardStore()->ledgersPerShard() * - ((numberOfDownloads * 3) + 1); - ++i) - { - env.close(); - } - - auto handler = env.app().getShardArchiveHandler(); - BEAST_EXPECT(handler); - BEAST_EXPECT( - dynamic_cast(handler) == nullptr); - - auto server = createServer(env); - auto host = server->local_endpoint().address().to_string(); - auto port = std::to_string(server->local_endpoint().port()); - server->stop(); - - Downloads const dl = [count = numberOfDownloads, &host, &port] { - Downloads ret; - - for (int i = 1; i <= count; ++i) - { - ret.push_back( - {i, - (boost::format("https://%s:%d/%d.tar.lz4") % host % - port % i) - .str()}); - } - - return ret; - }(); - - for (auto const& entry : dl) - { - parsedURL url; - parseUrl(url, entry.second); - handler->add(entry.first, {url, entry.second}); - } - - BEAST_EXPECT(!handler->start()); - auto stateDir = RPC::ShardArchiveHandler::getDownloadDirectory( - env.app().config()); - - handler->release(); - BEAST_EXPECT(!boost::filesystem::exists(stateDir)); - } - - auto const expectedErrorMessage2 = - "shard 1 maximum number of historical shards reached"; - BEAST_EXPECT( - capturedLogs.find(expectedErrorMessage2) != std::string::npos); - } - - // Ensure that downloads fail when the shard - // database has already stored one of the - // queued shards - void - testRedundantShardFailure() - { - testcase("testRedundantShardFailure"); - std::string capturedLogs; - - { - beast::temp_dir tempDir; - - auto c = jtx::envconfig(); - { - auto& section{c->section(ConfigSection::shardDatabase())}; - section.set("path", tempDir.path()); - section.set("max_historical_shards", "1"); - section.set("ledgers_per_shard", "256"); - section.set("earliest_seq", "257"); - } - { - auto& section{c->section(ConfigSection::nodeDatabase())}; - section.set("ledgers_per_shard", "256"); - section.set("earliest_seq", "257"); - } - c->setupControl(true, true, true); - - std::unique_ptr logs(new CaptureLogs(&capturedLogs)); - jtx::Env env( - *this, - std::move(c), - std::move(logs), - beast::severities::kDebug); - - std::uint8_t const numberOfDownloads = 10; - - // Create some ledgers so that the ShardArchiveHandler - // can verify the last ledger hash for the shard - // downloads. - for (int i = 0; i < env.app().getShardStore()->ledgersPerShard() * - (numberOfDownloads + 1); - ++i) - { - env.close(); - } - - BEAST_EXPECT(env.app().getShardStore()->prepareShards({1})); - - auto handler = env.app().getShardArchiveHandler(); - BEAST_EXPECT(handler); - BEAST_EXPECT( - dynamic_cast(handler) == nullptr); - - auto server = createServer(env); - auto host = server->local_endpoint().address().to_string(); - auto port = std::to_string(server->local_endpoint().port()); - server->stop(); - - Downloads const dl = [count = numberOfDownloads, &host, &port] { - Downloads ret; - - for (int i = 1; i <= count; ++i) - { - ret.push_back( - {i, - (boost::format("https://%s:%d/%d.tar.lz4") % host % - port % i) - .str()}); - } - - return ret; - }(); - - for (auto const& entry : dl) - { - parsedURL url; - parseUrl(url, entry.second); - handler->add(entry.first, {url, entry.second}); - } - - BEAST_EXPECT(!handler->start()); - auto stateDir = RPC::ShardArchiveHandler::getDownloadDirectory( - env.app().config()); - - handler->release(); - BEAST_EXPECT(!boost::filesystem::exists(stateDir)); - } - - auto const expectedErrorMessage = - "shard 1 is already queued for import"; - BEAST_EXPECT( - capturedLogs.find(expectedErrorMessage) != std::string::npos); - } - - void - run() override - { - testSingleDownloadAndStateDB(); - testDownloadsAndStateDB(); - testDownloadsAndFileSystem(); - testDownloadsAndRestart(); - testShardCountFailure(); - testRedundantShardFailure(); - } -}; - -BEAST_DEFINE_TESTSUITE_PRIO(ShardArchiveHandler, app, ripple, 3); - -} // namespace test -} // namespace ripple diff --git a/src/test/shamap/common.h b/src/test/shamap/common.h index 2280c77d4a1..db5a2c40acf 100644 --- a/src/test/shamap/common.h +++ b/src/test/shamap/common.h @@ -20,7 +20,6 @@ #ifndef RIPPLE_SHAMAP_TESTS_COMMON_H_INCLUDED #define RIPPLE_SHAMAP_TESTS_COMMON_H_INCLUDED -#include #include #include #include @@ -81,12 +80,14 @@ class TestNodeFamily : public Family return j_; } - std::shared_ptr getFullBelowCache(std::uint32_t) override + std::shared_ptr + getFullBelowCache() override { return fbCache_; } - std::shared_ptr getTreeNodeCache(std::uint32_t) override + std::shared_ptr + getTreeNodeCache() override { return tnCache_; } @@ -98,12 +99,6 @@ class TestNodeFamily : public Family tnCache_->sweep(); } - bool - isShardBacked() const override - { - return true; - } - void missingNodeAcquireBySeq(std::uint32_t refNum, uint256 const& nodeHash) override diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index 7b061e10b31..69fb371578a 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include diff --git a/src/xrpld/app/ledger/InboundLedger.h b/src/xrpld/app/ledger/InboundLedger.h index 62b6925d59c..13f603e79d0 100644 --- a/src/xrpld/app/ledger/InboundLedger.h +++ b/src/xrpld/app/ledger/InboundLedger.h @@ -42,7 +42,6 @@ class InboundLedger final : public TimeoutCounter, // These are the reasons we might acquire a ledger enum class Reason { HISTORY, // Acquiring past ledger - SHARD, // Acquiring for shard GENERIC, // Generic other reasons CONSENSUS // We believe the consensus round requires this ledger }; diff --git a/src/xrpld/app/ledger/LedgerMaster.h b/src/xrpld/app/ledger/LedgerMaster.h index 7921773e3f0..5149424e285 100644 --- a/src/xrpld/app/ledger/LedgerMaster.h +++ b/src/xrpld/app/ledger/LedgerMaster.h @@ -357,9 +357,6 @@ class LedgerMaster : public AbstractFetchPackContainer // The last ledger we handled fetching history std::shared_ptr mHistLedger; - // The last ledger we handled fetching for a shard - std::shared_ptr mShardLedger; - // Fully validated ledger, whether or not we have the ledger resident. std::pair mLastValidLedger{uint256(), 0}; diff --git a/src/xrpld/app/ledger/detail/InboundLedger.cpp b/src/xrpld/app/ledger/detail/InboundLedger.cpp index b98f24aed43..5b0c3469111 100644 --- a/src/xrpld/app/ledger/detail/InboundLedger.cpp +++ b/src/xrpld/app/ledger/detail/InboundLedger.cpp @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -112,40 +111,6 @@ InboundLedger::init(ScopedLockType& collectionLock) if (failed_) return; - if (!complete_) - { - auto shardStore = app_.getShardStore(); - if (mReason == Reason::SHARD) - { - if (!shardStore) - { - JLOG(journal_.error()) - << "Acquiring shard with no shard store available"; - failed_ = true; - return; - } - - mHaveHeader = false; - mHaveTransactions = false; - mHaveState = false; - mLedger.reset(); - - tryDB(app_.getShardFamily()->db()); - if (failed_) - return; - } - else if (shardStore && mSeq >= shardStore->earliestLedgerSeq()) - { - if (auto l = shardStore->fetchLedger(hash_, mSeq)) - { - mHaveHeader = true; - mHaveTransactions = true; - mHaveState = true; - complete_ = true; - mLedger = std::move(l); - } - } - } if (!complete_) { addPeers(); @@ -160,7 +125,7 @@ InboundLedger::init(ScopedLockType& collectionLock) mLedger->read(keylet::fees())); mLedger->setImmutable(); - if (mReason == Reason::HISTORY || mReason == Reason::SHARD) + if (mReason == Reason::HISTORY) return; app_.getLedgerMaster().storeLedger(mLedger); @@ -200,8 +165,6 @@ InboundLedger::checkLocal() { if (mLedger) tryDB(mLedger->stateMap().family().db()); - else if (mReason == Reason::SHARD) - tryDB(app_.getShardFamily()->db()); else tryDB(app_.getNodeFamily().db()); if (failed_ || complete_) @@ -283,8 +246,7 @@ InboundLedger::tryDB(NodeStore::Database& srcDB) mLedger = std::make_shared( deserializePrefixedHeader(makeSlice(data)), app_.config(), - mReason == Reason::SHARD ? *app_.getShardFamily() - : app_.getNodeFamily()); + app_.getNodeFamily()); if (mLedger->info().hash != hash_ || (mSeq != 0 && mSeq != mLedger->info().seq)) { @@ -495,9 +457,6 @@ InboundLedger::done() mLedger->setImmutable(); switch (mReason) { - case Reason::SHARD: - app_.getShardStore()->setStored(mLedger); - [[fallthrough]]; case Reason::HISTORY: app_.getInboundLedgers().onLedgerFetched(); break; @@ -551,9 +510,7 @@ InboundLedger::trigger(std::shared_ptr const& peer, TriggerReason reason) if (!mHaveHeader) { - tryDB( - mReason == Reason::SHARD ? app_.getShardFamily()->db() - : app_.getNodeFamily().db()); + tryDB(app_.getNodeFamily().db()); if (failed_) { JLOG(journal_.warn()) << " failed local for " << hash_; @@ -854,8 +811,7 @@ InboundLedger::takeHeader(std::string const& data) if (complete_ || failed_ || mHaveHeader) return true; - auto* f = mReason == Reason::SHARD ? app_.getShardFamily() - : &app_.getNodeFamily(); + auto* f = &app_.getNodeFamily(); mLedger = std::make_shared( deserializeHeader(makeSlice(data)), app_.config(), *f); if (mLedger->info().hash != hash_ || diff --git a/src/xrpld/app/ledger/detail/InboundLedgers.cpp b/src/xrpld/app/ledger/detail/InboundLedgers.cpp index 04964d2a921..2b4d2161b63 100644 --- a/src/xrpld/app/ledger/detail/InboundLedgers.cpp +++ b/src/xrpld/app/ledger/detail/InboundLedgers.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -70,9 +69,6 @@ class InboundLedgersImp : public InboundLedgers InboundLedger::Reason reason) override { assert(hash.isNonZero()); - assert( - reason != InboundLedger::Reason::SHARD || - (seq != 0 && app_.getShardStore())); // probably not the right rule if (app_.getOPs().isNeedNetworkLedger() && @@ -119,25 +115,6 @@ class InboundLedgersImp : public InboundLedgers if (!inbound->isComplete()) return {}; - if (reason == InboundLedger::Reason::HISTORY) - { - if (inbound->getLedger()->stateMap().family().isShardBacked()) - app_.getNodeStore().storeLedger(inbound->getLedger()); - } - else if (reason == InboundLedger::Reason::SHARD) - { - auto shardStore = app_.getShardStore(); - if (!shardStore) - { - JLOG(j_.error()) - << "Acquiring shard with no shard store available"; - return {}; - } - if (inbound->getLedger()->stateMap().family().isShardBacked()) - shardStore->setStored(inbound->getLedger()); - else - shardStore->storeLedger(inbound->getLedger()); - } return inbound->getLedger(); } @@ -285,7 +262,7 @@ class InboundLedgersImp : public InboundLedgers } // Should only be called with an inboundledger that has - // a reason of history or shard + // a reason of history void onLedgerFetched() override { diff --git a/src/xrpld/app/ledger/detail/LedgerMaster.cpp b/src/xrpld/app/ledger/detail/LedgerMaster.cpp index f03004fd14c..dab8f838249 100644 --- a/src/xrpld/app/ledger/detail/LedgerMaster.cpp +++ b/src/xrpld/app/ledger/detail/LedgerMaster.cpp @@ -39,7 +39,6 @@ #include #include #include -#include #include #include #include @@ -830,38 +829,13 @@ LedgerMaster::tryFill(std::shared_ptr ledger) void LedgerMaster::getFetchPack(LedgerIndex missing, InboundLedger::Reason reason) { - LedgerIndex const ledgerIndex([&]() { - if (reason == InboundLedger::Reason::SHARD) - { - // Do not acquire a ledger sequence greater - // than the last ledger in the shard - auto const shardStore{app_.getShardStore()}; - auto const shardIndex{shardStore->seqToShardIndex(missing)}; - return std::min(missing + 1, shardStore->lastLedgerSeq(shardIndex)); - } - return missing + 1; - }()); + LedgerIndex const ledgerIndex = missing + 1; auto const haveHash{getLedgerHashForHistory(ledgerIndex, reason)}; if (!haveHash || haveHash->isZero()) { - if (reason == InboundLedger::Reason::SHARD) - { - auto const shardStore{app_.getShardStore()}; - auto const shardIndex{shardStore->seqToShardIndex(missing)}; - if (missing < shardStore->lastLedgerSeq(shardIndex)) - { - JLOG(m_journal.error()) - << "No hash for fetch pack. " - << "Missing ledger sequence " << missing - << " while acquiring shard " << shardIndex; - } - } - else - { - JLOG(m_journal.error()) - << "No hash for fetch pack. Missing Index " << missing; - } + JLOG(m_journal.error()) + << "No hash for fetch pack. Missing Index " << missing; return; } @@ -1342,8 +1316,7 @@ LedgerMaster::getLedgerHashForHistory( { // Try to get the hash of a ledger we need to fetch for history std::optional ret; - auto const& l{ - reason == InboundLedger::Reason::SHARD ? mShardLedger : mHistLedger}; + auto const& l{mHistLedger}; if (l && l->info().seq >= index) { @@ -2001,54 +1974,35 @@ LedgerMaster::fetchForHistory( auto seq = ledger->info().seq; assert(seq == missing); JLOG(m_journal.trace()) << "fetchForHistory acquired " << seq; - if (reason == InboundLedger::Reason::SHARD) + setFullLedger(ledger, false, false); + int fillInProgress; { - ledger->setFull(); - { - std::lock_guard lock(m_mutex); - mShardLedger = ledger; - } - if (!ledger->stateMap().family().isShardBacked()) - app_.getShardStore()->storeLedger(ledger); + std::lock_guard lock(m_mutex); + mHistLedger = ledger; + fillInProgress = mFillInProgress; } - else + if (fillInProgress == 0 && + app_.getRelationalDatabase().getHashByIndex(seq - 1) == + ledger->info().parentHash) { - setFullLedger(ledger, false, false); - int fillInProgress; { + // Previous ledger is in DB std::lock_guard lock(m_mutex); - mHistLedger = ledger; - fillInProgress = mFillInProgress; - } - if (fillInProgress == 0 && - app_.getRelationalDatabase().getHashByIndex(seq - 1) == - ledger->info().parentHash) - { - { - // Previous ledger is in DB - std::lock_guard lock(m_mutex); - mFillInProgress = seq; - } - app_.getJobQueue().addJob( - jtADVANCE, "tryFill", [this, ledger]() { - tryFill(ledger); - }); + mFillInProgress = seq; } + app_.getJobQueue().addJob( + jtADVANCE, "tryFill", [this, ledger]() { + tryFill(ledger); + }); } progress = true; } else { std::uint32_t fetchSz; - if (reason == InboundLedger::Reason::SHARD) - // Do not fetch ledger sequences lower - // than the shard's first ledger sequence - fetchSz = app_.getShardStore()->firstLedgerSeq( - app_.getShardStore()->seqToShardIndex(missing)); - else - // Do not fetch ledger sequences lower - // than the earliest ledger sequence - fetchSz = app_.getNodeStore().earliestLedgerSeq(); + // Do not fetch ledger sequences lower + // than the earliest ledger sequence + fetchSz = app_.getNodeStore().earliestLedgerSeq(); fetchSz = missing >= fetchSz ? std::min(ledger_fetch_size_, (missing - fetchSz) + 1) : 0; @@ -2081,7 +2035,8 @@ LedgerMaster::fetchForHistory( << "Ledgers: " << app_.getLedgerMaster().getCompleteLedgers(); JLOG(m_journal.fatal()) << "Acquire reason: " - << (reason == InboundLedger::Reason::HISTORY ? "HISTORY" : "SHARD"); + << (reason == InboundLedger::Reason::HISTORY ? "HISTORY" + : "NOT HISTORY"); clearLedger(missing + 1); progress = true; } @@ -2133,15 +2088,6 @@ LedgerMaster::doAdvance(std::unique_lock& sl) else missing = std::nullopt; } - if (!missing && mFillInProgress == 0) - { - if (auto shardStore = app_.getShardStore()) - { - missing = shardStore->prepareLedger(mValidLedgerSeq); - if (missing) - reason = InboundLedger::Reason::SHARD; - } - } if (missing) { fetchForHistory(*missing, progress, reason, sl); @@ -2156,7 +2102,6 @@ LedgerMaster::doAdvance(std::unique_lock& sl) else { mHistLedger.reset(); - mShardLedger.reset(); JLOG(m_journal.trace()) << "tryAdvance not fetching history"; } } diff --git a/src/xrpld/app/main/Application.cpp b/src/xrpld/app/main/Application.cpp index ff6cc0584ca..f3308a091dc 100644 --- a/src/xrpld/app/main/Application.cpp +++ b/src/xrpld/app/main/Application.cpp @@ -50,17 +50,14 @@ #include #include #include -#include #include #include #include #include #include #include -#include #include #include -#include #include #include #include @@ -192,9 +189,6 @@ class ApplicationImp : public Application, public BasicApp std::unique_ptr m_nodeStore; NodeFamily nodeFamily_; - std::unique_ptr shardStore_; - std::unique_ptr shardFamily_; - std::unique_ptr shardArchiveHandler_; // VFALCO TODO Make OrderBookDB abstract OrderBookDB m_orderBookDB; std::unique_ptr m_pathRequests; @@ -361,13 +355,6 @@ class ApplicationImp : public Application, public BasicApp , nodeFamily_(*this, *m_collectorManager) - // The shard store is optional and make_ShardStore can return null. - , shardStore_(make_ShardStore( - *this, - m_nodeStoreScheduler, - 4, - logs_->journal("ShardStore"))) - , m_orderBookDB(*this) , m_pathRequests(std::make_unique( @@ -565,14 +552,6 @@ class ApplicationImp : public Application, public BasicApp return nodeFamily_; } - // The shard store is an optional feature. If the sever is configured for - // shards, this function will return a valid pointer, otherwise a nullptr. - Family* - getShardFamily() override - { - return shardFamily_.get(); - } - TimeKeeper& timeKeeper() override { @@ -696,72 +675,6 @@ class ApplicationImp : public Application, public BasicApp return *m_nodeStore; } - // The shard store is an optional feature. If the sever is configured for - // shards, this function will return a valid pointer, otherwise a nullptr. - NodeStore::DatabaseShard* - getShardStore() override - { - return shardStore_.get(); - } - - RPC::ShardArchiveHandler* - getShardArchiveHandler(bool tryRecovery) override - { - static std::mutex handlerMutex; - std::lock_guard lock(handlerMutex); - - // After constructing the handler, try to - // initialize it. Log on error; set the - // member variable on success. - auto initAndSet = - [this](std::unique_ptr&& handler) { - if (!handler) - return false; - - if (!handler->init()) - { - JLOG(m_journal.error()) - << "Failed to initialize ShardArchiveHandler."; - - return false; - } - - shardArchiveHandler_ = std::move(handler); - return true; - }; - - // Need to resume based on state from a previous - // run. - if (tryRecovery) - { - if (shardArchiveHandler_ != nullptr) - { - JLOG(m_journal.error()) - << "ShardArchiveHandler already created at startup."; - - return nullptr; - } - - auto handler = - RPC::ShardArchiveHandler::tryMakeRecoveryHandler(*this); - - if (!initAndSet(std::move(handler))) - return nullptr; - } - - // Construct the ShardArchiveHandler - if (shardArchiveHandler_ == nullptr) - { - auto handler = - RPC::ShardArchiveHandler::makeShardArchiveHandler(*this); - - if (!initAndSet(std::move(handler))) - return nullptr; - } - - return shardArchiveHandler_.get(); - } - Application::MutexType& getMasterMutex() override { @@ -1075,10 +988,10 @@ class ApplicationImp : public Application, public BasicApp { std::shared_ptr const fullBelowCache = - nodeFamily_.getFullBelowCache(0); + nodeFamily_.getFullBelowCache(); std::shared_ptr const treeNodeCache = - nodeFamily_.getTreeNodeCache(0); + nodeFamily_.getTreeNodeCache(); std::size_t const oldFullBelowSize = fullBelowCache->size(); std::size_t const oldTreeNodeSize = treeNodeCache->size(); @@ -1094,25 +1007,6 @@ class ApplicationImp : public Application, public BasicApp << "NodeFamily::TreeNodeCache sweep. Size before: " << oldTreeNodeSize << "; size after: " << treeNodeCache->size(); } - if (shardFamily_) - { - std::size_t const oldFullBelowSize = - shardFamily_->getFullBelowCacheSize(); - std::size_t const oldTreeNodeSize = - shardFamily_->getTreeNodeCacheSize().second; - - shardFamily_->sweep(); - - JLOG(m_journal.debug()) - << "ShardFamily::FullBelowCache sweep. Size before: " - << oldFullBelowSize - << "; size after: " << shardFamily_->getFullBelowCacheSize(); - - JLOG(m_journal.debug()) - << "ShardFamily::TreeNodeCache sweep. Size before: " - << oldTreeNodeSize << "; size after: " - << shardFamily_->getTreeNodeCacheSize().second; - } { TaggedCache const& masterTxCache = getMasterTransaction().getCache(); @@ -1129,11 +1023,6 @@ class ApplicationImp : public Application, public BasicApp // Does not appear to have an associated cache. getNodeStore().sweep(); } - if (shardStore_) - { - // Does not appear to have an associated cache. - shardStore_->sweep(); - } { std::size_t const oldLedgerMasterCacheSize = getLedgerMaster().getFetchPackCacheSize(); @@ -1266,9 +1155,6 @@ class ApplicationImp : public Application, public BasicApp // and new validations must be greater than this. std::atomic maxDisallowedLedger_{0}; - bool - nodeToShards(); - void startGenesisLedger(); @@ -1348,15 +1234,6 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline) if (!initRelationalDatabase() || !initNodeStore()) return false; - if (shardStore_) - { - shardFamily_ = - std::make_unique(*this, *m_collectorManager); - - if (!shardStore_->init()) - return false; - } - if (!peerReservations_->load(getWalletDB())) { JLOG(m_journal.fatal()) << "Cannot find peer reservations!"; @@ -1543,13 +1420,6 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline) add(*overlay_); // add to PropertyStream } - if (!config_->standalone()) - { - // NodeStore import into the ShardStore requires the SQLite database - if (config_->nodeToShard && !nodeToShards()) - return false; - } - // start first consensus round if (!config_->reporting() && !m_networkOPs->beginConsensus( @@ -1664,38 +1534,6 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline) } } - RPC::ShardArchiveHandler* shardArchiveHandler = nullptr; - if (shardStore_) - { - try - { - // Create a ShardArchiveHandler if recovery - // is needed (there's a state database left - // over from a previous run). - auto handler = getShardArchiveHandler(true); - - // Recovery is needed. - if (handler) - shardArchiveHandler = handler; - } - catch (std::exception const& e) - { - JLOG(m_journal.fatal()) - << "Exception when starting ShardArchiveHandler from " - "state database: " - << e.what(); - - return false; - } - } - - if (shardArchiveHandler && !shardArchiveHandler->start()) - { - JLOG(m_journal.fatal()) << "Failed to start ShardArchiveHandler."; - - return false; - } - validatorSites_->start(); if (reportingETL_) @@ -1807,12 +1645,8 @@ ApplicationImp::run() m_loadManager->stop(); m_shaMapStore->stop(); m_jobQueue->stop(); - if (shardArchiveHandler_) - shardArchiveHandler_->stop(); if (overlay_) overlay_->stop(); - if (shardStore_) - shardStore_->stop(); grpcServer_->stop(); m_networkOPs->stop(); serverHandler_->stop(); @@ -1876,9 +1710,6 @@ ApplicationImp::fdRequired() const // doubled if online delete is enabled). needed += std::max(5, m_shaMapStore->fdRequired()); - if (shardStore_) - needed += shardStore_->fdRequired(); - // One fd per incoming connection a port can accept, or // if no limit is set, assume it'll handle 256 clients. for (auto const& p : serverHandler_->setup().ports) @@ -2345,27 +2176,6 @@ ApplicationImp::journal(std::string const& name) return logs_->journal(name); } -bool -ApplicationImp::nodeToShards() -{ - assert(overlay_); - assert(!config_->standalone()); - - if (config_->section(ConfigSection::shardDatabase()).empty()) - { - JLOG(m_journal.fatal()) - << "The [shard_db] configuration setting must be set"; - return false; - } - if (!shardStore_) - { - JLOG(m_journal.fatal()) << "Invalid [shard_db] configuration"; - return false; - } - shardStore_->importDatabase(getNodeStore()); - return true; -} - void ApplicationImp::setMaxDisallowedLedger() { diff --git a/src/xrpld/app/main/Application.h b/src/xrpld/app/main/Application.h index 57e6f1730e5..d4871317e73 100644 --- a/src/xrpld/app/main/Application.h +++ b/src/xrpld/app/main/Application.h @@ -42,14 +42,10 @@ class Manager; } namespace NodeStore { class Database; -class DatabaseShard; } // namespace NodeStore namespace perf { class PerfLog; } -namespace RPC { -class ShardArchiveHandler; -} // VFALCO TODO Fix forward declares required for header dependency loops class AmendmentTable; @@ -172,8 +168,6 @@ class Application : public beast::PropertyStream::Source getCollectorManager() = 0; virtual Family& getNodeFamily() = 0; - virtual Family* - getShardFamily() = 0; virtual TimeKeeper& timeKeeper() = 0; virtual JobQueue& @@ -210,10 +204,6 @@ class Application : public beast::PropertyStream::Source getValidations() = 0; virtual NodeStore::Database& getNodeStore() = 0; - virtual NodeStore::DatabaseShard* - getShardStore() = 0; - virtual RPC::ShardArchiveHandler* - getShardArchiveHandler(bool tryRecovery = false) = 0; virtual InboundLedgers& getInboundLedgers() = 0; virtual InboundTransactions& diff --git a/src/xrpld/app/main/DBInit.h b/src/xrpld/app/main/DBInit.h index 3d2f42717b2..528f2e8105e 100644 --- a/src/xrpld/app/main/DBInit.h +++ b/src/xrpld/app/main/DBInit.h @@ -124,96 +124,6 @@ inline constexpr std::array TxDBInit{ //////////////////////////////////////////////////////////////////////////////// -// The Ledger Meta database maps ledger hashes to shard indexes -inline constexpr auto LgrMetaDBName{"ledger_meta.db"}; - -// In C++17 omitting the explicit template parameters caused -// a crash -inline constexpr std::array LgrMetaDBPragma -{ - "PRAGMA page_size=4096;", "PRAGMA journal_size_limit=1582080;", - "PRAGMA max_page_count=2147483646;", - -#if (ULONG_MAX > UINT_MAX) && !defined(NO_SQLITE_MMAP) - "PRAGMA mmap_size=17179869184;" -#else - - // Provide an explicit `no-op` SQL statement - // in order to keep the size of the array - // constant regardless of the preprocessor - // condition evaluation - "PRAGMA sqlite_noop_statement;" -#endif -}; - -inline constexpr std::array LgrMetaDBInit{ - {"BEGIN TRANSACTION;", - - "CREATE TABLE IF NOT EXISTS LedgerMeta ( \ - LedgerHash CHARACTER(64) PRIMARY KEY, \ - ShardIndex INTEGER \ - );", - - "END TRANSACTION;"}}; - -//////////////////////////////////////////////////////////////////////////////// - -// Transaction Meta database maps transaction IDs to shard indexes -inline constexpr auto TxMetaDBName{"transaction_meta.db"}; - -// In C++17 omitting the explicit template parameters caused -// a crash -inline constexpr std::array TxMetaDBPragma -{ - "PRAGMA page_size=4096;", "PRAGMA journal_size_limit=1582080;", - "PRAGMA max_page_count=2147483646;", - -#if (ULONG_MAX > UINT_MAX) && !defined(NO_SQLITE_MMAP) - "PRAGMA mmap_size=17179869184;" -#else - - // Provide an explicit `no-op` SQL statement - // in order to keep the size of the array - // constant regardless of the preprocessor - // condition evaluation - "PRAGMA sqlite_noop_statement;" -#endif -}; - -inline constexpr std::array TxMetaDBInit{ - {"BEGIN TRANSACTION;", - - "CREATE TABLE IF NOT EXISTS TransactionMeta ( \ - TransID CHARACTER(64) PRIMARY KEY, \ - ShardIndex INTEGER \ - );", - - "END TRANSACTION;"}}; - -//////////////////////////////////////////////////////////////////////////////// - -// Temporary database used with an incomplete shard that is being acquired -inline constexpr auto AcquireShardDBName{"acquire.db"}; - -inline constexpr std::array AcquireShardDBPragma{ - {"PRAGMA journal_size_limit=1582080;"}}; - -inline constexpr std::array AcquireShardDBInit{ - {"CREATE TABLE IF NOT EXISTS Shard ( \ - ShardIndex INTEGER PRIMARY KEY, \ - LastLedgerHash CHARACTER(64), \ - StoredLedgerSeqs BLOB \ - );"}}; - -//////////////////////////////////////////////////////////////////////////////// - -// Pragma for Ledger and Transaction databases with final shards -// These override the CommonDBPragma values defined above. -inline constexpr std::array FinalShardDBPragma{ - {"PRAGMA synchronous=OFF;", "PRAGMA journal_mode=OFF;"}}; - -//////////////////////////////////////////////////////////////////////////////// - inline constexpr auto WalletDBName{"wallet.db"}; inline constexpr std::array WalletDBInit{ @@ -247,36 +157,6 @@ inline constexpr std::array WalletDBInit{ "END TRANSACTION;"}}; -//////////////////////////////////////////////////////////////////////////////// - -static constexpr auto stateDBName{"state.db"}; - -// These override the CommonDBPragma values defined above. -static constexpr std::array DownloaderDBPragma{ - {"PRAGMA synchronous=FULL;", "PRAGMA journal_mode=DELETE;"}}; - -static constexpr std::array ShardArchiveHandlerDBInit{ - {"BEGIN TRANSACTION;", - - "CREATE TABLE IF NOT EXISTS State ( \ - ShardIndex INTEGER PRIMARY KEY, \ - URL TEXT \ - );", - - "END TRANSACTION;"}}; - -static constexpr std::array DatabaseBodyDBInit{ - {"BEGIN TRANSACTION;", - - "CREATE TABLE IF NOT EXISTS download ( \ - Path TEXT, \ - Data BLOB, \ - Size BIGINT UNSIGNED, \ - Part BIGINT UNSIGNED PRIMARY KEY \ - );", - - "END TRANSACTION;"}}; - } // namespace ripple #endif diff --git a/src/xrpld/app/main/Main.cpp b/src/xrpld/app/main/Main.cpp index 059d9758d39..799911f63dd 100644 --- a/src/xrpld/app/main/Main.cpp +++ b/src/xrpld/app/main/Main.cpp @@ -144,7 +144,6 @@ printHelp(const po::options_description& desc) " consensus_info\n" " deposit_authorized " "[]\n" - " download_shard [[ ]]\n" " feature [ [accept|reject]]\n" " fetch_info [clear]\n" " gateway_balances [] [ [ " @@ -160,7 +159,6 @@ printHelp(const po::options_description& desc) " log_level [[] ]\n" " logrotate\n" " manifest \n" - " node_to_shard [status|start|stop]\n" " peers\n" " ping\n" " random\n" @@ -398,7 +396,6 @@ run(int argc, char** argv) "Load the specified ledger file.")( "load", "Load the current ledger from the local DB.")( "net", "Get the initial ledger from the network.")( - "nodetoshard", "Import node store into shards")( "replay", "Replay a ledger close.")( "trap_tx_hash", po::value(), @@ -676,9 +673,6 @@ run(int argc, char** argv) if (vm.count("import")) config->doImport = true; - if (vm.count("nodetoshard")) - config->nodeToShard = true; - if (vm.count("ledger")) { config->START_LEDGER = vm["ledger"].as(); diff --git a/src/xrpld/app/misc/NetworkOPs.cpp b/src/xrpld/app/misc/NetworkOPs.cpp index 9cf5d097099..a7ee935f102 100644 --- a/src/xrpld/app/misc/NetworkOPs.cpp +++ b/src/xrpld/app/misc/NetworkOPs.cpp @@ -44,7 +44,6 @@ #include #include #include -#include #include #include #include @@ -2472,10 +2471,7 @@ NetworkOPsImp::getServerInfo(bool human, bool admin, bool counters) info[jss::counters] = app_.getPerfLog().countersJson(); Json::Value nodestore(Json::objectValue); - if (app_.getShardStore()) - app_.getShardStore()->getCountsJson(nodestore); - else - app_.getNodeStore().getCountsJson(nodestore); + app_.getNodeStore().getCountsJson(nodestore); info[jss::counters][jss::nodestore] = nodestore; info[jss::current_activities] = app_.getPerfLog().currentJson(); } diff --git a/src/xrpld/app/misc/SHAMapStoreImp.cpp b/src/xrpld/app/misc/SHAMapStoreImp.cpp index d32556a4b29..9344463295b 100644 --- a/src/xrpld/app/misc/SHAMapStoreImp.cpp +++ b/src/xrpld/app/misc/SHAMapStoreImp.cpp @@ -290,8 +290,8 @@ SHAMapStoreImp::run() LedgerIndex lastRotated = state_db_.getState().lastRotated; netOPs_ = &app_.getOPs(); ledgerMaster_ = &app_.getLedgerMaster(); - fullBelowCache_ = &(*app_.getNodeFamily().getFullBelowCache(0)); - treeNodeCache_ = &(*app_.getNodeFamily().getTreeNodeCache(0)); + fullBelowCache_ = &(*app_.getNodeFamily().getFullBelowCache()); + treeNodeCache_ = &(*app_.getNodeFamily().getTreeNodeCache()); if (advisoryDelete_) canDelete_ = state_db_.getCanDelete(); @@ -329,27 +329,8 @@ SHAMapStoreImp::run() validatedSeq >= lastRotated + deleteInterval_ && canDelete_ >= lastRotated - 1 && healthWait() == keepGoing; - // Make sure we don't delete ledgers currently being - // imported into the ShardStore - bool const waitForImport = readyToRotate && [this, lastRotated] { - if (auto shardStore = app_.getShardStore()) - { - if (auto sequence = shardStore->getDatabaseImportSequence()) - return sequence <= lastRotated - 1; - } - - return false; - }(); - - if (waitForImport) - { - JLOG(journal_.info()) - << "NOT rotating validatedSeq " << validatedSeq - << " as rotation would interfere with ShardStore import"; - } - // will delete up to (not including) lastRotated - if (readyToRotate && !waitForImport) + if (readyToRotate) { JLOG(journal_.warn()) << "rotating validatedSeq " << validatedSeq << " lastRotated " diff --git a/src/xrpld/app/rdb/Download.h b/src/xrpld/app/rdb/Download.h deleted file mode 100644 index 6ee02d4c100..00000000000 --- a/src/xrpld/app/rdb/Download.h +++ /dev/null @@ -1,79 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2021 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_APP_RDB_DOWNLOAD_H_INCLUDED -#define RIPPLE_APP_RDB_DOWNLOAD_H_INCLUDED - -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { - -/** - * @brief openDatabaseBodyDb Opens a database that will store the contents of a - * file being downloaded, returns its descriptor, and starts a new - * download process or continues an existing one. - * @param setup Path to the database and other opening parameters. - * @param path Path of the new file to download. - * @return Pair containing a unique pointer to the database and the amount of - * bytes already downloaded if a download is being continued. - */ -std::pair, std::optional> -openDatabaseBodyDb( - DatabaseCon::Setup const& setup, - boost::filesystem::path const& path); - -/** - * @brief databaseBodyDoPut Saves a new fragment of a downloaded file. - * @param session Session with the database. - * @param data Downloaded fragment of file data to save. - * @param path Path to the file currently being downloaded. - * @param fileSize Size of the portion of the file already downloaded. - * @param part The index of the most recently updated database row. - * @param maxRowSizePad A constant padding value that accounts for other data - * stored in each row of the database. - * @return Index of the most recently updated database row. - */ -std::uint64_t -databaseBodyDoPut( - soci::session& session, - std::string const& data, - std::string const& path, - std::uint64_t fileSize, - std::uint64_t part, - std::uint16_t maxRowSizePad); - -/** - * @brief databaseBodyFinish Finishes the download process and writes the file - * to disk. - * @param session Session with the database. - * @param fout Opened file into which the downloaded data from the database will - * be written. - */ -void -databaseBodyFinish(soci::session& session, std::ofstream& fout); - -} // namespace ripple - -#endif diff --git a/src/xrpld/app/rdb/README.md b/src/xrpld/app/rdb/README.md index 1a68a1ae5e3..f4cb5f203a4 100644 --- a/src/xrpld/app/rdb/README.md +++ b/src/xrpld/app/rdb/README.md @@ -2,9 +2,8 @@ The guiding principles of the Relational Database Interface are summarized below: -* All hard-coded SQL statements should be stored in the [files](#source-files) under the `ripple/app/rdb` directory. With the exception of test modules, no hard-coded SQL should be added to any other file in rippled. +* All hard-coded SQL statements should be stored in the [files](#source-files) under the `xrpld/app/rdb` directory. With the exception of test modules, no hard-coded SQL should be added to any other file in rippled. * The base class `RelationalDatabase` is inherited by derived classes that each provide an interface for operating on distinct relational database systems. -* For future use, the shard store will be used if the node store is absent. ## Overview @@ -12,7 +11,7 @@ Firstly, the interface `RelationalDatabase` is inherited by the classes `SQLiteD ## Configuration -The config section `[relational_db]` has a property named `backend` whose value designates which database implementation will be used for node or shard databases. Presently the only valid value for this property is `sqlite`: +The config section `[relational_db]` has a property named `backend` whose value designates which database implementation will be used for node databases. Presently the only valid value for this property is `sqlite`: ``` [relational_db] @@ -24,35 +23,25 @@ backend=sqlite The Relational Database Interface consists of the following directory structure (as of November 2021): ``` -src/ripple/app/rdb/ +src/xrpld/app/rdb/ ├── backend │   ├── detail -│   │   ├── impl -│   │   │   ├── Node.cpp -│   │   │   └── Shard.cpp +│   │   ├── Node.cpp │   │   ├── Node.h -│   │   └── Shard.h -│   ├── impl │   │   ├── PostgresDatabase.cpp │   │   └── SQLiteDatabase.cpp │   ├── PostgresDatabase.h │   └── SQLiteDatabase.h -├── impl -│   ├── Download.cpp +├── detail │   ├── PeerFinder.cpp │   ├── RelationalDatabase.cpp -│   ├── ShardArchive.cpp │   ├── State.cpp -│   ├── UnitaryShard.cpp │   ├── Vacuum.cpp │   └── Wallet.cpp -├── Download.h ├── PeerFinder.h ├── RelationalDatabase.h ├── README.md -├── ShardArchive.h ├── State.h -├── UnitaryShard.h ├── Vacuum.h └── Wallet.h ``` @@ -61,16 +50,12 @@ src/ripple/app/rdb/ | File | Contents | | ----------- | ----------- | | `Node.[h\|cpp]` | Defines/Implements methods used by `SQLiteDatabase` for interacting with SQLite node databases| -| `Shard.[h\|cpp]` | Defines/Implements methods used by `SQLiteDatabase` for interacting with SQLite shard databases | | `PostgresDatabase.[h\|cpp]` | Defines/Implements the class `PostgresDatabase`/`PostgresDatabaseImp` which inherits from `RelationalDatabase` and is used to operate on the main stores | |`SQLiteDatabase.[h\|cpp]`| Defines/Implements the class `SQLiteDatabase`/`SQLiteDatabaseImp` which inherits from `RelationalDatabase` and is used to operate on the main stores | -| `Download.[h\|cpp]` | Defines/Implements methods for persisting file downloads to a SQLite database | | `PeerFinder.[h\|cpp]` | Defines/Implements methods for interacting with the PeerFinder SQLite database | |`RelationalDatabase.cpp`| Implements the static method `RelationalDatabase::init` which is used to initialize an instance of `RelationalDatabase` | | `RelationalDatabase.h` | Defines the abstract class `RelationalDatabase`, the primary class of the Relational Database Interface | -| `ShardArchive.[h\|cpp]` | Defines/Implements methods used by `ShardArchiveHandler` for interacting with SQLite databases containing metadata regarding shard downloads | | `State.[h\|cpp]` | Defines/Implements methods for interacting with the State SQLite database which concerns ledger deletion and database rotation | -| `UnitaryShard.[h\|cpp]` | Defines/Implements methods used by a unitary instance of `Shard` for interacting with the various SQLite databases thereof. These files are distinct from `Shard.[h\|cpp]` which contain methods used by `SQLiteDatabaseImp` | | `Vacuum.[h\|cpp]` | Defines/Implements a method for performing the `VACUUM` operation on SQLite databases | | `Wallet.[h\|cpp]` | Defines/Implements methods for interacting with Wallet SQLite databases | @@ -84,19 +69,15 @@ The Relational Database Interface provides three categories of methods for inter * Free functions for interacting with SQLite databases used by various components of the software. These methods feature a `soci::session` parameter which facilitates connecting to SQLite databases, and are defined and implemented in the following files: - * `Download.[h\|cpp]` * `PeerFinder.[h\|cpp]` - * `ShardArchive.[h\|cpp]` * `State.[h\|cpp]` - * `UnitaryShard.[h\|cpp]` * `Vacuum.[h\|cpp]` * `Wallet.[h\|cpp]` -* Free functions used exclusively by `SQLiteDatabaseImp` for interacting with SQLite databases owned by the node store or shard store. Unlike the free functions in the files listed above, these are not intended to be invoked directly by clients. Rather, these methods are invoked by derived instances of `RelationalDatabase`. These methods are defined in the following files: +* Free functions used exclusively by `SQLiteDatabaseImp` for interacting with SQLite databases owned by the node store. Unlike the free functions in the files listed above, these are not intended to be invoked directly by clients. Rather, these methods are invoked by derived instances of `RelationalDatabase`. These methods are defined in the following files: * `Node.[h|cpp]` - * `Shard.[h|cpp]` -* Member functions of `RelationalDatabase`, `SQLiteDatabase`, and `PostgresDatabase` which are used to access the main stores (node store, shard store). The `SQLiteDatabase` class will access the node store by default, but will use shard databases if the node store is not present and the shard store is available. The class `PostgresDatabase` uses only the node store. +* Member functions of `RelationalDatabase`, `SQLiteDatabase`, and `PostgresDatabase` which are used to access the node store. diff --git a/src/xrpld/app/rdb/ShardArchive.h b/src/xrpld/app/rdb/ShardArchive.h deleted file mode 100644 index 44f990ab5f5..00000000000 --- a/src/xrpld/app/rdb/ShardArchive.h +++ /dev/null @@ -1,78 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2021 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_APP_RDB_SHARDARCHIVE_H_INCLUDED -#define RIPPLE_APP_RDB_SHARDARCHIVE_H_INCLUDED - -#include -#include - -namespace ripple { - -/** - * @brief makeArchiveDB Opens the shard archive database and returns its - * descriptor. - * @param dir Path to the database to open. - * @param dbName Name of the database. - * @return Unique pointer to the opened database. - */ -std::unique_ptr -makeArchiveDB(boost::filesystem::path const& dir, std::string const& dbName); - -/** - * @brief readArchiveDB Reads entries from the shard archive database and - * invokes the given callback for each entry. - * @param db Session with the database. - * @param func Callback to invoke for each entry. - */ -void -readArchiveDB( - DatabaseCon& db, - std::function const& func); - -/** - * @brief insertArchiveDB Adds an entry to the shard archive database. - * @param db Session with the database. - * @param shardIndex Shard index to add. - * @param url Shard download url to add. - */ -void -insertArchiveDB( - DatabaseCon& db, - std::uint32_t shardIndex, - std::string const& url); - -/** - * @brief deleteFromArchiveDB Deletes an entry from the shard archive database. - * @param db Session with the database. - * @param shardIndex Shard index to remove from the database. - */ -void -deleteFromArchiveDB(DatabaseCon& db, std::uint32_t shardIndex); - -/** - * @brief dropArchiveDB Removes a table in the shard archive database. - * @param db Session with the database. - */ -void -dropArchiveDB(DatabaseCon& db); - -} // namespace ripple - -#endif diff --git a/src/xrpld/app/rdb/State.h b/src/xrpld/app/rdb/State.h index b245270cda2..e65e9d4d57a 100644 --- a/src/xrpld/app/rdb/State.h +++ b/src/xrpld/app/rdb/State.h @@ -24,7 +24,6 @@ #include #include #include -#include #include #include diff --git a/src/xrpld/app/rdb/UnitaryShard.h b/src/xrpld/app/rdb/UnitaryShard.h deleted file mode 100644 index e848000221d..00000000000 --- a/src/xrpld/app/rdb/UnitaryShard.h +++ /dev/null @@ -1,155 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2021 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_APP_RDB_UNITARYSHARD_H_INCLUDED -#define RIPPLE_APP_RDB_UNITARYSHARD_H_INCLUDED - -#include -#include -#include -#include -#include - -namespace ripple { - -struct DatabasePair -{ - std::unique_ptr ledgerDb; - std::unique_ptr transactionDb; -}; - -/** - * @brief makeShardCompleteLedgerDBs Opens shard databases for verified shards - * and returns their descriptors. - * @param config Config object. - * @param setup Path to the databases and other opening parameters. - * @return Pair of unique pointers to the opened ledger and transaction - * databases. - */ -DatabasePair -makeShardCompleteLedgerDBs( - Config const& config, - DatabaseCon::Setup const& setup); - -/** - * @brief makeShardIncompleteLedgerDBs Opens shard databases for partially - * downloaded or unverified shards and returns their descriptors. - * @param config Config object. - * @param setup Path to the databases and other opening parameters. - * @param checkpointerSetup Checkpointer parameters. - * @return Pair of unique pointers to the opened ledger and transaction - * databases. - */ -DatabasePair -makeShardIncompleteLedgerDBs( - Config const& config, - DatabaseCon::Setup const& setup, - DatabaseCon::CheckpointerSetup const& checkpointerSetup); - -/** - * @brief updateLedgerDBs Saves the given ledger to shard databases. - * @param txdb Session with the transaction databases. - * @param lgrdb Session with the ledger databases. - * @param ledger Ledger to save. - * @param index Index of the shard that owns the ledger. - * @param stop Reference to an atomic flag that can stop the process if raised. - * @param j Journal - * @return True if the ledger was successfully saved. - */ -bool -updateLedgerDBs( - soci::session& txdb, - soci::session& lgrdb, - std::shared_ptr const& ledger, - std::uint32_t index, - std::atomic& stop, - beast::Journal j); - -/** - * @brief makeAcquireDB Opens the shard acquire database and returns its - * descriptor. - * @param setup Path to the database and other opening parameters. - * @param checkpointerSetup Checkpointer parameters. - * @return Unique pointer to the opened database. - */ -std::unique_ptr -makeAcquireDB( - DatabaseCon::Setup const& setup, - DatabaseCon::CheckpointerSetup const& checkpointerSetup); - -/** - * @brief insertAcquireDBIndex Adds a new shard index to the shard acquire - * database. - * @param session Session with the database. - * @param index Index to add. - */ -void -insertAcquireDBIndex(soci::session& session, std::uint32_t index); - -/** - * @brief selectAcquireDBLedgerSeqs Returns the set of acquired ledgers for - * the given shard. - * @param session Session with the database. - * @param index Shard index. - * @return Pair which contains true if such an index was found in the database, - * and a string which contains the set of ledger sequences. - * If no sequences were saved then the optional will have no value. - */ -std::pair> -selectAcquireDBLedgerSeqs(soci::session& session, std::uint32_t index); - -struct AcquireShardSeqsHash -{ - std::optional sequences; - std::optional hash; -}; - -/** - * @brief selectAcquireDBLedgerSeqsHash Returns the set of acquired ledger - * sequences and the last ledger hash for the shard with the provided - * index. - * @param session Session with the database. - * @param index Shard index. - * @return Pair which contains true if such an index was found in the database - * and the AcquireShardSeqsHash structure which contains a string with - * the ledger sequences and a string with last ledger hash. If the set - * of sequences or hash were not saved then no value is returned. - */ -std::pair -selectAcquireDBLedgerSeqsHash(soci::session& session, std::uint32_t index); - -/** - * @brief updateAcquireDB Updates information in the acquire DB. - * @param session Session with the database. - * @param ledger Ledger to save into the database. - * @param index Shard index. - * @param lastSeq Last acquired ledger sequence. - * @param seqs Current set of acquired ledger sequences if it's not empty. - */ -void -updateAcquireDB( - soci::session& session, - std::shared_ptr const& ledger, - std::uint32_t index, - std::uint32_t lastSeq, - std::optional const& seqs); - -} // namespace ripple - -#endif diff --git a/src/xrpld/app/rdb/backend/detail/detail/Node.cpp b/src/xrpld/app/rdb/backend/detail/Node.cpp similarity index 93% rename from src/xrpld/app/rdb/backend/detail/detail/Node.cpp rename to src/xrpld/app/rdb/backend/detail/Node.cpp index 67a80b43cf3..70e42b0ae85 100644 --- a/src/xrpld/app/rdb/backend/detail/detail/Node.cpp +++ b/src/xrpld/app/rdb/backend/detail/Node.cpp @@ -622,8 +622,7 @@ getTxHistory( soci::session& session, Application& app, LedgerIndex startIndex, - int quantity, - bool count) + int quantity) { std::string sql = boost::str( boost::format( @@ -663,13 +662,6 @@ getTxHistory( txs.push_back(trans); } } - - if (!total && count) - { - session << "SELECT COUNT(*) FROM Transactions;", soci::into(total); - - total = -total; - } } return {txs, total}; @@ -685,9 +677,6 @@ getTxHistory( * the account, the ledger search range, the offset of the first entry to * return, the number of transactions to return, and a flag if this * number is unlimited. - * @param limit_used Number of transactions already returned in calls - * to other shard databases, if shard databases are used. - * No value if the node database is used. * @param descending True for descending order, false for ascending. * @param binary True for binary form, false for decoded. * @param count True for counting the number of transactions, false for @@ -700,7 +689,6 @@ transactionsSQL( Application& app, std::string selection, RelationalDatabase::AccountTxOptions const& options, - std::optional const& limit_used, bool descending, bool binary, bool count, @@ -729,14 +717,6 @@ transactionsSQL( numberOfResults = options.limit; } - if (limit_used) - { - if (numberOfResults <= *limit_used) - return ""; - else - numberOfResults -= *limit_used; - } - std::string maxClause = ""; std::string minClause = ""; @@ -790,9 +770,6 @@ transactionsSQL( * the account, the ledger search range, the offset of the first entry to * return, the number of transactions to return, and a flag if this * number is unlimited. - * @param limit_used Number of transactions already returned in calls - * to other shard databases, if shard databases are used. - * No value if the node database is used. * @param descending True for descending order, false for ascending. * @param j Journal. * @return Vector of pairs of found transactions and their metadata sorted by @@ -809,7 +786,6 @@ getAccountTxs( Application& app, LedgerMaster& ledgerMaster, RelationalDatabase::AccountTxOptions const& options, - std::optional const& limit_used, bool descending, beast::Journal j) { @@ -819,7 +795,6 @@ getAccountTxs( app, "AccountTransactions.LedgerSeq,Status,RawTxn,TxnMeta", options, - limit_used, descending, false, false, @@ -880,18 +855,6 @@ getAccountTxs( total++; } } - - if (!total && limit_used) - { - RelationalDatabase::AccountTxOptions opt = options; - opt.offset = 0; - std::string sql1 = transactionsSQL( - app, "COUNT(*)", opt, limit_used, descending, false, false, j); - - session << sql1, soci::into(total); - - total = -total; - } } return {ret, total}; @@ -903,11 +866,9 @@ getOldestAccountTxs( Application& app, LedgerMaster& ledgerMaster, RelationalDatabase::AccountTxOptions const& options, - std::optional const& limit_used, beast::Journal j) { - return getAccountTxs( - session, app, ledgerMaster, options, limit_used, false, j); + return getAccountTxs(session, app, ledgerMaster, options, false, j); } std::pair @@ -916,11 +877,9 @@ getNewestAccountTxs( Application& app, LedgerMaster& ledgerMaster, RelationalDatabase::AccountTxOptions const& options, - std::optional const& limit_used, beast::Journal j) { - return getAccountTxs( - session, app, ledgerMaster, options, limit_used, true, j); + return getAccountTxs(session, app, ledgerMaster, options, true, j); } /** @@ -933,9 +892,6 @@ getNewestAccountTxs( * the account, the ledger search range, the offset of the first entry to * return, the number of transactions to return, and a flag if this * number is unlimited. - * @param limit_used Number of transactions already returned in calls to other - * shard databases, if shard databases are used. No value if the node - * database is used. * @param descending True for descending order, false for ascending. * @param j Journal. * @return Vector of tuples each containing (the found transactions, their @@ -951,7 +907,6 @@ getAccountTxsB( soci::session& session, Application& app, RelationalDatabase::AccountTxOptions const& options, - std::optional const& limit_used, bool descending, beast::Journal j) { @@ -961,7 +916,6 @@ getAccountTxsB( app, "AccountTransactions.LedgerSeq,Status,RawTxn,TxnMeta", options, - limit_used, descending, true /*binary*/, false, @@ -1001,18 +955,6 @@ getAccountTxsB( ret.emplace_back(std::move(rawTxn), std::move(txnMeta), seq); total++; } - - if (!total && limit_used) - { - RelationalDatabase::AccountTxOptions opt = options; - opt.offset = 0; - std::string sql1 = transactionsSQL( - app, "COUNT(*)", opt, limit_used, descending, true, false, j); - - session << sql1, soci::into(total); - - total = -total; - } } return {ret, total}; @@ -1023,10 +965,9 @@ getOldestAccountTxsB( soci::session& session, Application& app, RelationalDatabase::AccountTxOptions const& options, - std::optional const& limit_used, beast::Journal j) { - return getAccountTxsB(session, app, options, limit_used, false, j); + return getAccountTxsB(session, app, options, false, j); } std::pair, int> @@ -1034,10 +975,9 @@ getNewestAccountTxsB( soci::session& session, Application& app, RelationalDatabase::AccountTxOptions const& options, - std::optional const& limit_used, beast::Journal j) { - return getAccountTxsB(session, app, options, limit_used, true, j); + return getAccountTxsB(session, app, options, true, j); } /** @@ -1052,8 +992,6 @@ getNewestAccountTxsB( * match: the account, the ledger search range, the marker of the first * returned entry, the number of transactions to return, and a flag if * this number unlimited. - * @param limit_used Number of transactions already returned in calls - * to other shard databases. * @param page_length Total number of transactions to return. * @param forward True for ascending order, false for descending. * @return Vector of tuples of found transactions, their metadata and account @@ -1069,7 +1007,6 @@ accountTxPage( void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const& onTransaction, RelationalDatabase::AccountTxPageOptions const& options, - int limit_used, std::uint32_t page_length, bool forward) { @@ -1085,10 +1022,6 @@ accountTxPage( else numberOfResults = options.limit; - if (numberOfResults < limit_used) - return {options.marker, -1}; - numberOfResults -= limit_used; - // As an account can have many thousands of transactions, there is a limit // placed on the amount of transactions returned. If the limit is reached // before the result set has been exhausted (we always query for one more @@ -1104,8 +1037,6 @@ accountTxPage( } std::optional newmarker; - if (limit_used > 0) - newmarker = options.marker; static std::string const prefix( R"(SELECT AccountTransactions.LedgerSeq,AccountTransactions.TxnSeq, @@ -1251,17 +1182,10 @@ oldestAccountTxPage( void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const& onTransaction, RelationalDatabase::AccountTxPageOptions const& options, - int limit_used, std::uint32_t page_length) { return accountTxPage( - session, - onUnsavedLedger, - onTransaction, - options, - limit_used, - page_length, - true); + session, onUnsavedLedger, onTransaction, options, page_length, true); } std::pair, int> @@ -1272,17 +1196,10 @@ newestAccountTxPage( void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const& onTransaction, RelationalDatabase::AccountTxPageOptions const& options, - int limit_used, std::uint32_t page_length) { return accountTxPage( - session, - onUnsavedLedger, - onTransaction, - options, - limit_used, - page_length, - false); + session, onUnsavedLedger, onTransaction, options, page_length, false); } std::variant diff --git a/src/xrpld/app/rdb/backend/detail/Node.h b/src/xrpld/app/rdb/backend/detail/Node.h index d7c170663a5..5564adf8954 100644 --- a/src/xrpld/app/rdb/backend/detail/Node.h +++ b/src/xrpld/app/rdb/backend/detail/Node.h @@ -249,7 +249,6 @@ getHashesByIndex( * @param app Application object. * @param startIndex Offset of first returned entry. * @param quantity Number of returned entries. - * @param count True if counting of all transaction in that shard required. * @return Vector of shared pointers to transactions sorted in * descending order by ledger sequence. Also number of transactions * if count == true. @@ -259,8 +258,7 @@ getTxHistory( soci::session& session, Application& app, LedgerIndex startIndex, - int quantity, - bool count); + int quantity); /** * @brief getOldestAccountTxs Returns oldest transactions for given @@ -272,9 +270,6 @@ getTxHistory( * the account, minimum and maximum ledger numbers to search, * offset of first entry to return, number of transactions to return, * flag if this number unlimited. - * @param limit_used Number or transactions already returned in calls - * to another shard databases, if shard databases are used. - * None if node database is used. * @param j Journal. * @return Vector of pairs of found transactions and their metadata * sorted in ascending order by account sequence. @@ -290,7 +285,6 @@ getOldestAccountTxs( Application& app, LedgerMaster& ledgerMaster, RelationalDatabase::AccountTxOptions const& options, - std::optional const& limit_used, beast::Journal j); /** @@ -303,9 +297,6 @@ getOldestAccountTxs( * the account, minimum and maximum ledger numbers to search, * offset of first entry to return, number of transactions to return, * flag if this number unlimited. - * @param limit_used Number or transactions already returned in calls - * to another shard databases, if shard databases are used. - * None if node database is used. * @param j Journal. * @return Vector of pairs of found transactions and their metadata * sorted in descending order by account sequence. @@ -321,7 +312,6 @@ getNewestAccountTxs( Application& app, LedgerMaster& ledgerMaster, RelationalDatabase::AccountTxOptions const& options, - std::optional const& limit_used, beast::Journal j); /** @@ -334,9 +324,6 @@ getNewestAccountTxs( * the account, minimum and maximum ledger numbers to search, * offset of first entry to return, number of transactions to return, * flag if this number unlimited. - * @param limit_used Number or transactions already returned in calls - * to another shard databases, if shard databases are used. - * None if node database is used. * @param j Journal. * @return Vector of tuples of found transactions, their metadata and * account sequences sorted in ascending order by account @@ -351,7 +338,6 @@ getOldestAccountTxsB( soci::session& session, Application& app, RelationalDatabase::AccountTxOptions const& options, - std::optional const& limit_used, beast::Journal j); /** @@ -364,9 +350,6 @@ getOldestAccountTxsB( * the account, minimum and maximum ledger numbers to search, * offset of first entry to return, number of transactions to return, * flag if this number unlimited. - * @param limit_used Number or transactions already returned in calls - * to another shard databases, if shard databases are used. - * None if node database is used. * @param j Journal. * @return Vector of tuples of found transactions, their metadata and * account sequences sorted in descending order by account @@ -381,7 +364,6 @@ getNewestAccountTxsB( soci::session& session, Application& app, RelationalDatabase::AccountTxOptions const& options, - std::optional const& limit_used, beast::Journal j); /** @@ -396,8 +378,6 @@ getNewestAccountTxsB( * match: the account, minimum and maximum ledger numbers to search, * marker of first returned entry, number of transactions to return, * flag if this number unlimited. - * @param limit_used Number or transactions already returned in calls - * to another shard databases. * @param page_length Total number of transactions to return. * @return Vector of tuples of found transactions, their metadata and * account sequences sorted in ascending order by account @@ -412,7 +392,6 @@ oldestAccountTxPage( void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const& onTransaction, RelationalDatabase::AccountTxPageOptions const& options, - int limit_used, std::uint32_t page_length); /** @@ -427,8 +406,6 @@ oldestAccountTxPage( * match: the account, minimum and maximum ledger numbers to search, * marker of first returned entry, number of transactions to return, * flag if this number unlimited. - * @param limit_used Number or transactions already returned in calls - * to another shard databases. * @param page_length Total number of transactions to return. * @return Vector of tuples of found transactions, their metadata and * account sequences sorted in descending order by account @@ -443,7 +420,6 @@ newestAccountTxPage( void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const& onTransaction, RelationalDatabase::AccountTxPageOptions const& options, - int limit_used, std::uint32_t page_length); /** diff --git a/src/xrpld/app/rdb/backend/detail/PostgresDatabase.cpp b/src/xrpld/app/rdb/backend/detail/PostgresDatabase.cpp index ac998991a6d..ac1a9813c2b 100644 --- a/src/xrpld/app/rdb/backend/detail/PostgresDatabase.cpp +++ b/src/xrpld/app/rdb/backend/detail/PostgresDatabase.cpp @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include diff --git a/src/xrpld/app/rdb/backend/detail/SQLiteDatabase.cpp b/src/xrpld/app/rdb/backend/detail/SQLiteDatabase.cpp index 3edc5d163d8..bc5992d03d0 100644 --- a/src/xrpld/app/rdb/backend/detail/SQLiteDatabase.cpp +++ b/src/xrpld/app/rdb/backend/detail/SQLiteDatabase.cpp @@ -25,10 +25,8 @@ #include #include #include -#include #include #include -#include #include #include #include @@ -59,19 +57,6 @@ class SQLiteDatabaseImp final : public SQLiteDatabase JLOG(j_.fatal()) << error; Throw(error.data()); } - - if (app.getShardStore() && - !makeMetaDBs( - config, - setup, - DatabaseCon::CheckpointerSetup{&jobQueue, &app_.logs()})) - { - std::string_view constexpr error = - "Failed to create metadata databases"; - - JLOG(j_.fatal()) << error; - Throw(error.data()); - } } std::optional @@ -195,7 +180,6 @@ class SQLiteDatabaseImp final : public SQLiteDatabase bool const useTxTables_; beast::Journal j_; std::unique_ptr lgrdb_, txdb_; - std::unique_ptr lgrMetaDB_, txMetaDB_; /** * @brief makeLedgerDBs Opens ledger and transaction databases for the node @@ -211,56 +195,6 @@ class SQLiteDatabaseImp final : public SQLiteDatabase DatabaseCon::Setup const& setup, DatabaseCon::CheckpointerSetup const& checkpointerSetup); - /** - * @brief makeMetaDBs Opens shard index lookup databases, and stores - * their descriptors in private member variables. - * @param config Config object. - * @param setup Path to the databases and other opening parameters. - * @param checkpointerSetup Checkpointer parameters. - * @return True if node databases opened successfully. - */ - bool - makeMetaDBs( - Config const& config, - DatabaseCon::Setup const& setup, - DatabaseCon::CheckpointerSetup const& checkpointerSetup); - - /** - * @brief seqToShardIndex Provides the index of the shard that stores the - * ledger with the given sequence. - * @param ledgerSeq Ledger sequence. - * @return Shard index. - */ - std::uint32_t - seqToShardIndex(LedgerIndex ledgerSeq) - { - return app_.getShardStore()->seqToShardIndex(ledgerSeq); - } - - /** - * @brief firstLedgerSeq Returns the sequence of the first ledger stored in - * the shard specified by the shard index parameter. - * @param shardIndex Shard Index. - * @return First ledger sequence. - */ - LedgerIndex - firstLedgerSeq(std::uint32_t shardIndex) - { - return app_.getShardStore()->firstLedgerSeq(shardIndex); - } - - /** - * @brief lastLedgerSeq Returns the sequence of the last ledger stored in - * the shard specified by the shard index parameter. - * @param shardIndex Shard Index. - * @return Last ledger sequence. - */ - LedgerIndex - lastLedgerSeq(std::uint32_t shardIndex) - { - return app_.getShardStore()->lastLedgerSeq(shardIndex); - } - /** * @brief existsLedger Checks if the node store ledger database exists. * @return True if the node store ledger database exists. @@ -282,16 +216,6 @@ class SQLiteDatabaseImp final : public SQLiteDatabase return static_cast(txdb_); } - /** - * shardStoreExists Checks whether the shard store exists - * @return True if the shard store exists - */ - bool - shardStoreExists() - { - return app_.getShardStore() != nullptr; - } - /** * @brief checkoutTransaction Checks out and returns node store ledger * database. @@ -313,131 +237,6 @@ class SQLiteDatabaseImp final : public SQLiteDatabase { return txdb_->checkoutDb(); } - - /** - * @brief doLedger Checks out the ledger database owned by the shard - * containing the given ledger, and invokes the provided callback - * with a session to that database. - * @param ledgerSeq Ledger sequence. - * @param callback Callback function to call. - * @return Value returned by callback function. - */ - bool - doLedger( - LedgerIndex ledgerSeq, - std::function const& callback) - { - return app_.getShardStore()->callForLedgerSQLByLedgerSeq( - ledgerSeq, callback); - } - - /** - * @brief doTransaction Checks out the transaction database owned by the - * shard containing the given ledger, and invokes the provided - * callback with a session to that database. - * @param ledgerSeq Ledger sequence. - * @param callback Callback function to call. - * @return Value returned by callback function. - */ - bool - doTransaction( - LedgerIndex ledgerSeq, - std::function const& callback) - { - return app_.getShardStore()->callForTransactionSQLByLedgerSeq( - ledgerSeq, callback); - } - - /** - * @brief iterateLedgerForward Checks out ledger databases for all shards in - * ascending order starting from the given shard index, until all - * shards in range have been visited or the callback returns false. - * For each visited shard, we invoke the provided callback with a - * session to the database and the current shard index. - * @param firstIndex First shard index to visit or no value if all shards - * should be visited. - * @param callback Callback function to call. - * @return True if each callback function returned true, false otherwise. - */ - bool - iterateLedgerForward( - std::optional firstIndex, - std::function< - bool(soci::session& session, std::uint32_t shardIndex)> const& - callback) - { - return app_.getShardStore()->iterateLedgerSQLsForward( - firstIndex, callback); - } - - /** - * @brief iterateTransactionForward Checks out transaction databases for all - * shards in ascending order starting from the given shard index, - * until all shards in range have been visited or the callback - * returns false. For each visited shard, we invoke the provided - * callback with a session to the database and the current shard - * index. - * @param firstIndex First shard index to visit or no value if all shards - * should be visited. - * @param callback Callback function to call. - * @return True if each callback function returned true, false otherwise. - */ - bool - iterateTransactionForward( - std::optional firstIndex, - std::function< - bool(soci::session& session, std::uint32_t shardIndex)> const& - callback) - { - return app_.getShardStore()->iterateLedgerSQLsForward( - firstIndex, callback); - } - - /** - * @brief iterateLedgerBack Checks out ledger databases for all - * shards in descending order starting from the given shard index, - * until all shards in range have been visited or the callback - * returns false. For each visited shard, we invoke the provided - * callback with a session to the database and the current shard - * index. - * @param firstIndex First shard index to visit or no value if all shards - * should be visited. - * @param callback Callback function to call. - * @return True if each callback function returned true, false otherwise. - */ - bool - iterateLedgerBack( - std::optional firstIndex, - std::function< - bool(soci::session& session, std::uint32_t shardIndex)> const& - callback) - { - return app_.getShardStore()->iterateLedgerSQLsBack( - firstIndex, callback); - } - - /** - * @brief iterateTransactionBack Checks out transaction databases for all - * shards in descending order starting from the given shard index, - * until all shards in range have been visited or the callback - * returns false. For each visited shard, we invoke the provided - * callback with a session to the database and the current shard - * index. - * @param firstIndex First shard index to visit or no value if all shards - * should be visited. - * @param callback Callback function to call. - * @return True if each callback function returned true, false otherwise. - */ - bool - iterateTransactionBack( - std::optional firstIndex, - std::function< - bool(soci::session& session, std::uint32_t shardIndex)> const& - callback) - { - return app_.getShardStore()->iterateLedgerSQLsBack( - firstIndex, callback); - } }; bool @@ -453,21 +252,6 @@ SQLiteDatabaseImp::makeLedgerDBs( return res; } -bool -SQLiteDatabaseImp::makeMetaDBs( - Config const& config, - DatabaseCon::Setup const& setup, - DatabaseCon::CheckpointerSetup const& checkpointerSetup) -{ - auto [lgrMetaDB, txMetaDB] = - detail::makeMetaDBs(config, setup, checkpointerSetup); - - txMetaDB_ = std::move(txMetaDB); - lgrMetaDB_ = std::move(lgrMetaDB); - - return true; -} - std::optional SQLiteDatabaseImp::getMinLedgerSeq() { @@ -478,19 +262,6 @@ SQLiteDatabaseImp::getMinLedgerSeq() return detail::getMinLedgerSeq(*db, detail::TableType::Ledgers); } - /* else use shard databases, if available */ - if (shardStoreExists()) - { - std::optional res; - iterateLedgerForward( - {}, [&](soci::session& session, std::uint32_t shardIndex) { - res = detail::getMinLedgerSeq( - session, detail::TableType::Ledgers); - return !res; - }); - return res; - } - /* else return empty value */ return {}; } @@ -507,18 +278,6 @@ SQLiteDatabaseImp::getTransactionsMinLedgerSeq() return detail::getMinLedgerSeq(*db, detail::TableType::Transactions); } - if (shardStoreExists()) - { - std::optional res; - iterateTransactionForward( - {}, [&](soci::session& session, std::uint32_t shardIndex) { - res = detail::getMinLedgerSeq( - session, detail::TableType::Transactions); - return !res; - }); - return res; - } - return {}; } @@ -535,18 +294,6 @@ SQLiteDatabaseImp::getAccountTransactionsMinLedgerSeq() *db, detail::TableType::AccountTransactions); } - if (shardStoreExists()) - { - std::optional res; - iterateTransactionForward( - {}, [&](soci::session& session, std::uint32_t shardIndex) { - res = detail::getMinLedgerSeq( - session, detail::TableType::AccountTransactions); - return !res; - }); - return res; - } - return {}; } @@ -559,18 +306,6 @@ SQLiteDatabaseImp::getMaxLedgerSeq() return detail::getMaxLedgerSeq(*db, detail::TableType::Ledgers); } - if (shardStoreExists()) - { - std::optional res; - iterateLedgerBack( - {}, [&](soci::session& session, std::uint32_t shardIndex) { - res = detail::getMaxLedgerSeq( - session, detail::TableType::Ledgers); - return !res; - }); - return res; - } - return {}; } @@ -587,15 +322,6 @@ SQLiteDatabaseImp::deleteTransactionByLedgerSeq(LedgerIndex ledgerSeq) *db, detail::TableType::Transactions, ledgerSeq); return; } - - if (shardStoreExists()) - { - doTransaction(ledgerSeq, [&](soci::session& session) { - detail::deleteByLedgerSeq( - session, detail::TableType::Transactions, ledgerSeq); - return true; - }); - } } void @@ -608,17 +334,6 @@ SQLiteDatabaseImp::deleteBeforeLedgerSeq(LedgerIndex ledgerSeq) *db, detail::TableType::Ledgers, ledgerSeq); return; } - - if (shardStoreExists()) - { - iterateLedgerBack( - seqToShardIndex(ledgerSeq), - [&](soci::session& session, std::uint32_t shardIndex) { - detail::deleteBeforeLedgerSeq( - session, detail::TableType::Ledgers, ledgerSeq); - return true; - }); - } } void @@ -634,17 +349,6 @@ SQLiteDatabaseImp::deleteTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) *db, detail::TableType::Transactions, ledgerSeq); return; } - - if (shardStoreExists()) - { - iterateTransactionBack( - seqToShardIndex(ledgerSeq), - [&](soci::session& session, std::uint32_t shardIndex) { - detail::deleteBeforeLedgerSeq( - session, detail::TableType::Transactions, ledgerSeq); - return true; - }); - } } void @@ -661,17 +365,6 @@ SQLiteDatabaseImp::deleteAccountTransactionsBeforeLedgerSeq( *db, detail::TableType::AccountTransactions, ledgerSeq); return; } - - if (shardStoreExists()) - { - iterateTransactionBack( - seqToShardIndex(ledgerSeq), - [&](soci::session& session, std::uint32_t shardIndex) { - detail::deleteBeforeLedgerSeq( - session, detail::TableType::AccountTransactions, ledgerSeq); - return true; - }); - } } std::size_t @@ -686,18 +379,6 @@ SQLiteDatabaseImp::getTransactionCount() return detail::getRows(*db, detail::TableType::Transactions); } - if (shardStoreExists()) - { - std::size_t rows = 0; - iterateTransactionForward( - {}, [&](soci::session& session, std::uint32_t shardIndex) { - rows += - detail::getRows(session, detail::TableType::Transactions); - return true; - }); - return rows; - } - return 0; } @@ -713,18 +394,6 @@ SQLiteDatabaseImp::getAccountTransactionCount() return detail::getRows(*db, detail::TableType::AccountTransactions); } - if (shardStoreExists()) - { - std::size_t rows = 0; - iterateTransactionForward( - {}, [&](soci::session& session, std::uint32_t shardIndex) { - rows += detail::getRows( - session, detail::TableType::AccountTransactions); - return true; - }); - return rows; - } - return 0; } @@ -737,25 +406,6 @@ SQLiteDatabaseImp::getLedgerCountMinMax() return detail::getRowsMinMax(*db, detail::TableType::Ledgers); } - if (shardStoreExists()) - { - CountMinMax res{0, 0, 0}; - iterateLedgerForward( - {}, [&](soci::session& session, std::uint32_t shardIndex) { - auto r = - detail::getRowsMinMax(session, detail::TableType::Ledgers); - if (r.numberOfRows) - { - res.numberOfRows += r.numberOfRows; - if (res.minLedgerSequence == 0) - res.minLedgerSequence = r.minLedgerSequence; - res.maxLedgerSequence = r.maxLedgerSequence; - } - return true; - }); - return res; - } - return {0, 0, 0}; } @@ -771,27 +421,6 @@ SQLiteDatabaseImp::saveValidatedLedger( return false; } - if (auto shardStore = app_.getShardStore(); shardStore) - { - if (ledger->info().seq < shardStore->earliestLedgerSeq()) - // For the moment return false only when the ShardStore - // should accept the ledger, but fails when attempting - // to do so, i.e. when saveLedgerMeta fails. Later when - // the ShardStore supercedes the NodeStore, change this - // line to return false if the ledger is too early. - return true; - - auto lgrMetaSession = lgrMetaDB_->checkoutDb(); - auto txMetaSession = txMetaDB_->checkoutDb(); - - return detail::saveLedgerMeta( - ledger, - app_, - *lgrMetaSession, - *txMetaSession, - shardStore->seqToShardIndex(ledger->info().seq)); - } - return true; } @@ -807,16 +436,6 @@ SQLiteDatabaseImp::getLedgerInfoByIndex(LedgerIndex ledgerSeq) return res; } - if (shardStoreExists()) - { - std::optional res; - doLedger(ledgerSeq, [&](soci::session& session) { - res = detail::getLedgerInfoByIndex(session, ledgerSeq, j_); - return true; - }); - return res; - } - return {}; } @@ -832,22 +451,6 @@ SQLiteDatabaseImp::getNewestLedgerInfo() return res; } - if (shardStoreExists()) - { - std::optional res; - iterateLedgerBack( - {}, [&](soci::session& session, std::uint32_t shardIndex) { - if (auto info = detail::getNewestLedgerInfo(session, j_)) - { - res = info; - return false; - } - return true; - }); - - return res; - } - return {}; } @@ -864,24 +467,6 @@ SQLiteDatabaseImp::getLimitedOldestLedgerInfo(LedgerIndex ledgerFirstIndex) return res; } - if (shardStoreExists()) - { - std::optional res; - iterateLedgerForward( - seqToShardIndex(ledgerFirstIndex), - [&](soci::session& session, std::uint32_t shardIndex) { - if (auto info = detail::getLimitedOldestLedgerInfo( - session, ledgerFirstIndex, j_)) - { - res = info; - return false; - } - return true; - }); - - return res; - } - return {}; } @@ -898,23 +483,6 @@ SQLiteDatabaseImp::getLimitedNewestLedgerInfo(LedgerIndex ledgerFirstIndex) return res; } - if (shardStoreExists()) - { - std::optional res; - iterateLedgerBack( - {}, [&](soci::session& session, std::uint32_t shardIndex) { - if (auto info = detail::getLimitedNewestLedgerInfo( - session, ledgerFirstIndex, j_)) - { - res = info; - return false; - } - return shardIndex >= seqToShardIndex(ledgerFirstIndex); - }); - - return res; - } - return {}; } @@ -930,24 +498,6 @@ SQLiteDatabaseImp::getLedgerInfoByHash(uint256 const& ledgerHash) return res; } - if (auto shardStore = app_.getShardStore()) - { - std::optional res; - auto lgrMetaSession = lgrMetaDB_->checkoutDb(); - - if (auto const shardIndex = - detail::getShardIndexforLedger(*lgrMetaSession, ledgerHash)) - { - shardStore->callForLedgerSQLByShardIndex( - *shardIndex, [&](soci::session& session) { - res = detail::getLedgerInfoByHash(session, ledgerHash, j_); - return false; // unused - }); - } - - return res; - } - return {}; } @@ -963,16 +513,6 @@ SQLiteDatabaseImp::getHashByIndex(LedgerIndex ledgerIndex) return res; } - if (shardStoreExists()) - { - uint256 hash; - doLedger(ledgerIndex, [&](soci::session& session) { - hash = detail::getHashByIndex(session, ledgerIndex); - return true; - }); - return hash; - } - return uint256(); } @@ -988,16 +528,6 @@ SQLiteDatabaseImp::getHashesByIndex(LedgerIndex ledgerIndex) return res; } - if (shardStoreExists()) - { - std::optional res; - doLedger(ledgerIndex, [&](soci::session& session) { - res = detail::getHashesByIndex(session, ledgerIndex, j_); - return true; - }); - return res; - } - return {}; } @@ -1013,26 +543,6 @@ SQLiteDatabaseImp::getHashesByIndex(LedgerIndex minSeq, LedgerIndex maxSeq) return res; } - if (shardStoreExists()) - { - std::map res; - while (minSeq <= maxSeq) - { - LedgerIndex shardMaxSeq = lastLedgerSeq(seqToShardIndex(minSeq)); - if (shardMaxSeq > maxSeq) - shardMaxSeq = maxSeq; - doLedger(minSeq, [&](soci::session& session) { - auto r = - detail::getHashesByIndex(session, minSeq, shardMaxSeq, j_); - res.insert(r.begin(), r.end()); - return true; - }); - minSeq = shardMaxSeq + 1; - } - - return res; - } - return {}; } @@ -1045,39 +555,12 @@ SQLiteDatabaseImp::getTxHistory(LedgerIndex startIndex) if (existsTransaction()) { auto db = checkoutTransaction(); - auto const res = - detail::getTxHistory(*db, app_, startIndex, 20, false).first; + auto const res = detail::getTxHistory(*db, app_, startIndex, 20).first; if (!res.empty()) return res; } - if (shardStoreExists()) - { - std::vector> txs; - int quantity = 20; - iterateTransactionBack( - {}, [&](soci::session& session, std::uint32_t shardIndex) { - auto [tx, total] = detail::getTxHistory( - session, app_, startIndex, quantity, true); - txs.insert(txs.end(), tx.begin(), tx.end()); - if (total > 0) - { - quantity -= total; - if (quantity <= 0) - return false; - startIndex = 0; - } - else - { - startIndex += total; - } - return true; - }); - - return txs; - } - return {}; } @@ -1092,52 +575,10 @@ SQLiteDatabaseImp::getOldestAccountTxs(AccountTxOptions const& options) if (existsTransaction()) { auto db = checkoutTransaction(); - return detail::getOldestAccountTxs( - *db, app_, ledgerMaster, options, {}, j_) + return detail::getOldestAccountTxs(*db, app_, ledgerMaster, options, j_) .first; } - if (shardStoreExists()) - { - AccountTxs ret; - AccountTxOptions opt = options; - int limit_used = 0; - iterateTransactionForward( - opt.minLedger ? seqToShardIndex(opt.minLedger) - : std::optional(), - [&](soci::session& session, std::uint32_t shardIndex) { - if (opt.maxLedger && - shardIndex > seqToShardIndex(opt.maxLedger)) - return false; - auto [r, total] = detail::getOldestAccountTxs( - session, app_, ledgerMaster, opt, limit_used, j_); - ret.insert(ret.end(), r.begin(), r.end()); - if (!total) - return false; - if (total > 0) - { - limit_used += total; - opt.offset = 0; - } - else - { - /* - * If total < 0, then -total means number of transactions - * skipped, see definition of return value of function - * ripple::getOldestAccountTxs(). - */ - total = -total; - if (opt.offset <= total) - opt.offset = 0; - else - opt.offset -= total; - } - return true; - }); - - return ret; - } - return {}; } @@ -1152,52 +593,10 @@ SQLiteDatabaseImp::getNewestAccountTxs(AccountTxOptions const& options) if (existsTransaction()) { auto db = checkoutTransaction(); - return detail::getNewestAccountTxs( - *db, app_, ledgerMaster, options, {}, j_) + return detail::getNewestAccountTxs(*db, app_, ledgerMaster, options, j_) .first; } - if (shardStoreExists()) - { - AccountTxs ret; - AccountTxOptions opt = options; - int limit_used = 0; - iterateTransactionBack( - opt.maxLedger ? seqToShardIndex(opt.maxLedger) - : std::optional(), - [&](soci::session& session, std::uint32_t shardIndex) { - if (opt.minLedger && - shardIndex < seqToShardIndex(opt.minLedger)) - return false; - auto [r, total] = detail::getNewestAccountTxs( - session, app_, ledgerMaster, opt, limit_used, j_); - ret.insert(ret.end(), r.begin(), r.end()); - if (!total) - return false; - if (total > 0) - { - limit_used += total; - opt.offset = 0; - } - else - { - /* - * If total < 0, then -total means number of transactions - * skipped, see definition of return value of function - * ripple::getNewestAccountTxs(). - */ - total = -total; - if (opt.offset <= total) - opt.offset = 0; - else - opt.offset -= total; - } - return true; - }); - - return ret; - } - return {}; } @@ -1210,48 +609,7 @@ SQLiteDatabaseImp::getOldestAccountTxsB(AccountTxOptions const& options) if (existsTransaction()) { auto db = checkoutTransaction(); - return detail::getOldestAccountTxsB(*db, app_, options, {}, j_).first; - } - - if (shardStoreExists()) - { - MetaTxsList ret; - AccountTxOptions opt = options; - int limit_used = 0; - iterateTransactionForward( - opt.minLedger ? seqToShardIndex(opt.minLedger) - : std::optional(), - [&](soci::session& session, std::uint32_t shardIndex) { - if (opt.maxLedger && - shardIndex > seqToShardIndex(opt.maxLedger)) - return false; - auto [r, total] = detail::getOldestAccountTxsB( - session, app_, opt, limit_used, j_); - ret.insert(ret.end(), r.begin(), r.end()); - if (!total) - return false; - if (total > 0) - { - limit_used += total; - opt.offset = 0; - } - else - { - /* - * If total < 0, then -total means number of transactions - * skipped, see definition of return value of function - * ripple::getOldestAccountTxsB(). - */ - total = -total; - if (opt.offset <= total) - opt.offset = 0; - else - opt.offset -= total; - } - return true; - }); - - return ret; + return detail::getOldestAccountTxsB(*db, app_, options, j_).first; } return {}; @@ -1266,48 +624,7 @@ SQLiteDatabaseImp::getNewestAccountTxsB(AccountTxOptions const& options) if (existsTransaction()) { auto db = checkoutTransaction(); - return detail::getNewestAccountTxsB(*db, app_, options, {}, j_).first; - } - - if (shardStoreExists()) - { - MetaTxsList ret; - AccountTxOptions opt = options; - int limit_used = 0; - iterateTransactionBack( - opt.maxLedger ? seqToShardIndex(opt.maxLedger) - : std::optional(), - [&](soci::session& session, std::uint32_t shardIndex) { - if (opt.minLedger && - shardIndex < seqToShardIndex(opt.minLedger)) - return false; - auto [r, total] = detail::getNewestAccountTxsB( - session, app_, opt, limit_used, j_); - ret.insert(ret.end(), r.begin(), r.end()); - if (!total) - return false; - if (total > 0) - { - limit_used += total; - opt.offset = 0; - } - else - { - /* - * If total < 0, then -total means number of transactions - * skipped, see definition of return value of function - * ripple::getNewestAccountTxsB(). - */ - total = -total; - if (opt.offset <= total) - opt.offset = 0; - else - opt.offset -= total; - } - return true; - }); - - return ret; + return detail::getNewestAccountTxsB(*db, app_, options, j_).first; } return {}; @@ -1339,39 +656,11 @@ SQLiteDatabaseImp::oldestAccountTxPage(AccountTxPageOptions const& options) auto db = checkoutTransaction(); auto newmarker = detail::oldestAccountTxPage( - *db, onUnsavedLedger, onTransaction, options, 0, page_length) + *db, onUnsavedLedger, onTransaction, options, page_length) .first; return {ret, newmarker}; } - if (shardStoreExists()) - { - AccountTxPageOptions opt = options; - int limit_used = 0; - iterateTransactionForward( - opt.minLedger ? seqToShardIndex(opt.minLedger) - : std::optional(), - [&](soci::session& session, std::uint32_t shardIndex) { - if (opt.maxLedger != UINT32_MAX && - shardIndex > seqToShardIndex(opt.minLedger)) - return false; - auto [marker, total] = detail::oldestAccountTxPage( - session, - onUnsavedLedger, - onTransaction, - opt, - limit_used, - page_length); - opt.marker = marker; - if (total < 0) - return false; - limit_used += total; - return true; - }); - - return {ret, opt.marker}; - } - return {}; } @@ -1401,39 +690,11 @@ SQLiteDatabaseImp::newestAccountTxPage(AccountTxPageOptions const& options) auto db = checkoutTransaction(); auto newmarker = detail::newestAccountTxPage( - *db, onUnsavedLedger, onTransaction, options, 0, page_length) + *db, onUnsavedLedger, onTransaction, options, page_length) .first; return {ret, newmarker}; } - if (shardStoreExists()) - { - AccountTxPageOptions opt = options; - int limit_used = 0; - iterateTransactionBack( - opt.maxLedger != UINT32_MAX ? seqToShardIndex(opt.maxLedger) - : std::optional(), - [&](soci::session& session, std::uint32_t shardIndex) { - if (opt.minLedger && - shardIndex < seqToShardIndex(opt.minLedger)) - return false; - auto [marker, total] = detail::newestAccountTxPage( - session, - onUnsavedLedger, - onTransaction, - opt, - limit_used, - page_length); - opt.marker = marker; - if (total < 0) - return false; - limit_used += total; - return true; - }); - - return {ret, opt.marker}; - } - return {}; } @@ -1462,39 +723,11 @@ SQLiteDatabaseImp::oldestAccountTxPageB(AccountTxPageOptions const& options) auto db = checkoutTransaction(); auto newmarker = detail::oldestAccountTxPage( - *db, onUnsavedLedger, onTransaction, options, 0, page_length) + *db, onUnsavedLedger, onTransaction, options, page_length) .first; return {ret, newmarker}; } - if (shardStoreExists()) - { - AccountTxPageOptions opt = options; - int limit_used = 0; - iterateTransactionForward( - opt.minLedger ? seqToShardIndex(opt.minLedger) - : std::optional(), - [&](soci::session& session, std::uint32_t shardIndex) { - if (opt.maxLedger != UINT32_MAX && - shardIndex > seqToShardIndex(opt.minLedger)) - return false; - auto [marker, total] = detail::oldestAccountTxPage( - session, - onUnsavedLedger, - onTransaction, - opt, - limit_used, - page_length); - opt.marker = marker; - if (total < 0) - return false; - limit_used += total; - return true; - }); - - return {ret, opt.marker}; - } - return {}; } @@ -1523,39 +756,11 @@ SQLiteDatabaseImp::newestAccountTxPageB(AccountTxPageOptions const& options) auto db = checkoutTransaction(); auto newmarker = detail::newestAccountTxPage( - *db, onUnsavedLedger, onTransaction, options, 0, page_length) + *db, onUnsavedLedger, onTransaction, options, page_length) .first; return {ret, newmarker}; } - if (shardStoreExists()) - { - AccountTxPageOptions opt = options; - int limit_used = 0; - iterateTransactionBack( - opt.maxLedger != UINT32_MAX ? seqToShardIndex(opt.maxLedger) - : std::optional(), - [&](soci::session& session, std::uint32_t shardIndex) { - if (opt.minLedger && - shardIndex < seqToShardIndex(opt.minLedger)) - return false; - auto [marker, total] = detail::newestAccountTxPage( - session, - onUnsavedLedger, - onTransaction, - opt, - limit_used, - page_length); - opt.marker = marker; - if (total < 0) - return false; - limit_used += total; - return true; - }); - - return {ret, opt.marker}; - } - return {}; } @@ -1574,37 +779,6 @@ SQLiteDatabaseImp::getTransaction( return detail::getTransaction(*db, app_, id, range, ec); } - if (auto shardStore = app_.getShardStore(); shardStore) - { - std::variant res(TxSearched::unknown); - auto txMetaSession = txMetaDB_->checkoutDb(); - - if (auto const shardIndex = - detail::getShardIndexforTransaction(*txMetaSession, id)) - { - shardStore->callForTransactionSQLByShardIndex( - *shardIndex, [&](soci::session& session) { - std::optional> range1; - if (range) - { - std::uint32_t const low = std::max( - range->lower(), firstLedgerSeq(*shardIndex)); - std::uint32_t const high = std::min( - range->upper(), lastLedgerSeq(*shardIndex)); - if (low <= high) - range1 = ClosedInterval(low, high); - } - res = detail::getTransaction(session, app_, id, range1, ec); - - return res.index() == 1 && - std::get(res) != - TxSearched::unknown; // unused - }); - } - - return res; - } - return TxSearched::unknown; } @@ -1617,14 +791,6 @@ SQLiteDatabaseImp::ledgerDbHasSpace(Config const& config) return detail::dbHasSpace(*db, config, j_); } - if (shardStoreExists()) - { - return iterateLedgerBack( - {}, [&](soci::session& session, std::uint32_t shardIndex) { - return detail::dbHasSpace(session, config, j_); - }); - } - return true; } @@ -1640,14 +806,6 @@ SQLiteDatabaseImp::transactionDbHasSpace(Config const& config) return detail::dbHasSpace(*db, config, j_); } - if (shardStoreExists()) - { - return iterateTransactionBack( - {}, [&](soci::session& session, std::uint32_t shardIndex) { - return detail::dbHasSpace(session, config, j_); - }); - } - return true; } @@ -1659,17 +817,6 @@ SQLiteDatabaseImp::getKBUsedAll() return ripple::getKBUsedAll(lgrdb_->getSession()); } - if (shardStoreExists()) - { - std::uint32_t sum = 0; - iterateLedgerBack( - {}, [&](soci::session& session, std::uint32_t shardIndex) { - sum += ripple::getKBUsedAll(session); - return true; - }); - return sum; - } - return 0; } @@ -1681,17 +828,6 @@ SQLiteDatabaseImp::getKBUsedLedger() return ripple::getKBUsedDB(lgrdb_->getSession()); } - if (shardStoreExists()) - { - std::uint32_t sum = 0; - iterateLedgerBack( - {}, [&](soci::session& session, std::uint32_t shardIndex) { - sum += ripple::getKBUsedDB(session); - return true; - }); - return sum; - } - return 0; } @@ -1706,17 +842,6 @@ SQLiteDatabaseImp::getKBUsedTransaction() return ripple::getKBUsedDB(txdb_->getSession()); } - if (shardStoreExists()) - { - std::uint32_t sum = 0; - iterateTransactionBack( - {}, [&](soci::session& session, std::uint32_t shardIndex) { - sum += ripple::getKBUsedDB(session); - return true; - }); - return sum; - } - return 0; } diff --git a/src/xrpld/app/rdb/backend/detail/Shard.h b/src/xrpld/app/rdb/backend/detail/Shard.h deleted file mode 100644 index 870b6b82fe4..00000000000 --- a/src/xrpld/app/rdb/backend/detail/Shard.h +++ /dev/null @@ -1,90 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_APP_RDB_BACKEND_DETAIL_SHARD_H_INCLUDED -#define RIPPLE_APP_RDB_BACKEND_DETAIL_SHARD_H_INCLUDED - -#include -#include -#include -#include -#include -#include - -namespace ripple { -namespace detail { - -/** - * @brief makeMetaDBs Opens ledger and transaction 'meta' databases which - * map ledger hashes and transaction IDs to the index of the shard - * that holds the ledger or transaction. - * @param config Config object. - * @param setup Path to database and opening parameters. - * @param checkpointerSetup Database checkpointer setup. - * @return Struct DatabasePair which contains unique pointers to the ledger - * and transaction databases. - */ -DatabasePair -makeMetaDBs( - Config const& config, - DatabaseCon::Setup const& setup, - DatabaseCon::CheckpointerSetup const& checkpointerSetup); - -/** - * @brief saveLedgerMeta Stores (transaction ID -> shard index) and - * (ledger hash -> shard index) mappings in the meta databases. - * @param ledger The ledger. - * @param app Application object. - * @param lgrMetaSession Session to ledger meta database. - * @param txnMetaSession Session to transaction meta database. - * @param shardIndex The index of the shard that contains this ledger. - * @return True on success. - */ -bool -saveLedgerMeta( - std::shared_ptr const& ledger, - Application& app, - soci::session& lgrMetaSession, - soci::session& txnMetaSession, - std::uint32_t shardIndex); - -/** - * @brief getShardIndexforLedger Queries the ledger meta database to - * retrieve the index of the shard that contains this ledger. - * @param session Session to the database. - * @param hash Hash of the ledger. - * @return The index of the shard on success, otherwise an unseated value. - */ -std::optional -getShardIndexforLedger(soci::session& session, LedgerHash const& hash); - -/** - * @brief getShardIndexforTransaction Queries the transaction meta database to - * retrieve the index of the shard that contains this transaction. - * @param session Session to the database. - * @param id ID of the transaction. - * @return The index of the shard on success, otherwise an unseated value. - */ -std::optional -getShardIndexforTransaction(soci::session& session, TxID const& id); - -} // namespace detail -} // namespace ripple - -#endif diff --git a/src/xrpld/app/rdb/backend/detail/detail/Shard.cpp b/src/xrpld/app/rdb/backend/detail/detail/Shard.cpp deleted file mode 100644 index 6db64b1249b..00000000000 --- a/src/xrpld/app/rdb/backend/detail/detail/Shard.cpp +++ /dev/null @@ -1,147 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { -namespace detail { - -DatabasePair -makeMetaDBs( - Config const& config, - DatabaseCon::Setup const& setup, - DatabaseCon::CheckpointerSetup const& checkpointerSetup) -{ - // ledger meta database - auto lgrMetaDB{std::make_unique( - setup, - LgrMetaDBName, - LgrMetaDBPragma, - LgrMetaDBInit, - checkpointerSetup)}; - - if (!config.useTxTables()) - return {std::move(lgrMetaDB), nullptr}; - - // transaction meta database - auto txMetaDB{std::make_unique( - setup, TxMetaDBName, TxMetaDBPragma, TxMetaDBInit, checkpointerSetup)}; - - return {std::move(lgrMetaDB), std::move(txMetaDB)}; -} - -bool -saveLedgerMeta( - std::shared_ptr const& ledger, - Application& app, - soci::session& lgrMetaSession, - soci::session& txnMetaSession, - std::uint32_t const shardIndex) -{ - std::string_view constexpr lgrSQL = - R"sql(INSERT OR REPLACE INTO LedgerMeta VALUES - (:ledgerHash,:shardIndex);)sql"; - - auto const hash = to_string(ledger->info().hash); - lgrMetaSession << lgrSQL, soci::use(hash), soci::use(shardIndex); - - if (!app.config().useTxTables()) - return true; - - auto const aLedger = [&app, ledger]() -> std::shared_ptr { - try - { - auto aLedger = - app.getAcceptedLedgerCache().fetch(ledger->info().hash); - if (!aLedger) - { - aLedger = std::make_shared(ledger, app); - app.getAcceptedLedgerCache().canonicalize_replace_client( - ledger->info().hash, aLedger); - } - - return aLedger; - } - catch (std::exception const&) - { - JLOG(app.journal("Ledger").warn()) - << "An accepted ledger was missing nodes"; - } - - return {}; - }(); - - if (!aLedger) - return false; - - soci::transaction tr(txnMetaSession); - - for (auto const& acceptedLedgerTx : *aLedger) - { - std::string_view constexpr txnSQL = - R"sql(INSERT OR REPLACE INTO TransactionMeta VALUES - (:transactionID,:shardIndex);)sql"; - - auto const transactionID = - to_string(acceptedLedgerTx->getTransactionID()); - - txnMetaSession << txnSQL, soci::use(transactionID), - soci::use(shardIndex); - } - - tr.commit(); - return true; -} - -std::optional -getShardIndexforLedger(soci::session& session, LedgerHash const& hash) -{ - std::uint32_t shardIndex; - session << "SELECT ShardIndex FROM LedgerMeta WHERE LedgerHash = '" << hash - << "';", - soci::into(shardIndex); - - if (!session.got_data()) - return std::nullopt; - - return shardIndex; -} - -std::optional -getShardIndexforTransaction(soci::session& session, TxID const& id) -{ - std::uint32_t shardIndex; - session << "SELECT ShardIndex FROM TransactionMeta WHERE TransID = '" << id - << "';", - soci::into(shardIndex); - - if (!session.got_data()) - return std::nullopt; - - return shardIndex; -} - -} // namespace detail -} // namespace ripple diff --git a/src/xrpld/app/rdb/detail/Download.cpp b/src/xrpld/app/rdb/detail/Download.cpp deleted file mode 100644 index 012d60b3734..00000000000 --- a/src/xrpld/app/rdb/detail/Download.cpp +++ /dev/null @@ -1,152 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2021 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include - -namespace ripple { - -std::pair, std::optional> -openDatabaseBodyDb( - DatabaseCon::Setup const& setup, - boost::filesystem::path const& path) -{ - // SOCI requires boost::optional (not std::optional) as the parameter. - boost::optional pathFromDb; - boost::optional size; - - auto conn = std::make_unique( - setup, "Download", DownloaderDBPragma, DatabaseBodyDBInit); - - auto& session = *conn->checkoutDb(); - - session << "SELECT Path FROM Download WHERE Part=0;", - soci::into(pathFromDb); - - // Try to reuse preexisting - // database. - if (pathFromDb) - { - // Can't resuse - database was - // from a different file download. - if (pathFromDb != path.string()) - { - session << "DROP TABLE Download;"; - } - - // Continuing a file download. - else - { - session << "SELECT SUM(LENGTH(Data)) FROM Download;", - soci::into(size); - } - } - - return {std::move(conn), (size ? *size : std::optional())}; -} - -std::uint64_t -databaseBodyDoPut( - soci::session& session, - std::string const& data, - std::string const& path, - std::uint64_t fileSize, - std::uint64_t part, - std::uint16_t maxRowSizePad) -{ - std::uint64_t rowSize = 0; - soci::indicator rti; - - std::uint64_t remainingInRow = 0; - - auto be = - dynamic_cast(session.get_backend()); - BOOST_ASSERT(be); - - // This limits how large we can make the blob - // in each row. Also subtract a pad value to - // account for the other values in the row. - auto const blobMaxSize = - sqlite_api::sqlite3_limit(be->conn_, SQLITE_LIMIT_LENGTH, -1) - - maxRowSizePad; - - std::string newpath; - - auto rowInit = [&] { - session << "INSERT INTO Download VALUES (:path, zeroblob(0), 0, :part)", - soci::use(newpath), soci::use(part); - - remainingInRow = blobMaxSize; - rowSize = 0; - }; - - session << "SELECT Path,Size,Part FROM Download ORDER BY Part DESC " - "LIMIT 1", - soci::into(newpath), soci::into(rowSize), soci::into(part, rti); - - if (!session.got_data()) - { - newpath = path; - rowInit(); - } - else - remainingInRow = blobMaxSize - rowSize; - - auto insert = [&session, &rowSize, &part, &fs = fileSize]( - auto const& data) { - std::uint64_t updatedSize = rowSize + data.size(); - - session << "UPDATE Download SET Data = CAST(Data || :data AS blob), " - "Size = :size WHERE Part = :part;", - soci::use(data), soci::use(updatedSize), soci::use(part); - - fs += data.size(); - }; - - size_t currentBase = 0; - - while (currentBase + remainingInRow < data.size()) - { - if (remainingInRow) - { - insert(data.substr(currentBase, remainingInRow)); - currentBase += remainingInRow; - } - - ++part; - rowInit(); - } - - insert(data.substr(currentBase)); - - return part; -} - -void -databaseBodyFinish(soci::session& session, std::ofstream& fout) -{ - soci::rowset rs = - (session.prepare << "SELECT Data FROM Download ORDER BY PART ASC;"); - - // iteration through the resultset: - for (auto it = rs.begin(); it != rs.end(); ++it) - fout.write(it->data(), it->size()); -} - -} // namespace ripple diff --git a/src/xrpld/app/rdb/detail/RelationalDatabase.cpp b/src/xrpld/app/rdb/detail/RelationalDatabase.cpp index 874550abd97..07dc27fd1d3 100644 --- a/src/xrpld/app/rdb/detail/RelationalDatabase.cpp +++ b/src/xrpld/app/rdb/detail/RelationalDatabase.cpp @@ -20,7 +20,6 @@ #include #include #include -#include namespace ripple { diff --git a/src/xrpld/app/rdb/detail/ShardArchive.cpp b/src/xrpld/app/rdb/detail/ShardArchive.cpp deleted file mode 100644 index 81b99348cb4..00000000000 --- a/src/xrpld/app/rdb/detail/ShardArchive.cpp +++ /dev/null @@ -1,68 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2021 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include - -namespace ripple { - -std::unique_ptr -makeArchiveDB(boost::filesystem::path const& dir, std::string const& dbName) -{ - return std::make_unique( - dir, dbName, DownloaderDBPragma, ShardArchiveHandlerDBInit); -} - -void -readArchiveDB( - DatabaseCon& db, - std::function const& func) -{ - soci::rowset rs = - (db.getSession().prepare << "SELECT * FROM State;"); - - for (auto it = rs.begin(); it != rs.end(); ++it) - { - func(it->get(1), it->get(0)); - } -} - -void -insertArchiveDB( - DatabaseCon& db, - std::uint32_t shardIndex, - std::string const& url) -{ - db.getSession() << "INSERT INTO State VALUES (:index, :url);", - soci::use(shardIndex), soci::use(url); -} - -void -deleteFromArchiveDB(DatabaseCon& db, std::uint32_t shardIndex) -{ - db.getSession() << "DELETE FROM State WHERE ShardIndex = :index;", - soci::use(shardIndex); -} - -void -dropArchiveDB(DatabaseCon& db) -{ - db.getSession() << "DROP TABLE State;"; -} - -} // namespace ripple diff --git a/src/xrpld/app/rdb/detail/UnitaryShard.cpp b/src/xrpld/app/rdb/detail/UnitaryShard.cpp deleted file mode 100644 index dd64759f4a8..00000000000 --- a/src/xrpld/app/rdb/detail/UnitaryShard.cpp +++ /dev/null @@ -1,320 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2021 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include - -namespace ripple { - -DatabasePair -makeShardCompleteLedgerDBs( - Config const& config, - DatabaseCon::Setup const& setup) -{ - auto tx{std::make_unique( - setup, TxDBName, FinalShardDBPragma, TxDBInit)}; - tx->getSession() << boost::str( - boost::format("PRAGMA cache_size=-%d;") % - kilobytes(config.getValueFor(SizedItem::txnDBCache, std::nullopt))); - - auto lgr{std::make_unique( - setup, LgrDBName, FinalShardDBPragma, LgrDBInit)}; - lgr->getSession() << boost::str( - boost::format("PRAGMA cache_size=-%d;") % - kilobytes(config.getValueFor(SizedItem::lgrDBCache, std::nullopt))); - - return {std::move(lgr), std::move(tx)}; -} - -DatabasePair -makeShardIncompleteLedgerDBs( - Config const& config, - DatabaseCon::Setup const& setup, - DatabaseCon::CheckpointerSetup const& checkpointerSetup) -{ - // transaction database - auto tx{std::make_unique( - setup, TxDBName, TxDBPragma, TxDBInit, checkpointerSetup)}; - tx->getSession() << boost::str( - boost::format("PRAGMA cache_size=-%d;") % - kilobytes(config.getValueFor(SizedItem::txnDBCache))); - - // ledger database - auto lgr{std::make_unique( - setup, LgrDBName, LgrDBPragma, LgrDBInit, checkpointerSetup)}; - lgr->getSession() << boost::str( - boost::format("PRAGMA cache_size=-%d;") % - kilobytes(config.getValueFor(SizedItem::lgrDBCache))); - - return {std::move(lgr), std::move(tx)}; -} - -bool -updateLedgerDBs( - soci::session& txsession, - soci::session& lgrsession, - std::shared_ptr const& ledger, - std::uint32_t index, - std::atomic& stop, - beast::Journal j) -{ - auto const ledgerSeq{ledger->info().seq}; - - // Update the transactions database - { - auto& session{txsession}; - soci::transaction tr(session); - - session << "DELETE FROM Transactions " - "WHERE LedgerSeq = :seq;", - soci::use(ledgerSeq); - session << "DELETE FROM AccountTransactions " - "WHERE LedgerSeq = :seq;", - soci::use(ledgerSeq); - - if (ledger->info().txHash.isNonZero()) - { - auto const sSeq{std::to_string(ledgerSeq)}; - if (!ledger->txMap().isValid()) - { - JLOG(j.error()) - << "shard " << index << " has an invalid transaction map" - << " on sequence " << sSeq; - return false; - } - - for (auto const& item : ledger->txs) - { - if (stop.load(std::memory_order_relaxed)) - return false; - - TxMeta const txMeta{ - item.first->getTransactionID(), - ledger->seq(), - *item.second}; - - auto const sTxID = to_string(txMeta.getTxID()); - - session << "DELETE FROM AccountTransactions " - "WHERE TransID = :txID;", - soci::use(sTxID); - - auto const& accounts = txMeta.getAffectedAccounts(); - if (!accounts.empty()) - { - auto const sTxnSeq{std::to_string(txMeta.getIndex())}; - auto const s{boost::str( - boost::format("('%s','%s',%s,%s)") % sTxID % "%s" % - sSeq % sTxnSeq)}; - std::string sql; - sql.reserve((accounts.size() + 1) * 128); - sql = - "INSERT INTO AccountTransactions " - "(TransID, Account, LedgerSeq, TxnSeq) VALUES "; - sql += boost::algorithm::join( - accounts | - boost::adaptors::transformed( - [&](AccountID const& accountID) { - return boost::str( - boost::format(s) % - ripple::toBase58(accountID)); - }), - ","); - sql += ';'; - session << sql; - - JLOG(j.trace()) - << "shard " << index << " account transaction: " << sql; - } - else if (!isPseudoTx(*item.first)) - { - // It's okay for pseudo transactions to not affect any - // accounts. But otherwise... - JLOG(j.warn()) - << "shard " << index << " transaction in ledger " - << sSeq << " affects no accounts"; - } - - Serializer s; - item.second->add(s); - session - << (STTx::getMetaSQLInsertReplaceHeader() + - item.first->getMetaSQL( - ledgerSeq, sqlBlobLiteral(s.modData())) + - ';'); - } - } - - tr.commit(); - } - - auto const sHash{to_string(ledger->info().hash)}; - - // Update the ledger database - { - auto& session{lgrsession}; - soci::transaction tr(session); - - auto const sParentHash{to_string(ledger->info().parentHash)}; - auto const sDrops{to_string(ledger->info().drops)}; - auto const closingTime{ - ledger->info().closeTime.time_since_epoch().count()}; - auto const prevClosingTime{ - ledger->info().parentCloseTime.time_since_epoch().count()}; - auto const closeTimeRes{ledger->info().closeTimeResolution.count()}; - auto const sAccountHash{to_string(ledger->info().accountHash)}; - auto const sTxHash{to_string(ledger->info().txHash)}; - - session << "DELETE FROM Ledgers " - "WHERE LedgerSeq = :seq;", - soci::use(ledgerSeq); - session << "INSERT OR REPLACE INTO Ledgers (" - "LedgerHash, LedgerSeq, PrevHash, TotalCoins, ClosingTime," - "PrevClosingTime, CloseTimeRes, CloseFlags, AccountSetHash," - "TransSetHash)" - "VALUES (" - ":ledgerHash, :ledgerSeq, :prevHash, :totalCoins," - ":closingTime, :prevClosingTime, :closeTimeRes," - ":closeFlags, :accountSetHash, :transSetHash);", - soci::use(sHash), soci::use(ledgerSeq), soci::use(sParentHash), - soci::use(sDrops), soci::use(closingTime), - soci::use(prevClosingTime), soci::use(closeTimeRes), - soci::use(ledger->info().closeFlags), soci::use(sAccountHash), - soci::use(sTxHash); - - tr.commit(); - } - - return true; -} - -std::unique_ptr -makeAcquireDB( - DatabaseCon::Setup const& setup, - DatabaseCon::CheckpointerSetup const& checkpointerSetup) -{ - return std::make_unique( - setup, - AcquireShardDBName, - AcquireShardDBPragma, - AcquireShardDBInit, - checkpointerSetup); -} - -void -insertAcquireDBIndex(soci::session& session, std::uint32_t index) -{ - session << "INSERT INTO Shard (ShardIndex) " - "VALUES (:shardIndex);", - soci::use(index); -} - -std::pair> -selectAcquireDBLedgerSeqs(soci::session& session, std::uint32_t index) -{ - // resIndex and must be boost::optional (not std) because that's - // what SOCI expects in its interface. - boost::optional resIndex; - soci::blob sociBlob(session); - soci::indicator blobPresent; - - session << "SELECT ShardIndex, StoredLedgerSeqs " - "FROM Shard " - "WHERE ShardIndex = :index;", - soci::into(resIndex), soci::into(sociBlob, blobPresent), - soci::use(index); - - if (!resIndex || index != resIndex) - return {false, {}}; - - if (blobPresent != soci::i_ok) - return {true, {}}; - - std::string s; - convert(sociBlob, s); - - return {true, s}; -} - -std::pair -selectAcquireDBLedgerSeqsHash(soci::session& session, std::uint32_t index) -{ - // resIndex and sHash0 must be boost::optional (not std) because that's - // what SOCI expects in its interface. - boost::optional resIndex; - boost::optional sHash0; - soci::blob sociBlob(session); - soci::indicator blobPresent; - - session << "SELECT ShardIndex, LastLedgerHash, StoredLedgerSeqs " - "FROM Shard " - "WHERE ShardIndex = :index;", - soci::into(resIndex), soci::into(sHash0), - soci::into(sociBlob, blobPresent), soci::use(index); - - std::optional sHash = - (sHash0 ? *sHash0 : std::optional()); - - if (!resIndex || index != resIndex) - return {false, {{}, {}}}; - - if (blobPresent != soci::i_ok) - return {true, {{}, sHash}}; - - std::string s; - convert(sociBlob, s); - - return {true, {s, sHash}}; -} - -void -updateAcquireDB( - soci::session& session, - std::shared_ptr const& ledger, - std::uint32_t index, - std::uint32_t lastSeq, - std::optional const& seqs) -{ - soci::blob sociBlob(session); - auto const sHash{to_string(ledger->info().hash)}; - - if (seqs) - convert(*seqs, sociBlob); - - if (ledger->info().seq == lastSeq) - { - // Store shard's last ledger hash - session << "UPDATE Shard " - "SET LastLedgerHash = :lastLedgerHash," - "StoredLedgerSeqs = :storedLedgerSeqs " - "WHERE ShardIndex = :shardIndex;", - soci::use(sHash), soci::use(sociBlob), soci::use(index); - } - else - { - session << "UPDATE Shard " - "SET StoredLedgerSeqs = :storedLedgerSeqs " - "WHERE ShardIndex = :shardIndex;", - soci::use(sociBlob), soci::use(index); - } -} - -} // namespace ripple diff --git a/src/xrpld/core/Config.h b/src/xrpld/core/Config.h index 2872193f0ee..e63e6d2f356 100644 --- a/src/xrpld/core/Config.h +++ b/src/xrpld/core/Config.h @@ -146,7 +146,6 @@ class Config : public BasicConfig public: bool doImport = false; - bool nodeToShard = false; bool ELB_SUPPORT = false; // Entries from [ips] config stanza diff --git a/src/xrpld/core/ConfigSections.h b/src/xrpld/core/ConfigSections.h index b4e460f1cfc..8685d29a4d0 100644 --- a/src/xrpld/core/ConfigSections.h +++ b/src/xrpld/core/ConfigSections.h @@ -35,11 +35,6 @@ struct ConfigSection return "node_db"; } static std::string - shardDatabase() - { - return "shard_db"; - } - static std::string importNodeDatabase() { return "import_db"; @@ -56,7 +51,6 @@ struct ConfigSection #define SECTION_ELB_SUPPORT "elb_support" #define SECTION_FEE_DEFAULT "fee_default" #define SECTION_FETCH_DEPTH "fetch_depth" -#define SECTION_HISTORICAL_SHARD_PATHS "historical_shard_paths" #define SECTION_INSIGHT "insight" #define SECTION_IO_WORKERS "io_workers" #define SECTION_IPS "ips" diff --git a/src/xrpld/core/Job.h b/src/xrpld/core/Job.h index c5926ae2e08..76d26c39e72 100644 --- a/src/xrpld/core/Job.h +++ b/src/xrpld/core/Job.h @@ -47,7 +47,6 @@ enum JobType { jtCLIENT_FEE_CHANGE, // Subscription for fee change by a client jtCLIENT_CONSENSUS, // Subscription for consensus state change by a client jtCLIENT_ACCT_HIST, // Subscription for account history by a client - jtCLIENT_SHARD, // Client request for shard archiving jtCLIENT_RPC, // Client RPC request jtCLIENT_WEBSOCKET, // Client websocket request jtRPC, // A websocket command from the client diff --git a/src/xrpld/core/JobTypes.h b/src/xrpld/core/JobTypes.h index 2dbc45ca1b5..3b41ce7ff47 100644 --- a/src/xrpld/core/JobTypes.h +++ b/src/xrpld/core/JobTypes.h @@ -84,7 +84,6 @@ class JobTypes add(jtCLIENT_FEE_CHANGE, "clientFeeChange", maxLimit, 2000ms, 5000ms); add(jtCLIENT_CONSENSUS, "clientConsensus", maxLimit, 2000ms, 5000ms); add(jtCLIENT_ACCT_HIST, "clientAccountHistory", maxLimit, 2000ms, 5000ms); - add(jtCLIENT_SHARD, "clientShardArchive", maxLimit, 2000ms, 5000ms); add(jtCLIENT_RPC, "clientRPC", maxLimit, 2000ms, 5000ms); add(jtCLIENT_WEBSOCKET, "clientWebsocket", maxLimit, 2000ms, 5000ms); add(jtRPC, "RPC", maxLimit, 0ms, 0ms); diff --git a/src/xrpld/net/DatabaseBody.h b/src/xrpld/net/DatabaseBody.h deleted file mode 100644 index 15780ced313..00000000000 --- a/src/xrpld/net/DatabaseBody.h +++ /dev/null @@ -1,179 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_NET_DATABASEBODY_H -#define RIPPLE_NET_DATABASEBODY_H - -#include -#include -#include -#include -#include - -namespace ripple { - -// DatabaseBody needs to meet requirements -// from asio which is why some conventions -// used elsewhere in this code base are not -// followed. -struct DatabaseBody -{ - // Algorithm for storing buffers when parsing. - class reader; - - // The type of the @ref message::body member. - class value_type; - - /** Returns the size of the body - - @param body The database body to use - */ - static std::uint64_t - size(value_type const& body); -}; - -class DatabaseBody::value_type -{ - // This body container holds a connection to the - // database, and also caches the size when set. - - friend class reader; - friend struct DatabaseBody; - - // The cached file size - std::uint64_t fileSize_ = 0; - boost::filesystem::path path_; - std::unique_ptr conn_; - std::string batch_; - std::shared_ptr strand_; - std::mutex m_; - std::condition_variable c_; - std::uint64_t handlerCount_ = 0; - std::uint64_t part_ = 0; - bool closing_ = false; - -public: - /// Destructor - ~value_type() = default; - - /// Constructor - value_type() = default; - - /// Returns `true` if the file is open - bool - is_open() const - { - return static_cast(conn_); - } - - /// Returns the size of the file if open - std::uint64_t - size() const - { - return fileSize_; - } - - /// Close the file if open - void - close(); - - /** Open a file at the given path with the specified mode - - @param path The utf-8 encoded path to the file - - @param config The configuration settings - - @param io_service The asio context for running a strand. - - @param ec Set to the error, if any occurred - */ - void - open( - boost::filesystem::path const& path, - Config const& config, - boost::asio::io_service& io_service, - boost::system::error_code& ec); -}; - -/** Algorithm for storing buffers when parsing. - - Objects of this type are created during parsing - to store incoming buffers representing the body. -*/ -class DatabaseBody::reader -{ - value_type& body_; // The body we are writing to - - static constexpr std::uint32_t FLUSH_SIZE = 50000000; - static constexpr std::uint8_t MAX_HANDLERS = 3; - static constexpr std::uint16_t MAX_ROW_SIZE_PAD = 500; - -public: - // Constructor. - // - // This is called after the header is parsed and - // indicates that a non-zero sized body may be present. - // `h` holds the received message headers. - // `b` is an instance of `DatabaseBody`. - // - template - explicit reader( - boost::beast::http::header& h, - value_type& b); - - // Initializer - // - // This is called before the body is parsed and - // gives the reader a chance to do something that might - // need to return an error code. It informs us of - // the payload size (`content_length`) which we can - // optionally use for optimization. - // - // Note: boost::Beast calls init() and requires a - // boost::optional (not a std::optional) as the - // parameter. - void - init(boost::optional const&, boost::system::error_code& ec); - - // This function is called one or more times to store - // buffer sequences corresponding to the incoming body. - // - template - std::size_t - put(ConstBufferSequence const& buffers, boost::system::error_code& ec); - - void - do_put(std::string const& data); - - // This function is called when writing is complete. - // It is an opportunity to perform any final actions - // which might fail, in order to return an error code. - // Operations that might fail should not be attempted in - // destructors, since an exception thrown from there - // would terminate the program. - // - void - finish(boost::system::error_code& ec); -}; - -} // namespace ripple - -#include - -#endif // RIPPLE_NET_DATABASEBODY_H diff --git a/src/xrpld/net/DatabaseDownloader.h b/src/xrpld/net/DatabaseDownloader.h deleted file mode 100644 index 490e7c62e16..00000000000 --- a/src/xrpld/net/DatabaseDownloader.h +++ /dev/null @@ -1,76 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_NET_DATABASEDOWNLOADER_H -#define RIPPLE_NET_DATABASEDOWNLOADER_H - -#include -#include - -namespace ripple { - -class DatabaseDownloader : public HTTPDownloader -{ -public: - virtual ~DatabaseDownloader() = default; - -private: - DatabaseDownloader( - boost::asio::io_service& io_service, - Config const& config, - beast::Journal j); - - static const std::uint8_t MAX_PATH_LEN = - std::numeric_limits::max(); - - std::shared_ptr - getParser( - boost::filesystem::path dstPath, - std::function complete, - boost::system::error_code& ec) override; - - bool - checkPath(boost::filesystem::path const& dstPath) override; - - void - closeBody(std::shared_ptr p) override; - - std::uint64_t - size(std::shared_ptr p) override; - - Config const& config_; - boost::asio::io_service& io_service_; - - friend std::shared_ptr - make_DatabaseDownloader( - boost::asio::io_service& io_service, - Config const& config, - beast::Journal j); -}; - -// DatabaseDownloader must be a shared_ptr because it uses shared_from_this -std::shared_ptr -make_DatabaseDownloader( - boost::asio::io_service& io_service, - Config const& config, - beast::Journal j); - -} // namespace ripple - -#endif // RIPPLE_NET_DATABASEDOWNLOADER_H diff --git a/src/xrpld/net/HTTPDownloader.h b/src/xrpld/net/HTTPDownloader.h deleted file mode 100644 index f96fb8e572b..00000000000 --- a/src/xrpld/net/HTTPDownloader.h +++ /dev/null @@ -1,130 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2018 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_NET_HTTPDOWNLOADER_H_INCLUDED -#define RIPPLE_NET_HTTPDOWNLOADER_H_INCLUDED - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace ripple { - -/** Provides an asynchronous HTTP[S] file downloader - */ -class HTTPDownloader : public std::enable_shared_from_this -{ -public: - using error_code = boost::system::error_code; - - bool - download( - std::string const& host, - std::string const& port, - std::string const& target, - int version, - boost::filesystem::path const& dstPath, - std::function complete, - bool ssl = true); - - void - stop(); - - virtual ~HTTPDownloader() = default; - - bool - sessionIsActive() const; - - bool - isStopping() const; - -protected: - // must be accessed through a shared_ptr - // use make_XXX functions to create - HTTPDownloader( - boost::asio::io_service& io_service, - Config const& config, - beast::Journal j); - - using parser = boost::beast::http::basic_parser; - - beast::Journal const j_; - - void - fail( - boost::filesystem::path dstPath, - boost::system::error_code const& ec, - std::string const& errMsg, - std::shared_ptr parser); - -private: - Config const& config_; - boost::asio::io_service::strand strand_; - std::unique_ptr stream_; - boost::beast::flat_buffer read_buf_; - std::atomic stop_; - - // Used to protect sessionActive_ - mutable std::mutex m_; - bool sessionActive_; - std::condition_variable c_; - - void - do_session( - std::string host, - std::string port, - std::string target, - int version, - boost::filesystem::path dstPath, - std::function complete, - bool ssl, - boost::asio::yield_context yield); - - virtual std::shared_ptr - getParser( - boost::filesystem::path dstPath, - std::function complete, - boost::system::error_code& ec) = 0; - - virtual bool - checkPath(boost::filesystem::path const& dstPath) = 0; - - virtual void - closeBody(std::shared_ptr p) = 0; - - virtual uint64_t - size(std::shared_ptr p) = 0; -}; - -} // namespace ripple - -#endif diff --git a/src/xrpld/net/HTTPStream.h b/src/xrpld/net/HTTPStream.h deleted file mode 100644 index 275d8ca9544..00000000000 --- a/src/xrpld/net/HTTPStream.h +++ /dev/null @@ -1,165 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_NET_HTTPSTREAM_H_INCLUDED -#define RIPPLE_NET_HTTPSTREAM_H_INCLUDED - -#include -#include - -#include -#include -#include -#include - -#include - -namespace ripple { - -class HTTPStream -{ -public: - using request = boost::beast::http::request; - using parser = boost::beast::http::basic_parser; - - virtual ~HTTPStream() = default; - - [[nodiscard]] virtual boost::asio::ip::tcp::socket& - getStream() = 0; - - [[nodiscard]] virtual bool - connect( - std::string& errorOut, - std::string const& host, - std::string const& port, - boost::asio::yield_context& yield) = 0; - - virtual void - asyncWrite( - request& req, - boost::asio::yield_context& yield, - boost::system::error_code& ec) = 0; - - virtual void - asyncRead( - boost::beast::flat_buffer& buf, - parser& p, - boost::asio::yield_context& yield, - boost::system::error_code& ec) = 0; - - virtual void - asyncReadSome( - boost::beast::flat_buffer& buf, - parser& p, - boost::asio::yield_context& yield, - boost::system::error_code& ec) = 0; -}; - -class SSLStream : public HTTPStream -{ -public: - SSLStream( - Config const& config, - boost::asio::io_service::strand& strand, - beast::Journal j); - - virtual ~SSLStream() = default; - - boost::asio::ip::tcp::socket& - getStream() override; - - bool - connect( - std::string& errorOut, - std::string const& host, - std::string const& port, - boost::asio::yield_context& yield) override; - - void - asyncWrite( - request& req, - boost::asio::yield_context& yield, - boost::system::error_code& ec) override; - - void - asyncRead( - boost::beast::flat_buffer& buf, - parser& p, - boost::asio::yield_context& yield, - boost::system::error_code& ec) override; - - void - asyncReadSome( - boost::beast::flat_buffer& buf, - parser& p, - boost::asio::yield_context& yield, - boost::system::error_code& ec) override; - -private: - HTTPClientSSLContext ssl_ctx_; - std::optional> - stream_; - boost::asio::io_service::strand& strand_; -}; - -class RawStream : public HTTPStream -{ -public: - RawStream(boost::asio::io_service::strand& strand); - - virtual ~RawStream() = default; - - boost::asio::ip::tcp::socket& - getStream() override; - - bool - connect( - std::string& errorOut, - std::string const& host, - std::string const& port, - boost::asio::yield_context& yield) override; - - void - asyncWrite( - request& req, - boost::asio::yield_context& yield, - boost::system::error_code& ec) override; - - void - asyncRead( - boost::beast::flat_buffer& buf, - parser& p, - boost::asio::yield_context& yield, - boost::system::error_code& ec) override; - - void - asyncReadSome( - boost::beast::flat_buffer& buf, - parser& p, - boost::asio::yield_context& yield, - boost::system::error_code& ec) override; - -private: - std::optional stream_; - boost::asio::io_service::strand& strand_; -}; - -} // namespace ripple - -#endif // RIPPLE_NET_HTTPSTREAM_H diff --git a/src/xrpld/net/ShardDownloader.md b/src/xrpld/net/ShardDownloader.md deleted file mode 100644 index d961df61c65..00000000000 --- a/src/xrpld/net/ShardDownloader.md +++ /dev/null @@ -1,311 +0,0 @@ -# Shard Downloader - -## Overview - -This document describes mechanics of the `HTTPDownloader`, a class that performs -the task of downloading shards from remote web servers via HTTP. The downloader -utilizes a strand (`boost::asio::io_service::strand`) to ensure that downloads -are never executed concurrently. Hence, if a download is in progress when -another download is initiated, the second download will be queued and invoked -only when the first download is completed. - -## Motivation - -In March 2020 the downloader was modified to include some key features: - -- The ability to stop downloads during a graceful shutdown. -- The ability to resume partial downloads after a crash or shutdown. - -This document was created to document the changes introduced by this change. - -## Classes - -Much of the shard downloading process concerns the following classes: - -- `HTTPDownloader` - - This is a generic class designed for serially executing downloads via HTTP. - -- `ShardArchiveHandler` - - This class uses the `HTTPDownloader` to fetch shards from remote web servers. - Additionally, the archive handler performs validity checks on the downloaded - files and imports the validated files into the local shard store. - - The `ShardArchiveHandler` exposes a simple public interface: - - ```C++ - /** Add an archive to be downloaded and imported. - @param shardIndex the index of the shard to be imported. - @param url the location of the archive. - @return `true` if successfully added. - @note Returns false if called while downloading. - */ - bool - add(std::uint32_t shardIndex, std::pair&& url); - - /** Starts downloading and importing archives. */ - bool - start(); - ``` - - When a client submits a `download_shard` command via the RPC interface, each - of the requested files is registered with the handler via the `add` method. - After all the files have been registered, the handler's `start` method is - invoked, which in turn creates an instance of the `HTTPDownloader` and begins - the first download. When the download is completed, the downloader invokes - the handler's `complete` method, which will initiate the download of the next - file, or simply return if there are no more downloads to process. When - `complete` is invoked with no remaining files to be downloaded, the handler - and downloader are not destroyed automatically, but persist for the duration - of the application to assist with graceful shutdowns. - -- `DatabaseBody` - - This class defines a custom message body type, allowing an - `http::response_parser` to write to an SQLite database rather than to a flat - file. This class is discussed in further detail in the Recovery section. - -## Graceful Shutdowns & Recovery - -This section describes in greater detail how the shutdown and recovery features -of the downloader are implemented in C++ using the `boost::asio` framework. - -##### Member Variables: - -The variables shown here are members of the `HTTPDownloader` class and -will be used in the following code examples. - -```c++ -std::unique_ptr stream_; -std::condition_variable c_; -std::atomic stop_; -``` - -### Graceful Shutdowns - -##### Thread 1: - -A graceful shutdown begins when the `stop()` method of the -`ShardArchiveHandler` is invoked: - -```c++ -void -ShardArchiveHandler::stop() -{ - std::lock_guard lock(m_); - - if (downloader_) - { - downloader_->stop(); - downloader_.reset(); - } - - stopped(); -} -``` - -Inside of `HTTPDownloader::stop()`, if a download is currently in progress, -the `stop_` member variable is set and the thread waits for the -download to stop: - -```c++ -void -HTTPDownloader::stop() -{ - std::unique_lock lock(m_); - - stop_ = true; - - if(sessionActive_) - { - // Wait for the handler to exit. - c_.wait(lock, - [this]() - { - return !sessionActive_; - }); - } -} -``` - -##### Thread 2: - -The graceful shutdown is realized when the thread executing the download polls -`stop_` after this variable has been set to `true`. Polling occurs -while the file is being downloaded, in between calls to `async_read_some()`. The -stop takes effect when the socket is closed and the handler function ( -`do_session()` ) is exited. - -```c++ -void HTTPDownloader::do_session() -{ - - // (Connection initialization logic) . . . - - - // (In between calls to async_read_some): - if(stop_.load()) - { - close(p); - return exit(); - } - - // . . . - - break; -} -``` - -### Recovery - -Persisting the current state of both the archive handler and the downloader is -achieved by leveraging an SQLite database rather than flat files, as the -database protects against data corruption that could result from a system crash. - -##### ShardArchiveHandler - -Although `HTTPDownloader` is a generic class that could be used to download a -variety of file types, currently it is used exclusively by the -`ShardArchiveHandler` to download shards. In order to provide resilience, the -`ShardArchiveHandler` will use an SQLite database to preserve its current state -whenever there are active, paused, or queued downloads. The `shard_db` section -in the configuration file allows users to specify the location of the database -to use for this purpose. - -###### SQLite Table Format - -| Index | URL | -|:-----:|:-----------------------------------:| -| 1 | https://example.com/1.tar.lz4 | -| 2 | https://example.com/2.tar.lz4 | -| 5 | https://example.com/5.tar.lz4 | - -##### HTTPDownloader - -While the archive handler maintains a list of all partial and queued downloads, -the `HTTPDownloader` stores the raw bytes of the file currently being -downloaded. The partially downloaded file will be represented as one or more -`BLOB` entries in an SQLite database. As the maximum size of a `BLOB` entry is -currently limited to roughly 2.1 GB, a 5 GB shard file for instance will occupy -three database entries upon completion. - -###### SQLite Table Format - -Since downloads execute serially by design, the entries in this table always -correspond to the contents of a single file. - -| Bytes | size | Part | -|:------:|:----------:|:----:| -| 0x... | 2147483647 | 0 | -| 0x... | 2147483647 | 1 | -| 0x... | 705032706 | 2 | - -##### Config File Entry -The `download_path` field of the `shard_db` entry is used to determine where to -store the recovery database. If this field is omitted, the `path` field will be -used instead. - -```dosini -# This is the persistent datastore for shards. It is important for the health -# of the network that rippled operators shard as much as practical. -# NuDB requires SSD storage. Helpful information can be found on -# https://xrpl.org/history-sharding.html -[shard_db] -type=NuDB -path=/var/lib/rippled/db/shards/nudb -download_path=/var/lib/rippled/db/shards/ -max_historical_shards=50 -``` - -##### Resuming Partial Downloads -When resuming downloads after a shutdown, crash, or other interruption, the -`HTTPDownloader` will utilize the `range` field of the HTTP header to download -only the remainder of the partially downloaded file. - -```C++ -auto downloaded = getPartialFileSize(); -auto total = getTotalFileSize(); - -http::request req {http::verb::head, - target, - version}; - -if (downloaded < total) -{ - // If we already downloaded 1000 bytes to the database, - // the range header will look like: - // Range: "bytes=1000-" - req.set(http::field::range, "bytes=" + to_string(downloaded) + "-"); -} -else if(downloaded == total) -{ - // Download is already complete. (Interruption must - // have occurred after file was downloaded but before - // the state file was updated.) -} -else -{ - // The size of the partially downloaded file exceeds - // the total download size. Error condition. Handle - // appropriately. -} -``` - -##### DatabaseBody - -Previously, the `HTTPDownloader` leveraged an `http::response_parser` -instantiated with an `http::file_body`. The `file_body` class declares a nested -type, `reader`, which does the task of writing HTTP message payloads -(constituting a requested file) to the filesystem. In order for the -`http::response_parser` to interface with the database, we implement a custom -body type that declares a nested `reader` type which has been outfitted to -persist octects received from the remote host to a local SQLite database. The -code snippet below illustrates the customization points available to -user-defined body types: - -```C++ -/// Defines a Body type -struct body -{ - /// This determines the return type of the `message::body` member function - using value_type = ...; - - /// An optional function, returns the body's payload size (which may be - /// zero) - static - std::uint64_t - size(value_type const& v); - - /// The algorithm used for extracting buffers - class reader; - - /// The algorithm used for inserting buffers - class writer; -} - -``` -Note that the `DatabaseBody` class is specifically designed to work with `asio` -and follows `asio` conventions. - -The method invoked to write data to the filesystem (or SQLite database in our -case) has the following signature: - -```C++ -std::size_t -body::reader::put(ConstBufferSequence const& buffers, error_code& ec); -``` - -## Sequence Diagram - -This sequence diagram demonstrates a scenario wherein the `ShardArchiveHandler` -leverages the state persisted in the database to recover from a crash and resume -the requested downloads. - -![alt_text](./images/interrupt_sequence.png "Resuming downloads post abort") - -## State Diagram - -This diagram illustrates the various states of the Shard Downloader module. - -![alt_text](./images/states.png "Shard Downloader states") diff --git a/src/xrpld/net/detail/DatabaseBody.ipp b/src/xrpld/net/detail/DatabaseBody.ipp deleted file mode 100644 index 76223ca6a35..00000000000 --- a/src/xrpld/net/detail/DatabaseBody.ipp +++ /dev/null @@ -1,231 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include - -namespace ripple { - -inline void -DatabaseBody::value_type::close() -{ - { - std::unique_lock lock(m_); - - // Stop all scheduled and currently - // executing handlers before closing. - if (handlerCount_) - { - closing_ = true; - - auto predicate = [&] { return !handlerCount_; }; - c_.wait(lock, predicate); - } - - conn_.reset(); - } -} - -inline void -DatabaseBody::value_type::open( - boost::filesystem::path const& path, - Config const& config, - boost::asio::io_service& io_service, - boost::system::error_code& ec) -{ - strand_.reset(new boost::asio::io_service::strand(io_service)); - path_ = path; - - auto setup = setup_DatabaseCon(config); - setup.dataDir = path.parent_path(); - setup.useGlobalPragma = false; - - auto [conn, size] = openDatabaseBodyDb(setup, path); - conn_ = std::move(conn); - if (size) - fileSize_ = *size; -} - -// This is called from message::payload_size -inline std::uint64_t -DatabaseBody::size(value_type const& body) -{ - // Forward the call to the body - return body.size(); -} - -// We don't do much in the reader constructor since the -// database is already open. -// -template -DatabaseBody::reader::reader( - boost::beast::http::header&, - value_type& body) - : body_(body) -{ -} - -// We don't do anything with content_length but a sophisticated -// application might check available space on the device -// to see if there is enough room to store the body. -inline void -DatabaseBody::reader::init( - boost::optional const& /*content_length*/, - boost::system::error_code& ec) -{ - // The connection must already be available for writing - assert(body_.conn_); - - // The error_code specification requires that we - // either set the error to some value, or set it - // to indicate no error. - // - // We don't do anything fancy so set "no error" - ec = {}; -} - -// This will get called one or more times with body buffers -// -template -std::size_t -DatabaseBody::reader::put( - ConstBufferSequence const& buffers, - boost::system::error_code& ec) -{ - // This function must return the total number of - // bytes transferred from the input buffers. - std::size_t nwritten = 0; - - // Loop over all the buffers in the sequence, - // and write each one to the database. - for (auto it = buffer_sequence_begin(buffers); - it != buffer_sequence_end(buffers); - ++it) - { - boost::asio::const_buffer buffer = *it; - - body_.batch_.append( - static_cast(buffer.data()), buffer.size()); - - // Write this buffer to the database - if (body_.batch_.size() > FLUSH_SIZE) - { - bool post = true; - - { - std::lock_guard lock(body_.m_); - - if (body_.handlerCount_ >= MAX_HANDLERS) - post = false; - else - ++body_.handlerCount_; - } - - if (post) - { - body_.strand_->post( - [data = body_.batch_, this] { this->do_put(data); }); - - body_.batch_.clear(); - } - } - - nwritten += it->size(); - } - - // Indicate success - // This is required by the error_code specification - ec = {}; - - return nwritten; -} - -inline void -DatabaseBody::reader::do_put(std::string const& data) -{ - using namespace boost::asio; - - { - std::unique_lock lock(body_.m_); - - // The download is being halted. - if (body_.closing_) - { - if (--body_.handlerCount_ == 0) - { - lock.unlock(); - body_.c_.notify_one(); - } - - return; - } - } - - auto path = body_.path_.string(); - - { - auto db = body_.conn_->checkoutDb(); - body_.part_ = databaseBodyDoPut( - *db, data, path, body_.fileSize_, body_.part_, MAX_ROW_SIZE_PAD); - } - - bool const notify = [this] { - std::lock_guard lock(body_.m_); - return --body_.handlerCount_ == 0; - }(); - - if (notify) - body_.c_.notify_one(); -} - -// Called after writing is done when there's no error. -inline void -DatabaseBody::reader::finish(boost::system::error_code& ec) -{ - { - std::unique_lock lock(body_.m_); - - // Wait for scheduled DB writes - // to complete. - if (body_.handlerCount_) - { - auto predicate = [&] { return !body_.handlerCount_; }; - body_.c_.wait(lock, predicate); - } - } - - std::ofstream fout; - fout.open(body_.path_.string(), std::ios::binary | std::ios::out); - - { - auto db = body_.conn_->checkoutDb(); - databaseBodyFinish(*db, fout); - } - - // Flush any pending data that hasn't - // been been written to the DB. - if (body_.batch_.size()) - { - fout.write(body_.batch_.data(), body_.batch_.size()); - body_.batch_.clear(); - } - - fout.close(); -} - -} // namespace ripple diff --git a/src/xrpld/net/detail/DatabaseDownloader.cpp b/src/xrpld/net/detail/DatabaseDownloader.cpp deleted file mode 100644 index b39e6904c46..00000000000 --- a/src/xrpld/net/detail/DatabaseDownloader.cpp +++ /dev/null @@ -1,92 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include - -namespace ripple { - -std::shared_ptr -make_DatabaseDownloader( - boost::asio::io_service& io_service, - Config const& config, - beast::Journal j) -{ - return std::shared_ptr( - new DatabaseDownloader(io_service, config, j)); -} - -DatabaseDownloader::DatabaseDownloader( - boost::asio::io_service& io_service, - Config const& config, - beast::Journal j) - : HTTPDownloader(io_service, config, j) - , config_(config) - , io_service_(io_service) -{ -} - -auto -DatabaseDownloader::getParser( - boost::filesystem::path dstPath, - std::function complete, - boost::system::error_code& ec) -> std::shared_ptr -{ - using namespace boost::beast; - - auto p = std::make_shared>(); - p->body_limit(std::numeric_limits::max()); - p->get().body().open(dstPath, config_, io_service_, ec); - - if (ec) - p->get().body().close(); - - return p; -} - -bool -DatabaseDownloader::checkPath(boost::filesystem::path const& dstPath) -{ - return dstPath.string().size() <= MAX_PATH_LEN; -} - -void -DatabaseDownloader::closeBody(std::shared_ptr p) -{ - using namespace boost::beast; - - auto databaseBodyParser = - std::dynamic_pointer_cast>(p); - assert(databaseBodyParser); - - databaseBodyParser->get().body().close(); -} - -std::uint64_t -DatabaseDownloader::size(std::shared_ptr p) -{ - using namespace boost::beast; - - auto databaseBodyParser = - std::dynamic_pointer_cast>(p); - assert(databaseBodyParser); - - return databaseBodyParser->get().body().size(); -} - -} // namespace ripple diff --git a/src/xrpld/net/detail/HTTPDownloader.cpp b/src/xrpld/net/detail/HTTPDownloader.cpp deleted file mode 100644 index 760aa020e4a..00000000000 --- a/src/xrpld/net/detail/HTTPDownloader.cpp +++ /dev/null @@ -1,340 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include - -namespace ripple { - -HTTPDownloader::HTTPDownloader( - boost::asio::io_service& io_service, - Config const& config, - beast::Journal j) - : j_(j) - , config_(config) - , strand_(io_service) - , stop_(false) - , sessionActive_(false) -{ -} - -bool -HTTPDownloader::download( - std::string const& host, - std::string const& port, - std::string const& target, - int version, - boost::filesystem::path const& dstPath, - std::function complete, - bool ssl) -{ - if (!checkPath(dstPath)) - return false; - - if (stop_) - return true; - - { - std::lock_guard lock(m_); - sessionActive_ = true; - } - - if (!strand_.running_in_this_thread()) - strand_.post(std::bind( - &HTTPDownloader::download, - shared_from_this(), - host, - port, - target, - version, - dstPath, - complete, - ssl)); - else - boost::asio::spawn( - strand_, - std::bind( - &HTTPDownloader::do_session, - shared_from_this(), - host, - port, - target, - version, - dstPath, - complete, - ssl, - std::placeholders::_1)); - return true; -} - -void -HTTPDownloader::do_session( - std::string const host, - std::string const port, - std::string const target, - int version, - boost::filesystem::path dstPath, - std::function complete, - bool ssl, - boost::asio::yield_context yield) -{ - using namespace boost::asio; - using namespace boost::beast; - - boost::system::error_code ec; - bool skip = false; - - ////////////////////////////////////////////// - // Define lambdas for encapsulating download - // operations: - auto close = [&](auto p) { - closeBody(p); - - // Gracefully close the stream - stream_->getStream().shutdown(socket_base::shutdown_both, ec); - if (ec == boost::asio::error::eof) - ec.assign(0, ec.category()); - if (ec) - { - // Most web servers don't bother with performing - // the SSL shutdown handshake, for speed. - JLOG(j_.trace()) << "shutdown: " << ec.message(); - } - - // The stream cannot be reused - stream_.reset(); - }; - - // When the downloader is being stopped - // because the server is shutting down, - // this method notifies a caller of `onStop` - // (`RPC::ShardArchiveHandler` to be specific) - // that the session has ended. - auto exit = [this, &dstPath, complete] { - if (!stop_) - complete(std::move(dstPath)); - - std::lock_guard lock(m_); - sessionActive_ = false; - c_.notify_one(); - }; - - auto failAndExit = [&exit, &dstPath, complete, &ec, this]( - std::string const& errMsg, auto p) { - fail(dstPath, ec, errMsg, p); - exit(); - }; - // end lambdas - //////////////////////////////////////////////////////////// - - if (stop_.load()) - return exit(); - - auto p = this->getParser(dstPath, complete, ec); - if (ec) - return failAndExit("getParser", p); - - ////////////////////////////////////////////// - // Prepare for download and establish the - // connection: - if (ssl) - stream_ = std::make_unique(config_, strand_, j_); - else - stream_ = std::make_unique(strand_); - - std::string error; - if (!stream_->connect(error, host, port, yield)) - return failAndExit(error, p); - - // Set up an HTTP HEAD request message to find the file size - http::request req{http::verb::head, target, version}; - req.set(http::field::host, host); - req.set(http::field::user_agent, BOOST_BEAST_VERSION_STRING); - - std::uint64_t const rangeStart = size(p); - - // Requesting a portion of the file - if (rangeStart) - { - req.set( - http::field::range, - (boost::format("bytes=%llu-") % rangeStart).str()); - } - - stream_->asyncWrite(req, yield, ec); - if (ec) - return failAndExit("async_write", p); - - { - // Read the response - http::response_parser connectParser; - connectParser.skip(true); - stream_->asyncRead(read_buf_, connectParser, yield, ec); - if (ec) - return failAndExit("async_read", p); - - // Range request was rejected - if (connectParser.get().result() == http::status::range_not_satisfiable) - { - req.erase(http::field::range); - - stream_->asyncWrite(req, yield, ec); - if (ec) - return failAndExit("async_write_range_verify", p); - - http::response_parser rangeParser; - rangeParser.skip(true); - - stream_->asyncRead(read_buf_, rangeParser, yield, ec); - if (ec) - return failAndExit("async_read_range_verify", p); - - // The entire file is downloaded already. - if (rangeParser.content_length() == rangeStart) - skip = true; - else - return failAndExit("range_not_satisfiable", p); - } - else if ( - rangeStart && - connectParser.get().result() != http::status::partial_content) - { - ec.assign( - boost::system::errc::not_supported, - boost::system::generic_category()); - - return failAndExit("Range request ignored", p); - } - else if (auto len = connectParser.content_length()) - { - try - { - // Ensure sufficient space is available - if (*len > space(dstPath.parent_path()).available) - { - return failAndExit( - "Insufficient disk space for download", p); - } - } - catch (std::exception const& e) - { - return failAndExit(std::string("exception: ") + e.what(), p); - } - } - } - - if (!skip) - { - // Set up an HTTP GET request message to download the file - req.method(http::verb::get); - - if (rangeStart) - { - req.set( - http::field::range, - (boost::format("bytes=%llu-") % rangeStart).str()); - } - } - - stream_->asyncWrite(req, yield, ec); - if (ec) - return failAndExit("async_write", p); - - // end prepare and connect - //////////////////////////////////////////////////////////// - - if (skip) - p->skip(true); - - // Download the file - while (!p->is_done()) - { - if (stop_.load()) - { - close(p); - return exit(); - } - - stream_->asyncReadSome(read_buf_, *p, yield, ec); - } - - JLOG(j_.trace()) << "download completed: " << dstPath.string(); - - close(p); - exit(); -} - -void -HTTPDownloader::stop() -{ - stop_ = true; - - std::unique_lock lock(m_); - if (sessionActive_) - { - // Wait for the handler to exit. - c_.wait(lock, [this]() { return !sessionActive_; }); - } -} - -bool -HTTPDownloader::sessionIsActive() const -{ - std::lock_guard lock(m_); - return sessionActive_; -} - -bool -HTTPDownloader::isStopping() const -{ - std::lock_guard lock(m_); - return stop_; -} - -void -HTTPDownloader::fail( - boost::filesystem::path dstPath, - boost::system::error_code const& ec, - std::string const& errMsg, - std::shared_ptr parser) -{ - if (!ec) - { - JLOG(j_.error()) << errMsg; - } - else if (ec != boost::asio::error::operation_aborted) - { - JLOG(j_.error()) << errMsg << ": " << ec.message(); - } - - if (parser) - closeBody(parser); - - try - { - remove(dstPath); - } - catch (std::exception const& e) - { - JLOG(j_.error()) << "exception: " << e.what() - << " in function: " << __func__; - } -} - -} // namespace ripple diff --git a/src/xrpld/net/detail/HTTPStream.cpp b/src/xrpld/net/detail/HTTPStream.cpp deleted file mode 100644 index b94f8959ec9..00000000000 --- a/src/xrpld/net/detail/HTTPStream.cpp +++ /dev/null @@ -1,203 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include - -namespace ripple { - -SSLStream::SSLStream( - Config const& config, - boost::asio::io_service::strand& strand, - beast::Journal j) - : ssl_ctx_(config, j, boost::asio::ssl::context::tlsv12_client) - , strand_(strand) -{ -} - -boost::asio::ip::tcp::socket& -SSLStream::getStream() -{ - assert(stream_); - return stream_->next_layer(); -} - -bool -SSLStream::connect( - std::string& errorOut, - std::string const& host, - std::string const& port, - boost::asio::yield_context& yield) -{ - using namespace boost::asio; - using namespace boost::beast; - - boost::system::error_code ec; - - auto fail = [&errorOut, &ec]( - std::string const& errorIn, - std::string const& message = "") { - errorOut = errorIn + ": " + (message.empty() ? ec.message() : message); - return false; - }; - - ip::tcp::resolver resolver{strand_.context()}; - auto const endpoints = resolver.async_resolve(host, port, yield[ec]); - if (ec) - return fail("async_resolve"); - - try - { - stream_.emplace(strand_.context(), ssl_ctx_.context()); - } - catch (std::exception const& e) - { - return fail("exception", e.what()); - } - - ec = ssl_ctx_.preConnectVerify(*stream_, host); - if (ec) - return fail("preConnectVerify"); - - boost::asio::async_connect( - stream_->next_layer(), endpoints.begin(), endpoints.end(), yield[ec]); - if (ec) - return fail("async_connect"); - - ec = ssl_ctx_.postConnectVerify(*stream_, host); - if (ec) - return fail("postConnectVerify"); - - stream_->async_handshake(ssl::stream_base::client, yield[ec]); - if (ec) - return fail("async_handshake"); - - return true; -} - -void -SSLStream::asyncWrite( - request& req, - boost::asio::yield_context& yield, - boost::system::error_code& ec) -{ - boost::beast::http::async_write(*stream_, req, yield[ec]); -} - -void -SSLStream::asyncRead( - boost::beast::flat_buffer& buf, - parser& p, - boost::asio::yield_context& yield, - boost::system::error_code& ec) -{ - boost::beast::http::async_read(*stream_, buf, p, yield[ec]); -} - -void -SSLStream::asyncReadSome( - boost::beast::flat_buffer& buf, - parser& p, - boost::asio::yield_context& yield, - boost::system::error_code& ec) -{ - boost::beast::http::async_read_some(*stream_, buf, p, yield[ec]); -} - -RawStream::RawStream(boost::asio::io_service::strand& strand) : strand_(strand) -{ -} - -boost::asio::ip::tcp::socket& -RawStream::getStream() -{ - assert(stream_); - return *stream_; -} - -bool -RawStream::connect( - std::string& errorOut, - std::string const& host, - std::string const& port, - boost::asio::yield_context& yield) -{ - using namespace boost::asio; - using namespace boost::beast; - - boost::system::error_code ec; - - auto fail = [&errorOut, &ec]( - std::string const& errorIn, - std::string const& message = "") { - errorOut = errorIn + ": " + (message.empty() ? ec.message() : message); - return false; - }; - - ip::tcp::resolver resolver{strand_.context()}; - auto const endpoints = resolver.async_resolve(host, port, yield[ec]); - if (ec) - return fail("async_resolve"); - - try - { - stream_.emplace(strand_.context()); - } - catch (std::exception const& e) - { - return fail("exception", e.what()); - } - - boost::asio::async_connect( - *stream_, endpoints.begin(), endpoints.end(), yield[ec]); - if (ec) - return fail("async_connect"); - - return true; -} - -void -RawStream::asyncWrite( - request& req, - boost::asio::yield_context& yield, - boost::system::error_code& ec) -{ - boost::beast::http::async_write(*stream_, req, yield[ec]); -} - -void -RawStream::asyncRead( - boost::beast::flat_buffer& buf, - parser& p, - boost::asio::yield_context& yield, - boost::system::error_code& ec) -{ - boost::beast::http::async_read(*stream_, buf, p, yield[ec]); -} - -void -RawStream::asyncReadSome( - boost::beast::flat_buffer& buf, - parser& p, - boost::asio::yield_context& yield, - boost::system::error_code& ec) -{ - boost::beast::http::async_read_some(*stream_, buf, p, yield[ec]); -} - -} // namespace ripple diff --git a/src/xrpld/net/detail/RPCCall.cpp b/src/xrpld/net/detail/RPCCall.cpp index 533878ab1b0..997d6463f23 100644 --- a/src/xrpld/net/detail/RPCCall.cpp +++ b/src/xrpld/net/detail/RPCCall.cpp @@ -189,36 +189,6 @@ class RPCParser return v; } - Json::Value - parseDownloadShard(Json::Value const& jvParams) - { - Json::Value jvResult(Json::objectValue); - unsigned int sz{jvParams.size()}; - unsigned int i{0}; - - // If odd number of params then 'novalidate' may have been specified - if (sz & 1) - { - if (boost::iequals(jvParams[0u].asString(), "novalidate")) - ++i; - else if (!boost::iequals(jvParams[--sz].asString(), "novalidate")) - return rpcError(rpcINVALID_PARAMS); - } - - // Create the 'shards' array - Json::Value shards(Json::arrayValue); - for (; i < sz; i += 2) - { - Json::Value shard(Json::objectValue); - shard[jss::index] = jvParams[i].asUInt(); - shard[jss::url] = jvParams[i + 1].asString(); - shards.append(std::move(shard)); - } - jvResult[jss::shards] = std::move(shards); - - return jvResult; - } - Json::Value parseInternal(Json::Value const& jvParams) { @@ -870,15 +840,6 @@ class RPCParser return jvRequest; } - Json::Value - parseNodeToShard(Json::Value const& jvParams) - { - Json::Value jvRequest; - jvRequest[jss::action] = jvParams[0u].asString(); - - return jvRequest; - } - // peer_reservations_add [] Json::Value parsePeerReservationsAdd(Json::Value const& jvParams) @@ -1200,9 +1161,7 @@ class RPCParser {"channel_verify", &RPCParser::parseChannelVerify, 4, 4}, {"connect", &RPCParser::parseConnect, 1, 2}, {"consensus_info", &RPCParser::parseAsIs, 0, 0}, - {"crawl_shards", &RPCParser::parseAsIs, 0, 2}, {"deposit_authorized", &RPCParser::parseDepositAuthorized, 2, 3}, - {"download_shard", &RPCParser::parseDownloadShard, 2, -1}, {"feature", &RPCParser::parseFeature, 0, 2}, {"fetch_info", &RPCParser::parseFetchInfo, 0, 1}, {"gateway_balances", &RPCParser::parseGatewayBalances, 1, -1}, @@ -1220,7 +1179,6 @@ class RPCParser {"log_level", &RPCParser::parseLogLevel, 0, 2}, {"logrotate", &RPCParser::parseAsIs, 0, 0}, {"manifest", &RPCParser::parseManifest, 1, 1}, - {"node_to_shard", &RPCParser::parseNodeToShard, 1, 1}, {"owner_info", &RPCParser::parseAccountItems, 1, 3}, {"peers", &RPCParser::parseAsIs, 0, 0}, {"ping", &RPCParser::parseAsIs, 0, 0}, diff --git a/src/xrpld/net/uml/interrupt_sequence.pu b/src/xrpld/net/uml/interrupt_sequence.pu deleted file mode 100644 index ba046d084f8..00000000000 --- a/src/xrpld/net/uml/interrupt_sequence.pu +++ /dev/null @@ -1,233 +0,0 @@ -@startuml - - -skinparam shadowing false - -/' -skinparam sequence { - ArrowColor #e1e4e8 - ActorBorderColor #e1e4e8 - DatabaseBorderColor #e1e4e8 - LifeLineBorderColor Black - LifeLineBackgroundColor #d3d6d9 - - ParticipantBorderColor DeepSkyBlue - ParticipantBackgroundColor DodgerBlue - ParticipantFontName Impact - ParticipantFontSize 17 - ParticipantFontColor #A9DCDF - - NoteBackgroundColor #6a737d - - ActorBackgroundColor #f6f8fa - ActorFontColor #6a737d - ActorFontSize 17 - ActorFontName Aapex - - EntityBackgroundColor #f6f8fa - EntityFontColor #6a737d - EntityFontSize 17 - EntityFontName Aapex - - DatabaseBackgroundColor #f6f8fa - DatabaseFontColor #6a737d - DatabaseFontSize 17 - DatabaseFontName Aapex - - CollectionsBackgroundColor #f6f8fa - ActorFontColor #6a737d - ActorFontSize 17 - ActorFontName Aapex -} - -skinparam note { - BackgroundColor #fafbfc - BorderColor #e1e4e8 -} -'/ - -'skinparam monochrome true - -actor Client as c -entity RippleNode as rn -entity ShardArchiveHandler as sa -entity SSLHTTPDownloader as d -database Database as db -collections Fileserver as s - -c -> rn: Launch RippleNode -activate rn - -c -> rn: Issue download request - -note right of c - **Download Request:** - - { - "method": "download_shard", - "params": - [ - { - "shards": - [ - {"index": 1, "url": "https://example.com/1.tar.lz4"}, - {"index": 2, "url": "https://example.com/2.tar.lz4"}, - {"index": 5, "url": "https://example.com/5.tar.lz4"} - ] - } - ] - } -end note - -rn -> sa: Create instance of Handler -activate sa - -rn -> sa: Add three downloads -sa -> sa: Validate requested downloads - -rn -> sa: Initiate Downloads -sa -> rn: ACK: Initiating -rn -> c: Initiating requested downloads - -sa -> db: Save state to the database\n(Processing three downloads) - -note right of db - - **ArchiveHandler State (SQLite Table):** - - | Index | URL | - | 1 | https://example.com/1.tar.lz4 | - | 2 | https://example.com/2.tar.lz4 | - | 5 | https://example.com/5.tar.lz4 | - -end note - -sa -> d: Create instance of Downloader -activate d - -group Download 1 - - note over sa - **Download 1:** - - This encapsulates the download of the first file - at URL "https://example.com/1.tar.lz4". - - end note - - sa -> d: Start download - - d -> s: Connect and request file - s -> d: Send file - d -> sa: Invoke completion handler - -end - -sa -> sa: Import and validate shard - -sa -> db: Update persisted state\n(Remove download) - -note right of db - **ArchiveHandler State:** - - | Index | URL | - | 2 | https://example.com/2.tar.lz4 | - | 5 | https://example.com/5.tar.lz4 | - -end note - -group Download 2 - - sa -> d: Start download - - d -> s: Connect and request file - -end - -rn -> rn: **RippleNode crashes** - -deactivate sa -deactivate rn -deactivate d - -c -> rn: Restart RippleNode -activate rn - -rn -> db: Detect non-empty state database - -rn -> sa: Create instance of Handler -activate sa - -sa -> db: Load state - -note right of db - **ArchiveHandler State:** - - | Index | URL | - | 2 | https://example.com/2.tar.lz4 | - | 5 | https://example.com/5.tar.lz4 | - -end note - -sa -> d: Create instance of Downloader -activate d - -sa -> sa: Resume Download 2 - -group Download 2 - - sa -> d: Start download - - d -> s: Connect and request file - s -> d: Send file - d -> sa: Invoke completion handler - -end - -sa -> sa: Import and validate shard - -sa -> db: Update persisted state \n(Remove download) - -note right of db - **ArchiveHandler State:** - - | Index | URL | - | 5 | https://example.com/5.tar.lz4 | - -end note - -group Download 3 - - sa -> d: Start download - - d -> s: Connect and request file - s -> d: Send file - d -> sa: Invoke completion handler - -end - -sa -> sa: Import and validate shard - -sa -> db: Update persisted state \n(Remove download) - -note right of db - **ArchiveHandler State:** - - ***empty*** - -end note - -sa -> db: Remove empty database - -sa -> sa: Automatically destroyed -deactivate sa - -d -> d: Destroyed via reference\ncounting -deactivate d - -c -> rn: Poll RippleNode to verify successfull\nimport of all requested shards. -c -> rn: Shutdown RippleNode - -deactivate rn - -@enduml diff --git a/src/xrpld/net/uml/states.pu b/src/xrpld/net/uml/states.pu deleted file mode 100644 index b5db8ee48f4..00000000000 --- a/src/xrpld/net/uml/states.pu +++ /dev/null @@ -1,69 +0,0 @@ -@startuml - -state "Updating Database" as UD4 { - UD4: Update the database to reflect - UD4: the current state. -} -state "Initiating Download" as ID { - ID: Omit the range header to download - ID: the entire file. -} - -state "Evaluate Database" as ED { - ED: Determine the current state - ED: based on the contents of the - ED: database from a previous run. -} - -state "Remove Database" as RD { - RD: The database is destroyed when - RD: empty. -} - -state "Download in Progress" as DP - -state "Download Completed" as DC { - - state "Updating Database" as UD { - UD: Update the database to reflect - UD: the current state. - } - - state "Queue Check" as QC { - QC: Check the queue for any reamining - QC: downloads. - } - - [*] --> UD - UD --> QC -} - -state "Check Resume" as CR { - CR: Determine whether we're resuming - CR: a previous download or starting a - CR: new one. -} - -state "Resuming Download" as IPD { - IPD: Set the range header in the - IPD: HTTP request as needed. -} - -[*] --> ED : State DB is present at\nnode launch -ED --> RD : State DB is empty -ED --> CR : There are downloads queued -RD --> [*] - -[*] --> UD4 : Client invokes <>\ncommand -UD4 --> ID : Database updated -ID --> DP : Download started -DP --> DC : Download completed -DC --> ID : There **are** additional downloads\nqueued -DP --> [*] : A graceful shutdown is\nin progress -DC --> RD : There **are no** additional\ndownloads queued - -CR --> IPD : Resuming an interrupted\ndownload -IPD --> DP: Download started -CR --> ID : Initiating a new\ndownload - -@enduml diff --git a/src/xrpld/nodestore/Database.h b/src/xrpld/nodestore/Database.h index ad843c55d52..daf0483e890 100644 --- a/src/xrpld/nodestore/Database.h +++ b/src/xrpld/nodestore/Database.h @@ -31,8 +31,6 @@ namespace ripple { -class Ledger; - namespace NodeStore { /** Persistency layer for NodeObject @@ -153,7 +151,7 @@ class Database @note This can be called concurrently. @param hash The key of the object to retrieve @param ledgerSeq The sequence of the ledger where the - object is stored, used by the shard store. + object is stored. @param callback Callback function when read completes */ virtual void @@ -162,14 +160,6 @@ class Database std::uint32_t ledgerSeq, std::function const&)>&& callback); - /** Store a ledger from a different database. - - @param srcLedger The ledger to store. - @return true if the operation was successful - */ - virtual bool - storeLedger(std::shared_ptr const& srcLedger) = 0; - /** Remove expired entries from the positive and negative caches. */ virtual void sweep() = 0; @@ -224,14 +214,6 @@ class Database bool isStopping() const; - /** @return The maximum number of ledgers stored in a shard - */ - [[nodiscard]] std::uint32_t - ledgersPerShard() const noexcept - { - return ledgersPerShard_; - } - /** @return The earliest ledger sequence allowed */ [[nodiscard]] std::uint32_t @@ -240,63 +222,6 @@ class Database return earliestLedgerSeq_; } - /** @return The earliest shard index - */ - [[nodiscard]] std::uint32_t - earliestShardIndex() const noexcept - { - return earliestShardIndex_; - } - - /** Calculates the first ledger sequence for a given shard index - - @param shardIndex The shard index considered - @return The first ledger sequence pertaining to the shard index - */ - [[nodiscard]] std::uint32_t - firstLedgerSeq(std::uint32_t shardIndex) const noexcept - { - assert(shardIndex >= earliestShardIndex_); - if (shardIndex <= earliestShardIndex_) - return earliestLedgerSeq_; - return 1 + (shardIndex * ledgersPerShard_); - } - - /** Calculates the last ledger sequence for a given shard index - - @param shardIndex The shard index considered - @return The last ledger sequence pertaining to the shard index - */ - [[nodiscard]] std::uint32_t - lastLedgerSeq(std::uint32_t shardIndex) const noexcept - { - assert(shardIndex >= earliestShardIndex_); - return (shardIndex + 1) * ledgersPerShard_; - } - - /** Calculates the shard index for a given ledger sequence - - @param ledgerSeq ledger sequence - @return The shard index of the ledger sequence - */ - [[nodiscard]] std::uint32_t - seqToShardIndex(std::uint32_t ledgerSeq) const noexcept - { - assert(ledgerSeq >= earliestLedgerSeq_); - return (ledgerSeq - 1) / ledgersPerShard_; - } - - /** Calculates the maximum ledgers for a given shard index - - @param shardIndex The shard index considered - @return The maximum ledgers pertaining to the shard index - - @note The earliest shard may store less if the earliest ledger - sequence truncates its beginning - */ - [[nodiscard]] std::uint32_t - maxLedgers(std::uint32_t shardIndex) const noexcept; - protected: beast::Journal const j_; Scheduler& scheduler_; @@ -305,25 +230,14 @@ class Database std::atomic fetchHitCount_{0}; std::atomic fetchSz_{0}; - // The default is DEFAULT_LEDGERS_PER_SHARD (16384) to match the XRP ledger - // network. Can be set through the configuration file using the - // 'ledgers_per_shard' field under the 'node_db' and 'shard_db' stanzas. - // If specified, the value must be a multiple of 256 and equally assigned - // in both stanzas. Only unit tests or alternate networks should change - // this value. - std::uint32_t const ledgersPerShard_; - // The default is XRP_LEDGER_EARLIEST_SEQ (32570) to match the XRP ledger // network's earliest allowed ledger sequence. Can be set through the // configuration file using the 'earliest_seq' field under the 'node_db' - // and 'shard_db' stanzas. If specified, the value must be greater than zero - // and equally assigned in both stanzas. Only unit tests or alternate + // stanza. If specified, the value must be greater than zero. + // Only unit tests or alternate // networks should change this value. std::uint32_t const earliestLedgerSeq_; - // The earliest shard index - std::uint32_t const earliestShardIndex_; - // The maximum number of requests a thread extracts from the queue in an // attempt to minimize the overhead of mutex acquisition. This is an // advanced tunable, via the config file. The default value is 4. @@ -341,10 +255,6 @@ class Database void importInternal(Backend& dstBackend, Database& srcDB); - // Called by the public storeLedger function - bool - storeLedger(Ledger const& srcLedger, std::shared_ptr dstBackend); - void updateFetchMetrics(uint64_t fetches, uint64_t hits, uint64_t duration) { diff --git a/src/xrpld/nodestore/DatabaseShard.h b/src/xrpld/nodestore/DatabaseShard.h deleted file mode 100644 index 408ac3501d3..00000000000 --- a/src/xrpld/nodestore/DatabaseShard.h +++ /dev/null @@ -1,298 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2017 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_NODESTORE_DATABASESHARD_H_INCLUDED -#define RIPPLE_NODESTORE_DATABASESHARD_H_INCLUDED - -#include -#include -#include -#include -#include - -#include -#include - -namespace ripple { -namespace NodeStore { - -/** A collection of historical shards - */ -class DatabaseShard : public Database -{ -public: - /** Construct a shard store - - @param scheduler The scheduler to use for performing asynchronous tasks - @param readThreads The number of asynchronous read threads to create - @param config The shard configuration section for the database - @param journal Destination for logging output - */ - DatabaseShard( - Scheduler& scheduler, - int readThreads, - Section const& config, - beast::Journal journal) - : Database(scheduler, readThreads, config, journal) - { - } - - /** Initialize the database - - @return `true` if the database initialized without error - */ - [[nodiscard]] virtual bool - init() = 0; - - /** Prepare to store a new ledger in the shard being acquired - - @param validLedgerSeq The sequence of the maximum valid ledgers - @return If a ledger should be fetched and stored, then returns the - ledger sequence of the ledger to request. Otherwise returns - std::nullopt. - Some reasons this may return std::nullopt are: all shards are - stored and full, max allowed disk space would be exceeded, or a - ledger was recently requested and not enough time has passed - between requests. - @implNote adds a new writable shard if necessary - */ - [[nodiscard]] virtual std::optional - prepareLedger(std::uint32_t validLedgerSeq) = 0; - - /** Prepare one or more shard indexes to be imported into the database - - @param shardIndexes Shard indexes to be prepared for import - @return true if all shard indexes successfully prepared for import - */ - [[nodiscard]] virtual bool - prepareShards(std::vector const& shardIndexes) = 0; - - /** Remove a previously prepared shard index for import - - @param shardIndex Shard index to be removed from import - */ - virtual void - removePreShard(std::uint32_t shardIndex) = 0; - - /** Get shard indexes being imported - - @return a string representing the shards prepared for import - */ - [[nodiscard]] virtual std::string - getPreShards() = 0; - - /** Import a shard from the shard archive handler into the - shard database. This differs from 'importDatabase' which - imports the contents of the NodeStore - - @param shardIndex Shard index to import - @param srcDir The directory to import from - @return true If the shard was successfully imported - @implNote if successful, srcDir is moved to the database directory - */ - [[nodiscard]] virtual bool - importShard( - std::uint32_t shardIndex, - boost::filesystem::path const& srcDir) = 0; - - /** Fetch a ledger from the shard store - - @param hash The key of the ledger to retrieve - @param seq The sequence of the ledger - @return The ledger if found, nullptr otherwise - */ - [[nodiscard]] virtual std::shared_ptr - fetchLedger(uint256 const& hash, std::uint32_t seq) = 0; - - /** Notifies the database that the given ledger has been - fully acquired and stored. - - @param ledger The stored ledger to be marked as complete - */ - virtual void - setStored(std::shared_ptr const& ledger) = 0; - - /** Invoke a callback on the SQLite db holding the - corresponding ledger - - @return Value returned by callback function. - */ - virtual bool - callForLedgerSQLByLedgerSeq( - LedgerIndex ledgerSeq, - std::function const& callback) = 0; - - /** Invoke a callback on the ledger SQLite db for the - corresponding shard - - @return Value returned by callback function. - */ - virtual bool - callForLedgerSQLByShardIndex( - std::uint32_t shardIndex, - std::function const& callback) = 0; - - /** Invoke a callback on the transaction SQLite db - for the corresponding ledger - - @return Value returned by callback function. - */ - virtual bool - callForTransactionSQLByLedgerSeq( - LedgerIndex ledgerSeq, - std::function const& callback) = 0; - - /** Invoke a callback on the transaction SQLite db - for the corresponding shard - - @return Value returned by callback function. - */ - virtual bool - callForTransactionSQLByShardIndex( - std::uint32_t shardIndex, - std::function const& callback) = 0; - - /** - * @brief iterateLedgerSQLsForward Checks out ledger databases for all - * shards in ascending order starting from given shard index until - * shard with the largest index visited or callback returned false. - * For each visited shard calls given callback function passing - * shard index and session with the database to it. - * @param minShardIndex Start shard index to visit or none if all shards - * should be visited. - * @param callback Callback function to call. - * @return True if each callback function returns true, false otherwise. - */ - virtual bool - iterateLedgerSQLsForward( - std::optional minShardIndex, - std::function< - bool(soci::session& session, std::uint32_t shardIndex)> const& - callback) = 0; - - /** - * @brief iterateTransactionSQLsForward Checks out transaction databases for - * all shards in ascending order starting from given shard index - * until shard with the largest index visited or callback returned - * false. For each visited shard calls given callback function - * passing shard index and session with the database to it. - * @param minShardIndex Start shard index to visit or none if all shards - * should be visited. - * @param callback Callback function to call. - * @return True if each callback function returns true, false otherwise. - */ - virtual bool - iterateTransactionSQLsForward( - std::optional minShardIndex, - std::function< - bool(soci::session& session, std::uint32_t shardIndex)> const& - callback) = 0; - - /** - * @brief iterateLedgerSQLsBack Checks out ledger databases for - * all shards in descending order starting from given shard index - * until shard with the smallest index visited or callback returned - * false. For each visited shard calls given callback function - * passing shard index and session with the database to it. - * @param maxShardIndex Start shard index to visit or none if all shards - * should be visited. - * @param callback Callback function to call. - * @return True if each callback function returns true, false otherwise. - */ - virtual bool - iterateLedgerSQLsBack( - std::optional maxShardIndex, - std::function< - bool(soci::session& session, std::uint32_t shardIndex)> const& - callback) = 0; - - /** - * @brief iterateTransactionSQLsBack Checks out transaction databases for - * all shards in descending order starting from given shard index - * until shard with the smallest index visited or callback returned - * false. For each visited shard calls given callback function - * passing shard index and session with the database to it. - * @param maxShardIndex Start shard index to visit or none if all shards - * should be visited. - * @param callback Callback function to call. - * @return True if each callback function returns true, false otherwise. - */ - virtual bool - iterateTransactionSQLsBack( - std::optional maxShardIndex, - std::function< - bool(soci::session& session, std::uint32_t shardIndex)> const& - callback) = 0; - - /** Query information about shards held - - @return Information about shards held by this node - */ - [[nodiscard]] virtual std::unique_ptr - getShardInfo() const = 0; - - /** Returns the root database directory - */ - [[nodiscard]] virtual boost::filesystem::path const& - getRootDir() const = 0; - - /** Returns a JSON object detailing the status of an ongoing - database import if one is running, otherwise an error - object. - */ - virtual Json::Value - getDatabaseImportStatus() const = 0; - - /** Initiates a NodeStore to ShardStore import and returns - the result in a JSON object. - */ - virtual Json::Value - startNodeToShard() = 0; - - /** Terminates a NodeStore to ShardStore import and returns - the result in a JSON object. - */ - virtual Json::Value - stopNodeToShard() = 0; - - /** Returns the first ledger sequence of the shard currently being imported - from the NodeStore - - @return The ledger sequence or an unseated value if no import is running - */ - virtual std::optional - getDatabaseImportSequence() const = 0; - - /** Returns the number of queued tasks - */ - [[nodiscard]] virtual size_t - getNumTasks() const = 0; -}; - -extern std::unique_ptr -make_ShardStore( - Application& app, - Scheduler& scheduler, - int readThreads, - beast::Journal j); - -} // namespace NodeStore -} // namespace ripple - -#endif diff --git a/src/xrpld/nodestore/DeterministicShard.md b/src/xrpld/nodestore/DeterministicShard.md deleted file mode 100644 index 70d0584567b..00000000000 --- a/src/xrpld/nodestore/DeterministicShard.md +++ /dev/null @@ -1,162 +0,0 @@ -# Deterministic Database Shards - -This doc describes the standard way to assemble the database shard. -A shard assembled using this approach becomes deterministic i.e. -if two independent sides assemble a shard consisting of the same ledgers, -accounts and transactions, then they will obtain the same shard files -`nudb.dat` and `nudb.key`. The approach deals with the `NuDB` database -format only, refer to `https://github.com/vinniefalco/NuDB`. - - -## Headers - -Due to NuDB database definition, the following headers are used for -database files: - -nudb.key: -``` -char[8] Type The characters "nudb.key" -uint16 Version Holds the version number -uint64 UID Unique ID generated on creation -uint64 Appnum Application defined constant -uint16 KeySize Key size in bytes -uint64 Salt A random seed -uint64 Pepper The salt hashed -uint16 BlockSize size of a file block in bytes -uint16 LoadFactor Target fraction in 65536ths -uint8[56] Reserved Zeroes -uint8[] Reserved Zero-pad to block size -``` - -nudb.dat: -``` -char[8] Type The characters "nudb.dat" -uint16 Version Holds the version number -uint64 UID Unique ID generated on creation -uint64 Appnum Application defined constant -uint16 KeySize Key size in bytes -uint8[64] (reserved) Zeroes -``` -All of these fields are saved using network byte order -(bigendian: most significant byte first). - -To make the shard deterministic the following parameters are used -as values of header field both for `nudb.key` and `nudb.dat` files. -``` -Version 2 -UID digest(0) -Appnum digest(2) | 0x5348524400000000 /* 'SHRD' */ -KeySize 32 -Salt digest(1) -Pepper XXH64(Salt) -BlockSize 0x1000 (4096 bytes) -LoadFactor 0.5 (numeric 0x8000) -``` -Note: XXH64() is well-known hash algorithm. - -The `digest(i)` mentioned above defined as the follows: - -First, RIPEMD160 hash `H` calculated of the following structure -(the same as final Key of the shard): -``` -uint32 version Version of shard, 2 at the present -uint32 firstSeq Sequence number of first ledger in the shard -uint32 lastSeq Sequence number of last ledger in the shard -uint256 lastHash Hash of last ledger in shard -``` -there all 32-bit integers are hashed in network byte order -(bigendian: most significant byte first). - -Then, `digest(i)` is defined as the following part of the above hash `H`: -``` -digest(0) = H[0] << 56 | H[1] << 48 | ... | H[7] << 0, -digest(1) = H[8] << 56 | H[9] << 48 | ... | H[15] << 0, -digest(2) = H[16] << 24 | H[17] << 16 | ... | H[19] << 0, -``` -where `H[i]` denotes `i`-th byte of hash `H`. - - -## Contents - -After deterministic shard is created using the above mentioned headers, -it filled with objects using the following steps. - -1. All objects within the shard are visited in the order described in the -next section. Here the objects are: ledger headers, SHAmap tree nodes -including state and transaction nodes, final key. - -2. Set of all visited objects is divided into groups. Each group except of -the last contains 16384 objects in the order of their visiting. Last group -may contain less than 16384 objects. - -3. All objects within each group are sorted in according to their hashes. -Objects are sorted by increasing of their hashes, precisely, by increasing -of hex representations of hashes in lexicographic order. For example, -the following is an example of sorted hashes in their hex representation: -``` -0000000000000000000000000000000000000000000000000000000000000000 -154F29A919B30F50443A241C466691B046677C923EE7905AB97A4DBE8A5C2429 -2231553FC01D37A66C61BBEEACBB8C460994493E5659D118E19A8DDBB1444273 -272DCBFD8E4D5D786CF11A5444B30FB35435933B5DE6C660AA46E68CF0F5C441 -3C062FD9F0BCDCA31ACEBCD8E530D0BDAD1F1D1257B89C435616506A3EE6CB9E -58A0E5AE427CDDC1C7C06448E8C3E4BF718DE036D827881624B20465C3E1336F -... -``` - -4. Finally, objects added to the deterministic shard group by group in the -sorted order within each group from low to high hashes. - - -## Order of visiting objects - -The shard consists of 16384 ledgers and the final key with the hash 0. -Each ledger has the header object and two SMAmaps: state and transaction. -SHAmap is a rooted tree in which each node has maximum of 16 descendants -enumerating by indexes 0..15. Visiting each node in the SHAmap -is performing by functions visitNodes and visitDifferences implemented -in the file `ripple/shamap/impl/ShaMapSync.cpp`. - -Here is how the function visitNodes works: it visit the root at first. -Then it visit all nodes in the 1st layer, i. e. the nodes which are -immediately descendants of the root sequentially from index 0 to 15. -Then it visit all nodes in 2nd layer i.e. the nodes which are immediately -descendants the nodes from 1st layer. The order of visiting 2nd layer nodes -is the following. First, descendants of the 1st layer node with index 0 -are visited sequintially from index 0 to 15. Then descendents of 1st layer -node with index 1 are visited etc. After visiting all nodes of 2nd layer -the nodes from 3rd layer are visited etc. - -The function visitDifferences works similar to visitNodes with the following -exceptions. The first exception is that visitDifferences get 2 arguments: -current SHAmap and previous SHAmap and visit only the nodes from current -SHAmap which and not present in previous SHAmap. The second exception is -that visitDifferences visits all non-leaf nodes in the order of visitNodes -function, but all leaf nodes are visited immedeately after visiting of their -parent node sequentially from index 0 to 15. - -Finally, all objects within the shard are visited in the following order. -All ledgers are visited from the ledger with high index to the ledger with -low index in descending order. For each ledger the state SHAmap is visited -first using visitNode function for the ledger with highest index and -visitDifferences function for other ledgers. Then transaction SHAmap is visited -using visitNodes function. At last, the ledger header object is visited. -Final key of the shard is visited at the end. - - -## Tests - -To perform test to deterministic shards implementation one can enter -the following command: -``` -rippled --unittest ripple.NodeStore.DatabaseShard -``` - -The following is the right output of deterministic shards test: -``` -ripple.NodeStore.DatabaseShard DatabaseShard deterministic_shard -with backend nudb -Iteration 0: RIPEMD160[nudb.key] = F96BF2722AB2EE009FFAE4A36AAFC4F220E21951 -Iteration 0: RIPEMD160[nudb.dat] = FAE6AE84C15968B0419FDFC014931EA12A396C71 -Iteration 1: RIPEMD160[nudb.key] = F96BF2722AB2EE009FFAE4A36AAFC4F220E21951 -Iteration 1: RIPEMD160[nudb.dat] = FAE6AE84C15968B0419FDFC014931EA12A396C71 -``` diff --git a/src/xrpld/nodestore/Manager.h b/src/xrpld/nodestore/Manager.h index 5a4d46068be..89ed165b483 100644 --- a/src/xrpld/nodestore/Manager.h +++ b/src/xrpld/nodestore/Manager.h @@ -21,7 +21,6 @@ #define RIPPLE_NODESTORE_MANAGER_H_INCLUDED #include -#include #include namespace ripple { diff --git a/src/xrpld/nodestore/README.md b/src/xrpld/nodestore/README.md index 7b004f67a42..1549c1ef968 100644 --- a/src/xrpld/nodestore/README.md +++ b/src/xrpld/nodestore/README.md @@ -1,8 +1,6 @@ # Database Documentation * [NodeStore](#nodestore) * [Benchmarks](#benchmarks) -* [Downloaded Shard Validation](#downloaded-shard-validation) -* [Shard Storage Paths](#shard-storage-paths) # NodeStore @@ -176,195 +174,3 @@ instruction not being used. * Important point to note that is if this factory is tested with an existing set of sst files none of the old sst files will benefit from indexing changes until they are compacted at a future point in time. - -# Downloaded Shard Validation - -## Overview - -In order to validate shards that have been downloaded from file servers (as -opposed to shards acquired from peers), the application must confirm the -validity of the downloaded shard's last ledger. So before initiating the -download, we first confirm that we are able to retrieve the shard's last ledger -hash. The following sections describe this confirmation process in greater -detail. - -## Hash Verification - -### Flag Ledger - -Since the number of ledgers contained in each shard is always a multiple of 256, -a shard's last ledger is always a flag ledger. Conveniently, the skip list -stored within a ledger will provide us with a series of flag ledger hashes, -enabling the software to corroborate a shard's last ledger hash. We access the -skip list by calling `LedgerMaster::walkHashBySeq` and providing the sequence of -a shard's last ledger: - -```C++ -std::optional expectedHash; -expectedHash = - app_.getLedgerMaster().walkHashBySeq(lastLedgerSeq(shardIndex)); -``` - -When a user requests a shard download, the `ShardArchiveHandler` will first use -this function to retrieve the hash of the shard's last ledger. If the function -returns a hash, downloading the shard can proceed. Once the download completes, -the server can reliably retrieve this last ledger hash to complete validation of -the shard. - -### Caveats - -#### Later Ledger - -The `walkHashBySeq` function will provide the hash of a flag ledger only if the -application has stored a later ledger. When verifying the last ledger hash of a -pending shard download, if there is no later ledger stored, the download will be -deferred until a later ledger has been stored. - -We use the presence (or absence) of a validated ledger with a sequence number -later than the sequence of the shard's last ledger as a heuristic for -determining whether or not we should have the shard's last ledger hash. A later -ledger must be present in order to reliably retrieve the hash of the shard's -last ledger. The hash will only be retrieved when a later ledger is present. -Otherwise verification of the shard will be deferred. - -### Retries - -#### Retry Limit - -If the server must defer hash verification, the software will initiate a timer -that upon expiration, will re-attempt verifying the last ledger hash. We place -an upper limit on the number of attempts the server makes to achieve this -verification. When the maximum number of attempts has been reached, the download -request will fail, and the `ShardArchiveHandler` will proceed with any remaining -downloads. An attempt counts toward the limit only when we are able to get a -later validated ledger (implying a current view of the network), but are unable -to retrieve the last ledger hash. Retries that occur because no validated ledger -was available are not counted. - -# Shard Storage Paths - -## Overview - -The shard database stores validated ledgers in logical groups called shards. As -of June 2020, a shard stores 16384 ledgers by default. In order to allow users -to store shards on multiple devices, the shard database can be configured with -several file system paths. Each path provided should refer to a directory on a -distinct filesystem, and no two paths should ever correspond to the same -filesystem. Violating this restriction will cause the server to inaccurately -estimate the amount of space available for storing shards. In the absence of a -suitable platform agnostic solution, this requirement is enforced only on -Linux. However, on other platforms we employ a heuristic that issues a warning -if we suspect that this restriction is violated. - -## Configuration - -The `shard_db` and `historical_shard_paths` sections of the server's -configuration file will be used to determine where the server stores shards. -Minimally, the `shard_db` section must contain a single `path` key. -If this is the only storage path provided, all shards will be stored at this -location. If the configuration also lists one or more lines in the -`historical_shard_paths` section, all older shards will be stored at these -locations, and the `path` will be used only to store the current -and previous shards. The goal is to allow users to provide an efficient SSD for -storing recent shards, as these will be accessed more frequently, while using -large mechanical drives for storing older shards that will be accessed less -frequently. - -Below is a sample configuration snippet that provides a path for main storage -and several paths for historical storage: - -```dosini -# This is the persistent datastore for shards. It is important for the health -# of the network that server operators shard as much as practical. -# NuDB requires SSD storage. Helpful information can be found on -# https://xrpl.org/history-sharding.html -[shard_db] -type=NuDB - -# A single path for storing -# the current and previous -# shards: -# ------------------------- -path=/var/lib/rippled/db/shards/nudb - -# Path where shards are stored -# while being downloaded: -# ---------------------------- -download_path=/var/lib/rippled/db/shards/ - -# The number of historical shards to store. -# The default value is 0, which means that -# the server won't store any historical -# shards - only the current and previous -# shards will be stored. -# ------------------------------------ -max_historical_shards=100 - -# List of paths for storing older shards. -[historical_shard_paths] -/mnt/disk1 -/mnt/disk2 -/mnt/disk3 - -``` -## Shard Migration - -When a new shard (*current shard*) is confirmed by the network, the recent -shards will shift. The *previous shard* will become a *historical shard*, the -*current shard* will become the *previous shard*, and the new shard will become -the *current shard*. These are just logical labels, and the shards themselves -don't change to reflect being current, previous, or historical. However, if the -server's configuration specifies one or more paths for historical storage, -during this shift the formerly *previous shard* will be migrated to one of the -historical paths. If multiple paths are provided, the server dynamically -chooses one with sufficient space for storing the shard. - -**Note:** As of June 2020, the shard database does not store the partial shard -currently being built by live network transactions, but this is planned to -change. When this feature is implemented, the *current shard* will refer to this -partial shard, and the *previous shard* will refer to the most recently -validated shard. - -### Selecting a Historical Storage Path - -When storing historical shards, if multiple historical paths are provided, the -path to use for each shard will be selected in a random fashion. By using all -available storage devices, we create a uniform distribution of disk utilization -for disks of equivalent size, (provided that the disks are used only to store -shards). In theory, selecting devices in this manner will also increase our -chances for concurrent access to stored shards, however as of June 2020 -concurrent shard access is not implemented. Lastly, a storage path is included -in the random distribution only if it has enough storage capacity to hold the -next shard. - -## Shard Acquisition - -When the server is acquiring shard history, these acquired shards will be stored -at a path designated for historical storage (`historical_storage_path`). If no -such path is provided, acquired shards will be stored at the -`path`. - -## Storage capacity - -### Filesystem Capacity - -When the shard database updates its record of disk utilization, it trusts that -the provided historical paths refer to distinct devices, or at least distinct -filesystems. If this requirement is violated, the database will operate with an -inaccurate view of how many shards it can store. Violation of this requirement -won't necessarily impede database operations, but the database will fail to -identify scenarios wherein storing the maximum number of historical shards (as -per the 'historical_shard_count' parameter in the configuration file) would -exceed the amount of storage space available. - -### Shard Migration - -During a "recent shard shift", if the server has already reached the configured -limit of stored historical shards, instead of moving the formerly *previous -shard* to a historical drive (or keeping it at the 'path') the -shard will be dropped and removed from the filesystem. - -### Shard Acquisition - -Once the configured limit of stored historical shards has been reached, shard -acquisition halts, and no additional shards will be acquired. diff --git a/src/xrpld/nodestore/ShardInfo.h b/src/xrpld/nodestore/ShardInfo.h deleted file mode 100644 index b894ddc34a3..00000000000 --- a/src/xrpld/nodestore/ShardInfo.h +++ /dev/null @@ -1,122 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_NODESTORE_SHARDINFO_H_INCLUDED -#define RIPPLE_NODESTORE_SHARDINFO_H_INCLUDED - -#include -#include -#include - -namespace ripple { -namespace NodeStore { - -/* Contains information on the status of shards for a node - */ -class ShardInfo -{ -private: - class Incomplete - { - public: - Incomplete() = delete; - Incomplete(ShardState state, std::uint32_t percentProgress) - : state_(state), percentProgress_(percentProgress) - { - } - - [[nodiscard]] ShardState - state() const noexcept - { - return state_; - } - - [[nodiscard]] std::uint32_t - percentProgress() const noexcept - { - return percentProgress_; - } - - private: - ShardState state_; - std::uint32_t percentProgress_; - }; - -public: - [[nodiscard]] NetClock::time_point const& - msgTimestamp() const - { - return msgTimestamp_; - } - - void - setMsgTimestamp(NetClock::time_point const& timestamp) - { - msgTimestamp_ = timestamp; - } - - [[nodiscard]] std::string - finalizedToString() const; - - [[nodiscard]] bool - setFinalizedFromString(std::string const& str) - { - return from_string(finalized_, str); - } - - [[nodiscard]] RangeSet const& - finalized() const - { - return finalized_; - } - - [[nodiscard]] std::string - incompleteToString() const; - - [[nodiscard]] std::map const& - incomplete() const - { - return incomplete_; - } - - // Returns true if successful or false because of a duplicate index - bool - update( - std::uint32_t shardIndex, - ShardState state, - std::uint32_t percentProgress); - - [[nodiscard]] protocol::TMPeerShardInfoV2 - makeMessage(Application& app); - -private: - // Finalized immutable shards - RangeSet finalized_; - - // Incomplete shards being acquired or finalized - std::map incomplete_; - - // Message creation time - NetClock::time_point msgTimestamp_; -}; - -} // namespace NodeStore -} // namespace ripple - -#endif diff --git a/src/xrpld/nodestore/ShardPool.md b/src/xrpld/nodestore/ShardPool.md deleted file mode 100644 index 2079feabb4e..00000000000 --- a/src/xrpld/nodestore/ShardPool.md +++ /dev/null @@ -1,43 +0,0 @@ -# Open Shard Management - -## Overview - -Shard NuDB and SQLite databases consume server resources. This can be unnecessarily taxing on servers with many shards. The open shard management feature aims to improve the situation by managing a limited number of open shard database connections. The feature, which is integrated into the existing DatabaseShardImp and Shard classes, maintains a limited pool of open databases prioritized by their last use time stamp. The following sections describe the feature in greater detail. - -### Open Shard Management - -The open shard management feature is integrated into the DatabaseShardImp and Shard classes. As the DatabaseShardImp sweep function is periodically called, the number of finalized open shards, which constitutes the open pool, are examined. Upon the pool exceeding a pool limit, an attempt is made to close enough open shards to remain within the limit. Shards to be closed are selected based on their last use time stamp, which is automatically updated on database access. If necessary, shards will automatically open their databases when accessed. - -```C++ - if (openFinals.size() > openFinalLimit_) - { - // Try to close enough shards to be within the limit. - // Sort on largest elapsed time since last use. - std::sort( - openFinals.begin(), - openFinals.end(), - [&](std::shared_ptr const& lhsShard, - std::shared_ptr const& rhsShard) { - return lhsShard->getLastUse() > rhsShard->getLastUse(); - }); - - for (auto it{openFinals.cbegin()}; - it != openFinals.cend() && openFinals.size() > openFinalLimit_;) - { - if ((*it)->tryClose()) - it = openFinals.erase(it); - else - ++it; - } - } -``` - -### Shard - -When closing an open shard, DatabaseShardImp will call the Shard 'tryClose' function. This function will only close the shard databases if there are no outstanding references. - -DatabaseShardImp will use the Shard 'isOpen' function to determine the state of a shard's database. - -### Caveats - -The Shard class must check the state of its databases before use. Prior use assumed databases were always open, that is no longer the case with the open shard management feature. diff --git a/src/xrpld/nodestore/ShardSizeTuning.md b/src/xrpld/nodestore/ShardSizeTuning.md deleted file mode 100644 index bded73c43c5..00000000000 --- a/src/xrpld/nodestore/ShardSizeTuning.md +++ /dev/null @@ -1,213 +0,0 @@ -# Shard size Tuning - -The purpose of this document is to compare the sizes of shards containing -varying amounts of ledgers. - -## Methodology - -One can see visually from a block explorer that a typical mainnet ledger -consists of about 30 offer transactions issued by about 8 different accounts, -and several transactions of other types. To simulate this situation and -similar situations we have constructed deterministic shards of differenet -sizes, with varying amounts of offers per ledger and varying amounts of -accounts issuing these offers. - -In the following results table, the number of ledgers per shard ranges from 256 -to 16K with the size doubling the size at each step. We considered the -following numbers of offers per ledger: 0, 1, 5, 10 and 30. Also we considered -case of 1 and 8 accounts issuing offers. For each constructed deterministic -shard we counted its size. Finally we compared doubled size of the shard with -N ledgers and the size of a shard with 2*N ledgers where othere parameters such -as number of offers and accounts are the same. This comparison is sufficient to -determine which number of ledgers per shard leads to less storage size on the -disk. - -Note that we minimize total storage size on the disk, but not the size of each -shard because data below shows that the size of a typical shard is not larger -than 10G, but sizes of modern disks, even SSDs, start from 250G. So there is -no problem to fit a single shard to a disk, even small. - - -## Raw results table - -All sizes of constructed shards are shown in the following table. -Rows corresponds to shard sizes (S) counted in ledgers, columns corresponds -to numbers of offers (O) per ledger. In each cell there are two numbers: -first number corresponds to the case of 1 account issuing offers, the second -number corresponds to 8 accounts. Each number is a size of the shard with -given parameters measured in megabytes. - -|S\O|0|1|5|10|30| -|---|---|---|---|---|---| -|256|2.2/2.2|3.4/3.3|5.3/7.3|7.7/10.9|17.1/21.9| -|512|4.4/4.5|7.0/7.0|11.2/15.6|16.4/23.7|36.9/47.9| -|1K|8.9/9.0|14.7/14.6|23.7/33.6|35.0/51.0|78.2/ 102.9| -|2K|17.8/18.0|30.5/30.7|50.4/72.2|74.3/ 111.9|166.2/ 221.0| -|4K|35.5/35.9|63.6/64.2|106.2/ 154.8|156.1/ 238.7|354.7/ 476.0| -|8K|71.1/71.9|133.4/ 134.5|222.2/ 328.1|329.1/ 511.2|754.6/ 1021.0| -|16K|142.3/ 143.9|279/9 280.8|465.7/ 698.1|696.4/ 1094.2|1590.5/ 2166.6| - -## Preliminary conclusion - -If one compares a doubled size of shard with N ledgers and a size of shard -with 2*N ledgers anywhere in the above table than the conlusion will be that -the second number is greater. For example, the following table shows the -percentage by which the second number is greater for the most interesting case -of 30 offers per ledger. The first row corresponds to the case of 1 account -issuing offers, and the second row corresponds to the case of 8 issuing -accounts. - -|A\N|256|512|1K|2K|4K|8K| -|---|---|---|---|---|---|---| -|1|8%|6%|6%|6%|7%|6%|5%| -|8|9%|7%|7%|8%|6%|7%|6%| - -The common conclusion in this model is that if one doubled the number of -the ledgers in a shard then the total disk space utilized will raise by 5-9%. - -## Adding accounts into consideration - -Previous model does not take into account that there are large number of -XRP accounts in the mainnet, and each shard should contain information -about each of these accounts. As of January 2020, there were about 1.9 million -XRP accounts, and stored information about each of them is not less than 133 -bytes. The constant 133 was obtained from debug print of rippled program when -it saves account object to the database. So the actual size of each shard from -raw table should be increased by at least 1.9M * 133 = 252.7M. Thus we obtained -the following table of shard sizes for the most interesting case (30 offers -per ledger and 8 issuing accounts) where S is shard size in ledgers and M is -shard size in megabytes - -|S|256|512|1K|2K|4K|8K|16K| -|---|---|---|---|---|---|---|---| -|M|274.6|300.6|355.6|473.7|728.7|1273.7|2419.3| - -Now we can see from the last table that even considering minimum assumption -about number of accounts and corresponding additional size of a shard, -doubled size of shard with N ledgers is larger than size of a shard with -2*N ledgers. If number of accounts increase then this inequality will be -even stronger. - -## Using mainnet data - -Next idea to improve model is to count real shard sizes from mainnet. -We used real 16k-ledgers shards with indexes from 2600 to 3600 with step 100, -and corresponding real 8k-ledgers shards. Each 16k-ledgers shard consists -of two 8k-ledgers shards which are called "corresponding". For example, -16k-ledgers shard with index 2600 consists of two 8k-ledgers shards with -indexes 5200 and 5201. - -In the following table we compare size of a 16k-ledgers shard with sum of sizes -of two corresponding 8k-ledgers shards. There we only count size of nudb.dat -file, sizes are in GB. Ratio is the size of two 8k-ledgers shards divided -to the size of 16k-ledgers shard. - -|Index|16k-ledgers|8k-ledgers sum|Ratio| -|---|---|---|---| -|2600|2.39|1.49 + 1.63 = 3.12|1.31| -|2700|2.95|1.77 + 1.94 = 3.71|1.26| -|2800|2.53|1.54 + 1.75 = 3.29|1.30| -|2900|3.83|2.26 + 2.35 = 4.61|1.20| -|3000|4.49|2.70 + 2.59 = 5.29|1.18| -|3100|3.79|2.35 + 2.25 = 4.60|1.21| -|3200|4.15|2.54 + 2.43 = 4.97|1.20| -|3300|5.19|3.23 + 2.80 = 6.03|1.16| -|3400|4.18|2.53 + 2.51 = 5.04|1.21| -|3500|5.06|2.90 + 3.04 = 5.94|1.17| -|3600|4.18|2.56 + 2.51 = 5.07|1.21| -|Average|3.89|2.35 + 2.35 = 4.70|1.21| - -Note that shard on the disk consists of 4 files each of which can be large too. -These files are nudb.dat, nudb.key, ledger.db, transaction.db. Next table is -similar to previous with the following exception: each number is total size -of these 2 files: nudb.dat and nudb.key. We decided not to count sizes of -ledger.db and transaction.db since these sizes are not permanent instead of -sizes of nudb.* which are permanent for deterministic shards. - -|Index|16k-ledgers|8k-ledgers sum|Ratio| -|---|---|---|---| -|2600|2.76|1.73 + 1.89 = 3.62|1.31| -|2700|3.40|2.05 + 2.25 = 4.30|1.26| -|2800|2.91|1.79 + 2.02 = 3.81|1.31| -|2900|4.40|2.62 + 2.71 = 5.33|1.21| -|3000|5.09|3.09 + 2.96 = 6.05|1.19| -|3100|4.29|2.69 + 2.57 = 5.26|1.23| -|3200|4.69|2.90 + 2.78 = 5.68|1.21| -|3300|5.92|3.72 + 3.21 = 6.93|1.17| -|3400|4.77|2.91 + 2.89 = 5.80|1.22| -|3500|5.73|3.31 + 3.47 = 6.78|1.18| -|3600|4.77|2.95 + 2.90 = 5.85|1.23| -|Average|4.43|2.70 + 2.70 = 5.40|1.22| - -We can see that in all tables ratio is greater then 1, so using shards with -16 ledgers is preferred. - -## Compare 16K shards and 32K shards - -To claim that shards with 16K ledgers are the best choice, we also assembled -shards with 32k ledgers per shard with indexes from 1300 to 1800 with step 50 -and corresponding shards with 16k ledgers per shard. For example, 32k-ledgers -shard 1800 correnspond to 16k-ledgers shards with indexes 3600 and 3601 etc. - -Here are result tables for these shards similar to tables from previous part. -In the first table we only take into consideration sizes of nudb.dat files. - -|Index|32k-ledgers|16k-ledgers sum|Ratio| -|---|---|---|---| -|1300|4.00|2.39 + 2.32 = 4.71|1.18| -|1350|5.23|2.95 + 3.02 = 5.97|1.14| -|1400|4.37|2.53 + 2.59 = 5.12|1.17| -|1450|7.02|3.83 + 3.98 = 7.81|1.11| -|1500|7.53|4.49 + 3.86 = 8.35|1.11| -|1550|6.85|3.79 + 3.89 = 7.68|1.12| -|1600|7.28|4.15 + 3.99 = 8.14|1.12| -|1650|8.10|5.19 + 3.76 = 8.95|1.10| -|1700|7.58|4.18 + 4.27 = 8.45|1.11| -|1750|8.95|5.06 + 4.77 = 9.83|1.10| -|1800|7.29|4.18 + 4.02 = 8.20|1.12| -|Average|6.75|3.88 + 3.68 = 7.56|1.12| - -In the second table we take into consideration total sizes of files nudb.dat -and nudb.key. - -|Index|32k-ledgers|16k-ledgers sum|Ratio| -|---|---|---|---| -|1300|4.59|2.76 + 2.68 = 5.44|1.19| -|1350|5.98|3.40 + 3.47 = 6.87|1.15| -|1400|4.99|2.91 + 2.98 = 5.89|1.18| -|1450|8.02|4.40 + 4.56 = 8.96|1.12| -|1500|8.51|5.09 + 4.39 = 9.48|1.11| -|1550|7.73|4.29 + 4.42 = 8.71|1.13| -|1600|8.20|4.69 + 4.52 = 9.21|1.12| -|1650|9.20|5.92 + 4.29 = 10.21|1.11| -|1700|8.61|4.77 + 4.87 = 9.64|1.12| -|1750|10.09|5.73 + 5.41 = 11.14|1.10| -|1800|8.27|4.77 + 4.59 = 9.36|1.13| -|Average|7.69|4.43 + 4.20 = 8.63|1.12| - -## Conclusion - -We showed that using shards with 8k ledgers leads to raising required disk size -by 22% in comparison with using shards with 16k ledgers. In the same way, -using shards with 16k ledgers leads to raising required disk space by 12% -in comparison with using shards with 32k ledgers. Note that increase ratio 12% -is much less than 22% so using 32k-ledgers shards will bring us not so much -economy in disk space. - -At the same time, size is one thing to compare but there are other aspects. -Smaller shards have an advantage that they take less time to acquire and -finalize. They also make for smaller archived shards which take less time to -download and import. Having more/smaller shards might also lead to better -database concurrency/performance. - -It is hard to maintain both size and time parameters by a single optimization -formulae because different choices for weights of size and time may lead to -different results. But using "common sense" arguments we can compare -16k-ledgers shards and 32k-ledgers as follows: using 32k-ledgers shards give us -12% advantage in size, and about 44% disadvantage in time, because average size -of 16k-ledgers shard is about 56% of average 32k-ledgers shard. At the same, -if we compare 16k-ledgers shards with 8k-ledgers, then the first has 22% -advantage in size and 39% disadvantage in time. So the balance of -advantages/disadvantages is better when we use 16k-ledgers shards. - -Thus we recommend use shards with 16K ledgers. diff --git a/src/xrpld/nodestore/Types.h b/src/xrpld/nodestore/Types.h index 39104f946e3..a5792fe7df3 100644 --- a/src/xrpld/nodestore/Types.h +++ b/src/xrpld/nodestore/Types.h @@ -56,15 +56,6 @@ using Batch = std::vector>; } // namespace NodeStore -/** Shard states. */ -enum class ShardState : std::uint32_t { - acquire, // Acquiring ledgers - complete, // Backend is ledger complete, database is unverified - finalizing, // Verifying database - finalized, // Database verified, shard is immutable - queued // Queued to be finalized -}; - } // namespace ripple #endif diff --git a/src/xrpld/nodestore/backend/NuDBFactory.cpp b/src/xrpld/nodestore/backend/NuDBFactory.cpp index 742bf05031b..14cd84a1ad7 100644 --- a/src/xrpld/nodestore/backend/NuDBFactory.cpp +++ b/src/xrpld/nodestore/backend/NuDBFactory.cpp @@ -38,11 +38,11 @@ namespace NodeStore { class NuDBBackend : public Backend { public: - static constexpr std::uint64_t currentType = 1; - static constexpr std::uint64_t deterministicMask = 0xFFFFFFFF00000000ull; - - /* "SHRD" in ASCII */ - static constexpr std::uint64_t deterministicType = 0x5348524400000000ull; + // "appnum" is an application-defined constant stored in the header of a + // NuDB database. We used it to identify shard databases before that code + // was removed. For now, its only use is a sanity check that the database + // was created by xrpld. + static constexpr std::uint64_t appnum = 1; beast::Journal const j_; size_t const keyBytes_; @@ -149,16 +149,7 @@ class NuDBBackend : public Backend if (ec) Throw(ec); - /** Old value currentType is accepted for appnum in traditional - * databases, new value is used for deterministic shard databases. - * New 64-bit value is constructed from fixed and random parts. - * Fixed part is bounded by bitmask deterministicMask, - * and the value of fixed part is deterministicType. - * Random part depends on the contents of the shard and may be any. - * The contents of appnum field should match either old or new rule. - */ - if (db_.appnum() != currentType && - (db_.appnum() & deterministicMask) != deterministicType) + if (db_.appnum() != appnum) Throw("nodestore: unknown appnum"); db_.set_burst(burstSize_); } @@ -172,7 +163,7 @@ class NuDBBackend : public Backend void open(bool createIfMissing) override { - open(createIfMissing, currentType, nudb::make_uid(), nudb::make_salt()); + open(createIfMissing, appnum, nudb::make_uid(), nudb::make_salt()); } void diff --git a/src/xrpld/nodestore/detail/Database.cpp b/src/xrpld/nodestore/detail/Database.cpp index 93468eb6084..60cfb35051c 100644 --- a/src/xrpld/nodestore/detail/Database.cpp +++ b/src/xrpld/nodestore/detail/Database.cpp @@ -17,7 +17,6 @@ */ //============================================================================== -#include #include #include #include @@ -36,21 +35,13 @@ Database::Database( beast::Journal journal) : j_(journal) , scheduler_(scheduler) - , ledgersPerShard_(get( - config, - "ledgers_per_shard", - DEFAULT_LEDGERS_PER_SHARD)) , earliestLedgerSeq_( get(config, "earliest_seq", XRP_LEDGER_EARLIEST_SEQ)) - , earliestShardIndex_((earliestLedgerSeq_ - 1) / ledgersPerShard_) , requestBundle_(get(config, "rq_bundle", 4)) , readThreads_(std::max(1, readThreads)) { assert(readThreads != 0); - if (ledgersPerShard_ == 0 || ledgersPerShard_ % 256 != 0) - Throw("Invalid ledgers_per_shard"); - if (earliestLedgerSeq_ < 1) Throw("Invalid earliest_seq"); @@ -148,19 +139,6 @@ Database::isStopping() const return readStopping_.load(std::memory_order_relaxed); } -std::uint32_t -Database::maxLedgers(std::uint32_t shardIndex) const noexcept -{ - if (shardIndex > earliestShardIndex_) - return ledgersPerShard_; - - if (shardIndex == earliestShardIndex_) - return lastLedgerSeq(shardIndex) - firstLedgerSeq(shardIndex) + 1; - - assert(!"Invalid shard index"); - return 0; -} - void Database::stop() { @@ -275,105 +253,6 @@ Database::fetchNodeObject( return nodeObject; } -bool -Database::storeLedger( - Ledger const& srcLedger, - std::shared_ptr dstBackend) -{ - auto fail = [&](std::string const& msg) { - JLOG(j_.error()) << "Source ledger sequence " << srcLedger.info().seq - << ". " << msg; - return false; - }; - - if (srcLedger.info().hash.isZero()) - return fail("Invalid hash"); - if (srcLedger.info().accountHash.isZero()) - return fail("Invalid account hash"); - - auto& srcDB = const_cast(srcLedger.stateMap().family().db()); - if (&srcDB == this) - return fail("Source and destination databases are the same"); - - Batch batch; - batch.reserve(batchWritePreallocationSize); - auto storeBatch = [&, fname = __func__]() { - std::uint64_t sz{0}; - for (auto const& nodeObject : batch) - sz += nodeObject->getData().size(); - - try - { - dstBackend->storeBatch(batch); - } - catch (std::exception const& e) - { - fail( - std::string("Exception caught in function ") + fname + - ". Error: " + e.what()); - return false; - } - - storeStats(batch.size(), sz); - batch.clear(); - return true; - }; - - // Store ledger header - { - Serializer s(sizeof(std::uint32_t) + sizeof(LedgerInfo)); - s.add32(HashPrefix::ledgerMaster); - addRaw(srcLedger.info(), s); - auto nObj = NodeObject::createObject( - hotLEDGER, std::move(s.modData()), srcLedger.info().hash); - batch.emplace_back(std::move(nObj)); - } - - bool error = false; - auto visit = [&](SHAMapTreeNode& node) { - if (!isStopping()) - { - if (auto nodeObject = srcDB.fetchNodeObject( - node.getHash().as_uint256(), srcLedger.info().seq)) - { - batch.emplace_back(std::move(nodeObject)); - if (batch.size() < batchWritePreallocationSize || storeBatch()) - return true; - } - } - - error = true; - return false; - }; - - // Store the state map - if (srcLedger.stateMap().getHash().isNonZero()) - { - if (!srcLedger.stateMap().isValid()) - return fail("Invalid state map"); - - srcLedger.stateMap().snapShot(false)->visitNodes(visit); - if (error) - return fail("Failed to store state map"); - } - - // Store the transaction map - if (srcLedger.info().txHash.isNonZero()) - { - if (!srcLedger.txMap().isValid()) - return fail("Invalid transaction map"); - - srcLedger.txMap().snapShot(false)->visitNodes(visit); - if (error) - return fail("Failed to store transaction map"); - } - - if (!batch.empty() && !storeBatch()) - return fail("Failed to store"); - - return true; -} - void Database::getCountsJson(Json::Value& obj) { diff --git a/src/xrpld/nodestore/detail/DatabaseNodeImp.cpp b/src/xrpld/nodestore/detail/DatabaseNodeImp.cpp index d61c68e759a..85e5d3c0da9 100644 --- a/src/xrpld/nodestore/detail/DatabaseNodeImp.cpp +++ b/src/xrpld/nodestore/detail/DatabaseNodeImp.cpp @@ -17,7 +17,6 @@ */ //============================================================================== -#include #include #include diff --git a/src/xrpld/nodestore/detail/DatabaseNodeImp.h b/src/xrpld/nodestore/detail/DatabaseNodeImp.h index f5f5f64bd1d..c2bf237b943 100644 --- a/src/xrpld/nodestore/detail/DatabaseNodeImp.h +++ b/src/xrpld/nodestore/detail/DatabaseNodeImp.h @@ -128,12 +128,6 @@ class DatabaseNodeImp : public Database std::function const&)>&& callback) override; - bool - storeLedger(std::shared_ptr const& srcLedger) override - { - return Database::storeLedger(*srcLedger, backend_); - } - void sweep() override; diff --git a/src/xrpld/nodestore/detail/DatabaseRotatingImp.cpp b/src/xrpld/nodestore/detail/DatabaseRotatingImp.cpp index b1283d7de71..58cc3599dc6 100644 --- a/src/xrpld/nodestore/detail/DatabaseRotatingImp.cpp +++ b/src/xrpld/nodestore/detail/DatabaseRotatingImp.cpp @@ -17,7 +17,6 @@ */ //============================================================================== -#include #include #include @@ -79,17 +78,6 @@ DatabaseRotatingImp::importDatabase(Database& source) importInternal(*backend, source); } -bool -DatabaseRotatingImp::storeLedger(std::shared_ptr const& srcLedger) -{ - auto const backend = [&] { - std::lock_guard lock(mutex_); - return writableBackend_; - }(); - - return Database::storeLedger(*srcLedger, backend); -} - void DatabaseRotatingImp::sync() { diff --git a/src/xrpld/nodestore/detail/DatabaseRotatingImp.h b/src/xrpld/nodestore/detail/DatabaseRotatingImp.h index ec46fc687be..0c17dc59ceb 100644 --- a/src/xrpld/nodestore/detail/DatabaseRotatingImp.h +++ b/src/xrpld/nodestore/detail/DatabaseRotatingImp.h @@ -75,9 +75,6 @@ class DatabaseRotatingImp : public DatabaseRotating void sync() override; - bool - storeLedger(std::shared_ptr const& srcLedger) override; - void sweep() override; diff --git a/src/xrpld/nodestore/detail/DatabaseShardImp.cpp b/src/xrpld/nodestore/detail/DatabaseShardImp.cpp deleted file mode 100644 index c7e45641d7f..00000000000 --- a/src/xrpld/nodestore/detail/DatabaseShardImp.cpp +++ /dev/null @@ -1,2253 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2017 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#if BOOST_OS_LINUX -#include -#endif - -namespace ripple { - -namespace NodeStore { - -DatabaseShardImp::DatabaseShardImp( - Application& app, - Scheduler& scheduler, - int readThreads, - beast::Journal j) - : DatabaseShard( - scheduler, - readThreads, - app.config().section(ConfigSection::shardDatabase()), - j) - , app_(app) - , avgShardFileSz_(ledgersPerShard_ * kilobytes(192ull)) - , openFinalLimit_( - app.config().getValueFor(SizedItem::openFinalLimit, std::nullopt)) -{ - if (app.config().reporting()) - { - Throw( - "Attempted to create DatabaseShardImp in reporting mode. Reporting " - "does not support shards. Remove shards info from config"); - } -} - -bool -DatabaseShardImp::init() -{ - { - std::lock_guard lock(mutex_); - if (init_) - { - JLOG(j_.error()) << "already initialized"; - return false; - } - - if (!initConfig(lock)) - { - JLOG(j_.error()) << "invalid configuration file settings"; - return false; - } - - try - { - using namespace boost::filesystem; - - // Consolidate the main storage path and all historical paths - std::vector paths{dir_}; - paths.insert( - paths.end(), historicalPaths_.begin(), historicalPaths_.end()); - - for (auto const& path : paths) - { - if (exists(path)) - { - if (!is_directory(path)) - { - JLOG(j_.error()) << path << " must be a directory"; - return false; - } - } - else if (!create_directories(path)) - { - JLOG(j_.error()) - << "failed to create path: " + path.string(); - return false; - } - } - - if (!app_.config().standalone() && !historicalPaths_.empty()) - { - // Check historical paths for duplicated file systems - if (!checkHistoricalPaths(lock)) - return false; - } - - ctx_ = std::make_unique(); - ctx_->start(); - - // Find shards - std::uint32_t openFinals{0}; - for (auto const& path : paths) - { - for (auto const& it : directory_iterator(path)) - { - // Ignore files - if (!is_directory(it)) - continue; - - // Ignore nonnumerical directory names - auto const shardDir{it.path()}; - auto dirName{shardDir.stem().string()}; - if (!std::all_of( - dirName.begin(), dirName.end(), [](auto c) { - return ::isdigit(static_cast(c)); - })) - { - continue; - } - - // Ignore values below the earliest shard index - auto const shardIndex{std::stoul(dirName)}; - if (shardIndex < earliestShardIndex_) - { - JLOG(j_.debug()) - << "shard " << shardIndex - << " ignored, comes before earliest shard index " - << earliestShardIndex_; - continue; - } - - // Check if a previous database import failed - if (is_regular_file(shardDir / databaseImportMarker_)) - { - JLOG(j_.warn()) - << "shard " << shardIndex - << " previously failed database import, removing"; - remove_all(shardDir); - continue; - } - - auto shard{std::make_shared( - app_, *this, shardIndex, shardDir.parent_path(), j_)}; - if (!shard->init(scheduler_, *ctx_)) - { - // Remove corrupted or legacy shard - shard->removeOnDestroy(); - JLOG(j_.warn()) - << "shard " << shardIndex << " removed, " - << (shard->isLegacy() ? "legacy" : "corrupted") - << " shard"; - continue; - } - - switch (shard->getState()) - { - case ShardState::finalized: - if (++openFinals > openFinalLimit_) - shard->tryClose(); - shards_.emplace(shardIndex, std::move(shard)); - break; - - case ShardState::complete: - finalizeShard( - shards_.emplace(shardIndex, std::move(shard)) - .first->second, - true, - std::nullopt); - break; - - case ShardState::acquire: - if (acquireIndex_ != 0) - { - JLOG(j_.error()) - << "more than one shard being acquired"; - return false; - } - - shards_.emplace(shardIndex, std::move(shard)); - acquireIndex_ = shardIndex; - break; - - default: - JLOG(j_.error()) - << "shard " << shardIndex << " invalid state"; - return false; - } - } - } - } - catch (std::exception const& e) - { - JLOG(j_.fatal()) << "Exception caught in function " << __func__ - << ". Error: " << e.what(); - return false; - } - - init_ = true; - } - - updateFileStats(); - return true; -} - -std::optional -DatabaseShardImp::prepareLedger(std::uint32_t validLedgerSeq) -{ - std::optional shardIndex; - - { - std::lock_guard lock(mutex_); - assert(init_); - - if (acquireIndex_ != 0) - { - if (auto const it{shards_.find(acquireIndex_)}; it != shards_.end()) - return it->second->prepare(); - - // Should never get here - assert(false); - return std::nullopt; - } - - if (!canAdd_) - return std::nullopt; - - shardIndex = findAcquireIndex(validLedgerSeq, lock); - } - - if (!shardIndex) - { - JLOG(j_.debug()) << "no new shards to add"; - { - std::lock_guard lock(mutex_); - canAdd_ = false; - } - return std::nullopt; - } - - auto const pathDesignation = [this, shardIndex = *shardIndex]() { - std::lock_guard lock(mutex_); - return prepareForNewShard(shardIndex, numHistoricalShards(lock), lock); - }(); - - if (!pathDesignation) - return std::nullopt; - - auto const needsHistoricalPath = - *pathDesignation == PathDesignation::historical; - - auto shard = [this, shardIndex, needsHistoricalPath] { - std::lock_guard lock(mutex_); - return std::make_unique( - app_, - *this, - *shardIndex, - (needsHistoricalPath ? chooseHistoricalPath(lock) : ""), - j_); - }(); - - if (!shard->init(scheduler_, *ctx_)) - return std::nullopt; - - auto const ledgerSeq{shard->prepare()}; - { - std::lock_guard lock(mutex_); - shards_.emplace(*shardIndex, std::move(shard)); - acquireIndex_ = *shardIndex; - updatePeers(lock); - } - - return ledgerSeq; -} - -bool -DatabaseShardImp::prepareShards(std::vector const& shardIndexes) -{ - auto fail = [j = j_, &shardIndexes]( - std::string const& msg, - std::optional shardIndex = std::nullopt) { - auto multipleIndexPrequel = [&shardIndexes] { - std::vector indexesAsString(shardIndexes.size()); - std::transform( - shardIndexes.begin(), - shardIndexes.end(), - indexesAsString.begin(), - [](uint32_t const index) { return std::to_string(index); }); - - return std::string("shard") + - (shardIndexes.size() > 1 ? "s " : " ") + - boost::algorithm::join(indexesAsString, ", "); - }; - - JLOG(j.error()) << (shardIndex ? "shard " + std::to_string(*shardIndex) - : multipleIndexPrequel()) - << " " << msg; - return false; - }; - - if (shardIndexes.empty()) - return fail("invalid shard indexes"); - - std::lock_guard lock(mutex_); - assert(init_); - - if (!canAdd_) - return fail("cannot be stored at this time"); - - auto historicalShardsToPrepare = 0; - - for (auto const shardIndex : shardIndexes) - { - if (shardIndex < earliestShardIndex_) - { - return fail( - "comes before earliest shard index " + - std::to_string(earliestShardIndex_), - shardIndex); - } - - // If we are synced to the network, check if the shard index is - // greater or equal to the current or validated shard index. - auto seqCheck = [&](std::uint32_t ledgerSeq) { - if (ledgerSeq >= earliestLedgerSeq_ && - shardIndex >= seqToShardIndex(ledgerSeq)) - { - return fail("invalid index", shardIndex); - } - return true; - }; - if (!seqCheck(app_.getLedgerMaster().getValidLedgerIndex() + 1) || - !seqCheck(app_.getLedgerMaster().getCurrentLedgerIndex())) - { - return fail("invalid index", shardIndex); - } - - if (shards_.find(shardIndex) != shards_.end()) - return fail("is already stored", shardIndex); - - if (preparedIndexes_.find(shardIndex) != preparedIndexes_.end()) - return fail( - "is already queued for import from the shard archive handler", - shardIndex); - - if (databaseImportStatus_) - { - if (auto shard = databaseImportStatus_->currentShard.lock(); shard) - { - if (shard->index() == shardIndex) - return fail( - "is being imported from the nodestore", shardIndex); - } - } - - // Any shard earlier than the two most recent shards - // is a historical shard - if (shardIndex < shardBoundaryIndex()) - ++historicalShardsToPrepare; - } - - auto const numHistShards = numHistoricalShards(lock); - - // Check shard count and available storage space - if (numHistShards + historicalShardsToPrepare > maxHistoricalShards_) - return fail("maximum number of historical shards reached"); - - if (historicalShardsToPrepare) - { - // Check available storage space for historical shards - if (!sufficientStorage( - historicalShardsToPrepare, PathDesignation::historical, lock)) - return fail("insufficient storage space available"); - } - - if (auto const recentShardsToPrepare = - shardIndexes.size() - historicalShardsToPrepare; - recentShardsToPrepare) - { - // Check available storage space for recent shards - if (!sufficientStorage( - recentShardsToPrepare, PathDesignation::none, lock)) - return fail("insufficient storage space available"); - } - - for (auto const shardIndex : shardIndexes) - preparedIndexes_.emplace(shardIndex); - - updatePeers(lock); - return true; -} - -void -DatabaseShardImp::removePreShard(std::uint32_t shardIndex) -{ - std::lock_guard lock(mutex_); - assert(init_); - - if (preparedIndexes_.erase(shardIndex)) - updatePeers(lock); -} - -std::string -DatabaseShardImp::getPreShards() -{ - RangeSet rs; - { - std::lock_guard lock(mutex_); - assert(init_); - - for (auto const& shardIndex : preparedIndexes_) - rs.insert(shardIndex); - } - - if (rs.empty()) - return {}; - - return ripple::to_string(rs); -}; - -bool -DatabaseShardImp::importShard( - std::uint32_t shardIndex, - boost::filesystem::path const& srcDir) -{ - auto fail = [&](std::string const& msg, - std::lock_guard const& lock) { - JLOG(j_.error()) << "shard " << shardIndex << " " << msg; - - // Remove the failed import shard index so it can be retried - preparedIndexes_.erase(shardIndex); - updatePeers(lock); - return false; - }; - - using namespace boost::filesystem; - try - { - if (!is_directory(srcDir) || is_empty(srcDir)) - { - return fail( - "invalid source directory " + srcDir.string(), - std::lock_guard(mutex_)); - } - } - catch (std::exception const& e) - { - return fail( - std::string(". Exception caught in function ") + __func__ + - ". Error: " + e.what(), - std::lock_guard(mutex_)); - } - - auto const expectedHash{app_.getLedgerMaster().walkHashBySeq( - lastLedgerSeq(shardIndex), InboundLedger::Reason::GENERIC)}; - if (!expectedHash) - return fail("expected hash not found", std::lock_guard(mutex_)); - - path dstDir; - { - std::lock_guard lock(mutex_); - if (shards_.find(shardIndex) != shards_.end()) - return fail("already exists", lock); - - // Check shard was prepared for import - if (preparedIndexes_.find(shardIndex) == preparedIndexes_.end()) - return fail("was not prepared for import", lock); - - auto const pathDesignation{ - prepareForNewShard(shardIndex, numHistoricalShards(lock), lock)}; - if (!pathDesignation) - return fail("failed to import", lock); - - if (*pathDesignation == PathDesignation::historical) - dstDir = chooseHistoricalPath(lock); - else - dstDir = dir_; - } - dstDir /= std::to_string(shardIndex); - - auto renameDir = [&, fname = __func__](path const& src, path const& dst) { - try - { - rename(src, dst); - } - catch (std::exception const& e) - { - return fail( - std::string(". Exception caught in function ") + fname + - ". Error: " + e.what(), - std::lock_guard(mutex_)); - } - return true; - }; - - // Rename source directory to the shard database directory - if (!renameDir(srcDir, dstDir)) - return false; - - // Create the new shard - auto shard{std::make_unique( - app_, *this, shardIndex, dstDir.parent_path(), j_)}; - - if (!shard->init(scheduler_, *ctx_) || - shard->getState() != ShardState::complete) - { - shard.reset(); - renameDir(dstDir, srcDir); - return fail("failed to import", std::lock_guard(mutex_)); - } - - auto const [it, inserted] = [&]() { - std::lock_guard lock(mutex_); - preparedIndexes_.erase(shardIndex); - return shards_.emplace(shardIndex, std::move(shard)); - }(); - - if (!inserted) - { - shard.reset(); - renameDir(dstDir, srcDir); - return fail("failed to import", std::lock_guard(mutex_)); - } - - finalizeShard(it->second, true, expectedHash); - return true; -} - -std::shared_ptr -DatabaseShardImp::fetchLedger(uint256 const& hash, std::uint32_t ledgerSeq) -{ - auto const shardIndex{seqToShardIndex(ledgerSeq)}; - { - std::shared_ptr shard; - { - std::lock_guard lock(mutex_); - assert(init_); - - auto const it{shards_.find(shardIndex)}; - if (it == shards_.end()) - return nullptr; - shard = it->second; - } - - // Ledger must be stored in a final or acquiring shard - switch (shard->getState()) - { - case ShardState::finalized: - break; - case ShardState::acquire: - if (shard->containsLedger(ledgerSeq)) - break; - [[fallthrough]]; - default: - return nullptr; - } - } - - auto const nodeObject{Database::fetchNodeObject(hash, ledgerSeq)}; - if (!nodeObject) - return nullptr; - - auto fail = [&](std::string const& msg) -> std::shared_ptr { - JLOG(j_.error()) << "shard " << shardIndex << " " << msg; - return nullptr; - }; - - auto ledger{std::make_shared( - deserializePrefixedHeader(makeSlice(nodeObject->getData())), - app_.config(), - *app_.getShardFamily())}; - - if (ledger->info().seq != ledgerSeq) - { - return fail( - "encountered invalid ledger sequence " + std::to_string(ledgerSeq)); - } - if (ledger->info().hash != hash) - { - return fail( - "encountered invalid ledger hash " + to_string(hash) + - " on sequence " + std::to_string(ledgerSeq)); - } - - ledger->setFull(); - if (!ledger->stateMap().fetchRoot( - SHAMapHash{ledger->info().accountHash}, nullptr)) - { - return fail( - "is missing root STATE node on hash " + to_string(hash) + - " on sequence " + std::to_string(ledgerSeq)); - } - - if (ledger->info().txHash.isNonZero()) - { - if (!ledger->txMap().fetchRoot( - SHAMapHash{ledger->info().txHash}, nullptr)) - { - return fail( - "is missing root TXN node on hash " + to_string(hash) + - " on sequence " + std::to_string(ledgerSeq)); - } - } - return ledger; -} - -void -DatabaseShardImp::setStored(std::shared_ptr const& ledger) -{ - auto const ledgerSeq{ledger->info().seq}; - if (ledger->info().hash.isZero()) - { - JLOG(j_.error()) << "zero ledger hash for ledger sequence " - << ledgerSeq; - return; - } - if (ledger->info().accountHash.isZero()) - { - JLOG(j_.error()) << "zero account hash for ledger sequence " - << ledgerSeq; - return; - } - if (ledger->stateMap().getHash().isNonZero() && - !ledger->stateMap().isValid()) - { - JLOG(j_.error()) << "invalid state map for ledger sequence " - << ledgerSeq; - return; - } - if (ledger->info().txHash.isNonZero() && !ledger->txMap().isValid()) - { - JLOG(j_.error()) << "invalid transaction map for ledger sequence " - << ledgerSeq; - return; - } - - auto const shardIndex{seqToShardIndex(ledgerSeq)}; - std::shared_ptr shard; - { - std::lock_guard lock(mutex_); - assert(init_); - - if (shardIndex != acquireIndex_) - { - JLOG(j_.trace()) - << "shard " << shardIndex << " is not being acquired"; - return; - } - - auto const it{shards_.find(shardIndex)}; - if (it == shards_.end()) - { - JLOG(j_.error()) - << "shard " << shardIndex << " is not being acquired"; - return; - } - shard = it->second; - } - - if (shard->containsLedger(ledgerSeq)) - { - JLOG(j_.trace()) << "shard " << shardIndex << " ledger already stored"; - return; - } - - setStoredInShard(shard, ledger); -} - -std::unique_ptr -DatabaseShardImp::getShardInfo() const -{ - std::lock_guard lock(mutex_); - return getShardInfo(lock); -} - -void -DatabaseShardImp::stop() -{ - // Stop read threads in base before data members are destroyed - Database::stop(); - std::vector> shards; - { - std::lock_guard lock(mutex_); - shards.reserve(shards_.size()); - for (auto const& [_, shard] : shards_) - { - shards.push_back(shard); - shard->stop(); - } - shards_.clear(); - } - taskQueue_.stop(); - - // All shards should be expired at this point - for (auto const& wptr : shards) - { - if (auto const shard{wptr.lock()}) - { - JLOG(j_.warn()) << " shard " << shard->index() << " unexpired"; - } - } - - std::unique_lock lock(mutex_); - - // Notify the shard being imported - // from the node store to stop - if (databaseImportStatus_) - { - // A node store import is in progress - if (auto importShard = databaseImportStatus_->currentShard.lock(); - importShard) - importShard->stop(); - } - - // Wait for the node store import thread - // if necessary - if (databaseImporter_.joinable()) - { - // Tells the import function to halt - haltDatabaseImport_ = true; - - // Wait for the function to exit - while (databaseImportStatus_) - { - // Unlock just in case the import - // function is waiting on the mutex - lock.unlock(); - - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - lock.lock(); - } - - // Calling join while holding the mutex_ without - // first making sure that doImportDatabase has - // exited could lead to deadlock via the mutex - // acquisition that occurs in that function - if (databaseImporter_.joinable()) - databaseImporter_.join(); - } -} - -void -DatabaseShardImp::importDatabase(Database& source) -{ - std::lock_guard lock(mutex_); - assert(init_); - - // Only the application local node store can be imported - assert(&source == &app_.getNodeStore()); - - if (databaseImporter_.joinable()) - { - assert(false); - JLOG(j_.error()) << "database import already in progress"; - return; - } - - startDatabaseImportThread(lock); -} - -void -DatabaseShardImp::doImportDatabase() -{ - auto shouldHalt = [this] { - bool expected = true; - return haltDatabaseImport_.compare_exchange_strong(expected, false) || - isStopping(); - }; - - if (shouldHalt()) - return; - - auto loadLedger = - [this](char const* const sortOrder) -> std::optional { - std::shared_ptr ledger; - std::uint32_t ledgerSeq{0}; - std::optional info; - if (sortOrder == std::string("asc")) - { - info = dynamic_cast(&app_.getRelationalDatabase()) - ->getLimitedOldestLedgerInfo(earliestLedgerSeq()); - } - else - { - info = dynamic_cast(&app_.getRelationalDatabase()) - ->getLimitedNewestLedgerInfo(earliestLedgerSeq()); - } - if (info) - { - ledger = loadLedgerHelper(*info, app_, false); - ledgerSeq = info->seq; - } - if (!ledger || ledgerSeq == 0) - { - JLOG(j_.error()) << "no suitable ledgers were found in" - " the SQLite database to import"; - return std::nullopt; - } - return ledgerSeq; - }; - - // Find earliest ledger sequence stored - auto const earliestLedgerSeq{loadLedger("asc")}; - if (!earliestLedgerSeq) - return; - - auto const earliestIndex = [&] { - auto earliestIndex = seqToShardIndex(*earliestLedgerSeq); - - // Consider only complete shards - if (earliestLedgerSeq != firstLedgerSeq(earliestIndex)) - ++earliestIndex; - - return earliestIndex; - }(); - - // Find last ledger sequence stored - auto const latestLedgerSeq = loadLedger("desc"); - if (!latestLedgerSeq) - return; - - auto const latestIndex = [&] { - auto latestIndex = seqToShardIndex(*latestLedgerSeq); - - // Consider only complete shards - if (latestLedgerSeq != lastLedgerSeq(latestIndex)) - --latestIndex; - - return latestIndex; - }(); - - if (latestIndex < earliestIndex) - { - JLOG(j_.error()) << "no suitable ledgers were found in" - " the SQLite database to import"; - return; - } - - JLOG(j_.debug()) << "Importing ledgers for shards " << earliestIndex - << " through " << latestIndex; - - { - std::lock_guard lock(mutex_); - - assert(!databaseImportStatus_); - databaseImportStatus_ = std::make_unique( - earliestIndex, latestIndex, 0); - } - - // Import the shards - for (std::uint32_t shardIndex = earliestIndex; shardIndex <= latestIndex; - ++shardIndex) - { - if (shouldHalt()) - return; - - auto const pathDesignation = [this, shardIndex] { - std::lock_guard lock(mutex_); - - auto const numHistShards = numHistoricalShards(lock); - auto const pathDesignation = - prepareForNewShard(shardIndex, numHistShards, lock); - - return pathDesignation; - }(); - - if (!pathDesignation) - break; - - { - std::lock_guard lock(mutex_); - - // Skip if being acquired - if (shardIndex == acquireIndex_) - { - JLOG(j_.debug()) - << "shard " << shardIndex << " already being acquired"; - continue; - } - - // Skip if being imported from the shard archive handler - if (preparedIndexes_.find(shardIndex) != preparedIndexes_.end()) - { - JLOG(j_.debug()) - << "shard " << shardIndex << " already being imported"; - continue; - } - - // Skip if stored - if (shards_.find(shardIndex) != shards_.end()) - { - JLOG(j_.debug()) << "shard " << shardIndex << " already stored"; - continue; - } - } - - std::uint32_t const firstSeq = firstLedgerSeq(shardIndex); - std::uint32_t const lastSeq = - std::max(firstSeq, lastLedgerSeq(shardIndex)); - - // Verify SQLite ledgers are in the node store - { - auto const ledgerHashes{ - app_.getRelationalDatabase().getHashesByIndex( - firstSeq, lastSeq)}; - if (ledgerHashes.size() != maxLedgers(shardIndex)) - continue; - - auto& source = app_.getNodeStore(); - bool valid{true}; - - for (std::uint32_t n = firstSeq; n <= lastSeq; ++n) - { - if (!source.fetchNodeObject(ledgerHashes.at(n).ledgerHash, n)) - { - JLOG(j_.warn()) << "SQLite ledger sequence " << n - << " mismatches node store"; - valid = false; - break; - } - } - if (!valid) - continue; - } - - if (shouldHalt()) - return; - - bool const needsHistoricalPath = - *pathDesignation == PathDesignation::historical; - - auto const path = needsHistoricalPath - ? chooseHistoricalPath(std::lock_guard(mutex_)) - : dir_; - - // Create the new shard - auto shard{std::make_shared(app_, *this, shardIndex, path, j_)}; - if (!shard->init(scheduler_, *ctx_)) - continue; - - { - std::lock_guard lock(mutex_); - - if (shouldHalt()) - return; - - databaseImportStatus_->currentIndex = shardIndex; - databaseImportStatus_->currentShard = shard; - databaseImportStatus_->firstSeq = firstSeq; - databaseImportStatus_->lastSeq = lastSeq; - } - - // Create a marker file to signify a database import in progress - auto const shardDir{path / std::to_string(shardIndex)}; - auto const markerFile{shardDir / databaseImportMarker_}; - { - std::ofstream ofs{markerFile.string()}; - if (!ofs.is_open()) - { - JLOG(j_.error()) << "shard " << shardIndex - << " failed to create temp marker file"; - shard->removeOnDestroy(); - continue; - } - } - - // Copy the ledgers from node store - std::shared_ptr recentStored; - std::optional lastLedgerHash; - - while (auto const ledgerSeq = shard->prepare()) - { - if (shouldHalt()) - return; - - // Not const so it may be moved later - auto ledger{loadByIndex(*ledgerSeq, app_, false)}; - if (!ledger || ledger->info().seq != ledgerSeq) - break; - - auto const result{shard->storeLedger(ledger, recentStored)}; - storeStats(result.count, result.size); - if (result.error) - break; - - if (!shard->setLedgerStored(ledger)) - break; - - if (!lastLedgerHash && ledgerSeq == lastSeq) - lastLedgerHash = ledger->info().hash; - - recentStored = std::move(ledger); - } - - if (shouldHalt()) - return; - - using namespace boost::filesystem; - bool success{false}; - if (lastLedgerHash && shard->getState() == ShardState::complete) - { - // Store shard final key - Serializer s; - s.add32(Shard::version); - s.add32(firstLedgerSeq(shardIndex)); - s.add32(lastLedgerSeq(shardIndex)); - s.addBitString(*lastLedgerHash); - auto const nodeObject{NodeObject::createObject( - hotUNKNOWN, std::move(s.modData()), Shard::finalKey)}; - - if (shard->storeNodeObject(nodeObject)) - { - try - { - std::lock_guard lock(mutex_); - - // The database import process is complete and the - // marker file is no longer required - remove_all(markerFile); - - JLOG(j_.debug()) << "shard " << shardIndex - << " was successfully imported" - " from the NodeStore"; - finalizeShard( - shards_.emplace(shardIndex, std::move(shard)) - .first->second, - true, - std::nullopt); - - // This variable is meant to capture the success - // of everything up to the point of shard finalization. - // If the shard fails to finalize, this condition will - // be handled by the finalization function itself, and - // not here. - success = true; - } - catch (std::exception const& e) - { - JLOG(j_.fatal()) << "shard index " << shardIndex - << ". Exception caught in function " - << __func__ << ". Error: " << e.what(); - } - } - } - - if (!success) - { - JLOG(j_.error()) << "shard " << shardIndex - << " failed to import from the NodeStore"; - - if (shard) - shard->removeOnDestroy(); - } - } - - if (shouldHalt()) - return; - - updateFileStats(); -} - -std::int32_t -DatabaseShardImp::getWriteLoad() const -{ - std::shared_ptr shard; - { - std::lock_guard lock(mutex_); - assert(init_); - - auto const it{shards_.find(acquireIndex_)}; - if (it == shards_.end()) - return 0; - shard = it->second; - } - - return shard->getWriteLoad(); -} - -void -DatabaseShardImp::store( - NodeObjectType type, - Blob&& data, - uint256 const& hash, - std::uint32_t ledgerSeq) -{ - auto const shardIndex{seqToShardIndex(ledgerSeq)}; - std::shared_ptr shard; - { - std::lock_guard lock(mutex_); - if (shardIndex != acquireIndex_) - { - JLOG(j_.trace()) - << "shard " << shardIndex << " is not being acquired"; - return; - } - - auto const it{shards_.find(shardIndex)}; - if (it == shards_.end()) - { - JLOG(j_.error()) - << "shard " << shardIndex << " is not being acquired"; - return; - } - shard = it->second; - } - - auto const nodeObject{ - NodeObject::createObject(type, std::move(data), hash)}; - if (shard->storeNodeObject(nodeObject)) - storeStats(1, nodeObject->getData().size()); -} - -bool -DatabaseShardImp::storeLedger(std::shared_ptr const& srcLedger) -{ - auto const ledgerSeq{srcLedger->info().seq}; - auto const shardIndex{seqToShardIndex(ledgerSeq)}; - std::shared_ptr shard; - { - std::lock_guard lock(mutex_); - assert(init_); - - if (shardIndex != acquireIndex_) - { - JLOG(j_.trace()) - << "shard " << shardIndex << " is not being acquired"; - return false; - } - - auto const it{shards_.find(shardIndex)}; - if (it == shards_.end()) - { - JLOG(j_.error()) - << "shard " << shardIndex << " is not being acquired"; - return false; - } - shard = it->second; - } - - auto const result{shard->storeLedger(srcLedger, nullptr)}; - storeStats(result.count, result.size); - if (result.error || result.count == 0 || result.size == 0) - return false; - - return setStoredInShard(shard, srcLedger); -} - -void -DatabaseShardImp::sweep() -{ - std::vector> shards; - { - std::lock_guard lock(mutex_); - assert(init_); - - shards.reserve(shards_.size()); - for (auto const& e : shards_) - shards.push_back(e.second); - } - - std::vector> openFinals; - openFinals.reserve(openFinalLimit_); - - for (auto const& weak : shards) - { - if (auto const shard{weak.lock()}; shard && shard->isOpen()) - { - if (shard->getState() == ShardState::finalized) - openFinals.emplace_back(std::move(shard)); - } - } - - if (openFinals.size() > openFinalLimit_) - { - JLOG(j_.trace()) << "Open shards exceed configured limit of " - << openFinalLimit_ << " by " - << (openFinals.size() - openFinalLimit_); - - // Try to close enough shards to be within the limit. - // Sort ascending on last use so the oldest are removed first. - std::sort( - openFinals.begin(), - openFinals.end(), - [&](std::shared_ptr const& lhsShard, - std::shared_ptr const& rhsShard) { - return lhsShard->getLastUse() < rhsShard->getLastUse(); - }); - - for (auto it{openFinals.cbegin()}; - it != openFinals.cend() && openFinals.size() > openFinalLimit_;) - { - if ((*it)->tryClose()) - it = openFinals.erase(it); - else - ++it; - } - } -} - -Json::Value -DatabaseShardImp::getDatabaseImportStatus() const -{ - if (std::lock_guard lock(mutex_); databaseImportStatus_) - { - Json::Value ret(Json::objectValue); - - ret[jss::firstShardIndex] = databaseImportStatus_->earliestIndex; - ret[jss::lastShardIndex] = databaseImportStatus_->latestIndex; - ret[jss::currentShardIndex] = databaseImportStatus_->currentIndex; - - Json::Value currentShard(Json::objectValue); - currentShard[jss::firstSequence] = databaseImportStatus_->firstSeq; - currentShard[jss::lastSequence] = databaseImportStatus_->lastSeq; - - if (auto shard = databaseImportStatus_->currentShard.lock(); shard) - currentShard[jss::storedSeqs] = shard->getStoredSeqs(); - - ret[jss::currentShard] = currentShard; - - if (haltDatabaseImport_) - ret[jss::message] = "Database import halt initiated..."; - - return ret; - } - - return RPC::make_error(rpcINTERNAL, "Database import not running"); -} - -Json::Value -DatabaseShardImp::startNodeToShard() -{ - std::lock_guard lock(mutex_); - - if (!init_) - return RPC::make_error(rpcINTERNAL, "Shard store not initialized"); - - if (databaseImporter_.joinable()) - return RPC::make_error( - rpcINTERNAL, "Database import already in progress"); - - if (isStopping()) - return RPC::make_error(rpcINTERNAL, "Node is shutting down"); - - startDatabaseImportThread(lock); - - Json::Value result(Json::objectValue); - result[jss::message] = "Database import initiated..."; - - return result; -} - -Json::Value -DatabaseShardImp::stopNodeToShard() -{ - std::lock_guard lock(mutex_); - - if (!init_) - return RPC::make_error(rpcINTERNAL, "Shard store not initialized"); - - if (!databaseImporter_.joinable()) - return RPC::make_error(rpcINTERNAL, "Database import not running"); - - if (isStopping()) - return RPC::make_error(rpcINTERNAL, "Node is shutting down"); - - haltDatabaseImport_ = true; - - Json::Value result(Json::objectValue); - result[jss::message] = "Database import halt initiated..."; - - return result; -} - -std::optional -DatabaseShardImp::getDatabaseImportSequence() const -{ - std::lock_guard lock(mutex_); - - if (!databaseImportStatus_) - return {}; - - return databaseImportStatus_->firstSeq; -} - -bool -DatabaseShardImp::initConfig(std::lock_guard const&) -{ - auto fail = [j = j_](std::string const& msg) { - JLOG(j.error()) << "[" << ConfigSection::shardDatabase() << "] " << msg; - return false; - }; - - Config const& config{app_.config()}; - Section const& section{config.section(ConfigSection::shardDatabase())}; - - auto compare = [&](std::string const& name, std::uint32_t defaultValue) { - std::uint32_t shardDBValue{defaultValue}; - get_if_exists(section, name, shardDBValue); - - std::uint32_t nodeDBValue{defaultValue}; - get_if_exists( - config.section(ConfigSection::nodeDatabase()), name, nodeDBValue); - - return shardDBValue == nodeDBValue; - }; - - // If ledgers_per_shard or earliest_seq are specified, - // they must be equally assigned in 'node_db' - if (!compare("ledgers_per_shard", DEFAULT_LEDGERS_PER_SHARD)) - { - return fail( - "and [" + ConfigSection::nodeDatabase() + "] define different '" + - "ledgers_per_shard" + "' values"); - } - if (!compare("earliest_seq", XRP_LEDGER_EARLIEST_SEQ)) - { - return fail( - "and [" + ConfigSection::nodeDatabase() + "] define different '" + - "earliest_seq" + "' values"); - } - - using namespace boost::filesystem; - if (!get_if_exists(section, "path", dir_)) - return fail("'path' missing"); - - { - get_if_exists(section, "max_historical_shards", maxHistoricalShards_); - - Section const& historicalShardPaths = - config.section(SECTION_HISTORICAL_SHARD_PATHS); - - auto values = historicalShardPaths.values(); - - std::sort(values.begin(), values.end()); - values.erase(std::unique(values.begin(), values.end()), values.end()); - - for (auto const& s : values) - { - auto const dir = path(s); - if (dir_ == dir) - { - return fail( - "the 'path' cannot also be in the " - "'historical_shard_path' section"); - } - - historicalPaths_.push_back(s); - } - } - - // NuDB is the default and only supported permanent storage backend - backendName_ = get(section, "type", "nudb"); - if (!boost::iequals(backendName_, "NuDB")) - return fail("'type' value unsupported"); - - return true; -} - -std::shared_ptr -DatabaseShardImp::fetchNodeObject( - uint256 const& hash, - std::uint32_t ledgerSeq, - FetchReport& fetchReport, - bool duplicate) -{ - auto const shardIndex{seqToShardIndex(ledgerSeq)}; - std::shared_ptr shard; - { - std::lock_guard lock(mutex_); - auto const it{shards_.find(shardIndex)}; - if (it == shards_.end()) - return nullptr; - shard = it->second; - } - - return shard->fetchNodeObject(hash, fetchReport); -} - -std::optional -DatabaseShardImp::findAcquireIndex( - std::uint32_t validLedgerSeq, - std::lock_guard const&) -{ - if (validLedgerSeq < earliestLedgerSeq_) - return std::nullopt; - - auto const maxShardIndex{[this, validLedgerSeq]() { - auto shardIndex{seqToShardIndex(validLedgerSeq)}; - if (validLedgerSeq != lastLedgerSeq(shardIndex)) - --shardIndex; - return shardIndex; - }()}; - auto const maxNumShards{maxShardIndex - earliestShardIndex_ + 1}; - - // Check if the shard store has all shards - if (shards_.size() >= maxNumShards) - return std::nullopt; - - if (maxShardIndex < 1024 || - static_cast(shards_.size()) / maxNumShards > 0.5f) - { - // Small or mostly full index space to sample - // Find the available indexes and select one at random - std::vector available; - available.reserve(maxNumShards - shards_.size()); - - for (auto shardIndex = earliestShardIndex_; shardIndex <= maxShardIndex; - ++shardIndex) - { - if (shards_.find(shardIndex) == shards_.end() && - preparedIndexes_.find(shardIndex) == preparedIndexes_.end()) - { - available.push_back(shardIndex); - } - } - - if (available.empty()) - return std::nullopt; - - if (available.size() == 1) - return available.front(); - - return available[rand_int( - 0u, static_cast(available.size() - 1))]; - } - - // Large, sparse index space to sample - // Keep choosing indexes at random until an available one is found - // chances of running more than 30 times is less than 1 in a billion - for (int i = 0; i < 40; ++i) - { - auto const shardIndex{rand_int(earliestShardIndex_, maxShardIndex)}; - if (shards_.find(shardIndex) == shards_.end() && - preparedIndexes_.find(shardIndex) == preparedIndexes_.end()) - { - return shardIndex; - } - } - - assert(false); - return std::nullopt; -} - -void -DatabaseShardImp::finalizeShard( - std::shared_ptr& shard, - bool const writeSQLite, - std::optional const& expectedHash) -{ - taskQueue_.addTask([this, - wptr = std::weak_ptr(shard), - writeSQLite, - expectedHash]() { - if (isStopping()) - return; - - auto shard{wptr.lock()}; - if (!shard) - { - JLOG(j_.debug()) << "Shard removed before being finalized"; - return; - } - - if (!shard->finalize(writeSQLite, expectedHash)) - { - if (isStopping()) - return; - - // Invalid or corrupt shard, remove it - removeFailedShard(shard); - return; - } - - if (isStopping()) - return; - - { - auto const boundaryIndex{shardBoundaryIndex()}; - std::lock_guard lock(mutex_); - - if (shard->index() < boundaryIndex) - { - // This is a historical shard - if (!historicalPaths_.empty() && - shard->getDir().parent_path() == dir_) - { - // Shard wasn't placed at a separate historical path - JLOG(j_.warn()) << "shard " << shard->index() - << " is not stored at a historical path"; - } - } - else - { - // Not a historical shard. Shift recent shards if necessary - assert(!boundaryIndex || shard->index() - boundaryIndex <= 1); - relocateOutdatedShards(lock); - - // Set the appropriate recent shard index - if (shard->index() == boundaryIndex) - secondLatestShardIndex_ = shard->index(); - else - latestShardIndex_ = shard->index(); - - if (shard->getDir().parent_path() != dir_) - { - JLOG(j_.warn()) << "shard " << shard->index() - << " is not stored at the path"; - } - } - - updatePeers(lock); - } - - updateFileStats(); - }); -} - -void -DatabaseShardImp::updateFileStats() -{ - std::vector> shards; - { - std::lock_guard lock(mutex_); - if (shards_.empty()) - return; - - shards.reserve(shards_.size()); - for (auto const& e : shards_) - shards.push_back(e.second); - } - - std::uint64_t sumSz{0}; - std::uint32_t sumFd{0}; - std::uint32_t numShards{0}; - for (auto const& weak : shards) - { - if (auto const shard{weak.lock()}; shard) - { - auto const [sz, fd] = shard->getFileInfo(); - sumSz += sz; - sumFd += fd; - ++numShards; - } - } - - std::lock_guard lock(mutex_); - fileSz_ = sumSz; - fdRequired_ = sumFd; - avgShardFileSz_ = (numShards == 0 ? fileSz_ : fileSz_ / numShards); - - if (!canAdd_) - return; - - if (auto const count = numHistoricalShards(lock); - count >= maxHistoricalShards_) - { - if (maxHistoricalShards_) - { - // In order to avoid excessive output, don't produce - // this warning if the server isn't configured to - // store historical shards. - JLOG(j_.warn()) << "maximum number of historical shards reached"; - } - - canAdd_ = false; - } - else if (!sufficientStorage( - maxHistoricalShards_ - count, - PathDesignation::historical, - lock)) - { - JLOG(j_.warn()) - << "maximum shard store size exceeds available storage space"; - - canAdd_ = false; - } -} - -bool -DatabaseShardImp::sufficientStorage( - std::uint32_t numShards, - PathDesignation pathDesignation, - std::lock_guard const&) const -{ - try - { - std::vector capacities; - - if (pathDesignation == PathDesignation::historical && - !historicalPaths_.empty()) - { - capacities.reserve(historicalPaths_.size()); - - for (auto const& path : historicalPaths_) - { - // Get the available storage for each historical path - auto const availableSpace = - boost::filesystem::space(path).available; - - capacities.push_back(availableSpace); - } - } - else - { - // Get the available storage for the main shard path - capacities.push_back(boost::filesystem::space(dir_).available); - } - - for (std::uint64_t const capacity : capacities) - { - // Leverage all the historical shard paths to - // see if collectively they can fit the specified - // number of shards. For this to work properly, - // each historical path must correspond to a separate - // physical device or filesystem. - - auto const shardCap = capacity / avgShardFileSz_; - if (numShards <= shardCap) - return true; - - numShards -= shardCap; - } - } - catch (std::exception const& e) - { - JLOG(j_.fatal()) << "Exception caught in function " << __func__ - << ". Error: " << e.what(); - return false; - } - - return false; -} - -bool -DatabaseShardImp::setStoredInShard( - std::shared_ptr& shard, - std::shared_ptr const& ledger) -{ - if (!shard->setLedgerStored(ledger)) - { - // Invalid or corrupt shard, remove it - removeFailedShard(shard); - return false; - } - - if (shard->getState() == ShardState::complete) - { - std::lock_guard lock(mutex_); - if (auto const it{shards_.find(shard->index())}; it != shards_.end()) - { - if (shard->index() == acquireIndex_) - acquireIndex_ = 0; - - finalizeShard(it->second, false, std::nullopt); - } - else - { - JLOG(j_.debug()) - << "shard " << shard->index() << " is no longer being acquired"; - } - } - - updateFileStats(); - return true; -} - -void -DatabaseShardImp::removeFailedShard(std::shared_ptr& shard) -{ - { - std::lock_guard lock(mutex_); - - if (shard->index() == acquireIndex_) - acquireIndex_ = 0; - - if (shard->index() == latestShardIndex_) - latestShardIndex_ = std::nullopt; - - if (shard->index() == secondLatestShardIndex_) - secondLatestShardIndex_ = std::nullopt; - } - - shard->removeOnDestroy(); - - // Reset the shared_ptr to invoke the shard's - // destructor and remove it from the server - shard.reset(); - updateFileStats(); -} - -std::uint32_t -DatabaseShardImp::shardBoundaryIndex() const -{ - auto const validIndex = app_.getLedgerMaster().getValidLedgerIndex(); - - if (validIndex < earliestLedgerSeq_) - return 0; - - // Shards with an index earlier than the recent shard boundary index - // are considered historical. The three shards at or later than - // this index consist of the two most recently validated shards - // and the shard still in the process of being built by live - // transactions. - return seqToShardIndex(validIndex) - 1; -} - -std::uint32_t -DatabaseShardImp::numHistoricalShards( - std::lock_guard const& lock) const -{ - auto const boundaryIndex{shardBoundaryIndex()}; - return std::count_if( - shards_.begin(), shards_.end(), [boundaryIndex](auto const& entry) { - return entry.first < boundaryIndex; - }); -} - -void -DatabaseShardImp::relocateOutdatedShards( - std::lock_guard const& lock) -{ - auto& cur{latestShardIndex_}; - auto& prev{secondLatestShardIndex_}; - if (!cur && !prev) - return; - - auto const latestShardIndex = - seqToShardIndex(app_.getLedgerMaster().getValidLedgerIndex()); - auto const separateHistoricalPath = !historicalPaths_.empty(); - - auto const removeShard = [this](std::uint32_t const shardIndex) -> void { - canAdd_ = false; - - if (auto it = shards_.find(shardIndex); it != shards_.end()) - { - if (it->second) - removeFailedShard(it->second); - else - { - JLOG(j_.warn()) << "can't find shard to remove"; - } - } - else - { - JLOG(j_.warn()) << "can't find shard to remove"; - } - }; - - auto const keepShard = [this, &lock, removeShard, separateHistoricalPath]( - std::uint32_t const shardIndex) -> bool { - if (numHistoricalShards(lock) >= maxHistoricalShards_) - { - JLOG(j_.error()) << "maximum number of historical shards reached"; - removeShard(shardIndex); - return false; - } - if (separateHistoricalPath && - !sufficientStorage(1, PathDesignation::historical, lock)) - { - JLOG(j_.error()) << "insufficient storage space available"; - removeShard(shardIndex); - return false; - } - - return true; - }; - - // Move a shard from the main shard path to a historical shard - // path by copying the contents, and creating a new shard. - auto const moveShard = [this, - &lock](std::uint32_t const shardIndex) -> void { - auto it{shards_.find(shardIndex)}; - if (it == shards_.end()) - { - JLOG(j_.warn()) << "can't find shard to move to historical path"; - return; - } - - auto& shard{it->second}; - - // Close any open file descriptors before moving the shard - // directory. Don't call removeOnDestroy since that would - // attempt to close the fds after the directory has been moved. - if (!shard->tryClose()) - { - JLOG(j_.warn()) << "can't close shard to move to historical path"; - return; - } - - auto const dst{chooseHistoricalPath(lock)}; - try - { - // Move the shard directory to the new path - boost::filesystem::rename( - shard->getDir().string(), dst / std::to_string(shardIndex)); - } - catch (...) - { - JLOG(j_.error()) << "shard " << shardIndex - << " failed to move to historical storage"; - return; - } - - // Create a shard instance at the new location - shard = std::make_shared(app_, *this, shardIndex, dst, j_); - - // Open the new shard - if (!shard->init(scheduler_, *ctx_)) - { - JLOG(j_.error()) << "shard " << shardIndex - << " failed to open in historical storage"; - shard->removeOnDestroy(); - shard.reset(); - } - }; - - // See if either of the recent shards needs to be updated - bool const curNotSynched = - latestShardIndex_ && *latestShardIndex_ != latestShardIndex; - bool const prevNotSynched = secondLatestShardIndex_ && - *secondLatestShardIndex_ != latestShardIndex - 1; - - // A new shard has been published. Move outdated - // shards to historical storage as needed - if (curNotSynched || prevNotSynched) - { - if (prev) - { - // Move the formerly second latest shard to historical storage - if (keepShard(*prev) && separateHistoricalPath) - moveShard(*prev); - - prev = std::nullopt; - } - - if (cur) - { - // The formerly latest shard is now the second latest - if (cur == latestShardIndex - 1) - prev = cur; - - // The formerly latest shard is no longer a 'recent' shard - else - { - // Move the formerly latest shard to historical storage - if (keepShard(*cur) && separateHistoricalPath) - moveShard(*cur); - } - - cur = std::nullopt; - } - } -} - -auto -DatabaseShardImp::prepareForNewShard( - std::uint32_t shardIndex, - std::uint32_t numHistoricalShards, - std::lock_guard const& lock) -> std::optional -{ - // Any shard earlier than the two most recent shards is a historical shard - auto const boundaryIndex{shardBoundaryIndex()}; - auto const isHistoricalShard = shardIndex < boundaryIndex; - - auto const designation = isHistoricalShard && !historicalPaths_.empty() - ? PathDesignation::historical - : PathDesignation::none; - - // Check shard count and available storage space - if (isHistoricalShard && numHistoricalShards >= maxHistoricalShards_) - { - JLOG(j_.error()) << "maximum number of historical shards reached"; - canAdd_ = false; - return std::nullopt; - } - if (!sufficientStorage(1, designation, lock)) - { - JLOG(j_.error()) << "insufficient storage space available"; - canAdd_ = false; - return std::nullopt; - } - - return designation; -} - -boost::filesystem::path -DatabaseShardImp::chooseHistoricalPath(std::lock_guard const&) const -{ - // If not configured with separate historical paths, - // use the main path (dir_) by default. - if (historicalPaths_.empty()) - return dir_; - - boost::filesystem::path historicalShardPath; - std::vector potentialPaths; - - for (boost::filesystem::path const& path : historicalPaths_) - { - if (boost::filesystem::space(path).available >= avgShardFileSz_) - potentialPaths.push_back(path); - } - - if (potentialPaths.empty()) - { - JLOG(j_.error()) << "failed to select a historical shard path"; - return ""; - } - - std::sample( - potentialPaths.begin(), - potentialPaths.end(), - &historicalShardPath, - 1, - default_prng()); - - return historicalShardPath; -} - -bool -DatabaseShardImp::checkHistoricalPaths(std::lock_guard const&) const -{ -#if BOOST_OS_LINUX - // Each historical shard path must correspond - // to a directory on a distinct device or file system. - // Currently, this constraint is enforced only on Linux. - std::unordered_map> filesystemIDs( - historicalPaths_.size()); - - for (auto const& path : historicalPaths_) - { - struct statvfs buffer; - if (statvfs(path.c_str(), &buffer)) - { - JLOG(j_.error()) - << "failed to acquire stats for 'historical_shard_path': " - << path; - return false; - } - - filesystemIDs[buffer.f_fsid].push_back(path.string()); - } - - bool ret = true; - for (auto const& entry : filesystemIDs) - { - // Check to see if any of the paths are stored on the same file system - if (entry.second.size() > 1) - { - // Two or more historical storage paths - // correspond to the same file system. - JLOG(j_.error()) - << "The following paths correspond to the same filesystem: " - << boost::algorithm::join(entry.second, ", ") - << ". Each configured historical storage path should" - " be on a unique device or filesystem."; - - ret = false; - } - } - - return ret; - -#else - // The requirement that each historical storage path - // corresponds to a distinct device or file system is - // enforced only on Linux, so on other platforms - // keep track of the available capacities for each - // path. Issue a warning if we suspect any of the paths - // may violate this requirement. - - // Map byte counts to each path that shares that byte count. - std::unordered_map> - uniqueCapacities(historicalPaths_.size()); - - for (auto const& path : historicalPaths_) - uniqueCapacities[boost::filesystem::space(path).available].push_back( - path.string()); - - for (auto const& entry : uniqueCapacities) - { - // Check to see if any paths have the same amount of available bytes. - if (entry.second.size() > 1) - { - // Two or more historical storage paths may - // correspond to the same device or file system. - JLOG(j_.warn()) - << "Each of the following paths have " << entry.first - << " bytes free, and may be located on the same device" - " or file system: " - << boost::algorithm::join(entry.second, ", ") - << ". Each configured historical storage path should" - " be on a unique device or file system."; - } - } -#endif - - return true; -} - -bool -DatabaseShardImp::callForLedgerSQLByLedgerSeq( - LedgerIndex ledgerSeq, - std::function const& callback) -{ - if (ledgerSeq < earliestLedgerSeq_) - { - JLOG(j_.warn()) << "callForLedgerSQLByLedgerSeq ledger seq too early: " - << ledgerSeq; - return false; - } - - return callForLedgerSQLByShardIndex(seqToShardIndex(ledgerSeq), callback); -} - -bool -DatabaseShardImp::callForLedgerSQLByShardIndex( - const uint32_t shardIndex, - std::function const& callback) -{ - std::lock_guard lock(mutex_); - - auto const it{shards_.find(shardIndex)}; - - return it != shards_.end() && - it->second->getState() == ShardState::finalized && - it->second->callForLedgerSQL(callback); -} - -bool -DatabaseShardImp::callForTransactionSQLByLedgerSeq( - LedgerIndex ledgerSeq, - std::function const& callback) -{ - return callForTransactionSQLByShardIndex( - seqToShardIndex(ledgerSeq), callback); -} - -bool -DatabaseShardImp::callForTransactionSQLByShardIndex( - std::uint32_t const shardIndex, - std::function const& callback) -{ - std::lock_guard lock(mutex_); - - auto const it{shards_.find(shardIndex)}; - - return it != shards_.end() && - it->second->getState() == ShardState::finalized && - it->second->callForTransactionSQL(callback); -} - -bool -DatabaseShardImp::iterateShardsForward( - std::optional minShardIndex, - std::function const& visit) -{ - std::lock_guard lock(mutex_); - - std::map>::iterator it, eit; - - if (!minShardIndex) - it = shards_.begin(); - else - it = shards_.lower_bound(*minShardIndex); - - eit = shards_.end(); - - for (; it != eit; it++) - { - if (it->second->getState() == ShardState::finalized) - { - if (!visit(*it->second)) - return false; - } - } - - return true; -} - -bool -DatabaseShardImp::iterateLedgerSQLsForward( - std::optional minShardIndex, - std::function const& - callback) -{ - return iterateShardsForward( - minShardIndex, [&callback](Shard& shard) -> bool { - return shard.callForLedgerSQL(callback); - }); -} - -bool -DatabaseShardImp::iterateTransactionSQLsForward( - std::optional minShardIndex, - std::function const& - callback) -{ - return iterateShardsForward( - minShardIndex, [&callback](Shard& shard) -> bool { - return shard.callForTransactionSQL(callback); - }); -} - -bool -DatabaseShardImp::iterateShardsBack( - std::optional maxShardIndex, - std::function const& visit) -{ - std::lock_guard lock(mutex_); - - std::map>::reverse_iterator it, eit; - - if (!maxShardIndex) - it = shards_.rbegin(); - else - it = std::make_reverse_iterator(shards_.upper_bound(*maxShardIndex)); - - eit = shards_.rend(); - - for (; it != eit; it++) - { - if (it->second->getState() == ShardState::finalized && - (!maxShardIndex || it->first <= *maxShardIndex)) - { - if (!visit(*it->second)) - return false; - } - } - - return true; -} - -bool -DatabaseShardImp::iterateLedgerSQLsBack( - std::optional maxShardIndex, - std::function const& - callback) -{ - return iterateShardsBack(maxShardIndex, [&callback](Shard& shard) -> bool { - return shard.callForLedgerSQL(callback); - }); -} - -bool -DatabaseShardImp::iterateTransactionSQLsBack( - std::optional maxShardIndex, - std::function const& - callback) -{ - return iterateShardsBack(maxShardIndex, [&callback](Shard& shard) -> bool { - return shard.callForTransactionSQL(callback); - }); -} - -std::unique_ptr -DatabaseShardImp::getShardInfo(std::lock_guard const&) const -{ - auto shardInfo{std::make_unique()}; - for (auto const& [_, shard] : shards_) - { - shardInfo->update( - shard->index(), shard->getState(), shard->getPercentProgress()); - } - - for (auto const shardIndex : preparedIndexes_) - shardInfo->update(shardIndex, ShardState::queued, 0); - - return shardInfo; -} - -size_t -DatabaseShardImp::getNumTasks() const -{ - std::lock_guard lock(mutex_); - return taskQueue_.size(); -} - -void -DatabaseShardImp::updatePeers(std::lock_guard const& lock) const -{ - if (!app_.config().standalone() && - app_.getOPs().getOperatingMode() != OperatingMode::DISCONNECTED) - { - auto const message{getShardInfo(lock)->makeMessage(app_)}; - app_.overlay().foreach(send_always(std::make_shared( - message, protocol::mtPEER_SHARD_INFO_V2))); - } -} - -void -DatabaseShardImp::startDatabaseImportThread(std::lock_guard const&) -{ - // Run the lengthy node store import process in the background - // on a dedicated thread. - databaseImporter_ = std::thread([this] { - doImportDatabase(); - - std::lock_guard lock(mutex_); - - // Make sure to clear this in case the import - // exited early. - databaseImportStatus_.reset(); - - // Detach the thread so subsequent attempts - // to start the import won't get held up by - // the old thread of execution - databaseImporter_.detach(); - }); -} - -//------------------------------------------------------------------------------ - -std::unique_ptr -make_ShardStore( - Application& app, - Scheduler& scheduler, - int readThreads, - beast::Journal j) -{ - // The shard store is optional. Future changes will require it. - Section const& section{ - app.config().section(ConfigSection::shardDatabase())}; - if (section.empty()) - return nullptr; - - return std::make_unique(app, scheduler, readThreads, j); -} - -} // namespace NodeStore -} // namespace ripple diff --git a/src/xrpld/nodestore/detail/DatabaseShardImp.h b/src/xrpld/nodestore/detail/DatabaseShardImp.h deleted file mode 100644 index df740cf407c..00000000000 --- a/src/xrpld/nodestore/detail/DatabaseShardImp.h +++ /dev/null @@ -1,429 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2017 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_NODESTORE_DATABASESHARDIMP_H_INCLUDED -#define RIPPLE_NODESTORE_DATABASESHARDIMP_H_INCLUDED - -#include -#include -#include - -#include - -namespace ripple { -namespace NodeStore { - -class DatabaseShardImp : public DatabaseShard -{ -public: - DatabaseShardImp() = delete; - DatabaseShardImp(DatabaseShardImp const&) = delete; - DatabaseShardImp(DatabaseShardImp&&) = delete; - DatabaseShardImp& - operator=(DatabaseShardImp const&) = delete; - DatabaseShardImp& - operator=(DatabaseShardImp&&) = delete; - - DatabaseShardImp( - Application& app, - Scheduler& scheduler, - int readThreads, - beast::Journal j); - - ~DatabaseShardImp() - { - stop(); - } - - [[nodiscard]] bool - init() override; - - std::optional - prepareLedger(std::uint32_t validLedgerSeq) override; - - bool - prepareShards(std::vector const& shardIndexes) override; - - void - removePreShard(std::uint32_t shardIndex) override; - - std::string - getPreShards() override; - - bool - importShard(std::uint32_t shardIndex, boost::filesystem::path const& srcDir) - override; - - std::shared_ptr - fetchLedger(uint256 const& hash, std::uint32_t ledgerSeq) override; - - void - setStored(std::shared_ptr const& ledger) override; - - std::unique_ptr - getShardInfo() const override; - - size_t - getNumTasks() const override; - - boost::filesystem::path const& - getRootDir() const override - { - return dir_; - } - - std::string - getName() const override - { - return backendName_; - } - - void - stop() override; - - /** Import the application local node store - - @param source The application node store. - */ - void - importDatabase(Database& source) override; - - void - doImportDatabase(); - - std::int32_t - getWriteLoad() const override; - - bool - isSameDB(std::uint32_t s1, std::uint32_t s2) override - { - return seqToShardIndex(s1) == seqToShardIndex(s2); - } - - void - store( - NodeObjectType type, - Blob&& data, - uint256 const& hash, - std::uint32_t ledgerSeq) override; - - void - sync() override{}; - - bool - storeLedger(std::shared_ptr const& srcLedger) override; - - void - sweep() override; - - Json::Value - getDatabaseImportStatus() const override; - - Json::Value - startNodeToShard() override; - - Json::Value - stopNodeToShard() override; - - std::optional - getDatabaseImportSequence() const override; - - bool - callForLedgerSQLByLedgerSeq( - LedgerIndex ledgerSeq, - std::function const& callback) override; - - bool - callForLedgerSQLByShardIndex( - std::uint32_t const shardIndex, - std::function const& callback) override; - - bool - callForTransactionSQLByLedgerSeq( - LedgerIndex ledgerSeq, - std::function const& callback) override; - - bool - callForTransactionSQLByShardIndex( - std::uint32_t const shardIndex, - std::function const& callback) override; - - bool - iterateLedgerSQLsForward( - std::optional minShardIndex, - std::function< - bool(soci::session& session, std::uint32_t shardIndex)> const& - callback) override; - - bool - iterateTransactionSQLsForward( - std::optional minShardIndex, - std::function< - bool(soci::session& session, std::uint32_t shardIndex)> const& - callback) override; - - bool - iterateLedgerSQLsBack( - std::optional maxShardIndex, - std::function< - bool(soci::session& session, std::uint32_t shardIndex)> const& - callback) override; - - bool - iterateTransactionSQLsBack( - std::optional maxShardIndex, - std::function< - bool(soci::session& session, std::uint32_t shardIndex)> const& - callback) override; - -private: - enum class PathDesignation : uint8_t { - none, // No path specified - historical // Needs a historical path - }; - - struct DatabaseImportStatus - { - DatabaseImportStatus( - std::uint32_t const earliestIndex, - std::uint32_t const latestIndex, - std::uint32_t const currentIndex) - : earliestIndex(earliestIndex) - , latestIndex(latestIndex) - , currentIndex(currentIndex) - { - } - - // Index of the first shard to be imported - std::uint32_t earliestIndex{0}; - - // Index of the last shard to be imported - std::uint32_t latestIndex{0}; - - // Index of the shard currently being imported - std::uint32_t currentIndex{0}; - - // First ledger sequence of the current shard - std::uint32_t firstSeq{0}; - - // Last ledger sequence of the current shard - std::uint32_t lastSeq{0}; - - // The shard currently being imported - std::weak_ptr currentShard; - }; - - Application& app_; - mutable std::mutex mutex_; - bool init_{false}; - - // The context shared with all shard backend databases - std::unique_ptr ctx_; - - // Queue of background tasks to be performed - TaskQueue taskQueue_; - - // Shards held by this server - std::map> shards_; - - // Shard indexes being imported from the shard archive handler - std::set preparedIndexes_; - - // Shard index being acquired from the peer network - std::uint32_t acquireIndex_{0}; - - // The shard store root directory - boost::filesystem::path dir_; - - // If new shards can be stored - bool canAdd_{true}; - - // The name associated with the backend used with the shard store - std::string backendName_; - - // Maximum number of historical shards to store. - std::uint32_t maxHistoricalShards_{0}; - - // Contains historical shard paths - std::vector historicalPaths_; - - // Storage space utilized by the shard store (in bytes) - std::uint64_t fileSz_{0}; - - // Average storage space required by a shard (in bytes) - std::uint64_t avgShardFileSz_; - - // The limit of final shards with open databases at any time - std::uint32_t const openFinalLimit_; - - // File name used to mark shards being imported from node store - static constexpr auto databaseImportMarker_ = "database_import"; - - // latestShardIndex_ and secondLatestShardIndex hold the indexes - // of the shards most recently confirmed by the network. These - // values are not updated in real time and are modified only - // when adding shards to the database, in order to determine where - // pending shards will be stored on the filesystem. A value of - // std::nullopt indicates that the corresponding shard is not held - // by the database. - std::optional latestShardIndex_; - std::optional secondLatestShardIndex_; - - // Struct used for node store import progress - std::unique_ptr databaseImportStatus_; - - // Thread for running node store import - std::thread databaseImporter_; - - // Indicates whether the import should stop - std::atomic_bool haltDatabaseImport_{false}; - - // Initialize settings from the configuration file - // Lock must be held - bool - initConfig(std::lock_guard const&); - - std::shared_ptr - fetchNodeObject( - uint256 const& hash, - std::uint32_t ledgerSeq, - FetchReport& fetchReport, - bool duplicate) override; - - void - for_each(std::function)> f) override - { - Throw("Import from shard store not supported"); - } - - // Randomly select a shard index not stored - // Lock must be held - std::optional - findAcquireIndex( - std::uint32_t validLedgerSeq, - std::lock_guard const&); - - // Queue a task to finalize a shard by verifying its databases - // Lock must be held - void - finalizeShard( - std::shared_ptr& shard, - bool writeSQLite, - std::optional const& expectedHash); - - // Update storage and file descriptor usage stats - void - updateFileStats(); - - // Returns true if the file system has enough storage - // available to hold the specified number of shards. - // The value of pathDesignation determines whether - // the shard(s) in question are historical and thus - // meant to be stored at a path designated for historical - // shards. - bool - sufficientStorage( - std::uint32_t numShards, - PathDesignation pathDesignation, - std::lock_guard const&) const; - - bool - setStoredInShard( - std::shared_ptr& shard, - std::shared_ptr const& ledger); - - void - removeFailedShard(std::shared_ptr& shard); - - // Returns the index that represents the logical - // partition between historical and recent shards - std::uint32_t - shardBoundaryIndex() const; - - std::uint32_t - numHistoricalShards(std::lock_guard const& lock) const; - - // Shifts the recent and second most recent (by index) - // shards as new shards become available on the network. - // Older shards are moved to a historical shard path. - void - relocateOutdatedShards(std::lock_guard const& lock); - - // Checks whether the shard can be stored. If - // the new shard can't be stored, returns - // std::nullopt. Otherwise returns an enum - // indicating whether the new shard should be - // placed in a separate directory for historical - // shards. - std::optional - prepareForNewShard( - std::uint32_t shardIndex, - std::uint32_t numHistoricalShards, - std::lock_guard const& lock); - - boost::filesystem::path - chooseHistoricalPath(std::lock_guard const&) const; - - /** - * @brief iterateShardsForward Visits all shards starting from given - * in ascending order and calls given callback function to each - * of them passing shard as parameter. - * @param minShardIndex Start shard index to visit or none if all shards - * should be visited. - * @param visit Callback function to call. - * @return True if each callback function returned true, false otherwise. - */ - bool - iterateShardsForward( - std::optional minShardIndex, - std::function const& visit); - - /** - * @brief iterateShardsBack Visits all shards starting from given - * in descending order and calls given callback function to each - * of them passing shard as parameter. - * @param maxShardIndex Start shard index to visit or none if all shards - * should be visited. - * @param visit Callback function to call. - * @return True if each callback function returned true, false otherwise. - */ - bool - iterateShardsBack( - std::optional maxShardIndex, - std::function const& visit); - - bool - checkHistoricalPaths(std::lock_guard const&) const; - - std::unique_ptr - getShardInfo(std::lock_guard const&) const; - - // Update peers with the status of every complete and incomplete shard - void - updatePeers(std::lock_guard const& lock) const; - - // Start the node store import process - void - startDatabaseImportThread(std::lock_guard const&); -}; - -} // namespace NodeStore -} // namespace ripple - -#endif diff --git a/src/xrpld/nodestore/detail/DeterministicShard.cpp b/src/xrpld/nodestore/detail/DeterministicShard.cpp deleted file mode 100644 index c575a685ded..00000000000 --- a/src/xrpld/nodestore/detail/DeterministicShard.cpp +++ /dev/null @@ -1,216 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { -namespace NodeStore { - -DeterministicShard::DeterministicShard( - Application& app, - boost::filesystem::path const& dir, - std::uint32_t index, - beast::Journal j) - : app_(app) - , index_(index) - , dir_(dir / "tmp") - , ctx_(std::make_unique()) - , j_(j) - , curMemObjs_(0) - , maxMemObjs_( - app_.getShardStore()->ledgersPerShard() <= 256 ? maxMemObjsTest - : maxMemObjsDefault) -{ -} - -DeterministicShard::~DeterministicShard() -{ - close(true); -} - -bool -DeterministicShard::init(Serializer const& finalKey) -{ - auto db = app_.getShardStore(); - - auto fail = [&](std::string const& msg) { - JLOG(j_.error()) << "deterministic shard " << index_ - << " not created: " << msg; - backend_.reset(); - try - { - remove_all(dir_); - } - catch (std::exception const& e) - { - JLOG(j_.error()) << "deterministic shard " << index_ - << ". Exception caught in function " << __func__ - << ". Error: " << e.what(); - } - return false; - }; - - if (!db) - return fail("shard store not exists"); - - if (index_ < db->earliestShardIndex()) - return fail("Invalid shard index"); - - Config const& config{app_.config()}; - Section section{config.section(ConfigSection::shardDatabase())}; - auto const type{get(section, "type", "nudb")}; - auto const factory{Manager::instance().find(type)}; - if (!factory) - return fail("failed to find factory for " + type); - - section.set("path", dir_.string()); - backend_ = factory->createInstance( - NodeObject::keyBytes, section, 1, scheduler_, *ctx_, j_); - - if (!backend_) - return fail("failed to create database"); - - ripemd160_hasher h; - h(finalKey.data(), finalKey.size()); - auto const result{static_cast(h)}; - auto const hash{uint160::fromVoid(result.data())}; - - auto digest = [&](int n) { - auto const data{hash.data()}; - std::uint64_t result{0}; - - switch (n) - { - case 0: - case 1: - // Construct 64 bits from sequential eight bytes - for (int i = 0; i < 8; i++) - result = (result << 8) + data[n * 8 + i]; - break; - - case 2: - // Construct 64 bits using the last four bytes of data - result = (static_cast(data[16]) << 24) + - (static_cast(data[17]) << 16) + - (static_cast(data[18]) << 8) + - (static_cast(data[19])); - break; - } - - return result; - }; - auto const uid{digest(0)}; - auto const salt{digest(1)}; - auto const appType{digest(2) | deterministicType}; - - // Open or create the NuDB key/value store - try - { - if (exists(dir_)) - remove_all(dir_); - - backend_->open(true, appType, uid, salt); - } - catch (std::exception const& e) - { - return fail( - std::string(". Exception caught in function ") + __func__ + - ". Error: " + e.what()); - } - - return true; -} - -std::shared_ptr -make_DeterministicShard( - Application& app, - boost::filesystem::path const& shardDir, - std::uint32_t shardIndex, - Serializer const& finalKey, - beast::Journal j) -{ - std::shared_ptr dShard( - new DeterministicShard(app, shardDir, shardIndex, j)); - if (!dShard->init(finalKey)) - return {}; - return dShard; -} - -void -DeterministicShard::close(bool cancel) -{ - try - { - if (cancel) - { - backend_.reset(); - remove_all(dir_); - } - else - { - ctx_->flush(); - curMemObjs_ = 0; - backend_.reset(); - } - } - catch (std::exception const& e) - { - JLOG(j_.error()) << "deterministic shard " << index_ - << ". Exception caught in function " << __func__ - << ". Error: " << e.what(); - } -} - -bool -DeterministicShard::store(std::shared_ptr const& nodeObject) -{ - try - { - backend_->store(nodeObject); - - // Flush to the backend if at threshold - if (++curMemObjs_ >= maxMemObjs_) - { - ctx_->flush(); - curMemObjs_ = 0; - } - } - catch (std::exception const& e) - { - JLOG(j_.error()) << "deterministic shard " << index_ - << ". Exception caught in function " << __func__ - << ". Error: " << e.what(); - return false; - } - - return true; -} - -} // namespace NodeStore -} // namespace ripple diff --git a/src/xrpld/nodestore/detail/DeterministicShard.h b/src/xrpld/nodestore/detail/DeterministicShard.h deleted file mode 100644 index 3eb5eaa8144..00000000000 --- a/src/xrpld/nodestore/detail/DeterministicShard.h +++ /dev/null @@ -1,174 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_NODESTORE_DETERMINISTICSHARD_H_INCLUDED -#define RIPPLE_NODESTORE_DETERMINISTICSHARD_H_INCLUDED - -#include -#include -#include -#include - -namespace ripple { -namespace NodeStore { - -/** DeterministicShard class. - * - * 1. The init() method creates temporary folder dir_, - * and the deterministic shard is initialized in that folder. - * 2. The store() method adds object to memory pool. - * 3. The flush() method stores all objects from memory pool to the shard - * located in dir_ in sorted order. - * 4. The close(true) method closes the backend and removes the directory. - */ -class DeterministicShard -{ - constexpr static std::uint32_t maxMemObjsDefault = 16384u; - constexpr static std::uint32_t maxMemObjsTest = 16u; - - /* "SHRD" in ASCII */ - constexpr static std::uint64_t deterministicType = 0x5348524400000000ll; - -private: - DeterministicShard(DeterministicShard const&) = delete; - DeterministicShard& - operator=(DeterministicShard const&) = delete; - - /** Creates the object for shard database - * - * @param app Application object - * @param dir Directory where shard is located - * @param index Index of the shard - * @param j Journal to logging - */ - DeterministicShard( - Application& app, - boost::filesystem::path const& dir, - std::uint32_t index, - beast::Journal j); - - /** Initializes the deterministic shard. - * - * @param finalKey Serializer of shard's final key which consists of: - * shard version (32 bit) - * first ledger sequence in the shard (32 bit) - * last ledger sequence in the shard (32 bit) - * hash of last ledger (256 bits) - * @return true if no error, false if error - */ - bool - init(Serializer const& finalKey); - -public: - ~DeterministicShard(); - - /** Finalizes and closes the shard. - */ - void - close() - { - close(false); - } - - [[nodiscard]] boost::filesystem::path const& - getDir() const - { - return dir_; - } - - /** Store a node object in memory. - * - * @param nodeObject The node object to store - * @return true on success. - * @note Flushes all objects in memory to the backend when the number - * of node objects held in memory exceed a threshold - */ - [[nodiscard]] bool - store(std::shared_ptr const& nodeObject); - -private: - /** Finalizes and closes the shard. - * - * @param cancel True if reject the shard and delete all files, - * false if finalize the shard and store them - */ - void - close(bool cancel); - - // Application reference - Application& app_; - - // Shard Index - std::uint32_t const index_; - - // Path to temporary database files - boost::filesystem::path const dir_; - - // Dummy scheduler for deterministic write - DummyScheduler scheduler_; - - // NuDB context - std::unique_ptr ctx_; - - // NuDB key/value store for node objects - std::shared_ptr backend_; - - // Journal - beast::Journal const j_; - - // Current number of in-cache objects - std::uint32_t curMemObjs_; - - // Maximum number of in-cache objects - std::uint32_t const maxMemObjs_; - - friend std::shared_ptr - make_DeterministicShard( - Application& app, - boost::filesystem::path const& shardDir, - std::uint32_t shardIndex, - Serializer const& finalKey, - beast::Journal j); -}; - -/** Creates shared pointer to deterministic shard and initializes it. - * - * @param app Application object - * @param shardDir Directory where shard is located - * @param shardIndex Index of the shard - * @param finalKey Serializer of shard's ginal key which consists of: - * shard version (32 bit) - * first ledger sequence in the shard (32 bit) - * last ledger sequence in the shard (32 bit) - * hash of last ledger (256 bits) - * @param j Journal to logging - * @return Shared pointer to deterministic shard or {} in case of error. - */ -std::shared_ptr -make_DeterministicShard( - Application& app, - boost::filesystem::path const& shardDir, - std::uint32_t shardIndex, - Serializer const& finalKey, - beast::Journal j); - -} // namespace NodeStore -} // namespace ripple - -#endif diff --git a/src/xrpld/nodestore/detail/Shard.cpp b/src/xrpld/nodestore/detail/Shard.cpp deleted file mode 100644 index 8c2e9997fbf..00000000000 --- a/src/xrpld/nodestore/detail/Shard.cpp +++ /dev/null @@ -1,1272 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2017 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { -namespace NodeStore { - -uint256 const Shard::finalKey{0}; - -Shard::Shard( - Application& app, - DatabaseShard const& db, - std::uint32_t index, - beast::Journal j) - : Shard(app, db, index, "", j) -{ -} - -Shard::Shard( - Application& app, - DatabaseShard const& db, - std::uint32_t index, - boost::filesystem::path const& dir, - beast::Journal j) - : app_(app) - , j_(j) - , index_(index) - , firstSeq_(db.firstLedgerSeq(index)) - , lastSeq_(std::max(firstSeq_, db.lastLedgerSeq(index))) - , maxLedgers_(db.maxLedgers(index)) - , dir_((dir.empty() ? db.getRootDir() : dir) / std::to_string(index_)) -{ -} - -Shard::~Shard() -{ - if (!removeOnDestroy_) - return; - - if (backend_) - { - // Abort removal if the backend is in use - if (backendCount_ > 0) - { - JLOG(j_.error()) << "shard " << index_ - << " backend in use, unable to remove directory"; - return; - } - - // Release database files first otherwise remove_all may fail - backend_.reset(); - lgrSQLiteDB_.reset(); - txSQLiteDB_.reset(); - acquireInfo_.reset(); - } - - try - { - boost::filesystem::remove_all(dir_); - } - catch (std::exception const& e) - { - JLOG(j_.fatal()) << "shard " << index_ - << ". Exception caught in function " << __func__ - << ". Error: " << e.what(); - } -} - -bool -Shard::init(Scheduler& scheduler, nudb::context& context) -{ - Section section{app_.config().section(ConfigSection::shardDatabase())}; - std::string const type{get(section, "type", "nudb")}; - auto const factory{Manager::instance().find(type)}; - if (!factory) - { - JLOG(j_.error()) << "shard " << index_ << " failed to find factory for " - << type; - return false; - } - section.set("path", dir_.string()); - - std::lock_guard lock{mutex_}; - if (backend_) - { - JLOG(j_.error()) << "shard " << index_ << " already initialized"; - return false; - } - backend_ = factory->createInstance( - NodeObject::keyBytes, - section, - megabytes( - app_.config().getValueFor(SizedItem::burstSize, std::nullopt)), - scheduler, - context, - j_); - - return open(lock); -} - -bool -Shard::isOpen() const -{ - std::lock_guard lock(mutex_); - if (!backend_) - { - JLOG(j_.error()) << "shard " << index_ << " not initialized"; - return false; - } - - return backend_->isOpen(); -} - -bool -Shard::tryClose() -{ - // Keep database open if being acquired or finalized - if (state_ != ShardState::finalized) - return false; - - std::lock_guard lock(mutex_); - - // Keep database open if in use - if (backendCount_ > 0) - return false; - - if (!backend_) - { - JLOG(j_.error()) << "shard " << index_ << " not initialized"; - return false; - } - if (!backend_->isOpen()) - return false; - - try - { - backend_->close(); - } - catch (std::exception const& e) - { - JLOG(j_.fatal()) << "shard " << index_ - << ". Exception caught in function " << __func__ - << ". Error: " << e.what(); - return false; - } - - lgrSQLiteDB_.reset(); - txSQLiteDB_.reset(); - acquireInfo_.reset(); - - // Reset caches to reduce memory use - app_.getShardFamily()->getFullBelowCache(lastSeq_)->reset(); - app_.getShardFamily()->getTreeNodeCache(lastSeq_)->reset(); - - return true; -} - -std::optional -Shard::prepare() -{ - if (state_ != ShardState::acquire) - { - JLOG(j_.warn()) << "shard " << index_ - << " prepare called when not acquiring"; - return std::nullopt; - } - - std::lock_guard lock(mutex_); - if (!acquireInfo_) - { - JLOG(j_.error()) << "shard " << index_ - << " missing acquire SQLite database"; - return std::nullopt; - } - - if (acquireInfo_->storedSeqs.empty()) - return lastSeq_; - return prevMissing(acquireInfo_->storedSeqs, 1 + lastSeq_, firstSeq_); -} - -bool -Shard::storeNodeObject(std::shared_ptr const& nodeObject) -{ - if (state_ != ShardState::acquire) - { - // The import node store case is an exception - if (nodeObject->getHash() != finalKey) - { - // Ignore residual calls from InboundLedgers - JLOG(j_.trace()) << "shard " << index_ << " not acquiring"; - return false; - } - } - - auto const scopedCount{makeBackendCount()}; - if (!scopedCount) - return false; - - try - { - std::lock_guard lock(mutex_); - backend_->store(nodeObject); - } - catch (std::exception const& e) - { - JLOG(j_.fatal()) << "shard " << index_ - << ". Exception caught in function " << __func__ - << ". Error: " << e.what(); - return false; - } - - return true; -} - -std::shared_ptr -Shard::fetchNodeObject(uint256 const& hash, FetchReport& fetchReport) -{ - auto const scopedCount{makeBackendCount()}; - if (!scopedCount) - return nullptr; - - std::shared_ptr nodeObject; - - // Try the backend - Status status; - try - { - std::lock_guard lock(mutex_); - status = backend_->fetch(hash.data(), &nodeObject); - } - catch (std::exception const& e) - { - JLOG(j_.fatal()) << "shard " << index_ - << ". Exception caught in function " << __func__ - << ". Error: " << e.what(); - return nullptr; - } - - switch (status) - { - case ok: - case notFound: - break; - case dataCorrupt: { - JLOG(j_.fatal()) - << "shard " << index_ << ". Corrupt node object at hash " - << to_string(hash); - break; - } - default: { - JLOG(j_.warn()) - << "shard " << index_ << ". Unknown status=" << status - << " fetching node object at hash " << to_string(hash); - break; - } - } - - if (nodeObject) - fetchReport.wasFound = true; - - return nodeObject; -} - -Shard::StoreLedgerResult -Shard::storeLedger( - std::shared_ptr const& srcLedger, - std::shared_ptr const& next) -{ - StoreLedgerResult result; - if (state_ != ShardState::acquire) - { - // Ignore residual calls from InboundLedgers - JLOG(j_.trace()) << "shard " << index_ << ". Not acquiring"; - return result; - } - if (containsLedger(srcLedger->info().seq)) - { - JLOG(j_.trace()) << "shard " << index_ << ". Ledger already stored"; - return result; - } - - auto fail = [&](std::string const& msg) { - JLOG(j_.error()) << "shard " << index_ << ". Source ledger sequence " - << srcLedger->info().seq << ". " << msg; - result.error = true; - return result; - }; - - if (srcLedger->info().hash.isZero()) - return fail("Invalid hash"); - if (srcLedger->info().accountHash.isZero()) - return fail("Invalid account hash"); - - auto& srcDB{const_cast(srcLedger->stateMap().family().db())}; - if (&srcDB == &(app_.getShardFamily()->db())) - return fail("Source and destination databases are the same"); - - auto const scopedCount{makeBackendCount()}; - if (!scopedCount) - return fail("Failed to lock backend"); - - Batch batch; - batch.reserve(batchWritePreallocationSize); - auto storeBatch = [&]() { - std::uint64_t sz{0}; - for (auto const& nodeObject : batch) - sz += nodeObject->getData().size(); - - try - { - std::lock_guard lock(mutex_); - backend_->storeBatch(batch); - } - catch (std::exception const& e) - { - fail( - std::string(". Exception caught in function ") + __func__ + - ". Error: " + e.what()); - return false; - } - - result.count += batch.size(); - result.size += sz; - batch.clear(); - return true; - }; - - // Store ledger header - { - Serializer s(sizeof(std::uint32_t) + sizeof(LedgerInfo)); - s.add32(HashPrefix::ledgerMaster); - addRaw(srcLedger->info(), s); - auto nodeObject = NodeObject::createObject( - hotLEDGER, std::move(s.modData()), srcLedger->info().hash); - batch.emplace_back(std::move(nodeObject)); - } - - bool error = false; - auto visit = [&](SHAMapTreeNode const& node) { - if (!stop_) - { - if (auto nodeObject = srcDB.fetchNodeObject( - node.getHash().as_uint256(), srcLedger->info().seq)) - { - batch.emplace_back(std::move(nodeObject)); - if (batch.size() < batchWritePreallocationSize || storeBatch()) - return true; - } - } - - error = true; - return false; - }; - - // Store the state map - if (srcLedger->stateMap().getHash().isNonZero()) - { - if (!srcLedger->stateMap().isValid()) - return fail("Invalid state map"); - - if (next && next->info().parentHash == srcLedger->info().hash) - { - auto have = next->stateMap().snapShot(false); - srcLedger->stateMap().snapShot(false)->visitDifferences( - &(*have), visit); - } - else - srcLedger->stateMap().snapShot(false)->visitNodes(visit); - if (error) - return fail("Failed to store state map"); - } - - // Store the transaction map - if (srcLedger->info().txHash.isNonZero()) - { - if (!srcLedger->txMap().isValid()) - return fail("Invalid transaction map"); - - srcLedger->txMap().snapShot(false)->visitNodes(visit); - if (error) - return fail("Failed to store transaction map"); - } - - if (!batch.empty() && !storeBatch()) - return fail("Failed to store"); - - return result; -} - -bool -Shard::setLedgerStored(std::shared_ptr const& ledger) -{ - if (state_ != ShardState::acquire) - { - // Ignore residual calls from InboundLedgers - JLOG(j_.trace()) << "shard " << index_ << " not acquiring"; - return false; - } - - auto fail = [&](std::string const& msg) { - JLOG(j_.error()) << "shard " << index_ << ". " << msg; - return false; - }; - - auto const ledgerSeq{ledger->info().seq}; - if (ledgerSeq < firstSeq_ || ledgerSeq > lastSeq_) - return fail("Invalid ledger sequence " + std::to_string(ledgerSeq)); - - auto const scopedCount{makeBackendCount()}; - if (!scopedCount) - return false; - - // This lock is used as an optimization to prevent unneeded - // calls to storeSQLite before acquireInfo_ is updated - std::lock_guard storedLock(storedMutex_); - - { - std::lock_guard lock(mutex_); - if (!acquireInfo_) - return fail("Missing acquire SQLite database"); - - if (boost::icl::contains(acquireInfo_->storedSeqs, ledgerSeq)) - { - // Ignore redundant calls - JLOG(j_.debug()) << "shard " << index_ << " ledger sequence " - << ledgerSeq << " already stored"; - return true; - } - } - - if (!storeSQLite(ledger)) - return fail("Failed to store ledger"); - - std::lock_guard lock(mutex_); - - // Update the acquire database - acquireInfo_->storedSeqs.insert(ledgerSeq); - - try - { - auto session{acquireInfo_->SQLiteDB->checkoutDb()}; - soci::blob sociBlob(*session); - convert(to_string(acquireInfo_->storedSeqs), sociBlob); - if (ledgerSeq == lastSeq_) - { - // Store shard's last ledger hash - auto const sHash{to_string(ledger->info().hash)}; - *session << "UPDATE Shard " - "SET LastLedgerHash = :lastLedgerHash," - "StoredLedgerSeqs = :storedLedgerSeqs " - "WHERE ShardIndex = :shardIndex;", - soci::use(sHash), soci::use(sociBlob), soci::use(index_); - } - else - { - *session << "UPDATE Shard " - "SET StoredLedgerSeqs = :storedLedgerSeqs " - "WHERE ShardIndex = :shardIndex;", - soci::use(sociBlob), soci::use(index_); - } - } - catch (std::exception const& e) - { - acquireInfo_->storedSeqs.erase(ledgerSeq); - return fail( - std::string(". Exception caught in function ") + __func__ + - ". Error: " + e.what()); - } - - // Update progress - progress_ = boost::icl::length(acquireInfo_->storedSeqs); - if (progress_ == maxLedgers_) - state_ = ShardState::complete; - - setFileStats(lock); - JLOG(j_.trace()) << "shard " << index_ << " stored ledger sequence " - << ledgerSeq; - return true; -} - -bool -Shard::containsLedger(std::uint32_t ledgerSeq) const -{ - if (ledgerSeq < firstSeq_ || ledgerSeq > lastSeq_) - return false; - if (state_ != ShardState::acquire) - return true; - - std::lock_guard lock(mutex_); - if (!acquireInfo_) - { - JLOG(j_.error()) << "shard " << index_ - << " missing acquire SQLite database"; - return false; - } - return boost::icl::contains(acquireInfo_->storedSeqs, ledgerSeq); -} - -std::chrono::steady_clock::time_point -Shard::getLastUse() const -{ - std::lock_guard lock(mutex_); - return lastAccess_; -} - -std::pair -Shard::getFileInfo() const -{ - std::lock_guard lock(mutex_); - return {fileSz_, fdRequired_}; -} - -std::int32_t -Shard::getWriteLoad() -{ - auto const scopedCount{makeBackendCount()}; - if (!scopedCount) - return 0; - std::lock_guard lock(mutex_); - return backend_->getWriteLoad(); -} - -bool -Shard::isLegacy() const -{ - std::lock_guard lock(mutex_); - return legacy_; -} - -bool -Shard::finalize(bool writeSQLite, std::optional const& referenceHash) -{ - auto const scopedCount{makeBackendCount()}; - if (!scopedCount) - return false; - - uint256 hash{0}; - std::uint32_t ledgerSeq{0}; - auto fail = [&](std::string const& msg) { - JLOG(j_.fatal()) << "shard " << index_ << ". " << msg - << (hash.isZero() ? "" - : ". Ledger hash " + to_string(hash)) - << (ledgerSeq == 0 ? "" - : ". Ledger sequence " + - std::to_string(ledgerSeq)); - state_ = ShardState::finalizing; - progress_ = 0; - busy_ = false; - return false; - }; - - try - { - std::lock_guard lock(mutex_); - - state_ = ShardState::finalizing; - progress_ = 0; - - // Check if a final key has been stored - if (std::shared_ptr nodeObject; - backend_->fetch(finalKey.data(), &nodeObject) == Status::ok) - { - // Check final key's value - SerialIter sIt( - nodeObject->getData().data(), nodeObject->getData().size()); - if (sIt.get32() != version) - return fail("invalid version"); - - if (sIt.get32() != firstSeq_ || sIt.get32() != lastSeq_) - return fail("out of range ledger sequences"); - - if (hash = sIt.get256(); hash.isZero()) - return fail("invalid last ledger hash"); - } - else - { - // In the absence of a final key, an acquire SQLite database - // must be present in order to verify the shard - if (!acquireInfo_) - return fail("missing acquire SQLite database"); - - auto [res, seqshash] = selectAcquireDBLedgerSeqsHash( - *acquireInfo_->SQLiteDB->checkoutDb(), index_); - - if (!res) - return fail("missing or invalid ShardIndex"); - - if (!seqshash.hash) - return fail("missing LastLedgerHash"); - - if (!hash.parseHex(*seqshash.hash) || hash.isZero()) - return fail("invalid LastLedgerHash"); - - if (!seqshash.sequences) - return fail("missing StoredLedgerSeqs"); - - auto& storedSeqs{acquireInfo_->storedSeqs}; - if (!from_string(storedSeqs, *seqshash.sequences) || - boost::icl::first(storedSeqs) != firstSeq_ || - boost::icl::last(storedSeqs) != lastSeq_ || - storedSeqs.size() != maxLedgers_) - { - return fail("invalid StoredLedgerSeqs"); - } - } - } - catch (std::exception const& e) - { - return fail( - std::string(". Exception caught in function ") + __func__ + - ". Error: " + e.what()); - } - - // Verify the last ledger hash of a downloaded shard - // using a ledger hash obtained from the peer network - if (referenceHash && *referenceHash != hash) - return fail("invalid last ledger hash"); - - // Verify every ledger stored in the backend - Config const& config{app_.config()}; - std::shared_ptr ledger; - std::shared_ptr next; - auto const lastLedgerHash{hash}; - auto& shardFamily{*app_.getShardFamily()}; - auto const fullBelowCache{shardFamily.getFullBelowCache(lastSeq_)}; - auto const treeNodeCache{shardFamily.getTreeNodeCache(lastSeq_)}; - - // Reset caches to reduce memory usage - fullBelowCache->reset(); - treeNodeCache->reset(); - - Serializer s; - s.add32(version); - s.add32(firstSeq_); - s.add32(lastSeq_); - s.addBitString(lastLedgerHash); - - std::shared_ptr dShard{ - make_DeterministicShard(app_, dir_, index_, s, j_)}; - if (!dShard) - return fail("Failed to create deterministic shard"); - - // Start with the last ledger in the shard and walk backwards from - // child to parent until we reach the first ledger - ledgerSeq = lastSeq_; - while (ledgerSeq >= firstSeq_) - { - if (stop_) - return false; - - auto nodeObject{verifyFetch(hash)}; - if (!nodeObject) - return fail("invalid ledger"); - - ledger = std::make_shared( - deserializePrefixedHeader(makeSlice(nodeObject->getData())), - config, - shardFamily); - if (ledger->info().seq != ledgerSeq) - return fail("invalid ledger sequence"); - if (ledger->info().hash != hash) - return fail("invalid ledger hash"); - - ledger->stateMap().setLedgerSeq(ledgerSeq); - ledger->txMap().setLedgerSeq(ledgerSeq); - ledger->setImmutable(); - if (!ledger->stateMap().fetchRoot( - SHAMapHash{ledger->info().accountHash}, nullptr)) - { - return fail("missing root STATE node"); - } - if (ledger->info().txHash.isNonZero() && - !ledger->txMap().fetchRoot( - SHAMapHash{ledger->info().txHash}, nullptr)) - { - return fail("missing root TXN node"); - } - - if (!verifyLedger(ledger, next, dShard)) - return fail("failed to verify ledger"); - - if (!dShard->store(nodeObject)) - return fail("failed to store node object"); - - if (writeSQLite && !storeSQLite(ledger)) - return fail("failed storing to SQLite databases"); - - assert( - ledger->info().seq == ledgerSeq && - (ledger->info().seq < XRP_LEDGER_EARLIEST_FEES || - ledger->read(keylet::fees()))); - - hash = ledger->info().parentHash; - next = std::move(ledger); - - // Update progress - progress_ = maxLedgers_ - (ledgerSeq - firstSeq_); - - --ledgerSeq; - - fullBelowCache->reset(); - treeNodeCache->reset(); - } - - JLOG(j_.debug()) << "shard " << index_ << " is valid"; - - /* - TODO MP - SQLite VACUUM blocks all database access while processing. - Depending on the file size, that can take a while. Until we find - a non-blocking way of doing this, we cannot enable vacuum as - it can desync a server. - - try - { - // VACUUM the SQLite databases - auto const tmpDir {dir_ / "tmp_vacuum"}; - create_directory(tmpDir); - - auto vacuum = [&tmpDir](std::unique_ptr& sqliteDB) - { - auto session {sqliteDB->checkoutDb()}; - *session << "PRAGMA synchronous=OFF;"; - *session << "PRAGMA journal_mode=OFF;"; - *session << "PRAGMA temp_store_directory='" << - tmpDir.string() << "';"; - *session << "VACUUM;"; - }; - vacuum(lgrSQLiteDB_); - vacuum(txSQLiteDB_); - remove_all(tmpDir); - } - catch (std::exception const& e) - { - return fail( - std::string(". Exception caught in function ") + __func__ + - ". Error: " + e.what()); - } - */ - - auto const nodeObject{ - NodeObject::createObject(hotUNKNOWN, std::move(s.modData()), finalKey)}; - if (!dShard->store(nodeObject)) - return fail("failed to store node object"); - - try - { - { - // Store final key's value, may already be stored - std::lock_guard lock(mutex_); - backend_->store(nodeObject); - } - - // Do not allow all other threads work with the shard - busy_ = true; - - // Wait until all other threads leave the shard - while (backendCount_ > 1) - std::this_thread::yield(); - - std::lock_guard lock(mutex_); - - // Close original backend - backend_->close(); - - // Close SQL databases - lgrSQLiteDB_.reset(); - txSQLiteDB_.reset(); - - // Remove the acquire SQLite database - if (acquireInfo_) - { - acquireInfo_.reset(); - remove_all(dir_ / AcquireShardDBName); - } - - // Close deterministic backend - dShard->close(); - - // Replace original backend with deterministic backend - remove(dir_ / "nudb.key"); - remove(dir_ / "nudb.dat"); - rename(dShard->getDir() / "nudb.key", dir_ / "nudb.key"); - rename(dShard->getDir() / "nudb.dat", dir_ / "nudb.dat"); - - // Re-open deterministic shard - if (!open(lock)) - return fail("failed to open"); - - assert(state_ == ShardState::finalized); - - // Allow all other threads work with the shard - busy_ = false; - } - catch (std::exception const& e) - { - return fail( - std::string(". Exception caught in function ") + __func__ + - ". Error: " + e.what()); - } - - return true; -} - -bool -Shard::open(std::lock_guard const& lock) -{ - using namespace boost::filesystem; - Config const& config{app_.config()}; - auto preexist{false}; - auto fail = [this, &preexist](std::string const& msg) REQUIRES(mutex_) { - backend_->close(); - lgrSQLiteDB_.reset(); - txSQLiteDB_.reset(); - acquireInfo_.reset(); - - state_ = ShardState::acquire; - progress_ = 0; - - if (!preexist) - remove_all(dir_); - - if (!msg.empty()) - { - JLOG(j_.fatal()) << "shard " << index_ << " " << msg; - } - return false; - }; - auto createAcquireInfo = [this, &config]() REQUIRES(mutex_) { - DatabaseCon::Setup setup; - setup.startUp = config.standalone() ? config.LOAD : config.START_UP; - setup.standAlone = config.standalone(); - setup.dataDir = dir_; - setup.useGlobalPragma = true; - - acquireInfo_ = std::make_unique(); - acquireInfo_->SQLiteDB = makeAcquireDB( - setup, - DatabaseCon::CheckpointerSetup{&app_.getJobQueue(), &app_.logs()}); - - state_ = ShardState::acquire; - progress_ = 0; - }; - - try - { - // Open or create the NuDB key/value store - preexist = exists(dir_); - backend_->open(!preexist); - - if (!preexist) - { - // A new shard - createAcquireInfo(); - insertAcquireDBIndex(acquireInfo_->SQLiteDB->getSession(), index_); - } - else if (exists(dir_ / AcquireShardDBName)) - { - // A shard being acquired, backend is likely incomplete - createAcquireInfo(); - auto [res, s] = selectAcquireDBLedgerSeqs( - acquireInfo_->SQLiteDB->getSession(), index_); - - if (!res) - return fail("invalid acquire SQLite database"); - - if (s) - { - auto& storedSeqs{acquireInfo_->storedSeqs}; - if (!from_string(storedSeqs, *s)) - return fail("invalid StoredLedgerSeqs"); - - if (boost::icl::first(storedSeqs) < firstSeq_ || - boost::icl::last(storedSeqs) > lastSeq_) - { - return fail("invalid StoredLedgerSeqs"); - } - - // Check if backend is complete - progress_ = boost::icl::length(storedSeqs); - if (progress_ == maxLedgers_) - state_ = ShardState::complete; - } - } - else - { - // A shard with a finalized or complete state - std::shared_ptr nodeObject; - if (backend_->fetch(finalKey.data(), &nodeObject) != Status::ok) - { - legacy_ = true; - return fail("incompatible, missing backend final key"); - } - - // Check final key's value - SerialIter sIt( - nodeObject->getData().data(), nodeObject->getData().size()); - if (sIt.get32() != version) - return fail("invalid version"); - - if (sIt.get32() != firstSeq_ || sIt.get32() != lastSeq_) - return fail("out of range ledger sequences"); - - if (sIt.get256().isZero()) - return fail("invalid last ledger hash"); - - if (exists(dir_ / LgrDBName) && exists(dir_ / TxDBName)) - { - lastAccess_ = std::chrono::steady_clock::now(); - state_ = ShardState::finalized; - } - else - state_ = ShardState::complete; - - progress_ = maxLedgers_; - } - } - catch (std::exception const& e) - { - return fail( - std::string(". Exception caught in function ") + __func__ + - ". Error: " + e.what()); - } - - if (!initSQLite(lock)) - return fail({}); - - setFileStats(lock); - return true; -} - -bool -Shard::initSQLite(std::lock_guard const&) -{ - Config const& config{app_.config()}; - DatabaseCon::Setup const setup = [&]() { - DatabaseCon::Setup setup; - setup.startUp = config.standalone() ? config.LOAD : config.START_UP; - setup.standAlone = config.standalone(); - setup.dataDir = dir_; - setup.useGlobalPragma = (state_ != ShardState::complete); - return setup; - }(); - - try - { - if (lgrSQLiteDB_) - lgrSQLiteDB_.reset(); - - if (txSQLiteDB_) - txSQLiteDB_.reset(); - - switch (state_) - { - case ShardState::complete: - case ShardState::finalizing: - case ShardState::finalized: { - auto [lgr, tx] = makeShardCompleteLedgerDBs(config, setup); - - lgrSQLiteDB_ = std::move(lgr); - lgrSQLiteDB_->getSession() << boost::str( - boost::format("PRAGMA cache_size=-%d;") % - kilobytes(config.getValueFor( - SizedItem::lgrDBCache, std::nullopt))); - - txSQLiteDB_ = std::move(tx); - txSQLiteDB_->getSession() << boost::str( - boost::format("PRAGMA cache_size=-%d;") % - kilobytes(config.getValueFor( - SizedItem::txnDBCache, std::nullopt))); - break; - } - - // case ShardState::acquire: - // case ShardState::queued: - default: { - // Incomplete shards use a Write Ahead Log for performance - auto [lgr, tx] = makeShardIncompleteLedgerDBs( - config, - setup, - DatabaseCon::CheckpointerSetup{ - &app_.getJobQueue(), &app_.logs()}); - - lgrSQLiteDB_ = std::move(lgr); - lgrSQLiteDB_->getSession() << boost::str( - boost::format("PRAGMA cache_size=-%d;") % - kilobytes(config.getValueFor(SizedItem::lgrDBCache))); - - txSQLiteDB_ = std::move(tx); - txSQLiteDB_->getSession() << boost::str( - boost::format("PRAGMA cache_size=-%d;") % - kilobytes(config.getValueFor(SizedItem::txnDBCache))); - break; - } - } - } - catch (std::exception const& e) - { - JLOG(j_.fatal()) << "shard " << index_ - << ". Exception caught in function " << __func__ - << ". Error: " << e.what(); - return false; - } - - return true; -} - -bool -Shard::storeSQLite(std::shared_ptr const& ledger) -{ - if (stop_) - return false; - - try - { - std::lock_guard lock(mutex_); - - auto res = updateLedgerDBs( - *txSQLiteDB_->checkoutDb(), - *lgrSQLiteDB_->checkoutDb(), - ledger, - index_, - stop_, - j_); - - if (!res) - return false; - - // Update the acquire database if present - if (acquireInfo_) - { - std::optional s; - if (!acquireInfo_->storedSeqs.empty()) - s = to_string(acquireInfo_->storedSeqs); - - updateAcquireDB( - acquireInfo_->SQLiteDB->getSession(), - ledger, - index_, - lastSeq_, - s); - } - } - catch (std::exception const& e) - { - JLOG(j_.fatal()) << "shard " << index_ - << ". Exception caught in function " << __func__ - << ". Error: " << e.what(); - return false; - } - - return true; -} - -void -Shard::setFileStats(std::lock_guard const&) -{ - fileSz_ = 0; - fdRequired_ = 0; - try - { - using namespace boost::filesystem; - for (auto const& d : directory_iterator(dir_)) - { - if (is_regular_file(d)) - { - fileSz_ += file_size(d); - ++fdRequired_; - } - } - } - catch (std::exception const& e) - { - JLOG(j_.fatal()) << "shard " << index_ - << ". Exception caught in function " << __func__ - << ". Error: " << e.what(); - } -} - -bool -Shard::verifyLedger( - std::shared_ptr const& ledger, - std::shared_ptr const& next, - std::shared_ptr const& dShard) const -{ - auto fail = [j = j_, index = index_, &ledger](std::string const& msg) { - JLOG(j.error()) << "shard " << index << ". " << msg - << (ledger->info().hash.isZero() ? "" - : ". Ledger hash " + - to_string(ledger->info().hash)) - << (ledger->info().seq == 0 ? "" - : ". Ledger sequence " + - std::to_string(ledger->info().seq)); - return false; - }; - - if (ledger->info().hash.isZero()) - return fail("Invalid ledger hash"); - if (ledger->info().accountHash.isZero()) - return fail("Invalid ledger account hash"); - - bool error{false}; - auto visit = [this, &error, &dShard](SHAMapTreeNode const& node) { - if (stop_) - return false; - - auto nodeObject{verifyFetch(node.getHash().as_uint256())}; - if (!nodeObject || !dShard->store(nodeObject)) - error = true; - - return !error; - }; - - // Verify the state map - if (ledger->stateMap().getHash().isNonZero()) - { - if (!ledger->stateMap().isValid()) - return fail("Invalid state map"); - - try - { - if (next && next->info().parentHash == ledger->info().hash) - ledger->stateMap().visitDifferences(&next->stateMap(), visit); - else - ledger->stateMap().visitNodes(visit); - } - catch (std::exception const& e) - { - return fail( - std::string(". Exception caught in function ") + __func__ + - ". Error: " + e.what()); - } - - if (stop_) - return false; - if (error) - return fail("Invalid state map"); - } - - // Verify the transaction map - if (ledger->info().txHash.isNonZero()) - { - if (!ledger->txMap().isValid()) - return fail("Invalid transaction map"); - - try - { - ledger->txMap().visitNodes(visit); - } - catch (std::exception const& e) - { - return fail( - std::string(". Exception caught in function ") + __func__ + - ". Error: " + e.what()); - } - - if (stop_) - return false; - if (error) - return fail("Invalid transaction map"); - } - - return true; -} - -std::shared_ptr -Shard::verifyFetch(uint256 const& hash) const -{ - std::shared_ptr nodeObject; - auto fail = - [j = j_, index = index_, &hash, &nodeObject](std::string const& msg) { - JLOG(j.error()) << "shard " << index << ". " << msg - << ". Node object hash " << to_string(hash); - nodeObject.reset(); - return nodeObject; - }; - - try - { - std::lock_guard lock(mutex_); - switch (backend_->fetch(hash.data(), &nodeObject)) - { - case ok: - // Verify that the hash of node object matches the payload - if (nodeObject->getHash() != - sha512Half(makeSlice(nodeObject->getData()))) - return fail("Node object hash does not match payload"); - return nodeObject; - case notFound: - return fail("Missing node object"); - case dataCorrupt: - return fail("Corrupt node object"); - default: - return fail("Unknown error"); - } - } - catch (std::exception const& e) - { - return fail( - std::string(". Exception caught in function ") + __func__ + - ". Error: " + e.what()); - } -} - -Shard::Count -Shard::makeBackendCount() -{ - if (stop_ || busy_) - return Shard::Count{nullptr}; - - std::lock_guard lock(mutex_); - if (!backend_) - { - JLOG(j_.error()) << "shard " << index_ << " not initialized"; - return Shard::Count{nullptr}; - } - if (!backend_->isOpen()) - { - if (!open(lock)) - return Shard::Count{nullptr}; - } - else if (state_ == ShardState::finalized) - lastAccess_ = std::chrono::steady_clock::now(); - - return Shard::Count(&backendCount_); -} - -bool -Shard::doCallForSQL( - std::function const& callback, - LockedSociSession&& db) -{ - return callback(*db); -} - -bool -Shard::doCallForSQL( - std::function const& - callback, - LockedSociSession&& db) -{ - return callback(*db, index_); -} - -} // namespace NodeStore -} // namespace ripple diff --git a/src/xrpld/nodestore/detail/Shard.h b/src/xrpld/nodestore/detail/Shard.h deleted file mode 100644 index be11e538c77..00000000000 --- a/src/xrpld/nodestore/detail/Shard.h +++ /dev/null @@ -1,432 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2017 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_NODESTORE_SHARD_H_INCLUDED -#define RIPPLE_NODESTORE_SHARD_H_INCLUDED - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -namespace ripple { -namespace NodeStore { - -using PCache = TaggedCache; -using NCache = KeyCache; -class DatabaseShard; - -/* A range of historical ledgers backed by a node store. - Shards are indexed and store `ledgersPerShard`. - Shard `i` stores ledgers starting with sequence: `1 + (i * ledgersPerShard)` - and ending with sequence: `(i + 1) * ledgersPerShard`. - Once a shard has all its ledgers, it is never written to again. - - Public functions can be called concurrently from any thread. -*/ -class Shard final -{ -public: - /// Copy constructor (disallowed) - Shard(Shard const&) = delete; - - /// Move constructor (disallowed) - Shard(Shard&&) = delete; - - // Copy assignment (disallowed) - Shard& - operator=(Shard const&) = delete; - - // Move assignment (disallowed) - Shard& - operator=(Shard&&) = delete; - - Shard( - Application& app, - DatabaseShard const& db, - std::uint32_t index, - boost::filesystem::path const& dir, - beast::Journal j); - - Shard( - Application& app, - DatabaseShard const& db, - std::uint32_t index, - beast::Journal j); - - ~Shard(); - - /** Initialize shard. - - @param scheduler The scheduler to use for performing asynchronous tasks. - @param context The context to use for the backend. - */ - [[nodiscard]] bool - init(Scheduler& scheduler, nudb::context& context); - - /** Returns true if the database are open. - */ - [[nodiscard]] bool - isOpen() const; - - /** Try to close databases if not in use. - - @return true if databases were closed. - */ - bool - tryClose(); - - /** Notify shard to prepare for shutdown. - */ - void - stop() noexcept - { - stop_ = true; - } - - [[nodiscard]] std::optional - prepare(); - - [[nodiscard]] bool - storeNodeObject(std::shared_ptr const& nodeObject); - - [[nodiscard]] std::shared_ptr - fetchNodeObject(uint256 const& hash, FetchReport& fetchReport); - - /** Store a ledger. - - @param srcLedger The ledger to store. - @param next The ledger that immediately follows srcLedger, can be null. - @return StoreLedgerResult containing data about the store. - */ - struct StoreLedgerResult - { - std::uint64_t count{0}; // Number of storage calls - std::uint64_t size{0}; // Number of bytes stored - bool error{false}; - }; - - [[nodiscard]] StoreLedgerResult - storeLedger( - std::shared_ptr const& srcLedger, - std::shared_ptr const& next); - - [[nodiscard]] bool - setLedgerStored(std::shared_ptr const& ledger); - - [[nodiscard]] bool - containsLedger(std::uint32_t ledgerSeq) const; - - [[nodiscard]] std::uint32_t - index() const noexcept - { - return index_; - } - - [[nodiscard]] boost::filesystem::path const& - getDir() const noexcept - { - return dir_; - } - - [[nodiscard]] std::chrono::steady_clock::time_point - getLastUse() const; - - /** Returns a pair where the first item describes the storage space - utilized and the second item is the number of file descriptors required. - */ - [[nodiscard]] std::pair - getFileInfo() const; - - [[nodiscard]] ShardState - getState() const noexcept - { - return state_; - } - - /** Returns a percent signifying how complete - the current state of the shard is. - */ - [[nodiscard]] std::uint32_t - getPercentProgress() const noexcept - { - return calculatePercent(progress_, maxLedgers_); - } - - [[nodiscard]] std::int32_t - getWriteLoad(); - - /** Returns `true` if shard is older, without final key data - */ - [[nodiscard]] bool - isLegacy() const; - - /** Finalize shard by walking its ledgers, verifying each Merkle tree and - creating a deterministic backend. - - @param writeSQLite If true, SQLite entries will be rewritten using - verified backend data. - @param referenceHash If present, this hash must match the hash - of the last ledger in the shard. - */ - [[nodiscard]] bool - finalize(bool writeSQLite, std::optional const& referenceHash); - - /** Enables removal of the shard directory on destruction. - */ - void - removeOnDestroy() noexcept - { - removeOnDestroy_ = true; - } - - std::string - getStoredSeqs() - { - std::lock_guard lock(mutex_); - if (!acquireInfo_) - return ""; - - return to_string(acquireInfo_->storedSeqs); - } - - /** Invoke a callback on the ledger SQLite db - - @param callback Callback function to call. - @return Value returned by callback function. - */ - template - bool - callForLedgerSQL(std::function const& callback) - { - return callForSQL(callback, lgrSQLiteDB_->checkoutDb()); - } - - /** Invoke a callback on the transaction SQLite db - - @param callback Callback function to call. - @return Value returned by callback function. - */ - template - bool - callForTransactionSQL(std::function const& callback) - { - return callForSQL(callback, txSQLiteDB_->checkoutDb()); - } - - // Current shard version - static constexpr std::uint32_t version{2}; - - // The finalKey is a hard coded value of zero. It is used to store - // finalizing shard data to the backend. The data contains a version, - // last ledger's hash, and the first and last ledger sequences. - static uint256 const finalKey; - -private: - class Count final - { - public: - Count(Count const&) = delete; - Count& - operator=(Count const&) = delete; - Count& - operator=(Count&&) = delete; - - Count(Count&& other) noexcept : counter_(other.counter_) - { - other.counter_ = nullptr; - } - - explicit Count(std::atomic* counter) noexcept - : counter_(counter) - { - if (counter_) - ++(*counter_); - } - - ~Count() noexcept - { - if (counter_) - --(*counter_); - } - - explicit operator bool() const noexcept - { - return counter_ != nullptr; - } - - private: - std::atomic* counter_; - }; - - struct AcquireInfo - { - // SQLite database to track information about what has been acquired - std::unique_ptr SQLiteDB; - - // Tracks the sequences of ledgers acquired and stored in the backend - RangeSet storedSeqs; - }; - - Application& app_; - beast::Journal const j_; - mutable std::mutex mutex_; - mutable std::mutex storedMutex_; - - // Shard Index - std::uint32_t const index_; - - // First ledger sequence in the shard - std::uint32_t const firstSeq_; - - // Last ledger sequence in the shard - std::uint32_t const lastSeq_; - - // The maximum number of ledgers the shard can store - // The earliest shard may store fewer ledgers than subsequent shards - std::uint32_t const maxLedgers_; - - // Path to database files - boost::filesystem::path const dir_; - - // Storage space utilized by the shard - GUARDED_BY(mutex_) std::uint64_t fileSz_{0}; - - // Number of file descriptors required by the shard - GUARDED_BY(mutex_) std::uint32_t fdRequired_{0}; - - // NuDB key/value store for node objects - std::unique_ptr backend_ GUARDED_BY(mutex_); - - std::atomic backendCount_{0}; - - // Ledger SQLite database used for indexes - std::unique_ptr lgrSQLiteDB_ GUARDED_BY(mutex_); - - // Transaction SQLite database used for indexes - std::unique_ptr txSQLiteDB_ GUARDED_BY(mutex_); - - // Tracking information used only when acquiring a shard from the network. - // If the shard is finalized, this member will be null. - std::unique_ptr acquireInfo_ GUARDED_BY(mutex_); - ; - - // Older shard without an acquire database or final key - // Eventually there will be no need for this and should be removed - GUARDED_BY(mutex_) bool legacy_{false}; - - // Determines if the shard needs to stop processing for shutdown - std::atomic stop_{false}; - - // Determines if the shard busy with replacing by deterministic one - std::atomic busy_{false}; - - // State of the shard - std::atomic state_{ShardState::acquire}; - - // Number of ledgers processed for the current shard state - std::atomic progress_{0}; - - // Determines if the shard directory should be removed in the destructor - std::atomic removeOnDestroy_{false}; - - // The time of the last access of a shard with a finalized state - std::chrono::steady_clock::time_point lastAccess_ GUARDED_BY(mutex_); - ; - - // Open shard databases - [[nodiscard]] bool - open(std::lock_guard const& lock) REQUIRES(mutex_); - - // Open/Create SQLite databases - // Lock over mutex_ required - [[nodiscard]] bool - initSQLite(std::lock_guard const&) REQUIRES(mutex_); - - // Write SQLite entries for this ledger - [[nodiscard]] bool - storeSQLite(std::shared_ptr const& ledger); - - // Set storage and file descriptor usage stats - // Lock over mutex_ required - void - setFileStats(std::lock_guard const&) REQUIRES(mutex_); - - // Verify this ledger by walking its SHAMaps and verifying its Merkle trees - // Every node object verified will be stored in the deterministic shard - [[nodiscard]] bool - verifyLedger( - std::shared_ptr const& ledger, - std::shared_ptr const& next, - std::shared_ptr const& dShard) const; - - // Fetches from backend and log errors based on status codes - [[nodiscard]] std::shared_ptr - verifyFetch(uint256 const& hash) const; - - // Open databases if they are closed - [[nodiscard]] Shard::Count - makeBackendCount(); - - // Invoke a callback on the supplied session parameter - template - bool - callForSQL( - std::function const& callback, - LockedSociSession&& db) - { - auto const scopedCount{makeBackendCount()}; - if (!scopedCount) - return false; - - return doCallForSQL(callback, std::move(db)); - } - - // Invoke a callback that accepts a SQLite session parameter - bool - doCallForSQL( - std::function const& callback, - LockedSociSession&& db); - - // Invoke a callback that accepts a SQLite session and the - // shard index as parameters - bool - doCallForSQL( - std::function< - bool(soci::session& session, std::uint32_t shardIndex)> const& - callback, - LockedSociSession&& db); -}; - -} // namespace NodeStore -} // namespace ripple - -#endif diff --git a/src/xrpld/nodestore/detail/ShardInfo.cpp b/src/xrpld/nodestore/detail/ShardInfo.cpp deleted file mode 100644 index 43a9d484900..00000000000 --- a/src/xrpld/nodestore/detail/ShardInfo.cpp +++ /dev/null @@ -1,136 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include - -namespace ripple { -namespace NodeStore { - -std::string -ShardInfo::finalizedToString() const -{ - if (!finalized_.empty()) - return ripple::to_string(finalized_); - return {}; -} - -std::string -ShardInfo::incompleteToString() const -{ - std::string result; - if (!incomplete_.empty()) - { - for (auto const& [shardIndex, incomplete] : incomplete_) - { - result += std::to_string(shardIndex) + ":" + - std::to_string(incomplete.percentProgress()) + ","; - } - result.pop_back(); - } - - return result; -} - -bool -ShardInfo::update( - std::uint32_t shardIndex, - ShardState state, - std::uint32_t percentProgress) -{ - if (state == ShardState::finalized) - { - if (boost::icl::contains(finalized_, shardIndex)) - return false; - - finalized_.insert(shardIndex); - return true; - } - - return incomplete_.emplace(shardIndex, Incomplete(state, percentProgress)) - .second; -} - -protocol::TMPeerShardInfoV2 -ShardInfo::makeMessage(Application& app) -{ - protocol::TMPeerShardInfoV2 message; - Serializer s; - s.add32(HashPrefix::shardInfo); - - // Set the message creation time - msgTimestamp_ = app.timeKeeper().now(); - { - auto const timestamp{msgTimestamp_.time_since_epoch().count()}; - message.set_timestamp(timestamp); - s.add32(timestamp); - } - - if (!incomplete_.empty()) - { - message.mutable_incomplete()->Reserve(incomplete_.size()); - for (auto const& [shardIndex, incomplete] : incomplete_) - { - auto tmIncomplete{message.add_incomplete()}; - - tmIncomplete->set_shardindex(shardIndex); - s.add32(shardIndex); - - static_assert(std::is_same_v< - std::underlying_type_t, - std::uint32_t>); - auto const state{static_cast(incomplete.state())}; - tmIncomplete->set_state(state); - s.add32(state); - - // Set progress if greater than zero - auto const percentProgress{incomplete.percentProgress()}; - if (percentProgress > 0) - { - tmIncomplete->set_progress(percentProgress); - s.add32(percentProgress); - } - } - } - - if (!finalized_.empty()) - { - auto const str{ripple::to_string(finalized_)}; - message.set_finalized(str); - s.addRaw(str.data(), str.size()); - } - - // Set the public key - auto const& publicKey{app.nodeIdentity().first}; - message.set_publickey(publicKey.data(), publicKey.size()); - - // Create a digital signature using the node private key - auto const signature{sign(publicKey, app.nodeIdentity().second, s.slice())}; - - // Set the digital signature - message.set_signature(signature.data(), signature.size()); - - return message; -} - -} // namespace NodeStore -} // namespace ripple diff --git a/src/xrpld/nodestore/detail/TaskQueue.cpp b/src/xrpld/nodestore/detail/TaskQueue.cpp deleted file mode 100644 index 6062138c60f..00000000000 --- a/src/xrpld/nodestore/detail/TaskQueue.cpp +++ /dev/null @@ -1,76 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2019 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include - -#include - -namespace ripple { -namespace NodeStore { - -TaskQueue::TaskQueue() : workers_(*this, nullptr, "Shard store taskQueue", 1) -{ -} - -void -TaskQueue::stop() -{ - workers_.stop(); -} - -void -TaskQueue::addTask(std::function task) -{ - { - std::lock_guard lock{mutex_}; - tasks_.emplace(std::move(task)); - } - workers_.addTask(); -} - -size_t -TaskQueue::size() const -{ - std::lock_guard lock{mutex_}; - return tasks_.size() + processing_; -} - -void -TaskQueue::processTask(int instance) -{ - std::function task; - - { - std::lock_guard lock{mutex_}; - - assert(!tasks_.empty()); - task = std::move(tasks_.front()); - tasks_.pop(); - - ++processing_; - } - - task(); - - std::lock_guard lock{mutex_}; - --processing_; -} - -} // namespace NodeStore -} // namespace ripple diff --git a/src/xrpld/nodestore/detail/TaskQueue.h b/src/xrpld/nodestore/detail/TaskQueue.h deleted file mode 100644 index 8a743ff6016..00000000000 --- a/src/xrpld/nodestore/detail/TaskQueue.h +++ /dev/null @@ -1,64 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2019 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_NODESTORE_TASKQUEUE_H_INCLUDED -#define RIPPLE_NODESTORE_TASKQUEUE_H_INCLUDED - -#include - -#include -#include - -namespace ripple { -namespace NodeStore { - -class TaskQueue : private Workers::Callback -{ -public: - TaskQueue(); - - void - stop(); - - /** Adds a task to the queue - - @param task std::function with signature void() - */ - void - addTask(std::function task); - - /** Return the queue size - */ - [[nodiscard]] size_t - size() const; - -private: - mutable std::mutex mutex_; - Workers workers_; - std::queue> tasks_; - std::uint64_t processing_{0}; - - void - processTask(int instance) override; -}; - -} // namespace NodeStore -} // namespace ripple - -#endif diff --git a/src/xrpld/overlay/Overlay.h b/src/xrpld/overlay/Overlay.h index 1a3362d386f..b9550ba2ef4 100644 --- a/src/xrpld/overlay/Overlay.h +++ b/src/xrpld/overlay/Overlay.h @@ -219,15 +219,6 @@ class Overlay : public beast::PropertyStream::Source virtual std::uint64_t getPeerDisconnectCharges() const = 0; - /** Returns information reported to the crawl shard RPC command. - - @param includePublicKey include peer public keys in the result. - @param hops the maximum jumps the crawler will attempt. - The number of hops achieved is not guaranteed. - */ - virtual Json::Value - crawlShards(bool includePublicKey, std::uint32_t hops) = 0; - /** Returns the ID of the network this server is configured for, if any. The ID is just a numerical identifier, with the IDs 0, 1 and 2 used to diff --git a/src/xrpld/overlay/Peer.h b/src/xrpld/overlay/Peer.h index 81c04f7206c..82ed2c2481a 100644 --- a/src/xrpld/overlay/Peer.h +++ b/src/xrpld/overlay/Peer.h @@ -32,9 +32,6 @@ namespace Resource { class Charge; } -// Maximum hops to relay the peer shard info request -static constexpr std::uint32_t relayLimit = 3; - enum class ProtocolFeature { ValidatorListPropagation, ValidatorList2Propagation, diff --git a/src/xrpld/overlay/detail/Message.cpp b/src/xrpld/overlay/detail/Message.cpp index e19d718c73d..71917db0506 100644 --- a/src/xrpld/overlay/detail/Message.cpp +++ b/src/xrpld/overlay/detail/Message.cpp @@ -94,13 +94,9 @@ Message::compress() case protocol::mtSTATUS_CHANGE: case protocol::mtHAVE_SET: case protocol::mtVALIDATION: - case protocol::mtGET_PEER_SHARD_INFO: - case protocol::mtPEER_SHARD_INFO: case protocol::mtPROOF_PATH_REQ: case protocol::mtPROOF_PATH_RESPONSE: case protocol::mtREPLAY_DELTA_REQ: - case protocol::mtGET_PEER_SHARD_INFO_V2: - case protocol::mtPEER_SHARD_INFO_V2: case protocol::mtHAVE_TRANSACTIONS: break; } diff --git a/src/xrpld/overlay/detail/OverlayImpl.cpp b/src/xrpld/overlay/detail/OverlayImpl.cpp index 1978a2617aa..970873007c2 100644 --- a/src/xrpld/overlay/detail/OverlayImpl.cpp +++ b/src/xrpld/overlay/detail/OverlayImpl.cpp @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -679,103 +678,6 @@ OverlayImpl::reportTraffic( m_traffic.addCount(cat, isInbound, number); } -Json::Value -OverlayImpl::crawlShards(bool includePublicKey, std::uint32_t relays) -{ - using namespace std::chrono; - - Json::Value jv(Json::objectValue); - - // Add shard info from this server to json result - if (auto shardStore = app_.getShardStore()) - { - if (includePublicKey) - jv[jss::public_key] = - toBase58(TokenType::NodePublic, app_.nodeIdentity().first); - - auto const shardInfo{shardStore->getShardInfo()}; - if (!shardInfo->finalized().empty()) - jv[jss::complete_shards] = shardInfo->finalizedToString(); - if (!shardInfo->incomplete().empty()) - jv[jss::incomplete_shards] = shardInfo->incompleteToString(); - } - - if (relays == 0 || size() == 0) - return jv; - - { - protocol::TMGetPeerShardInfoV2 tmGPS; - tmGPS.set_relays(relays); - - // Wait if a request is in progress - std::unique_lock csLock{csMutex_}; - if (!csIDs_.empty()) - csCV_.wait(csLock); - - { - std::lock_guard lock{mutex_}; - for (auto const& id : ids_) - csIDs_.emplace(id.first); - } - - // Request peer shard info - foreach(send_always(std::make_shared( - tmGPS, protocol::mtGET_PEER_SHARD_INFO_V2))); - - if (csCV_.wait_for(csLock, seconds(60)) == std::cv_status::timeout) - { - csIDs_.clear(); - csCV_.notify_all(); - } - } - - // Combine shard info from peers - hash_map peerShardInfo; - for_each([&](std::shared_ptr&& peer) { - auto const psi{peer->getPeerShardInfos()}; - for (auto const& [publicKey, shardInfo] : psi) - { - auto const it{peerShardInfo.find(publicKey)}; - if (it == peerShardInfo.end()) - peerShardInfo.emplace(publicKey, shardInfo); - else if (shardInfo.msgTimestamp() > it->second.msgTimestamp()) - it->second = shardInfo; - } - }); - - // Add shard info to json result - if (!peerShardInfo.empty()) - { - auto& av = jv[jss::peers] = Json::Value(Json::arrayValue); - for (auto const& [publicKey, shardInfo] : peerShardInfo) - { - auto& pv{av.append(Json::Value(Json::objectValue))}; - if (includePublicKey) - { - pv[jss::public_key] = - toBase58(TokenType::NodePublic, publicKey); - } - - if (!shardInfo.finalized().empty()) - pv[jss::complete_shards] = shardInfo.finalizedToString(); - if (!shardInfo.incomplete().empty()) - pv[jss::incomplete_shards] = shardInfo.incompleteToString(); - } - } - - return jv; -} - -void -OverlayImpl::endOfPeerChain(std::uint32_t id) -{ - // Notify threads if all peers have received a reply from all peer chains - std::lock_guard csLock{csMutex_}; - csIDs_.erase(id); - if (csIDs_.empty()) - csCV_.notify_all(); -} - /** The number of active peers on the network Active peers are only those peers that have completed the handshake and are running the Ripple protocol. @@ -833,17 +735,6 @@ OverlayImpl::getOverlayInfo() if (minSeq != 0 || maxSeq != 0) pv[jss::complete_ledgers] = std::to_string(minSeq) + "-" + std::to_string(maxSeq); - - auto const peerShardInfos{sp->getPeerShardInfos()}; - auto const it{peerShardInfos.find(sp->getNodePublic())}; - if (it != peerShardInfos.end()) - { - auto const& shardInfo{it->second}; - if (!shardInfo.finalized().empty()) - pv[jss::complete_shards] = shardInfo.finalizedToString(); - if (!shardInfo.incomplete().empty()) - pv[jss::incomplete_shards] = shardInfo.incompleteToString(); - } }); return jv; diff --git a/src/xrpld/overlay/detail/OverlayImpl.h b/src/xrpld/overlay/detail/OverlayImpl.h index 1934a7c94c8..a50dfc5e905 100644 --- a/src/xrpld/overlay/detail/OverlayImpl.h +++ b/src/xrpld/overlay/detail/OverlayImpl.h @@ -119,12 +119,6 @@ class OverlayImpl : public Overlay, public reduce_relay::SquelchHandler std::atomic peerDisconnects_{0}; std::atomic peerDisconnectsCharges_{0}; - // 'cs' = crawl shards - std::mutex csMutex_; - std::condition_variable csCV_; - // Peer IDs expecting to receive a last link notification - std::set csIDs_; - reduce_relay::Slots slots_; // Transaction reduce-relay metrics @@ -392,16 +386,6 @@ class OverlayImpl : public Overlay, public reduce_relay::SquelchHandler return setup_.networkID; } - Json::Value - crawlShards(bool includePublicKey, std::uint32_t relays) override; - - /** Called when the reply from the last peer in a peer chain is received. - - @param id peer id that received the shard info. - */ - void - endOfPeerChain(std::uint32_t id); - /** Updates message count for validator/peer. Sends TMSquelch if the number * of messages for N peers reaches threshold T. A message is counted * if a peer receives the message for the first time and if diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index 96f793b8d80..86e336f850b 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -521,17 +520,6 @@ PeerImp::hasLedger(uint256 const& hash, std::uint32_t seq) const recentLedgers_.end()) return true; } - - if (seq >= app_.getNodeStore().earliestLedgerSeq()) - { - std::lock_guard lock{shardInfoMutex_}; - auto const it{shardInfos_.find(publicKey_)}; - if (it != shardInfos_.end()) - { - auto const shardIndex{app_.getNodeStore().seqToShardIndex(seq)}; - return boost::icl::contains(it->second.finalized(), shardIndex); - } - } return false; } @@ -626,13 +614,6 @@ PeerImp::fail(std::string const& name, error_code ec) close(); } -hash_map const -PeerImp::getPeerShardInfos() const -{ - std::lock_guard l{shardInfoMutex_}; - return shardInfos_; -} - void PeerImp::gracefulClose() { @@ -878,11 +859,6 @@ PeerImp::doProtocolStart() if (auto m = overlay_.getManifestsMessage()) send(m); - // Request shard info from peer - protocol::TMGetPeerShardInfoV2 tmGPS; - tmGPS.set_relays(0); - send(std::make_shared(tmGPS, protocol::mtGET_PEER_SHARD_INFO_V2)); - setTimer(); } @@ -1175,294 +1151,6 @@ PeerImp::onMessage(std::shared_ptr const& m) app_.getFeeTrack().setClusterFee(clusterFee); } -void -PeerImp::onMessage(std::shared_ptr const& m) -{ - // DEPRECATED -} - -void -PeerImp::onMessage(std::shared_ptr const& m) -{ - // DEPRECATED -} - -void -PeerImp::onMessage(std::shared_ptr const& m) -{ - auto badData = [&](std::string msg) { - fee_ = Resource::feeBadData; - JLOG(p_journal_.warn()) << msg; - }; - - // Verify relays - if (m->relays() > relayLimit) - return badData("Invalid relays"); - - // Verify peer chain - // The peer chain should not contain this node's public key - // nor the public key of the sending peer - std::set pubKeyChain; - pubKeyChain.insert(app_.nodeIdentity().first); - pubKeyChain.insert(publicKey_); - - auto const peerChainSz{m->peerchain_size()}; - if (peerChainSz > 0) - { - if (peerChainSz > relayLimit) - return badData("Invalid peer chain size"); - - if (peerChainSz + m->relays() > relayLimit) - return badData("Invalid relays and peer chain size"); - - for (int i = 0; i < peerChainSz; ++i) - { - auto const slice{makeSlice(m->peerchain(i).publickey())}; - - // Verify peer public key - if (!publicKeyType(slice)) - return badData("Invalid peer public key"); - - // Verify peer public key is unique in the peer chain - if (!pubKeyChain.emplace(slice).second) - return badData("Invalid peer public key"); - } - } - - // Reply with shard info this node may have - if (auto shardStore = app_.getShardStore()) - { - auto reply{shardStore->getShardInfo()->makeMessage(app_)}; - if (peerChainSz > 0) - *(reply.mutable_peerchain()) = m->peerchain(); - send(std::make_shared(reply, protocol::mtPEER_SHARD_INFO_V2)); - } - - if (m->relays() == 0) - return; - - // Charge originating peer a fee for requesting relays - if (peerChainSz == 0) - fee_ = Resource::feeMediumBurdenPeer; - - // Add peer to the peer chain - m->add_peerchain()->set_publickey(publicKey_.data(), publicKey_.size()); - - // Relay the request to peers, exclude the peer chain - m->set_relays(m->relays() - 1); - overlay_.foreach(send_if_not( - std::make_shared(*m, protocol::mtGET_PEER_SHARD_INFO_V2), - [&](std::shared_ptr const& peer) { - return pubKeyChain.find(peer->getNodePublic()) != pubKeyChain.end(); - })); -} - -void -PeerImp::onMessage(std::shared_ptr const& m) -{ - // Find the earliest and latest shard indexes - auto const& db{app_.getNodeStore()}; - auto const earliestShardIndex{db.earliestShardIndex()}; - auto const latestShardIndex{[&]() -> std::optional { - auto const curLedgerSeq{app_.getLedgerMaster().getCurrentLedgerIndex()}; - if (curLedgerSeq >= db.earliestLedgerSeq()) - return db.seqToShardIndex(curLedgerSeq); - return std::nullopt; - }()}; - - auto badData = [&](std::string msg) { - fee_ = Resource::feeBadData; - JLOG(p_journal_.warn()) << msg; - }; - - // Used to create a digest and verify the message signature - Serializer s; - s.add32(HashPrefix::shardInfo); - - // Verify message creation time - NodeStore::ShardInfo shardInfo; - { - auto const timestamp{ - NetClock::time_point{std::chrono::seconds{m->timestamp()}}}; - auto const now{app_.timeKeeper().now()}; - if (timestamp > (now + 5s)) - return badData("Invalid timestamp"); - - // Check if stale - using namespace std::chrono_literals; - if (timestamp < (now - 5min)) - return badData("Stale timestamp"); - - s.add32(m->timestamp()); - shardInfo.setMsgTimestamp(timestamp); - } - - // Verify incomplete shards - auto const numIncomplete{m->incomplete_size()}; - if (numIncomplete > 0) - { - if (latestShardIndex && numIncomplete > *latestShardIndex) - return badData("Invalid number of incomplete shards"); - - // Verify each incomplete shard - for (int i = 0; i < numIncomplete; ++i) - { - auto const& incomplete{m->incomplete(i)}; - auto const shardIndex{incomplete.shardindex()}; - - // Verify shard index - if (shardIndex < earliestShardIndex || - (latestShardIndex && shardIndex > latestShardIndex)) - { - return badData("Invalid incomplete shard index"); - } - s.add32(shardIndex); - - // Verify state - auto const state{static_cast(incomplete.state())}; - switch (state) - { - // Incomplete states - case ShardState::acquire: - case ShardState::complete: - case ShardState::finalizing: - case ShardState::queued: - break; - - // case ShardState::finalized: - default: - return badData("Invalid incomplete shard state"); - } - s.add32(incomplete.state()); - - // Verify progress - std::uint32_t progress{0}; - if (incomplete.has_progress()) - { - progress = incomplete.progress(); - if (progress < 1 || progress > 100) - return badData("Invalid incomplete shard progress"); - s.add32(progress); - } - - // Verify each incomplete shard is unique - if (!shardInfo.update(shardIndex, state, progress)) - return badData("Invalid duplicate incomplete shards"); - } - } - - // Verify finalized shards - if (m->has_finalized()) - { - auto const& str{m->finalized()}; - if (str.empty()) - return badData("Invalid finalized shards"); - - if (!shardInfo.setFinalizedFromString(str)) - return badData("Invalid finalized shard indexes"); - - auto const& finalized{shardInfo.finalized()}; - auto const numFinalized{boost::icl::length(finalized)}; - if (numFinalized == 0 || - boost::icl::first(finalized) < earliestShardIndex || - (latestShardIndex && - boost::icl::last(finalized) > latestShardIndex)) - { - return badData("Invalid finalized shard indexes"); - } - - if (latestShardIndex && - (numFinalized + numIncomplete) > *latestShardIndex) - { - return badData("Invalid number of finalized and incomplete shards"); - } - - s.addRaw(str.data(), str.size()); - } - - // Verify public key - auto slice{makeSlice(m->publickey())}; - if (!publicKeyType(slice)) - return badData("Invalid public key"); - - // Verify peer public key isn't this nodes's public key - PublicKey const publicKey(slice); - if (publicKey == app_.nodeIdentity().first) - return badData("Invalid public key"); - - // Verify signature - if (!verify(publicKey, s.slice(), makeSlice(m->signature()), false)) - return badData("Invalid signature"); - - // Forward the message if a peer chain exists - auto const peerChainSz{m->peerchain_size()}; - if (peerChainSz > 0) - { - // Verify peer chain - if (peerChainSz > relayLimit) - return badData("Invalid peer chain size"); - - // The peer chain should not contain this node's public key - // nor the public key of the sending peer - std::set pubKeyChain; - pubKeyChain.insert(app_.nodeIdentity().first); - pubKeyChain.insert(publicKey_); - - for (int i = 0; i < peerChainSz; ++i) - { - // Verify peer public key - slice = makeSlice(m->peerchain(i).publickey()); - if (!publicKeyType(slice)) - return badData("Invalid peer public key"); - - // Verify peer public key is unique in the peer chain - if (!pubKeyChain.emplace(slice).second) - return badData("Invalid peer public key"); - } - - // If last peer in the chain is connected, relay the message - PublicKey const peerPubKey( - makeSlice(m->peerchain(peerChainSz - 1).publickey())); - if (auto peer = overlay_.findPeerByPublicKey(peerPubKey)) - { - m->mutable_peerchain()->RemoveLast(); - peer->send( - std::make_shared(*m, protocol::mtPEER_SHARD_INFO_V2)); - JLOG(p_journal_.trace()) - << "Relayed TMPeerShardInfoV2 from peer IP " - << remote_address_.address().to_string() << " to peer IP " - << peer->getRemoteAddress().to_string(); - } - else - { - // Peer is no longer available so the relay ends - JLOG(p_journal_.info()) << "Unable to relay peer shard info"; - } - } - - JLOG(p_journal_.trace()) - << "Consumed TMPeerShardInfoV2 originating from public key " - << toBase58(TokenType::NodePublic, publicKey) << " finalized shards[" - << ripple::to_string(shardInfo.finalized()) << "] incomplete shards[" - << (shardInfo.incomplete().empty() ? "empty" - : shardInfo.incompleteToString()) - << "]"; - - // Consume the message - { - std::lock_guard lock{shardInfoMutex_}; - auto const it{shardInfos_.find(publicKey_)}; - if (it == shardInfos_.end()) - shardInfos_.emplace(publicKey, std::move(shardInfo)); - else if (shardInfo.msgTimestamp() > it->second.msgTimestamp()) - it->second = std::move(shardInfo); - } - - // Notify overlay a reply was received from the last peer in this chain - if (peerChainSz == 0) - overlay_.endOfPeerChain(id_); -} - void PeerImp::onMessage(std::shared_ptr const& m) { @@ -1659,13 +1347,6 @@ PeerImp::onMessage(std::shared_ptr const& m) if (m->has_ledgerseq()) { auto const ledgerSeq{m->ledgerseq()}; - // Verifying the network's earliest ledger only pertains to shards. - if (app_.getShardStore() && - ledgerSeq < app_.getNodeStore().earliestLedgerSeq()) - { - return badData( - "Invalid ledger sequence " + std::to_string(ledgerSeq)); - } // Check if within a reasonable range using namespace std::chrono_literals; @@ -1835,14 +1516,6 @@ PeerImp::onMessage(std::shared_ptr const& m) } else { - // Verifying the network's earliest ledger only pertains to shards. - if (app_.getShardStore() && - ledgerSeq < app_.getNodeStore().earliestLedgerSeq()) - { - return badData( - "Invalid ledger sequence " + std::to_string(ledgerSeq)); - } - // Check if within a reasonable range using namespace std::chrono_literals; if (app_.getLedgerMaster().getValidatedLedgerAge() <= 10s && @@ -2705,14 +2378,6 @@ PeerImp::onMessage(std::shared_ptr const& m) // need to inject the NodeStore interfaces. std::uint32_t seq{obj.has_ledgerseq() ? obj.ledgerseq() : 0}; auto nodeObject{app_.getNodeStore().fetchNodeObject(hash, seq)}; - if (!nodeObject) - { - if (auto shardStore = app_.getShardStore()) - { - if (seq >= shardStore->earliestLedgerSeq()) - nodeObject = shardStore->fetchNodeObject(hash, seq); - } - } if (nodeObject) { protocol::TMIndexedObject& newObj = *reply.add_objects(); @@ -3312,44 +2977,28 @@ PeerImp::getLedger(std::shared_ptr const& m) ledger = app_.getLedgerMaster().getLedgerByHash(ledgerHash); if (!ledger) { - if (m->has_ledgerseq()) + JLOG(p_journal_.trace()) + << "getLedger: Don't have ledger with hash " << ledgerHash; + + if (m->has_querytype() && !m->has_requestcookie()) { - // Attempt to find ledger by sequence in the shard store - if (auto shards = app_.getShardStore()) + // Attempt to relay the request to a peer + if (auto const peer = getPeerWithLedger( + overlay_, + ledgerHash, + m->has_ledgerseq() ? m->ledgerseq() : 0, + this)) { - if (m->ledgerseq() >= shards->earliestLedgerSeq()) - { - ledger = - shards->fetchLedger(ledgerHash, m->ledgerseq()); - } + m->set_requestcookie(id()); + peer->send( + std::make_shared(*m, protocol::mtGET_LEDGER)); + JLOG(p_journal_.debug()) + << "getLedger: Request relayed to peer"; + return ledger; } - } - if (!ledger) - { JLOG(p_journal_.trace()) - << "getLedger: Don't have ledger with hash " << ledgerHash; - - if (m->has_querytype() && !m->has_requestcookie()) - { - // Attempt to relay the request to a peer - if (auto const peer = getPeerWithLedger( - overlay_, - ledgerHash, - m->has_ledgerseq() ? m->ledgerseq() : 0, - this)) - { - m->set_requestcookie(id()); - peer->send(std::make_shared( - *m, protocol::mtGET_LEDGER)); - JLOG(p_journal_.debug()) - << "getLedger: Request relayed to peer"; - return ledger; - } - - JLOG(p_journal_.trace()) - << "getLedger: Failed to find peer to relay request"; - } + << "getLedger: Failed to find peer to relay request"; } } } diff --git a/src/xrpld/overlay/detail/PeerImp.h b/src/xrpld/overlay/detail/PeerImp.h index 1c25d8089b8..9c76ddb4db8 100644 --- a/src/xrpld/overlay/detail/PeerImp.h +++ b/src/xrpld/overlay/detail/PeerImp.h @@ -22,7 +22,6 @@ #include #include -#include #include #include #include @@ -163,10 +162,6 @@ class PeerImp : public Peer, // been sent to or received from this peer. hash_map publisherListSequences_; - // Any known shard info from this peer and its sub peers - hash_map shardInfos_; - std::mutex mutable shardInfoMutex_; - Compressed compressionEnabled_ = Compressed::Off; // Queue of transactions' hashes that have not been @@ -415,10 +410,6 @@ class PeerImp : public Peer, void fail(std::string const& reason); - // Return any known shard info from this peer and its sub peers - [[nodiscard]] hash_map const - getPeerShardInfos() const; - bool compressionEnabled() const override { @@ -541,14 +532,6 @@ class PeerImp : public Peer, void onMessage(std::shared_ptr const& m); void - onMessage(std::shared_ptr const& m); - void - onMessage(std::shared_ptr const& m); - void - onMessage(std::shared_ptr const& m); - void - onMessage(std::shared_ptr const& m); - void onMessage(std::shared_ptr const& m); void onMessage(std::shared_ptr const& m); diff --git a/src/xrpld/overlay/detail/ProtocolMessage.h b/src/xrpld/overlay/detail/ProtocolMessage.h index b8c6a2c1cf2..8a7512afb31 100644 --- a/src/xrpld/overlay/detail/ProtocolMessage.h +++ b/src/xrpld/overlay/detail/ProtocolMessage.h @@ -88,10 +88,6 @@ protocolMessageName(int type) return "validator_list_collection"; case protocol::mtVALIDATION: return "validation"; - case protocol::mtGET_PEER_SHARD_INFO: - return "get_peer_shard_info"; - case protocol::mtPEER_SHARD_INFO: - return "peer_shard_info"; case protocol::mtGET_OBJECTS: return "get_objects"; case protocol::mtHAVE_TRANSACTIONS: @@ -108,10 +104,6 @@ protocolMessageName(int type) return "replay_delta_request"; case protocol::mtREPLAY_DELTA_RESPONSE: return "replay_delta_response"; - case protocol::mtGET_PEER_SHARD_INFO_V2: - return "get_peer_shard_info_v2"; - case protocol::mtPEER_SHARD_INFO_V2: - return "peer_shard_info_v2"; default: break; } @@ -436,14 +428,6 @@ invokeProtocolMessage( success = detail::invoke( *header, buffers, handler); break; - case protocol::mtGET_PEER_SHARD_INFO: - success = detail::invoke( - *header, buffers, handler); - break; - case protocol::mtPEER_SHARD_INFO: - success = detail::invoke( - *header, buffers, handler); - break; case protocol::mtVALIDATORLIST: success = detail::invoke( *header, buffers, handler); @@ -484,14 +468,6 @@ invokeProtocolMessage( success = detail::invoke( *header, buffers, handler); break; - case protocol::mtGET_PEER_SHARD_INFO_V2: - success = detail::invoke( - *header, buffers, handler); - break; - case protocol::mtPEER_SHARD_INFO_V2: - success = detail::invoke( - *header, buffers, handler); - break; default: handler.onMessageUnknown(header->message_type); success = true; diff --git a/src/xrpld/overlay/detail/TrafficCount.cpp b/src/xrpld/overlay/detail/TrafficCount.cpp index f3e9c137fba..c64a033e3e3 100644 --- a/src/xrpld/overlay/detail/TrafficCount.cpp +++ b/src/xrpld/overlay/detail/TrafficCount.cpp @@ -39,12 +39,6 @@ TrafficCount::categorize( if (type == protocol::mtENDPOINTS) return TrafficCount::category::overlay; - if ((type == protocol::mtGET_PEER_SHARD_INFO) || - (type == protocol::mtPEER_SHARD_INFO) || - (type == protocol::mtGET_PEER_SHARD_INFO_V2) || - (type == protocol::mtPEER_SHARD_INFO_V2)) - return TrafficCount::category::shards; - if (type == protocol::mtTRANSACTION) return TrafficCount::category::transaction; diff --git a/src/xrpld/overlay/detail/TrafficCount.h b/src/xrpld/overlay/detail/TrafficCount.h index acd96695257..7dd5cbba901 100644 --- a/src/xrpld/overlay/detail/TrafficCount.h +++ b/src/xrpld/overlay/detail/TrafficCount.h @@ -74,7 +74,6 @@ class TrafficCount proposal, validation, validatorlist, - shards, // shard-related traffic // TMHaveSet message: get_set, // transaction sets we try to get @@ -208,7 +207,6 @@ class TrafficCount {"proposals"}, // category::proposal {"validations"}, // category::validation {"validator_lists"}, // category::validatorlist - {"shards"}, // category::shards {"set_get"}, // category::get_set {"set_share"}, // category::share_set {"ledger_data_Transaction_Set_candidate_get"}, // category::ld_tsc_get diff --git a/src/xrpld/perflog/detail/PerfLogImp.cpp b/src/xrpld/perflog/detail/PerfLogImp.cpp index b9691e05c23..a4773b33e10 100644 --- a/src/xrpld/perflog/detail/PerfLogImp.cpp +++ b/src/xrpld/perflog/detail/PerfLogImp.cpp @@ -20,7 +20,6 @@ #include #include -#include #include #include #include @@ -299,10 +298,7 @@ PerfLogImp::report() report[jss::hostid] = hostname_; report[jss::counters] = counters_.countersJson(); report[jss::nodestore] = Json::objectValue; - if (app_.getShardStore()) - app_.getShardStore()->getCountsJson(report[jss::nodestore]); - else - app_.getNodeStore().getCountsJson(report[jss::nodestore]); + app_.getNodeStore().getCountsJson(report[jss::nodestore]); report[jss::current_activities] = counters_.currentJson(); app_.getOPs().stateAccounting(report); diff --git a/src/xrpld/rpc/ShardArchiveHandler.h b/src/xrpld/rpc/ShardArchiveHandler.h deleted file mode 100644 index 3a407b37976..00000000000 --- a/src/xrpld/rpc/ShardArchiveHandler.h +++ /dev/null @@ -1,176 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_RPC_SHARDARCHIVEHANDLER_H_INCLUDED -#define RIPPLE_RPC_SHARDARCHIVEHANDLER_H_INCLUDED - -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace ripple { -#ifdef ENABLE_TESTS -namespace test { -class ShardArchiveHandler_test; -} -#endif // ENABLE_TESTS -namespace RPC { - -/** Handles the download and import of one or more shard archives. */ -class ShardArchiveHandler -{ -public: - using TimerOpCounter = - ClosureCounter; -#ifdef ENABLE_TESTS - friend class test::ShardArchiveHandler_test; -#endif // ENABLE_TESTS - - static boost::filesystem::path - getDownloadDirectory(Config const& config); - - static std::unique_ptr - makeShardArchiveHandler(Application& app); - - // Create a ShardArchiveHandler only if - // the state database is present, indicating - // that recovery is needed. - static std::unique_ptr - tryMakeRecoveryHandler(Application& app); - - explicit ShardArchiveHandler(Application& app); - - virtual ~ShardArchiveHandler() = default; - - [[nodiscard]] bool - init(); - - bool - add(std::uint32_t shardIndex, std::pair&& url); - - /** Starts downloading and importing archives. */ - bool - start(); - - void - stop(); - - void - release(); - -private: - ShardArchiveHandler() = delete; - ShardArchiveHandler(ShardArchiveHandler const&) = delete; - ShardArchiveHandler& - operator=(ShardArchiveHandler&&) = delete; - ShardArchiveHandler& - operator=(ShardArchiveHandler const&) = delete; - - [[nodiscard]] bool - initFromDB(std::lock_guard const&); - - /** Add an archive to be downloaded and imported. - @param shardIndex the index of the shard to be imported. - @param url the location of the archive. - @return `true` if successfully added. - @note Returns false if called while downloading. - */ - bool - add(std::uint32_t shardIndex, - parsedURL&& url, - std::lock_guard const&); - - // Begins the download and import of the next archive. - bool - next(std::lock_guard const& l); - - // Callback used by the downloader to notify completion of a download. - void - complete(boost::filesystem::path dstPath); - - // Extract a downloaded archive and import it into the shard store. - void - process(boost::filesystem::path const& dstPath); - - // Remove the archive being processed. - void - remove(std::lock_guard const&); - - void - doRelease(std::lock_guard const&); - - bool - onClosureFailed( - std::string const& errorMsg, - std::lock_guard const& lock); - - bool - removeAndProceed(std::lock_guard const& lock); - - ///////////////////////////////////////////////// - // m_ is used to protect access to downloader_, - // archives_, process_ and to protect setting and - // destroying sqlDB_. - ///////////////////////////////////////////////// - std::mutex mutable m_; - std::atomic_bool stopping_{false}; - std::shared_ptr downloader_; - std::map archives_; - bool process_; - std::unique_ptr sqlDB_; - ///////////////////////////////////////////////// - - Application& app_; - beast::Journal const j_; - boost::filesystem::path const downloadDir_; - boost::asio::basic_waitable_timer timer_; - JobCounter jobCounter_; - TimerOpCounter timerCounter_; - ShardVerificationScheduler verificationScheduler_; -}; - -//////////////////////////////////////////////////////////////////// -// The RecoveryHandler is an empty class that is constructed by -// the application when the ShardArchiveHandler's state database -// is present at application start, indicating that the handler -// needs to perform recovery. However, if recovery isn't needed -// at application start, and the user subsequently submits a request -// to download shards, we construct a ShardArchiveHandler rather -// than a RecoveryHandler to process the request. With this approach, -// type verification can be employed to determine whether the -// ShardArchiveHandler was constructed in recovery mode by the -// application, or as a response to a user submitting a request to -// download shards. -//////////////////////////////////////////////////////////////////// -class RecoveryHandler : public ShardArchiveHandler -{ -public: - explicit RecoveryHandler(Application& app); -}; - -} // namespace RPC -} // namespace ripple - -#endif diff --git a/src/xrpld/rpc/ShardVerificationScheduler.h b/src/xrpld/rpc/ShardVerificationScheduler.h deleted file mode 100644 index bc561381b3e..00000000000 --- a/src/xrpld/rpc/ShardVerificationScheduler.h +++ /dev/null @@ -1,84 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_RPC_SHARDVERIFICATIONSCHEDULER_H_INCLUDED -#define RIPPLE_RPC_SHARDVERIFICATIONSCHEDULER_H_INCLUDED - -#include -#include - -namespace ripple { -namespace RPC { - -class ShardVerificationScheduler -{ -public: - // This is the signature of the function that the client - // wants to have invoked upon timer expiration. The function - // should check the error code 'ec' and abort the function - // if the timer was cancelled: - // (ec == boost::asio::error::operation_aborted). - // In the body of the function, the client should perform - // the necessary verification. - using retryFunction = - std::function; - - ShardVerificationScheduler() = default; - - ShardVerificationScheduler( - std::chrono::seconds retryInterval, - std::uint32_t maxAttempts); - - bool - retry(Application& app, bool shouldHaveHash, retryFunction f); - - void - reset(); - -private: - using waitable_timer = - boost::asio::basic_waitable_timer; - - ///////////////////////////////////////////////////// - // NOTE: retryInterval_ and maxAttempts_ were chosen - // semi-arbitrarily and experimenting with other - // values might prove useful. - ///////////////////////////////////////////////////// - - static constexpr std::chrono::seconds defaultRetryInterval_{60}; - - static constexpr std::uint32_t defaultmaxAttempts_{5}; - - // The number of seconds to wait before retrying - // retrieval of a shard's last ledger hash - const std::chrono::seconds retryInterval_{defaultRetryInterval_}; - - // Maximum attempts to retrieve a shard's last ledger hash - const std::uint32_t maxAttempts_{defaultmaxAttempts_}; - - std::unique_ptr timer_; - - // Number of attempts to retrieve a shard's last ledger hash - std::uint32_t numAttempts_{0}; -}; - -} // namespace RPC -} // namespace ripple - -#endif // RIPPLE_RPC_SHARDVERIFICATIONSCHEDULER_H_INCLUDED diff --git a/src/xrpld/rpc/detail/Handler.cpp b/src/xrpld/rpc/detail/Handler.cpp index 4bac4610229..d4a3fda380f 100644 --- a/src/xrpld/rpc/detail/Handler.cpp +++ b/src/xrpld/rpc/detail/Handler.cpp @@ -99,12 +99,10 @@ Handler const handlerArray[]{ {"channel_verify", byRef(&doChannelVerify), Role::USER, NO_CONDITION}, {"connect", byRef(&doConnect), Role::ADMIN, NO_CONDITION}, {"consensus_info", byRef(&doConsensusInfo), Role::ADMIN, NO_CONDITION}, - {"crawl_shards", byRef(&doCrawlShards), Role::ADMIN, NO_CONDITION}, {"deposit_authorized", byRef(&doDepositAuthorized), Role::USER, NO_CONDITION}, - {"download_shard", byRef(&doDownloadShard), Role::ADMIN, NO_CONDITION}, {"feature", byRef(&doFeature), Role::USER, NO_CONDITION}, {"fee", byRef(&doFee), Role::USER, NEEDS_CURRENT_LEDGER}, {"fetch_info", byRef(&doFetchInfo), Role::ADMIN, NO_CONDITION}, @@ -140,7 +138,6 @@ Handler const handlerArray[]{ {"manifest", byRef(&doManifest), Role::USER, NO_CONDITION}, {"nft_buy_offers", byRef(&doNFTBuyOffers), Role::USER, NO_CONDITION}, {"nft_sell_offers", byRef(&doNFTSellOffers), Role::USER, NO_CONDITION}, - {"node_to_shard", byRef(&doNodeToShard), Role::ADMIN, NO_CONDITION}, {"noripple_check", byRef(&doNoRippleCheck), Role::USER, NO_CONDITION}, {"owner_info", byRef(&doOwnerInfo), Role::USER, NEEDS_CURRENT_LEDGER}, {"peers", byRef(&doPeers), Role::ADMIN, NO_CONDITION}, diff --git a/src/xrpld/rpc/detail/ShardArchiveHandler.cpp b/src/xrpld/rpc/detail/ShardArchiveHandler.cpp deleted file mode 100644 index 1ab8f7767b5..00000000000 --- a/src/xrpld/rpc/detail/ShardArchiveHandler.cpp +++ /dev/null @@ -1,585 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012-2014 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace ripple { -namespace RPC { - -using namespace boost::filesystem; -using namespace std::chrono_literals; - -boost::filesystem::path -ShardArchiveHandler::getDownloadDirectory(Config const& config) -{ - return boost::filesystem::path{ - get(config.section(ConfigSection::shardDatabase()), - "download_path", - get(config.section(ConfigSection::shardDatabase()), - "path", - ""))} / - "download"; -} - -std::unique_ptr -ShardArchiveHandler::makeShardArchiveHandler(Application& app) -{ - return std::make_unique(app); -} - -std::unique_ptr -ShardArchiveHandler::tryMakeRecoveryHandler(Application& app) -{ - auto const downloadDir(getDownloadDirectory(app.config())); - - // Create the handler iff the database - // is present. - if (exists(downloadDir / stateDBName) && - is_regular_file(downloadDir / stateDBName)) - { - return std::make_unique(app); - } - - return nullptr; -} - -ShardArchiveHandler::ShardArchiveHandler(Application& app) - : process_(false) - , app_(app) - , j_(app.journal("ShardArchiveHandler")) - , downloadDir_(getDownloadDirectory(app.config())) - , timer_(app_.getIOService()) - , verificationScheduler_( - std::chrono::seconds(get( - app.config().section(ConfigSection::shardDatabase()), - "shard_verification_retry_interval")), - - get( - app.config().section(ConfigSection::shardDatabase()), - "shard_verification_max_attempts")) -{ - assert(app_.getShardStore()); -} - -bool -ShardArchiveHandler::init() -{ - std::lock_guard lock(m_); - - if (process_ || downloader_ != nullptr || sqlDB_ != nullptr) - { - JLOG(j_.warn()) << "Archives already being processed"; - return false; - } - - // Initialize from pre-existing database - if (exists(downloadDir_ / stateDBName) && - is_regular_file(downloadDir_ / stateDBName)) - { - downloader_ = - make_DatabaseDownloader(app_.getIOService(), app_.config(), j_); - - return initFromDB(lock); - } - - // Fresh initialization - else - { - try - { - create_directories(downloadDir_); - - sqlDB_ = makeArchiveDB(downloadDir_, stateDBName); - } - catch (std::exception const& e) - { - JLOG(j_.error()) - << "exception: " << e.what() << " in function: " << __func__; - - return false; - } - } - - return true; -} - -bool -ShardArchiveHandler::initFromDB(std::lock_guard const& lock) -{ - try - { - using namespace boost::filesystem; - - assert( - exists(downloadDir_ / stateDBName) && - is_regular_file(downloadDir_ / stateDBName)); - - sqlDB_ = makeArchiveDB(downloadDir_, stateDBName); - - readArchiveDB(*sqlDB_, [&](std::string const& url_, int state) { - parsedURL url; - - if (!parseUrl(url, url_)) - { - JLOG(j_.error()) << "Failed to parse url: " << url_; - - return; - } - - add(state, std::move(url), lock); - }); - - // Failed to load anything - // from the state database. - if (archives_.empty()) - { - release(); - return false; - } - } - catch (std::exception const& e) - { - JLOG(j_.error()) << "exception: " << e.what() - << " in function: " << __func__; - - return false; - } - - return true; -} - -void -ShardArchiveHandler::stop() -{ - stopping_ = true; - { - std::lock_guard lock(m_); - - if (downloader_) - { - downloader_->stop(); - downloader_.reset(); - } - - timer_.cancel(); - } - - jobCounter_.join( - "ShardArchiveHandler", std::chrono::milliseconds(2000), j_); - - timerCounter_.join( - "ShardArchiveHandler", std::chrono::milliseconds(2000), j_); -} - -bool -ShardArchiveHandler::add( - std::uint32_t shardIndex, - std::pair&& url) -{ - std::lock_guard lock(m_); - - if (!add(shardIndex, std::forward(url.first), lock)) - return false; - - insertArchiveDB(*sqlDB_, shardIndex, url.second); - - return true; -} - -bool -ShardArchiveHandler::add( - std::uint32_t shardIndex, - parsedURL&& url, - std::lock_guard const&) -{ - if (process_) - { - JLOG(j_.error()) << "Download and import already in progress"; - return false; - } - - auto const it{archives_.find(shardIndex)}; - if (it != archives_.end()) - return url == it->second; - - archives_.emplace(shardIndex, std::move(url)); - - return true; -} - -bool -ShardArchiveHandler::start() -{ - std::lock_guard lock(m_); - if (!app_.getShardStore()) - { - JLOG(j_.error()) << "No shard store available"; - return false; - } - if (process_) - { - JLOG(j_.warn()) << "Archives already being processed"; - return false; - } - if (archives_.empty()) - { - JLOG(j_.warn()) << "No archives to process"; - return false; - } - - std::vector shardIndexes(archives_.size()); - std::transform( - archives_.begin(), - archives_.end(), - shardIndexes.begin(), - [](auto const& entry) { return entry.first; }); - - if (!app_.getShardStore()->prepareShards(shardIndexes)) - return false; - - try - { - // Create temp root download directory - create_directories(downloadDir_); - - if (!downloader_) - { - // will throw if can't initialize ssl context - downloader_ = - make_DatabaseDownloader(app_.getIOService(), app_.config(), j_); - } - } - catch (std::exception const& e) - { - JLOG(j_.error()) << "exception: " << e.what(); - return false; - } - - process_ = true; - return next(lock); -} - -void -ShardArchiveHandler::release() -{ - std::lock_guard lock(m_); - doRelease(lock); -} - -bool -ShardArchiveHandler::next(std::lock_guard const& l) -{ - if (stopping_) - return false; - - if (archives_.empty()) - { - doRelease(l); - return false; - } - - auto const shardIndex{archives_.begin()->first}; - - // We use the sequence of the last validated ledger - // to determine whether or not we have stored a ledger - // that comes after the last ledger in this shard. A - // later ledger must be present in order to reliably - // retrieve the hash of the shard's last ledger. - std::optional expectedHash; - bool shouldHaveHash = false; - if (auto const seq = app_.getShardStore()->lastLedgerSeq(shardIndex); - (shouldHaveHash = app_.getLedgerMaster().getValidLedgerIndex() > seq)) - { - expectedHash = app_.getLedgerMaster().walkHashBySeq( - seq, InboundLedger::Reason::GENERIC); - } - - if (!expectedHash) - { - auto wrapper = - timerCounter_.wrap([this](boost::system::error_code const& ec) { - if (ec != boost::asio::error::operation_aborted) - { - std::lock_guard lock(m_); - this->next(lock); - } - }); - - if (!wrapper) - return onClosureFailed( - "failed to wrap closure for last ledger confirmation timer", l); - - if (!verificationScheduler_.retry(app_, shouldHaveHash, *wrapper)) - { - JLOG(j_.error()) << "failed to find last ledger hash for shard " - << shardIndex << ", maximum attempts reached"; - - return removeAndProceed(l); - } - - return true; - } - - // Create a temp archive directory at the root - auto const dstDir{downloadDir_ / std::to_string(shardIndex)}; - try - { - create_directory(dstDir); - } - catch (std::exception const& e) - { - JLOG(j_.error()) << "exception: " << e.what(); - return removeAndProceed(l); - } - - // Download the archive. Process in another thread - // to prevent holding up the lock if the downloader - // sleeps. - auto const& url{archives_.begin()->second}; - auto wrapper = jobCounter_.wrap([this, url, dstDir]() { - auto const ssl = (url.scheme == "https"); - auto const defaultPort = ssl ? 443 : 80; - - if (!downloader_->download( - url.domain, - std::to_string(url.port.value_or(defaultPort)), - url.path, - 11, - dstDir / "archive.tar.lz4", - [this](path dstPath) { complete(dstPath); }, - ssl)) - { - std::lock_guard l(m_); - removeAndProceed(l); - } - }); - - if (!wrapper) - return onClosureFailed( - "failed to wrap closure for starting download", l); - - app_.getJobQueue().addJob(jtCLIENT_SHARD, "ShardArchiveHandler", *wrapper); - - return true; -} - -void -ShardArchiveHandler::complete(path dstPath) -{ - if (stopping_) - return; - - { - std::lock_guard lock(m_); - try - { - if (!is_regular_file(dstPath)) - { - auto ar{archives_.begin()}; - JLOG(j_.error()) - << "Downloading shard id " << ar->first << " from URL " - << ar->second.domain << ar->second.path; - removeAndProceed(lock); - return; - } - } - catch (std::exception const& e) - { - JLOG(j_.error()) << "exception: " << e.what(); - removeAndProceed(lock); - return; - } - } - - // Make lambdas mutable captured vars can be moved from - auto wrapper = - jobCounter_.wrap([=, this, dstPath = std::move(dstPath)]() mutable { - if (stopping_) - return; - - // If not synced then defer and retry - auto const mode{app_.getOPs().getOperatingMode()}; - if (mode != OperatingMode::FULL) - { - std::lock_guard lock(m_); - timer_.expires_from_now(static_cast( - (static_cast(OperatingMode::FULL) - - static_cast(mode)) * - 10)); - - auto wrapper = timerCounter_.wrap( - [=, this, dstPath = std::move(dstPath)]( - boost::system::error_code const& ec) mutable { - if (ec != boost::asio::error::operation_aborted) - complete(std::move(dstPath)); - }); - - if (!wrapper) - onClosureFailed( - "failed to wrap closure for operating mode timer", - lock); - else - timer_.async_wait(*wrapper); - } - else - { - process(dstPath); - std::lock_guard lock(m_); - removeAndProceed(lock); - } - }); - - if (!wrapper) - { - if (stopping_) - return; - - JLOG(j_.error()) << "failed to wrap closure for process()"; - - std::lock_guard lock(m_); - removeAndProceed(lock); - } - - // Process in another thread to not hold up the IO service - app_.getJobQueue().addJob(jtCLIENT_SHARD, "ShardArchiveHandler", *wrapper); -} - -void -ShardArchiveHandler::process(path const& dstPath) -{ - std::uint32_t shardIndex; - { - std::lock_guard lock(m_); - shardIndex = archives_.begin()->first; - } - - auto const shardDir{dstPath.parent_path() / std::to_string(shardIndex)}; - try - { - // Extract the downloaded archive - extractTarLz4(dstPath, dstPath.parent_path()); - - // The extracted root directory name must match the shard index - if (!is_directory(shardDir)) - { - JLOG(j_.error()) << "Shard " << shardIndex - << " mismatches archive shard directory"; - return; - } - } - catch (std::exception const& e) - { - JLOG(j_.error()) << "exception: " << e.what(); - return; - } - - // Import the shard into the shard store - if (!app_.getShardStore()->importShard(shardIndex, shardDir)) - { - JLOG(j_.error()) << "Importing shard " << shardIndex; - return; - } - - JLOG(j_.debug()) << "Shard " << shardIndex << " downloaded and imported"; -} - -void -ShardArchiveHandler::remove(std::lock_guard const&) -{ - verificationScheduler_.reset(); - - auto const shardIndex{archives_.begin()->first}; - app_.getShardStore()->removePreShard(shardIndex); - archives_.erase(shardIndex); - - deleteFromArchiveDB(*sqlDB_, shardIndex); - - auto const dstDir{downloadDir_ / std::to_string(shardIndex)}; - try - { - remove_all(dstDir); - } - catch (std::exception const& e) - { - JLOG(j_.error()) << "exception: " << e.what(); - } -} - -void -ShardArchiveHandler::doRelease(std::lock_guard const&) -{ - timer_.cancel(); - for (auto const& ar : archives_) - app_.getShardStore()->removePreShard(ar.first); - archives_.clear(); - - dropArchiveDB(*sqlDB_); - - sqlDB_.reset(); - - // Remove temp root download directory - try - { - remove_all(downloadDir_); - } - catch (std::exception const& e) - { - JLOG(j_.error()) << "exception: " << e.what() - << " in function: " << __func__; - } - - downloader_.reset(); - process_ = false; -} - -bool -ShardArchiveHandler::onClosureFailed( - std::string const& errorMsg, - std::lock_guard const& lock) -{ - if (stopping_) - return false; - - JLOG(j_.error()) << errorMsg; - - return removeAndProceed(lock); -} - -bool -ShardArchiveHandler::removeAndProceed(std::lock_guard const& lock) -{ - remove(lock); - return next(lock); -} - -RecoveryHandler::RecoveryHandler(Application& app) : ShardArchiveHandler(app) -{ -} - -} // namespace RPC -} // namespace ripple diff --git a/src/xrpld/rpc/detail/ShardVerificationScheduler.cpp b/src/xrpld/rpc/detail/ShardVerificationScheduler.cpp deleted file mode 100644 index f571e8b29cd..00000000000 --- a/src/xrpld/rpc/detail/ShardVerificationScheduler.cpp +++ /dev/null @@ -1,68 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include - -namespace ripple { -namespace RPC { - -ShardVerificationScheduler::ShardVerificationScheduler( - std::chrono::seconds retryInterval, - std::uint32_t maxAttempts) - : retryInterval_( - (retryInterval == std::chrono::seconds(0) ? defaultRetryInterval_ - : retryInterval)) - , maxAttempts_(maxAttempts == 0 ? defaultmaxAttempts_ : maxAttempts) -{ -} - -bool -ShardVerificationScheduler::retry( - Application& app, - bool shouldHaveHash, - retryFunction f) -{ - if (numAttempts_ >= maxAttempts_) - return false; - - // Retry attempts only count when we - // have a validated ledger with a - // sequence later than the shard's - // last ledger. - if (shouldHaveHash) - ++numAttempts_; - - if (!timer_) - timer_ = std::make_unique(app.getIOService()); - - timer_->expires_from_now(retryInterval_); - timer_->async_wait(f); - - return true; -} - -void -ShardVerificationScheduler::reset() -{ - numAttempts_ = 0; -} - -} // namespace RPC -} // namespace ripple diff --git a/src/xrpld/rpc/handlers/CrawlShards.cpp b/src/xrpld/rpc/handlers/CrawlShards.cpp deleted file mode 100644 index f586d750439..00000000000 --- a/src/xrpld/rpc/handlers/CrawlShards.cpp +++ /dev/null @@ -1,73 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2018 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { - -/** RPC command that reports stored shards by nodes. - { - // Determines if the result includes node public key. - // optional, default is false - public_key: - - // The maximum number of peer hops to attempt. - // optional, default is zero, maximum is 3 - limit: - } -*/ -Json::Value -doCrawlShards(RPC::JsonContext& context) -{ - if (context.app.config().reporting()) - return rpcError(rpcREPORTING_UNSUPPORTED); - - if (context.role != Role::ADMIN) - return rpcError(rpcNO_PERMISSION); - - std::uint32_t relays{0}; - if (auto const& jv = context.params[jss::limit]) - { - if (!(jv.isUInt() || (jv.isInt() && jv.asInt() >= 0))) - return RPC::expected_field_error(jss::limit, "unsigned integer"); - relays = std::min(jv.asUInt(), relayLimit); - context.loadType = Resource::feeHighBurdenRPC; - } - else - context.loadType = Resource::feeMediumBurdenRPC; - - // Collect shard info from server and peers - bool const includePublicKey{ - context.params.isMember(jss::public_key) && - context.params[jss::public_key].asBool()}; - Json::Value jvResult{ - context.app.overlay().crawlShards(includePublicKey, relays)}; - - return jvResult; -} - -} // namespace ripple diff --git a/src/xrpld/rpc/handlers/DownloadShard.cpp b/src/xrpld/rpc/handlers/DownloadShard.cpp deleted file mode 100644 index 1ec12e0fa66..00000000000 --- a/src/xrpld/rpc/handlers/DownloadShard.cpp +++ /dev/null @@ -1,176 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012-2014 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace ripple { - -/** RPC command that downloads and import shard archives. - { - shards: [{index: , url: }] - } - - example: - { - "command": "download_shard", - "shards": [ - {"index": 1, "url": "https://domain.com/1.tar.lz4"}, - {"index": 5, "url": "https://domain.com/5.tar.lz4"} - ] - } -*/ -Json::Value -doDownloadShard(RPC::JsonContext& context) -{ - if (context.app.config().reporting()) - return rpcError(rpcREPORTING_UNSUPPORTED); - - if (context.role != Role::ADMIN) - return rpcError(rpcNO_PERMISSION); - - // The shard store must be configured - auto shardStore{context.app.getShardStore()}; - if (!shardStore) - return rpcError(rpcNOT_ENABLED); - - // Return status update if already downloading - auto preShards{shardStore->getPreShards()}; - if (!preShards.empty()) - { - std::string s{"Download already in progress. Shard"}; - if (!std::all_of(preShards.begin(), preShards.end(), ::isdigit)) - s += "s"; - return RPC::makeObjectValue(s + " " + preShards); - } - - if (!context.params.isMember(jss::shards)) - return RPC::missing_field_error(jss::shards); - if (!context.params[jss::shards].isArray() || - context.params[jss::shards].size() == 0) - { - return RPC::expected_field_error(std::string(jss::shards), "an array"); - } - - // Validate shards - static const std::string ext{".tar.lz4"}; - std::map> archives; - for (auto& it : context.params[jss::shards]) - { - // Validate the index - if (!it.isMember(jss::index)) - return RPC::missing_field_error(jss::index); - auto& jv{it[jss::index]}; - if (!(jv.isUInt() || (jv.isInt() && jv.asInt() >= 0))) - { - return RPC::expected_field_error( - std::string(jss::index), "an unsigned integer"); - } - - // Validate the URL - if (!it.isMember(jss::url)) - return RPC::missing_field_error(jss::url); - parsedURL url; - auto unparsedURL = it[jss::url].asString(); - if (!parseUrl(url, unparsedURL) || url.domain.empty() || - url.path.empty()) - { - return RPC::invalid_field_error(jss::url); - } - if (url.scheme != "https" && url.scheme != "http") - return RPC::expected_field_error( - std::string(jss::url), "HTTPS or HTTP"); - - // URL must point to an lz4 compressed tar archive '.tar.lz4' - auto archiveName{url.path.substr(url.path.find_last_of("/\\") + 1)}; - if (archiveName.empty() || archiveName.size() <= ext.size()) - { - return RPC::make_param_error( - "Invalid field '" + std::string(jss::url) + - "', invalid archive name"); - } - if (!boost::iends_with(archiveName, ext)) - { - return RPC::make_param_error( - "Invalid field '" + std::string(jss::url) + - "', invalid archive extension"); - } - - // Check for duplicate indexes - if (!archives - .emplace( - jv.asUInt(), std::make_pair(std::move(url), unparsedURL)) - .second) - { - return RPC::make_param_error( - "Invalid field '" + std::string(jss::index) + - "', duplicate shard ids."); - } - } - - RPC::ShardArchiveHandler* handler = nullptr; - - try - { - handler = context.app.getShardArchiveHandler(); - - if (!handler) - return RPC::make_error( - rpcINTERNAL, "Failed to create ShardArchiveHandler."); - } - catch (std::exception const& e) - { - return RPC::make_error( - rpcINTERNAL, std::string("Failed to start download: ") + e.what()); - } - - for (auto& [index, url] : archives) - { - if (!handler->add(index, std::move(url))) - { - return RPC::make_param_error( - "Invalid field '" + std::string(jss::index) + "', shard id " + - std::to_string(index) + " exists or being acquired"); - } - } - - // Begin downloading. - if (!handler->start()) - { - handler->release(); - return rpcError(rpcINTERNAL); - } - - std::string s{"Downloading shard"}; - preShards = shardStore->getPreShards(); - if (!std::all_of(preShards.begin(), preShards.end(), ::isdigit)) - s += "s"; - return RPC::makeObjectValue(s + " " + preShards); -} - -} // namespace ripple diff --git a/src/xrpld/rpc/handlers/GetCounts.cpp b/src/xrpld/rpc/handlers/GetCounts.cpp index 0a2327e117a..035d698a5d4 100644 --- a/src/xrpld/rpc/handlers/GetCounts.cpp +++ b/src/xrpld/rpc/handlers/GetCounts.cpp @@ -25,9 +25,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -113,11 +111,11 @@ getCountsJson(Application& app, int minObjectCount) ret[jss::AL_hit_rate] = app.getAcceptedLedgerCache().getHitRate(); ret[jss::fullbelow_size] = - static_cast(app.getNodeFamily().getFullBelowCache(0)->size()); + static_cast(app.getNodeFamily().getFullBelowCache()->size()); ret[jss::treenode_cache_size] = - app.getNodeFamily().getTreeNodeCache(0)->getCacheSize(); + app.getNodeFamily().getTreeNodeCache()->getCacheSize(); ret[jss::treenode_track_size] = - app.getNodeFamily().getTreeNodeCache(0)->getTrackSize(); + app.getNodeFamily().getTreeNodeCache()->getTrackSize(); std::string uptime; auto s = UptimeClock::now(); @@ -129,27 +127,7 @@ getCountsJson(Application& app, int minObjectCount) textTime(uptime, s, "second", 1s); ret[jss::uptime] = uptime; - if (auto shardStore = app.getShardStore()) - { - auto shardFamily{dynamic_cast(app.getShardFamily())}; - auto const [cacheSz, trackSz] = shardFamily->getTreeNodeCacheSize(); - Json::Value& jv = (ret[jss::shards] = Json::objectValue); - - jv[jss::fullbelow_size] = shardFamily->getFullBelowCacheSize(); - jv[jss::treenode_cache_size] = cacheSz; - jv[jss::treenode_track_size] = trackSz; - ret[jss::write_load] = shardStore->getWriteLoad(); - jv[jss::node_writes] = std::to_string(shardStore->getStoreCount()); - jv[jss::node_reads_total] = shardStore->getFetchTotalCount(); - jv[jss::node_reads_hit] = shardStore->getFetchHitCount(); - jv[jss::node_written_bytes] = - std::to_string(shardStore->getStoreSize()); - jv[jss::node_read_bytes] = shardStore->getFetchSize(); - } - else - { - app.getNodeStore().getCountsJson(ret); - } + app.getNodeStore().getCountsJson(ret); return ret; } diff --git a/src/xrpld/rpc/handlers/Handlers.h b/src/xrpld/rpc/handlers/Handlers.h index 917ad38a741..0085f51465a 100644 --- a/src/xrpld/rpc/handlers/Handlers.h +++ b/src/xrpld/rpc/handlers/Handlers.h @@ -61,8 +61,6 @@ doConsensusInfo(RPC::JsonContext&); Json::Value doDepositAuthorized(RPC::JsonContext&); Json::Value -doDownloadShard(RPC::JsonContext&); -Json::Value doFeature(RPC::JsonContext&); Json::Value doFee(RPC::JsonContext&); @@ -101,8 +99,6 @@ doNFTBuyOffers(RPC::JsonContext&); Json::Value doNFTSellOffers(RPC::JsonContext&); Json::Value -doNodeToShard(RPC::JsonContext&); -Json::Value doNoRippleCheck(RPC::JsonContext&); Json::Value doOwnerInfo(RPC::JsonContext&); @@ -139,8 +135,6 @@ doSign(RPC::JsonContext&); Json::Value doSignFor(RPC::JsonContext&); Json::Value -doCrawlShards(RPC::JsonContext&); -Json::Value doStop(RPC::JsonContext&); Json::Value doSubmit(RPC::JsonContext&); diff --git a/src/xrpld/rpc/handlers/NodeToShard.cpp b/src/xrpld/rpc/handlers/NodeToShard.cpp deleted file mode 100644 index 917086ab0f4..00000000000 --- a/src/xrpld/rpc/handlers/NodeToShard.cpp +++ /dev/null @@ -1,86 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2021 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { - -// node_to_shard [status|start|stop] -Json::Value -doNodeToShard(RPC::JsonContext& context) -{ - if (context.app.config().reporting()) - return rpcError(rpcREPORTING_UNSUPPORTED); - - // Shard store must be enabled - auto const shardStore = context.app.getShardStore(); - if (!shardStore) - return RPC::make_error(rpcNOT_ENABLED); - - if (!context.params.isMember(jss::action)) - return RPC::missing_field_error(jss::action); - - // Obtain and normalize the action to perform - auto const action = [&context] { - auto value = context.params[jss::action].asString(); - boost::to_lower(value); - - return value; - }(); - - // Vector of allowed actions - std::vector const allowedActions = {"status", "start", "stop"}; - - // Validate the action - if (std::find(allowedActions.begin(), allowedActions.end(), action) == - allowedActions.end()) - return RPC::invalid_field_error(jss::action); - - // Perform the action - if (action == "status") - { - // Get the status of the database import - return shardStore->getDatabaseImportStatus(); - } - else if (action == "start") - { - // Kick off an import - return shardStore->startNodeToShard(); - } - else if (action == "stop") - { - // Halt an import - return shardStore->stopNodeToShard(); - } - else - { - // Shouldn't happen - assert(false); - return rpcError(rpcINTERNAL); - } -} - -} // namespace ripple diff --git a/src/xrpld/rpc/handlers/Tx.cpp b/src/xrpld/rpc/handlers/Tx.cpp index de95044b71f..e32d926e566 100644 --- a/src/xrpld/rpc/handlers/Tx.cpp +++ b/src/xrpld/rpc/handlers/Tx.cpp @@ -100,7 +100,6 @@ doTxPostgres(RPC::Context& context, TxArgs const& args) if (locator.isFound()) { auto start = std::chrono::system_clock::now(); - // The second argument of fetch is ignored when not using shards if (auto obj = context.app.getNodeFamily().db().fetchNodeObject( locator.getNodestoreHash(), locator.getLedgerSequence())) { diff --git a/src/xrpld/shamap/Family.h b/src/xrpld/shamap/Family.h index 730f83483a6..6559ce5059b 100644 --- a/src/xrpld/shamap/Family.h +++ b/src/xrpld/shamap/Family.h @@ -53,28 +53,17 @@ class Family virtual beast::Journal const& journal() = 0; - /** Return a pointer to the Family Full Below Cache - - @param ledgerSeq ledger sequence determines a corresponding shard cache - @note ledgerSeq is used by ShardFamily and ignored by NodeFamily - */ + /** Return a pointer to the Family Full Below Cache */ virtual std::shared_ptr - getFullBelowCache(std::uint32_t ledgerSeq) = 0; - - /** Return a pointer to the Family Tree Node Cache + getFullBelowCache() = 0; - @param ledgerSeq ledger sequence determines a corresponding shard cache - @note ledgerSeq is used by ShardFamily and ignored by NodeFamily - */ + /** Return a pointer to the Family Tree Node Cache */ virtual std::shared_ptr - getTreeNodeCache(std::uint32_t ledgerSeq) = 0; + getTreeNodeCache() = 0; virtual void sweep() = 0; - virtual bool - isShardBacked() const = 0; - /** Acquire ledger that has a missing node by ledger sequence * * Throw if in reporting mode. diff --git a/src/xrpld/shamap/NodeFamily.h b/src/xrpld/shamap/NodeFamily.h index c540172c374..4062ea23897 100644 --- a/src/xrpld/shamap/NodeFamily.h +++ b/src/xrpld/shamap/NodeFamily.h @@ -60,18 +60,14 @@ class NodeFamily : public Family return j_; } - bool - isShardBacked() const override - { - return false; - } - - std::shared_ptr getFullBelowCache(std::uint32_t) override + std::shared_ptr + getFullBelowCache() override { return fbCache_; } - std::shared_ptr getTreeNodeCache(std::uint32_t) override + std::shared_ptr + getTreeNodeCache() override { return tnCache_; } diff --git a/src/xrpld/shamap/ShardFamily.h b/src/xrpld/shamap/ShardFamily.h deleted file mode 100644 index 2e8bece6dcf..00000000000 --- a/src/xrpld/shamap/ShardFamily.h +++ /dev/null @@ -1,125 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_SHAMAP_SHARDFAMILY_H_INCLUDED -#define RIPPLE_SHAMAP_SHARDFAMILY_H_INCLUDED - -#include -#include - -namespace ripple { - -class Application; - -class ShardFamily : public Family -{ -public: - ShardFamily() = delete; - ShardFamily(ShardFamily const&) = delete; - ShardFamily(ShardFamily&&) = delete; - - ShardFamily& - operator=(ShardFamily const&) = delete; - - ShardFamily& - operator=(ShardFamily&&) = delete; - - ShardFamily(Application& app, CollectorManager& cm); - - NodeStore::Database& - db() override - { - return db_; - } - - NodeStore::Database const& - db() const override - { - return db_; - } - - beast::Journal const& - journal() override - { - return j_; - } - - bool - isShardBacked() const override - { - return true; - } - - std::shared_ptr - getFullBelowCache(std::uint32_t ledgerSeq) override; - - /** Return the number of entries in the cache */ - int - getFullBelowCacheSize(); - - std::shared_ptr - getTreeNodeCache(std::uint32_t ledgerSeq) override; - - /** Return a pair where the first item is the number of items cached - and the second item is the number of entries in the cached - */ - std::pair - getTreeNodeCacheSize(); - - void - sweep() override; - - void - reset() override; - - void - missingNodeAcquireBySeq(std::uint32_t seq, uint256 const& nodeHash) - override; - - void - missingNodeAcquireByHash(uint256 const& hash, std::uint32_t seq) override - { - acquire(hash, seq); - } - -private: - Application& app_; - NodeStore::Database& db_; - CollectorManager& cm_; - beast::Journal const j_; - - std::unordered_map> fbCache_; - std::mutex fbCacheMutex_; - - std::unordered_map> tnCache_; - std::mutex tnCacheMutex_; - int const tnTargetSize_; - std::chrono::seconds const tnTargetAge_; - - // Missing node handler - LedgerIndex maxSeq_{0}; - std::mutex maxSeqMutex_; - - void - acquire(uint256 const& hash, std::uint32_t seq); -}; - -} // namespace ripple - -#endif diff --git a/src/xrpld/shamap/detail/SHAMap.cpp b/src/xrpld/shamap/detail/SHAMap.cpp index e17f9346b85..d06ba2a153a 100644 --- a/src/xrpld/shamap/detail/SHAMap.cpp +++ b/src/xrpld/shamap/detail/SHAMap.cpp @@ -1166,7 +1166,7 @@ SHAMap::dump(bool hash) const std::shared_ptr SHAMap::cacheLookup(SHAMapHash const& hash) const { - auto ret = f_.getTreeNodeCache(ledgerSeq_)->fetch(hash.as_uint256()); + auto ret = f_.getTreeNodeCache()->fetch(hash.as_uint256()); assert(!ret || !ret->cowid()); return ret; } @@ -1180,8 +1180,7 @@ SHAMap::canonicalize( assert(node->cowid() == 0); assert(node->getHash() == hash); - f_.getTreeNodeCache(ledgerSeq_) - ->canonicalize_replace_client(hash.as_uint256(), node); + f_.getTreeNodeCache()->canonicalize_replace_client(hash.as_uint256(), node); } void diff --git a/src/xrpld/shamap/detail/SHAMapSync.cpp b/src/xrpld/shamap/detail/SHAMapSync.cpp index 02d548be24e..7235e526560 100644 --- a/src/xrpld/shamap/detail/SHAMapSync.cpp +++ b/src/xrpld/shamap/detail/SHAMapSync.cpp @@ -192,8 +192,7 @@ SHAMap::gmn_ProcessNodes(MissingNodes& mn, MissingNodes::StackEntry& se) } else if ( !backed_ || - !f_.getFullBelowCache(ledgerSeq_) - ->touch_if_exists(childHash.as_uint256())) + !f_.getFullBelowCache()->touch_if_exists(childHash.as_uint256())) { bool pending = false; auto d = descendAsync( @@ -251,8 +250,7 @@ SHAMap::gmn_ProcessNodes(MissingNodes& mn, MissingNodes::StackEntry& se) node->setFullBelowGen(mn.generation_); if (backed_) { - f_.getFullBelowCache(ledgerSeq_) - ->insert(node->getHash().as_uint256()); + f_.getFullBelowCache()->insert(node->getHash().as_uint256()); } } @@ -323,7 +321,7 @@ SHAMap::getMissingNodes(int max, SHAMapSyncFilter* filter) max, filter, 512, // number of async reads per pass - f_.getFullBelowCache(ledgerSeq_)->getGeneration()); + f_.getFullBelowCache()->getGeneration()); if (!root_->isInner() || std::static_pointer_cast(root_)->isFullBelow( @@ -580,7 +578,7 @@ SHAMap::addKnownNode( return SHAMapAddNode::duplicate(); } - auto const generation = f_.getFullBelowCache(ledgerSeq_)->getGeneration(); + auto const generation = f_.getFullBelowCache()->getGeneration(); SHAMapNodeID iNodeID; auto iNode = root_.get(); @@ -598,8 +596,7 @@ SHAMap::addKnownNode( } auto childHash = inner->getChildHash(branch); - if (f_.getFullBelowCache(ledgerSeq_) - ->touch_if_exists(childHash.as_uint256())) + if (f_.getFullBelowCache()->touch_if_exists(childHash.as_uint256())) { return SHAMapAddNode::duplicate(); } diff --git a/src/xrpld/shamap/detail/ShardFamily.cpp b/src/xrpld/shamap/detail/ShardFamily.cpp deleted file mode 100644 index aef4c6cde0a..00000000000 --- a/src/xrpld/shamap/detail/ShardFamily.cpp +++ /dev/null @@ -1,198 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include - -namespace ripple { - -static NodeStore::Database& -getShardStore(Application& app) -{ - auto const dbPtr = app.getShardStore(); - assert(dbPtr); - return *dbPtr; -} - -ShardFamily::ShardFamily(Application& app, CollectorManager& cm) - : app_(app) - , db_(getShardStore(app)) - , cm_(cm) - , j_(app.journal("ShardFamily")) - , tnTargetSize_(app.config().getValueFor(SizedItem::treeCacheSize, 0)) - , tnTargetAge_(app.config().getValueFor(SizedItem::treeCacheAge, 0)) -{ -} - -std::shared_ptr -ShardFamily::getFullBelowCache(std::uint32_t ledgerSeq) -{ - auto const shardIndex{app_.getShardStore()->seqToShardIndex(ledgerSeq)}; - std::lock_guard lock(fbCacheMutex_); - if (auto const it{fbCache_.find(shardIndex)}; it != fbCache_.end()) - return it->second; - - // Create a cache for the corresponding shard - auto fbCache{std::make_shared( - "Shard family full below cache shard " + std::to_string(shardIndex), - stopwatch(), - j_, - cm_.collector(), - fullBelowTargetSize, - fullBelowExpiration)}; - return fbCache_.emplace(shardIndex, std::move(fbCache)).first->second; -} - -int -ShardFamily::getFullBelowCacheSize() -{ - size_t sz{0}; - std::lock_guard lock(fbCacheMutex_); - for (auto const& e : fbCache_) - sz += e.second->size(); - return sz; -} - -std::shared_ptr -ShardFamily::getTreeNodeCache(std::uint32_t ledgerSeq) -{ - auto const shardIndex{app_.getShardStore()->seqToShardIndex(ledgerSeq)}; - std::lock_guard lock(tnCacheMutex_); - if (auto const it{tnCache_.find(shardIndex)}; it != tnCache_.end()) - return it->second; - - // Create a cache for the corresponding shard - auto tnCache{std::make_shared( - "Shard family tree node cache shard " + std::to_string(shardIndex), - tnTargetSize_, - tnTargetAge_, - stopwatch(), - j_)}; - return tnCache_.emplace(shardIndex, std::move(tnCache)).first->second; -} - -std::pair -ShardFamily::getTreeNodeCacheSize() -{ - int cacheSz{0}; - int trackSz{0}; - std::lock_guard lock(tnCacheMutex_); - for (auto const& e : tnCache_) - { - cacheSz += e.second->getCacheSize(); - trackSz += e.second->getTrackSize(); - } - return {cacheSz, trackSz}; -} - -void -ShardFamily::sweep() -{ - { - std::lock_guard lock(fbCacheMutex_); - for (auto it = fbCache_.cbegin(); it != fbCache_.cend();) - { - it->second->sweep(); - - // Remove cache if empty - if (it->second->size() == 0) - it = fbCache_.erase(it); - else - ++it; - } - } - - std::lock_guard lock(tnCacheMutex_); - for (auto it = tnCache_.cbegin(); it != tnCache_.cend();) - { - it->second->sweep(); - - // Remove cache if empty - if (it->second->getTrackSize() == 0) - it = tnCache_.erase(it); - else - ++it; - } -} - -void -ShardFamily::reset() -{ - { - std::lock_guard lock(maxSeqMutex_); - maxSeq_ = 0; - } - - { - std::lock_guard lock(fbCacheMutex_); - fbCache_.clear(); - } - - std::lock_guard lock(tnCacheMutex_); - tnCache_.clear(); -} - -void -ShardFamily::missingNodeAcquireBySeq(std::uint32_t seq, uint256 const& nodeHash) -{ - std::ignore = nodeHash; - JLOG(j_.error()) << "Missing node in ledger sequence " << seq; - - std::unique_lock lock(maxSeqMutex_); - if (maxSeq_ == 0) - { - maxSeq_ = seq; - - do - { - // Try to acquire the most recent missing ledger - seq = maxSeq_; - - lock.unlock(); - - // This can invoke the missing node handler - acquire(app_.getLedgerMaster().getHashBySeq(seq), seq); - - lock.lock(); - } while (maxSeq_ != seq); - } - else if (maxSeq_ < seq) - { - // We found a more recent ledger with a missing node - maxSeq_ = seq; - } -} - -void -ShardFamily::acquire(uint256 const& hash, std::uint32_t seq) -{ - if (hash.isNonZero()) - { - JLOG(j_.error()) << "Missing node in " << to_string(hash); - - app_.getInboundLedgers().acquire( - hash, seq, InboundLedger::Reason::SHARD); - } -} - -} // namespace ripple