From f36d1d5b8934aac60d3097047ecedeb58bae2185 Mon Sep 17 00:00:00 2001 From: Jeffrey Czyz Date: Wed, 5 Jun 2019 15:54:11 -0700 Subject: [PATCH 0001/1293] Use void* throughout support/lockedpool.h Replace uses of char* with void* in Arena's member variables. Instead, cast to char* where needed in the implementation. Certain compiler environments disallow std::hash specializations to prevent hashing the pointer's value instead of the string contents. Thus, compilation fails when std::unordered_map is keyed by char*. Explicitly using void* is a workaround in such environments. For consistency, void* is used throughout all member variables similarly to the public interface. --- src/support/lockedpool.cpp | 18 ++++++++++-------- src/support/lockedpool.h | 10 +++++----- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index 85e3351e72576..6d767ca210638 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -27,6 +27,7 @@ #include #include #endif +#include LockedPoolManager* LockedPoolManager::_instance = nullptr; std::once_flag LockedPoolManager::init_flag; @@ -44,12 +45,12 @@ static inline size_t align_up(size_t x, size_t align) // Implementation: Arena Arena::Arena(void *base_in, size_t size_in, size_t alignment_in): - base(static_cast(base_in)), end(static_cast(base_in) + size_in), alignment(alignment_in) + base(base_in), end(static_cast(base_in) + size_in), alignment(alignment_in) { // Start with one free chunk that covers the entire arena auto it = size_to_free_chunk.emplace(size_in, base); chunks_free.emplace(base, it); - chunks_free_end.emplace(base + size_in, it); + chunks_free_end.emplace(static_cast(base) + size_in, it); } Arena::~Arena() @@ -75,8 +76,9 @@ void* Arena::alloc(size_t size) // Create the used-chunk, taking its space from the end of the free-chunk const size_t size_remaining = size_ptr_it->first - size; - auto allocated = chunks_used.emplace(size_ptr_it->second + size_remaining, size).first; - chunks_free_end.erase(size_ptr_it->second + size_ptr_it->first); + char* const free_chunk = static_cast(size_ptr_it->second); + auto allocated = chunks_used.emplace(free_chunk + size_remaining, size).first; + chunks_free_end.erase(free_chunk + size_ptr_it->first); if (size_ptr_it->first == size) { // whole chunk is used up chunks_free.erase(size_ptr_it->second); @@ -84,11 +86,11 @@ void* Arena::alloc(size_t size) // still some memory left in the chunk auto it_remaining = size_to_free_chunk.emplace(size_remaining, size_ptr_it->second); chunks_free[size_ptr_it->second] = it_remaining; - chunks_free_end.emplace(size_ptr_it->second + size_remaining, it_remaining); + chunks_free_end.emplace(free_chunk + size_remaining, it_remaining); } size_to_free_chunk.erase(size_ptr_it); - return reinterpret_cast(allocated->first); + return allocated->first; } void Arena::free(void *ptr) @@ -99,11 +101,11 @@ void Arena::free(void *ptr) } // Remove chunk from used map - auto i = chunks_used.find(static_cast(ptr)); + auto i = chunks_used.find(ptr); if (i == chunks_used.end()) { throw std::runtime_error("Arena: invalid or double free"); } - std::pair freed = *i; + auto freed = std::make_pair(static_cast(i->first), i->second); chunks_used.erase(i); // coalesce freed with previous chunk diff --git a/src/support/lockedpool.h b/src/support/lockedpool.h index b420c909fc53a..ce6fedc8e8bf5 100644 --- a/src/support/lockedpool.h +++ b/src/support/lockedpool.h @@ -89,23 +89,23 @@ class Arena */ bool addressInArena(void *ptr) const { return ptr >= base && ptr < end; } private: - typedef std::multimap SizeToChunkSortedMap; + typedef std::multimap SizeToChunkSortedMap; /** Map to enable O(log(n)) best-fit allocation, as it's sorted by size */ SizeToChunkSortedMap size_to_free_chunk; - typedef std::unordered_map ChunkToSizeMap; + typedef std::unordered_map ChunkToSizeMap; /** Map from begin of free chunk to its node in size_to_free_chunk */ ChunkToSizeMap chunks_free; /** Map from end of free chunk to its node in size_to_free_chunk */ ChunkToSizeMap chunks_free_end; /** Map from begin of used chunk to its size */ - std::unordered_map chunks_used; + std::unordered_map chunks_used; /** Base address of arena */ - char* base; + void* base; /** End address of arena */ - char* end; + void* end; /** Minimum chunk alignment */ size_t alignment; }; From 98868633d1d5cd2591b9613545bd2ce2e81af212 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Wed, 12 Jan 2022 22:02:21 +0000 Subject: [PATCH 0002/1293] Bugfix: configure: bitcoin-{cli,tx,util} don't need UPnP, NAT-PMP, or ZMQ --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 9b080eb0c8037..b9567c9780560 100644 --- a/configure.ac +++ b/configure.ac @@ -1349,7 +1349,7 @@ if test "$use_usdt" != "no"; then ) fi -if test "$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests" = "nonononononono"; then +if test "$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests" = "nononono"; then use_upnp=no use_natpmp=no use_zmq=no From 0811cbfc2868ee80c522fd426f188f10b06cd421 Mon Sep 17 00:00:00 2001 From: brunoerg Date: Sat, 15 Jan 2022 15:37:59 -0300 Subject: [PATCH 0003/1293] doc: add info about status code 404 for some rest endpoints --- doc/REST-interface.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 51a73b89fcecf..1e0e701df552c 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -31,6 +31,7 @@ Supported API `GET /rest/tx/.` Given a transaction hash: returns a transaction in binary, hex-encoded binary, or JSON formats. +Responds with 404 if the transaction doesn't exist. By default, this endpoint will only search the mempool. To query for a confirmed transaction, enable the transaction index via "txindex=1" command line / configuration option. @@ -70,6 +71,7 @@ Responds with 404 if the block doesn't exist. `GET /rest/blockhashbyheight/.` Given a height: returns hash of block in best-block-chain at height provided. +Responds with 404 if block not found. #### Chaininfos `GET /rest/chaininfo.json` From 75347236f212f327a5bba10d8a900cc58ebe5de0 Mon Sep 17 00:00:00 2001 From: Pasta Date: Fri, 17 Dec 2021 23:40:06 -0500 Subject: [PATCH 0004/1293] docs: document c-style cast prohibition --- doc/developer-notes.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 188889785686f..76aebc55b7d56 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -104,6 +104,10 @@ code. - `++i` is preferred over `i++`. - `nullptr` is preferred over `NULL` or `(void*)0`. - `static_assert` is preferred over `assert` where possible. Generally; compile-time checking is preferred over run-time checking. + - Use a named cast or functional cast, not a C-Style cast. When casting + between integer types, use functional casts such as `int(x)` or `int{x}` + instead of `(int) x`. When casting between more complex types, use static_cast. + Use reinterpret_cast and const_cast as appropriate. Block style example: ```c++ From 0f40d653218789aa176ca2f844e3222d2ad890a3 Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Wed, 16 Feb 2022 21:17:21 -0500 Subject: [PATCH 0005/1293] refactor: remove duplicate code from BlockAssembler --- src/node/miner.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/node/miner.cpp b/src/node/miner.cpp index 6e9bde84d8eba..303738db4bc3f 100644 --- a/src/node/miner.cpp +++ b/src/node/miner.cpp @@ -266,13 +266,9 @@ int BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& already modtxiter mit = mapModifiedTx.find(desc); if (mit == mapModifiedTx.end()) { CTxMemPoolModifiedEntry modEntry(desc); - modEntry.nSizeWithAncestors -= it->GetTxSize(); - modEntry.nModFeesWithAncestors -= it->GetModifiedFee(); - modEntry.nSigOpCostWithAncestors -= it->GetSigOpCost(); - mapModifiedTx.insert(modEntry); - } else { - mapModifiedTx.modify(mit, update_for_parent_inclusion(it)); + mit = mapModifiedTx.insert(modEntry).first; } + mapModifiedTx.modify(mit, update_for_parent_inclusion(it)); } } return nDescendantsUpdated; From c456302d4258e3abc4b8afde20fba808632771b2 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Thu, 3 Mar 2022 11:56:20 +0100 Subject: [PATCH 0006/1293] doc: minor improvements in getutxos REST endpoint synopsis Describing an optional sub-path as in the synopsis could be misleading as the angle brackets normally indicate that the field has to be replaced a custom value. Clarify that by showing two variants instead, similar to the block endpoint with the notxdetails option. Further improvements: - uppercase and , to match the description of the other endpoints - s/getutxo command/getutxos endpoint/ - describe what the checkmempool option does - s/serialisation/serialization/ (the US spelling is more dominant than the UK spelling in the project, and there is indeed no other instance of the string "serialis*" in the source tree, except once in a release note) - link to BIP64 within the text instead of only showing bare URL - mention that BIP64 is only relevant for bin and hex output formats - show two endpoint formats of the block section as list --- doc/REST-interface.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 1f0a07a2840d9..02315c359c531 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -36,8 +36,8 @@ By default, this endpoint will only search the mempool. To query for a confirmed transaction, enable the transaction index via "txindex=1" command line / configuration option. #### Blocks -`GET /rest/block/.` -`GET /rest/block/notxdetails/.` +- `GET /rest/block/.` +- `GET /rest/block/notxdetails/.` Given a block hash: returns a block, in binary, hex-encoded binary or JSON formats. Responds with 404 if the block doesn't exist. @@ -89,11 +89,13 @@ Only supports JSON as output format. * softforks : (array) status of softforks in progress #### Query UTXO set -`GET /rest/getutxos//-/-/.../-.` +- `GET /rest/getutxos/-/-/.../-.` +- `GET /rest/getutxos/checkmempool/-/-/.../-.` -The getutxo command allows querying of the UTXO set given a set of outpoints. -See BIP64 for input and output serialisation: -https://github.com/bitcoin/bips/blob/master/bip-0064.mediawiki +The getutxos endpoint allows querying the UTXO set, given a set of outpoints. +With the `/checkmempool/` option, the mempool is also taken into account. +See [BIP64](https://github.com/bitcoin/bips/blob/master/bip-0064.mediawiki) for +input and output serialization (relevant for `bin` and `hex` output formats). Example: ``` From 511eb7fdeac36da9d6576c878a1c8d390b38f1bd Mon Sep 17 00:00:00 2001 From: Chris Geihsler Date: Wed, 13 Apr 2022 22:50:10 -0400 Subject: [PATCH 0007/1293] Ignore problematic blocks in DisconnectBlock When using checklevel=4, block verification fails because of duplicate coinbase transactions involving blocks 91812 and 91722. There was already a check in place for ConnectBlock to ignore the problematic blocks, but DisconnectBlock did not contain a similar check. This change ignores the blocks where these inconsistencies surface so that block verification will succeed at checklevel=4. --- src/validation.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/validation.cpp b/src/validation.cpp index f4b316f67aa94..6d87ab88b3243 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1837,6 +1837,15 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI return DISCONNECT_FAILED; } + // Ignore blocks that contain transactions which are 'overwritten' by later transactions, + // unless those are already completely spent. + // See https://github.com/bitcoin/bitcoin/issues/22596 for additional information. + // Note: the blocks specified here are different than the ones used in ConnectBlock because DisconnectBlock + // unwinds the blocks in reverse. As a result, the inconsistency is not discovered until the earlier + // blocks with the duplicate coinbase transactions are disconnected. + bool fEnforceBIP30 = !((pindex->nHeight==91722 && pindex->GetBlockHash() == uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e")) || + (pindex->nHeight==91812 && pindex->GetBlockHash() == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f"))); + // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { const CTransaction &tx = *(block.vtx[i]); @@ -1850,7 +1859,7 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI COutPoint out(hash, o); Coin coin; bool is_spent = view.SpendCoin(out, &coin); - if (!is_spent || tx.vout[o] != coin.out || pindex->nHeight != coin.nHeight || is_coinbase != coin.fCoinBase) { + if (fEnforceBIP30 && (!is_spent || tx.vout[o] != coin.out || pindex->nHeight != coin.nHeight || is_coinbase != coin.fCoinBase)) { fClean = false; // transaction output mismatch } } From e899d4ca6f44785b6e5481e200a6a9a8d2448612 Mon Sep 17 00:00:00 2001 From: Chris Geihsler Date: Thu, 14 Apr 2022 11:48:19 -0400 Subject: [PATCH 0008/1293] init: limit bip30 exceptions to coinbase txs Co-authored-by: James O'Beirne --- src/validation.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/validation.cpp b/src/validation.cpp index 6d87ab88b3243..ffad7e8402081 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1851,6 +1851,7 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI const CTransaction &tx = *(block.vtx[i]); uint256 hash = tx.GetHash(); bool is_coinbase = tx.IsCoinBase(); + bool is_bip30_exception = (is_coinbase && !fEnforceBIP30); // Check that all outputs are available and match the outputs in the block itself // exactly. @@ -1859,8 +1860,10 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI COutPoint out(hash, o); Coin coin; bool is_spent = view.SpendCoin(out, &coin); - if (fEnforceBIP30 && (!is_spent || tx.vout[o] != coin.out || pindex->nHeight != coin.nHeight || is_coinbase != coin.fCoinBase)) { - fClean = false; // transaction output mismatch + if (!is_spent || tx.vout[o] != coin.out || pindex->nHeight != coin.nHeight || is_coinbase != coin.fCoinBase) { + if (!is_bip30_exception) { + fClean = false; // transaction output mismatch + } } } } From 071eef1e974f128131afe6c6b5c68a430c64687a Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sun, 28 Nov 2021 19:25:01 +0200 Subject: [PATCH 0009/1293] build: Propagate user-defined flags to host packages --- depends/hosts/default.mk | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/depends/hosts/default.mk b/depends/hosts/default.mk index 258619a9d0590..57e71ad55f124 100644 --- a/depends/hosts/default.mk +++ b/depends/hosts/default.mk @@ -29,8 +29,13 @@ host_$1=$$($(host_arch)_$(host_os)_$1) endef define add_host_flags_func +ifeq ($(filter $(origin $1),undefined default),) +$(host_arch)_$(host_os)_$1 = +$(host_arch)_$(host_os)_$(release_type)_$1 = $($1) +else $(host_arch)_$(host_os)_$1 += $($(host_os)_$1) $(host_arch)_$(host_os)_$(release_type)_$1 += $($(host_os)_$(release_type)_$1) +endif host_$1 = $$($(host_arch)_$(host_os)_$1) host_$(release_type)_$1 = $$($(host_arch)_$(host_os)_$(release_type)_$1) endef From a3a2bd9e8ad360a63cc8bdfc365d8bfd25ecc720 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sun, 28 Nov 2021 19:30:09 +0200 Subject: [PATCH 0010/1293] ci: Drop no longer needed package-specific flags --- ci/test/00_setup_env_native_fuzz_with_msan.sh | 2 +- ci/test/00_setup_env_native_msan.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 071bac8fb3343..90c4d29737569 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -15,7 +15,7 @@ export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" export PACKAGES="clang-12 llvm-12 cmake" # BDB generates false-positives and will be removed in future -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 34a792ec8ff26..33e7a4e6645f2 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -15,7 +15,7 @@ export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" export PACKAGES="clang-12 llvm-12 cmake" # BDB generates false-positives and will be removed in future -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" From e7a5bf6be79e341e037305a4c2d8a1a510a8d709 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Mon, 21 Feb 2022 17:03:27 +0100 Subject: [PATCH 0011/1293] fees: make the class FeeFilterRounder thread-safe So that its methods can be called concurrently by different threads on the same object. Currently it has just one method (`round()`). Co-authored-by: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> --- src/policy/fees.cpp | 6 ++++-- src/policy/fees.h | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 6499dbd97f07a..9f576e738a558 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -1010,8 +1010,10 @@ FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) CAmount FeeFilterRounder::round(CAmount currentMinFee) { std::set::iterator it = feeset.lower_bound(currentMinFee); - if ((it != feeset.begin() && insecure_rand.rand32() % 3 != 0) || it == feeset.end()) { - it--; + if (it == feeset.end() || + (it != feeset.begin() && + WITH_LOCK(m_insecure_rand_mutex, return insecure_rand.rand32()) % 3 != 0)) { + --it; } return static_cast(*it); } diff --git a/src/policy/fees.h b/src/policy/fees.h index 6e25bb42b8af1..c7fa1a0b77793 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -299,12 +299,13 @@ class FeeFilterRounder /** Create new FeeFilterRounder */ explicit FeeFilterRounder(const CFeeRate& minIncrementalFee); - /** Quantize a minimum fee for privacy purpose before broadcast. Not thread-safe due to use of FastRandomContext */ + /** Quantize a minimum fee for privacy purpose before broadcast. */ CAmount round(CAmount currentMinFee); private: std::set feeset; - FastRandomContext insecure_rand; + Mutex m_insecure_rand_mutex; + FastRandomContext insecure_rand GUARDED_BY(m_insecure_rand_mutex); }; #endif // BITCOIN_POLICY_FEES_H From 8b4ad203d06c5ded6ecebbd7277b29a442d88bcf Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Mon, 21 Feb 2022 17:11:59 +0100 Subject: [PATCH 0012/1293] fees: make FeeFilterRounder::feeset const It is only set in the constructor, thus improve readability by marking it as `const` and setting it from the initializer list using a helper function to derive its value. The idea was suggested by Anthony Towns in https://github.com/bitcoin/bitcoin/pull/19268#discussion_r439929792 --- src/policy/fees.cpp | 17 ++++++++++++++--- src/policy/fees.h | 2 +- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 9f576e738a558..d64360e82d123 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -998,13 +998,24 @@ void CBlockPolicyEstimator::FlushUnconfirmed() { LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %gs\n", num_entries, (endclear - startclear)*0.000001); } -FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) +static std::set MakeFeeSet(const CFeeRate& minIncrementalFee, + double max_filter_fee_rate, + double fee_filter_spacing) { - CAmount minFeeLimit = std::max(CAmount(1), minIncrementalFee.GetFeePerK() / 2); + std::set feeset; + + const CAmount minFeeLimit{std::max(CAmount(1), minIncrementalFee.GetFeePerK() / 2)}; feeset.insert(0); - for (double bucketBoundary = minFeeLimit; bucketBoundary <= MAX_FILTER_FEERATE; bucketBoundary *= FEE_FILTER_SPACING) { + for (double bucketBoundary = minFeeLimit; bucketBoundary <= max_filter_fee_rate; bucketBoundary *= fee_filter_spacing) { feeset.insert(bucketBoundary); } + + return feeset; +} + +FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) + : feeset{MakeFeeSet(minIncrementalFee, MAX_FILTER_FEERATE, FEE_FILTER_SPACING)} +{ } CAmount FeeFilterRounder::round(CAmount currentMinFee) diff --git a/src/policy/fees.h b/src/policy/fees.h index c7fa1a0b77793..e7f45c3151e6c 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -303,7 +303,7 @@ class FeeFilterRounder CAmount round(CAmount currentMinFee); private: - std::set feeset; + const std::set feeset; Mutex m_insecure_rand_mutex; FastRandomContext insecure_rand GUARDED_BY(m_insecure_rand_mutex); }; From 8173f160e085186c9bcc7f3506205c309ee66af6 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Tue, 15 Mar 2022 16:57:05 +0100 Subject: [PATCH 0013/1293] style: rename variables to match coding style Rename the variables that were touched by the previous commit (split logical from style changes). minIncrementalFee -> min_incremental_fee minFeeLimit -> min_fee_limit bucketBoundary -> bucket_boundary feeset -> fee_set FeeFilterRounder::feeset -> FeeFilterRounder::m_fee_set --- src/policy/fees.cpp | 25 ++++++++++++++----------- src/policy/fees.h | 4 ++-- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index d64360e82d123..44024ce7acb1b 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -998,31 +998,34 @@ void CBlockPolicyEstimator::FlushUnconfirmed() { LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %gs\n", num_entries, (endclear - startclear)*0.000001); } -static std::set MakeFeeSet(const CFeeRate& minIncrementalFee, +static std::set MakeFeeSet(const CFeeRate& min_incremental_fee, double max_filter_fee_rate, double fee_filter_spacing) { - std::set feeset; + std::set fee_set; - const CAmount minFeeLimit{std::max(CAmount(1), minIncrementalFee.GetFeePerK() / 2)}; - feeset.insert(0); - for (double bucketBoundary = minFeeLimit; bucketBoundary <= max_filter_fee_rate; bucketBoundary *= fee_filter_spacing) { - feeset.insert(bucketBoundary); + const CAmount min_fee_limit{std::max(CAmount(1), min_incremental_fee.GetFeePerK() / 2)}; + fee_set.insert(0); + for (double bucket_boundary = min_fee_limit; + bucket_boundary <= max_filter_fee_rate; + bucket_boundary *= fee_filter_spacing) { + + fee_set.insert(bucket_boundary); } - return feeset; + return fee_set; } FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) - : feeset{MakeFeeSet(minIncrementalFee, MAX_FILTER_FEERATE, FEE_FILTER_SPACING)} + : m_fee_set{MakeFeeSet(minIncrementalFee, MAX_FILTER_FEERATE, FEE_FILTER_SPACING)} { } CAmount FeeFilterRounder::round(CAmount currentMinFee) { - std::set::iterator it = feeset.lower_bound(currentMinFee); - if (it == feeset.end() || - (it != feeset.begin() && + std::set::iterator it = m_fee_set.lower_bound(currentMinFee); + if (it == m_fee_set.end() || + (it != m_fee_set.begin() && WITH_LOCK(m_insecure_rand_mutex, return insecure_rand.rand32()) % 3 != 0)) { --it; } diff --git a/src/policy/fees.h b/src/policy/fees.h index e7f45c3151e6c..b44ba4dc7abe7 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -297,13 +297,13 @@ class FeeFilterRounder public: /** Create new FeeFilterRounder */ - explicit FeeFilterRounder(const CFeeRate& minIncrementalFee); + explicit FeeFilterRounder(const CFeeRate& min_incremental_fee); /** Quantize a minimum fee for privacy purpose before broadcast. */ CAmount round(CAmount currentMinFee); private: - const std::set feeset; + const std::set m_fee_set; Mutex m_insecure_rand_mutex; FastRandomContext insecure_rand GUARDED_BY(m_insecure_rand_mutex); }; From 304ece994504220c355577170409b9200941f2af Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 31 May 2021 16:55:56 +0200 Subject: [PATCH 0014/1293] rpc: document bools in FillPSBT() calls --- src/wallet/rpc/spend.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 07119133b7ad6..13df213065273 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -82,10 +82,10 @@ static UniValue FinishTransaction(const std::shared_ptr pwallet, const PartiallySignedTransaction psbtx(rawTx); // First fill transaction with our data without signing, - // so external signers are not asked sign more than once. + // so external signers are not asked to sign more than once. bool complete; - pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, false, true); - const TransactionError err{pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, true, false)}; + pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/false, /*bip32derivs=*/true); + const TransactionError err{pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/true, /*bip32derivs=*/false)}; if (err != TransactionError::OK) { throw JSONRPCTransactionError(err); } @@ -1100,7 +1100,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) } else { PartiallySignedTransaction psbtx(mtx); bool complete = false; - const TransactionError err = pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, false /* sign */, true /* bip32derivs */); + const TransactionError err = pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/false, /*bip32derivs=*/true); CHECK_NONFATAL(err == TransactionError::OK); CHECK_NONFATAL(!complete); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); @@ -1675,7 +1675,7 @@ RPCHelpMan walletcreatefundedpsbt() // Fill transaction with out data but don't sign bool bip32derivs = request.params[4].isNull() ? true : request.params[4].get_bool(); bool complete = true; - const TransactionError err{wallet.FillPSBT(psbtx, complete, 1, false, bip32derivs)}; + const TransactionError err{wallet.FillPSBT(psbtx, complete, 1, /*sign=*/false, /*bip32derivs=*/bip32derivs)}; if (err != TransactionError::OK) { throw JSONRPCTransactionError(err); } From 7e02a3329797211ed5d35e5f5e7b919c099b78ba Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 25 Apr 2022 18:13:23 +0200 Subject: [PATCH 0015/1293] rpc: bumpfee signer support --- src/wallet/feebumper.cpp | 17 ++++++++++++++++- src/wallet/rpc/spend.cpp | 5 ++++- test/functional/wallet_signer.py | 23 ++++++++++++++++++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 73042424ad498..c66754bfcb43b 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -239,7 +239,22 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo bool SignTransaction(CWallet& wallet, CMutableTransaction& mtx) { LOCK(wallet.cs_wallet); - return wallet.SignTransaction(mtx); + + if (wallet.IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)) { + // Make a blank psbt + PartiallySignedTransaction psbtx(mtx); + + // First fill transaction with our data without signing, + // so external signers are not asked to sign more than once. + bool complete; + wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, false /* sign */, true /* bip32derivs */); + const TransactionError err = wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, true /* sign */, false /* bip32derivs */); + if (err != TransactionError::OK) return false; + complete = FinalizeAndExtractPSBT(psbtx, mtx); + return complete; + } else { + return wallet.SignTransaction(mtx); + } } Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransaction&& mtx, std::vector& errors, uint256& bumped_txid) diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 13df213065273..602d7aec96e31 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -1010,7 +1010,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) std::shared_ptr const pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; - if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !want_psbt) { + if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !pwallet->IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER) && !want_psbt) { throw JSONRPCError(RPC_WALLET_ERROR, "bumpfee is not available with wallets that have private keys disabled. Use psbtbumpfee instead."); } @@ -1088,6 +1088,9 @@ static RPCHelpMan bumpfee_helper(std::string method_name) // For psbtbumpfee, return the base64-encoded unsigned PSBT of the new transaction. if (!want_psbt) { if (!feebumper::SignTransaction(*pwallet, mtx)) { + if (pwallet->IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Transaction incomplete. Try psbtbumpfee instead."); + } throw JSONRPCError(RPC_WALLET_ERROR, "Can't sign transaction."); } diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index 8e4e1f5d36b67..0839dbf9f4925 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -13,6 +13,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, + assert_greater_than, assert_raises_rpc_error, ) @@ -150,7 +151,7 @@ def test_valid_signer(self): assert_equal(result[1], {'success': True}) assert_equal(mock_wallet.getwalletinfo()["txcount"], 1) dest = self.nodes[0].getnewaddress(address_type='bech32') - mock_psbt = mock_wallet.walletcreatefundedpsbt([], {dest:0.5}, 0, {}, True)['psbt'] + mock_psbt = mock_wallet.walletcreatefundedpsbt([], {dest:0.5}, 0, {'replaceable': True}, True)['psbt'] mock_psbt_signed = mock_wallet.walletprocesspsbt(psbt=mock_psbt, sign=True, sighashtype="ALL", bip32derivs=True) mock_psbt_final = mock_wallet.finalizepsbt(mock_psbt_signed["psbt"]) mock_tx = mock_psbt_final["hex"] @@ -190,6 +191,7 @@ def test_valid_signer(self): self.log.info('Test send using hww1') + # Don't broadcast transaction yet so the RPC returns the raw hex res = hww.send(outputs={dest:0.5},options={"add_to_wallet": False}) assert(res["complete"]) assert_equal(res["hex"], mock_tx) @@ -199,6 +201,25 @@ def test_valid_signer(self): res = hww.sendall(recipients=[{dest:0.5}, hww.getrawchangeaddress()],options={"add_to_wallet": False}) assert(res["complete"]) assert_equal(res["hex"], mock_tx) + # Broadcast transaction so we can bump the fee + hww.sendrawtransaction(res["hex"]) + + self.log.info('Prepare fee bumped mock PSBT') + + # Now that the transaction is broadcast, bump fee in mock wallet: + orig_tx_id = res["txid"] + mock_psbt_bumped = mock_wallet.psbtbumpfee(orig_tx_id)["psbt"] + mock_psbt_bumped_signed = mock_wallet.walletprocesspsbt(psbt=mock_psbt_bumped, sign=True, sighashtype="ALL", bip32derivs=True) + + with open(os.path.join(self.nodes[1].cwd, "mock_psbt"), "w", encoding="utf8") as f: + f.write(mock_psbt_bumped_signed["psbt"]) + + self.log.info('Test bumpfee using hww1') + + # Bump fee + res = hww.bumpfee(orig_tx_id) + assert_greater_than(res["fee"], res["origfee"]) + assert_equal(res["errors"], []) # # Handle error thrown by script # self.set_mock_result(self.nodes[4], "2") From 2c07cfacd1745844a1d3c57f2e8617549b9815d7 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 25 Apr 2022 18:14:28 +0200 Subject: [PATCH 0016/1293] gui: bumpfee signer support Specifically this enables the Send button in the fee bump dialog for wallets with external signer support. Similar to 2efdfb88aab6496dcf2b98e0de30635bc6bade85. --- src/qt/walletmodel.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 5ee32e79d5d4e..8c2f8e568d7df 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -508,7 +508,9 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) questionString.append(tr("Warning: This may pay the additional fee by reducing change outputs or adding inputs, when necessary. It may add a new change output if one does not already exist. These changes may potentially leak privacy.")); } - auto confirmationDialog = new SendConfirmationDialog(tr("Confirm fee bump"), questionString, "", "", SEND_CONFIRM_DELAY, !m_wallet->privateKeysDisabled(), getOptionsModel()->getEnablePSBTControls(), nullptr); + const bool enable_send{!wallet().privateKeysDisabled() || wallet().hasExternalSigner()}; + const bool always_show_unsigned{getOptionsModel()->getEnablePSBTControls()}; + auto confirmationDialog = new SendConfirmationDialog(tr("Confirm fee bump"), questionString, "", "", SEND_CONFIRM_DELAY, enable_send, always_show_unsigned, nullptr); confirmationDialog->setAttribute(Qt::WA_DeleteOnClose); // TODO: Replace QDialog::exec() with safer QDialog::show(). const auto retval = static_cast(confirmationDialog->exec()); @@ -526,6 +528,7 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) // Short-circuit if we are returning a bumped transaction PSBT to clipboard if (retval == QMessageBox::Save) { + // "Create Unsigned" clicked PartiallySignedTransaction psbtx(mtx); bool complete = false; const TransactionError err = wallet().fillPSBT(SIGHASH_ALL, false /* sign */, true /* bip32derivs */, nullptr, psbtx, complete); @@ -541,7 +544,7 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) return true; } - assert(!m_wallet->privateKeysDisabled()); + assert(!m_wallet->privateKeysDisabled() || wallet().hasExternalSigner()); // sign bumped transaction if (!m_wallet->signBumpTransaction(mtx)) { From bcb0cacac28e98a39dc856c574a0872fe17059e9 Mon Sep 17 00:00:00 2001 From: mruddy <6440430+mruddy@users.noreply.github.com> Date: Thu, 14 Apr 2022 18:44:34 -0400 Subject: [PATCH 0017/1293] reindex, log, test: fixes #21379 This fixes a blk file size calculation made during reindex that results in increased blk file malformity. The fix is to avoid double counting the size of the serialization header during reindex. This adds a unit test to reproduce the bug before the fix and to ensure that it does not recur. These changes include a log message change also so as to not be as alarming. This is a common and recoverable data corruption. These messages can now be filtered by the debug log reindex category. --- src/Makefile.test.include | 1 + src/node/blockstorage.cpp | 13 ++++++---- src/node/blockstorage.h | 4 ++++ src/test/blockmanager_tests.cpp | 42 +++++++++++++++++++++++++++++++++ src/validation.cpp | 13 +++++++++- 5 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 src/test/blockmanager_tests.cpp diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 02a3f9ae7db9a..09148f9d36a6f 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -77,6 +77,7 @@ BITCOIN_TESTS =\ test/blockencodings_tests.cpp \ test/blockfilter_index_tests.cpp \ test/blockfilter_tests.cpp \ + test/blockmanager_tests.cpp \ test/bloom_tests.cpp \ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 17ab226a30d38..826e4150fe126 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -788,19 +788,24 @@ bool ReadRawBlockFromDisk(std::vector& block, const FlatFilePos& pos, c return true; } -/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); FlatFilePos blockPos; - if (dbp != nullptr) { + const auto position_known {dbp != nullptr}; + if (position_known) { blockPos = *dbp; + } else { + // when known, blockPos.nPos points at the offset of the block data in the blk file. that already accounts for + // the serialization header present in the file (the 4 magic message start bytes + the 4 length bytes = 8 bytes = BLOCK_SERIALIZATION_HEADER_SIZE). + // we add BLOCK_SERIALIZATION_HEADER_SIZE only for new blocks since they will have the serialization header added when written to disk. + nBlockSize += static_cast(BLOCK_SERIALIZATION_HEADER_SIZE); } - if (!FindBlockPos(blockPos, nBlockSize + 8, nHeight, active_chain, block.GetBlockTime(), dbp != nullptr)) { + if (!FindBlockPos(blockPos, nBlockSize, nHeight, active_chain, block.GetBlockTime(), position_known)) { error("%s: FindBlockPos failed", __func__); return FlatFilePos(); } - if (dbp == nullptr) { + if (!position_known) { if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { AbortNode("Failed to write block"); return FlatFilePos(); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 488713dbd8c1c..8e49254e76cec 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -44,6 +44,9 @@ static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB +/** Size of header written by WriteBlockToDisk before a serialized CBlock */ +static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = CMessageHeader::MESSAGE_START_SIZE + sizeof(unsigned int); + extern std::atomic_bool fImporting; extern std::atomic_bool fReindex; /** Pruning-related variables and constants */ @@ -171,6 +174,7 @@ class BlockManager bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** Store block on disk. If dbp is not nullptr, then it provides the known position of the block within a block file on disk. */ FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp); /** Calculate the amount of disk space the block & undo files currently use */ diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp new file mode 100644 index 0000000000000..dd7c32cc46c0e --- /dev/null +++ b/src/test/blockmanager_tests.cpp @@ -0,0 +1,42 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include + +#include +#include + +using node::BlockManager; +using node::BLOCK_SERIALIZATION_HEADER_SIZE; + +// use BasicTestingSetup here for the data directory configuration, setup, and cleanup +BOOST_FIXTURE_TEST_SUITE(blockmanager_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) +{ + const auto params {CreateChainParams(ArgsManager{}, CBaseChainParams::MAIN)}; + BlockManager blockman {}; + CChain chain {}; + // simulate adding a genesis block normally + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, *params, nullptr).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + // simulate what happens during reindex + // simulate a well-formed genesis block being found at offset 8 in the blk00000.dat file + // the block is found at offset 8 because there is an 8 byte serialization header + // consisting of 4 magic bytes + 4 length bytes before each block in a well-formed blk file. + FlatFilePos pos{0, BLOCK_SERIALIZATION_HEADER_SIZE}; + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, *params, &pos).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + // now simulate what happens after reindex for the first new block processed + // the actual block contents don't matter, just that it's a block. + // verify that the write position is at offset 0x12d. + // this is a check to make sure that https://github.com/bitcoin/bitcoin/issues/21379 does not recur + // 8 bytes (for serialization header) + 285 (for serialized genesis block) = 293 + // add another 8 bytes for the second block's serialization header and we get 293 + 8 = 301 + FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, chain, *params, nullptr)}; + BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(params->GenesisBlock(), CLIENT_VERSION) + BLOCK_SERIALIZATION_HEADER_SIZE); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/validation.cpp b/src/validation.cpp index b5d6a660886fe..e8cb92a1f1a7b 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -4378,7 +4378,18 @@ void CChainState::LoadExternalBlockFile(FILE* fileIn, FlatFilePos* dbp) } } } catch (const std::exception& e) { - LogPrintf("%s: Deserialize or I/O error - %s\n", __func__, e.what()); + // historical bugs added extra data to the block files that does not deserialize cleanly. + // commonly this data is between readable blocks, but it does not really matter. such data is not fatal to the import process. + // the code that reads the block files deals with invalid data by simply ignoring it. + // it continues to search for the next {4 byte magic message start bytes + 4 byte length + block} that does deserialize cleanly + // and passes all of the other block validation checks dealing with POW and the merkle root, etc... + // we merely note with this informational log message when unexpected data is encountered. + // we could also be experiencing a storage system read error, or a read of a previous bad write. these are possible, but + // less likely scenarios. we don't have enough information to tell a difference here. + // the reindex process is not the place to attempt to clean and/or compact the block files. if so desired, a studious node operator + // may use knowledge of the fact that the block files are not entirely pristine in order to prepare a set of pristine, and + // perhaps ordered, block files for later reindexing. + LogPrint(BCLog::REINDEX, "%s: unexpected data at file offset 0x%x - %s. continuing\n", __func__, (nRewind - 1), e.what()); } } } catch (const std::runtime_error& e) { From 7fa851fba8570ef256317f7d5759aa3de9088bf1 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Fri, 31 Dec 2021 16:25:35 +0100 Subject: [PATCH 0018/1293] rpc: Pruned nodes can not fetch unsynced blocks While a node is still catching up to the tip that it is aware of via the headers, the user can currently use to fetch blocks close to the tip. These blocks are stored in the current block/rev file which otherwise contains blocks the node is receiving as part of the syncing process. This creates a problem for pruned nodes: The files containing a fetched block are not pruned during syncing because they contain a block close to the tip. This means the entire file will not be pruned until the tip have moved on far enough from the fetched block. In extreme cases with heavy pruning (550) and multiple blocks being fetched this could mean that the disc usage far exceeds what the user expects, potentially running out of space. --- src/rpc/blockchain.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f2186c131fd36..4fbed3da60bc2 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -453,6 +453,12 @@ static RPCHelpMan getblockfrompeer() throw JSONRPCError(RPC_MISC_ERROR, "Block header missing"); } + // Fetching blocks before the node has syncing past their height can prevent block files from + // being pruned, so we avoid it if the node is in prune mode. + if (index->nHeight > chainman.ActiveChain().Tip()->nHeight && node::fPruneMode) { + throw JSONRPCError(RPC_MISC_ERROR, "In prune mode, only blocks that the node has already synced previously can be fetched from a peer"); + } + const bool block_has_data = WITH_LOCK(::cs_main, return index->nStatus & BLOCK_HAVE_DATA); if (block_has_data) { throw JSONRPCError(RPC_MISC_ERROR, "Block already downloaded"); From 5826bf546e83478947edbdf49978414f0b69eb1a Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Sun, 2 Jan 2022 17:04:58 +0100 Subject: [PATCH 0019/1293] test: Add test for getblockfrompeer on syncing pruned nodes --- test/functional/rpc_getblockfrompeer.py | 32 ++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/test/functional/rpc_getblockfrompeer.py b/test/functional/rpc_getblockfrompeer.py index a7628b55915a4..8986b65eacecf 100755 --- a/test/functional/rpc_getblockfrompeer.py +++ b/test/functional/rpc_getblockfrompeer.py @@ -5,7 +5,12 @@ """Test the getblockfrompeer RPC.""" from test_framework.authproxy import JSONRPCException -from test_framework.messages import NODE_WITNESS +from test_framework.messages import ( + CBlock, + from_hex, + msg_headers, + NODE_WITNESS, +) from test_framework.p2p import ( P2P_SERVICES, P2PInterface, @@ -16,6 +21,7 @@ assert_raises_rpc_error, ) + class GetBlockFromPeerTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 @@ -78,6 +84,30 @@ def run_test(self): self.log.info("Don't fetch blocks we already have") assert_raises_rpc_error(-1, "Block already downloaded", self.nodes[0].getblockfrompeer, short_tip, peer_0_peer_1_id) + self.log.info("Don't fetch blocks while the node has not synced past it yet") + # For this test we need node 1 in prune mode and as a side effect this also disconnects + # the nodes which is also necessary for the rest of the test. + self.restart_node(1, ["-prune=550"]) + + # Generate a block on the disconnected node that the pruning node is not connected to + blockhash = self.generate(self.nodes[0], 1, sync_fun=self.no_op)[0] + block_hex = self.nodes[0].getblock(blockhash=blockhash, verbosity=0) + block = from_hex(CBlock(), block_hex) + + # Connect a P2PInterface to the pruning node and have it submit only the header of the + # block that the pruning node has not seen + node1_interface = self.nodes[1].add_p2p_connection(P2PInterface()) + node1_interface.send_message(msg_headers([block])) + + # Get the peer id of the P2PInterface from the pruning node + node1_peers = self.nodes[1].getpeerinfo() + assert_equal(len(node1_peers), 1) + node1_interface_id = node1_peers[0]["id"] + + # Trying to fetch this block from the P2PInterface should not be possible + error_msg = "In prune mode, only blocks that the node has already synced previously can be fetched from a peer" + assert_raises_rpc_error(-1, error_msg, self.nodes[1].getblockfrompeer, blockhash, node1_interface_id) + if __name__ == '__main__': GetBlockFromPeerTest().main() From a413595c37f51557f9506e0a279cd80fc9a6fb36 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Thu, 9 Jun 2022 14:38:26 +0200 Subject: [PATCH 0020/1293] build: Fix `capnp` package build for Android --- depends/packages/capnp.mk | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk index 8a3a14810d6ed..cd9b91cf7cf27 100644 --- a/depends/packages/capnp.mk +++ b/depends/packages/capnp.mk @@ -6,8 +6,13 @@ $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) $(package)_dependencies=native_$(package) +define $(package)_set_vars := +$(package)_config_opts := --with-external-capnp +$(package)_config_opts_android := --disable-shared +endef + define $(package)_config_cmds - $($(package)_autoconf) --with-external-capnp + $($(package)_autoconf) endef define $(package)_build_cmds From 8b8edc25c13a3e613770bf38b21a2556192e6315 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Thu, 9 Jun 2022 15:49:54 +0200 Subject: [PATCH 0021/1293] build: Specify native binaries explicitly when building `capnp` package From `configure --help`: --with-external-capnp use the system capnp binary (or the one specified with $CAPNP) instead of compiling a new one (useful for cross-compiling) --- depends/packages/capnp.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk index cd9b91cf7cf27..f4778c1ecdc32 100644 --- a/depends/packages/capnp.mk +++ b/depends/packages/capnp.mk @@ -8,6 +8,8 @@ $(package)_dependencies=native_$(package) define $(package)_set_vars := $(package)_config_opts := --with-external-capnp +$(package)_config_opts += CAPNP="$$(native_capnp_prefixbin)/capnp" +$(package)_config_opts += CAPNP_CXX="$$(native_capnp_prefixbin)/capnp-c++" $(package)_config_opts_android := --disable-shared endef From 796b020c37c793674f9d614d5d70fd1ed65f0938 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Fri, 20 May 2022 09:54:41 +0200 Subject: [PATCH 0022/1293] wallet: add taproot support to external signer --- src/external_signer.cpp | 3 +++ test/functional/mocks/signer.py | 10 +++++++--- test/functional/wallet_signer.py | 15 +++++++++++---- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/external_signer.cpp b/src/external_signer.cpp index d125fe479b3a5..537e3a0b13e4d 100644 --- a/src/external_signer.cpp +++ b/src/external_signer.cpp @@ -81,6 +81,9 @@ bool ExternalSigner::SignTransaction(PartiallySignedTransaction& psbtx, std::str for (const auto& entry : input.hd_keypaths) { if (parsed_m_fingerprint == MakeUCharSpan(entry.second.fingerprint)) return true; } + for (const auto& entry : input.m_tap_bip32_paths) { + if (parsed_m_fingerprint == MakeUCharSpan(entry.second.second.fingerprint)) return true; + } return false; }; diff --git a/test/functional/mocks/signer.py b/test/functional/mocks/signer.py index b732b26a53d07..0b4f964c47955 100755 --- a/test/functional/mocks/signer.py +++ b/test/functional/mocks/signer.py @@ -27,12 +27,15 @@ def getdescriptors(args): "receive": [ "pkh([00000001/44'/1'/" + args.account + "']" + xpub + "/0/*)#vt6w3l3j", "sh(wpkh([00000001/49'/1'/" + args.account + "']" + xpub + "/0/*))#r0grqw5x", - "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/0/*)#x30uthjs" + "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/0/*)#x30uthjs", + "tr([00000001/86'/1'/" + args.account + "']" + xpub + "/0/*)#sng9rd4t" ], "internal": [ "pkh([00000001/44'/1'/" + args.account + "']" + xpub + "/1/*)#all0v2p2", "sh(wpkh([00000001/49'/1'/" + args.account + "']" + xpub + "/1/*))#kwx4c3pe", - "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/1/*)#h92akzzg" + "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/1/*)#h92akzzg", + "tr([00000001/86'/1'/" + args.account + "']" + xpub + "/1/*)#p8dy7c9n" + ] })) @@ -44,7 +47,8 @@ def displayaddress(args): return sys.stdout.write(json.dumps({"error": "Unexpected fingerprint", "fingerprint": args.fingerprint})) expected_desc = [ - "wpkh([00000001/84'/1'/0'/0/0]02c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#0yneg42r" + "wpkh([00000001/84'/1'/0'/0/0]02c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#0yneg42r", + "tr([00000001/86'/1'/0'/0/0]c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#4vdj9jqk", ] if args.desc not in expected_desc: return sys.stdout.write(json.dumps({"error": "Unexpected descriptor", "desc": args.desc})) diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index 8e4e1f5d36b67..4bb60a9f58eb5 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -89,7 +89,7 @@ def test_valid_signer(self): # ) # self.clear_mock_result(self.nodes[1]) - assert_equal(hww.getwalletinfo()["keypoolsize"], 30) + assert_equal(hww.getwalletinfo()["keypoolsize"], 40) address1 = hww.getnewaddress(address_type="bech32") assert_equal(address1, "bcrt1qm90ugl4d48jv8n6e5t9ln6t9zlpm5th68x4f8g") @@ -112,6 +112,13 @@ def test_valid_signer(self): assert_equal(address_info['ismine'], True) assert_equal(address_info['hdkeypath'], "m/44'/1'/0'/0/0") + address4 = hww.getnewaddress(address_type="bech32m") + assert_equal(address4, "bcrt1phw4cgpt6cd30kz9k4wkpwm872cdvhss29jga2xpmftelhqll62ms4e9sqj") + address_info = hww.getaddressinfo(address4) + assert_equal(address_info['solvable'], True) + assert_equal(address_info['ismine'], True) + assert_equal(address_info['hdkeypath'], "m/86'/1'/0'/0/0") + self.log.info('Test walletdisplayaddress') result = hww.walletdisplayaddress(address1) assert_equal(result, {"address": address1}) @@ -124,7 +131,7 @@ def test_valid_signer(self): self.clear_mock_result(self.nodes[1]) self.log.info('Prepare mock PSBT') - self.nodes[0].sendtoaddress(address1, 1) + self.nodes[0].sendtoaddress(address4, 1) self.generate(self.nodes[0], 1) # Load private key into wallet to generate a signed PSBT for the mock @@ -133,14 +140,14 @@ def test_valid_signer(self): assert mock_wallet.getwalletinfo()['private_keys_enabled'] result = mock_wallet.importdescriptors([{ - "desc": "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0/*)#rweraev0", + "desc": "tr([00000001/86'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0/*)#0jtt2jc9", "timestamp": 0, "range": [0,1], "internal": False, "active": True }, { - "desc": "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/*)#j6uzqvuh", + "desc": "tr([00000001/86'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/*)#7xw2h8ga", "timestamp": 0, "range": [0, 0], "internal": True, From 5db7d2ca0aa51ff25f97bf21ce0cbc9e6b741cbd Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Tue, 5 Jul 2022 12:15:59 +0200 Subject: [PATCH 0023/1293] moveonly: move IsSelectableSocket() from compat.h to sock.{h,cpp} To be converted to a method of the `Sock` class. --- src/compat/compat.h | 8 -------- src/util/sock.cpp | 8 ++++++++ src/util/sock.h | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/compat/compat.h b/src/compat/compat.h index a8e5552c0add2..cc3779757712a 100644 --- a/src/compat/compat.h +++ b/src/compat/compat.h @@ -109,14 +109,6 @@ typedef char* sockopt_arg_type; #define USE_POLL #endif -bool static inline IsSelectableSocket(const SOCKET& s) { -#if defined(USE_POLL) || defined(WIN32) - return true; -#else - return (s < FD_SETSIZE); -#endif -} - // MSG_NOSIGNAL is not available on some platforms, if it doesn't exist define it as 0 #if !defined(MSG_NOSIGNAL) #define MSG_NOSIGNAL 0 diff --git a/src/util/sock.cpp b/src/util/sock.cpp index 125dbc7f18725..fed50444a973e 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -117,6 +117,14 @@ int Sock::GetSockName(sockaddr* name, socklen_t* name_len) const return getsockname(m_socket, name, name_len); } +bool IsSelectableSocket(const SOCKET& s) { +#if defined(USE_POLL) || defined(WIN32) + return true; +#else + return (s < FD_SETSIZE); +#endif +} + bool Sock::Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred) const { // We need a `shared_ptr` owning `this` for `WaitMany()`, but don't want diff --git a/src/util/sock.h b/src/util/sock.h index 38a7dc80d6f88..9ec53ec91d17b 100644 --- a/src/util/sock.h +++ b/src/util/sock.h @@ -267,6 +267,8 @@ class Sock void Close(); }; +bool IsSelectableSocket(const SOCKET& s); + /** Return readable error string for a network error code */ std::string NetworkErrorString(int err); From b4bac556791b5bb8aa118d4c1fed42c3fe45550c Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Tue, 13 Apr 2021 14:29:14 +0200 Subject: [PATCH 0024/1293] net: convert standalone IsSelectableSocket() to Sock::IsSelectable() This makes the callers mockable. --- src/net.cpp | 3 +-- src/netbase.cpp | 2 +- src/test/fuzz/util.cpp | 7 ++++++- src/test/fuzz/util.h | 9 +++++++++ src/test/util/net.h | 2 ++ src/util/sock.cpp | 9 +++++---- src/util/sock.h | 8 ++++++-- 7 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index e9aa7ee43b30f..85bd0b529eca7 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -961,8 +961,7 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr&& sock, return; } - if (!IsSelectableSocket(sock->Get())) - { + if (!sock->IsSelectable()) { LogPrintf("connection from %s dropped: non-selectable socket\n", addr.ToString()); return; } diff --git a/src/netbase.cpp b/src/netbase.cpp index 4b8d2f8d0c8d7..22cf4415c80b6 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -503,7 +503,7 @@ std::unique_ptr CreateSockTCP(const CService& address_family) // Ensure that waiting for I/O on this socket won't result in undefined // behavior. - if (!IsSelectableSocket(sock->Get())) { + if (!sock->IsSelectable()) { LogPrintf("Cannot create connection: non-selectable socket created (fd >= FD_SETSIZE ?)\n"); return nullptr; } diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index fabcea22c3440..1d1985667126d 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -16,7 +16,7 @@ #include FuzzedSock::FuzzedSock(FuzzedDataProvider& fuzzed_data_provider) - : m_fuzzed_data_provider{fuzzed_data_provider} + : m_fuzzed_data_provider{fuzzed_data_provider}, m_selectable{fuzzed_data_provider.ConsumeBool()} { m_socket = fuzzed_data_provider.ConsumeIntegralInRange(INVALID_SOCKET - 1, INVALID_SOCKET); } @@ -254,6 +254,11 @@ int FuzzedSock::GetSockName(sockaddr* name, socklen_t* name_len) const return 0; } +bool FuzzedSock::IsSelectable() const +{ + return m_selectable; +} + bool FuzzedSock::Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred) const { constexpr std::array wait_errnos{ diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 60e2875953f6b..92ad08ec59aed 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -48,6 +48,13 @@ class FuzzedSock : public Sock */ mutable std::optional m_peek_data; + /** + * Whether to pretend that the socket is select(2)-able. This is randomly set in the + * constructor. It should remain constant so that repeated calls to `IsSelectable()` + * return the same value. + */ + const bool m_selectable; + public: explicit FuzzedSock(FuzzedDataProvider& fuzzed_data_provider); @@ -73,6 +80,8 @@ class FuzzedSock : public Sock int GetSockName(sockaddr* name, socklen_t* name_len) const override; + bool IsSelectable() const override; + bool Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred = nullptr) const override; bool WaitMany(std::chrono::milliseconds timeout, EventsPerSock& events_per_sock) const override; diff --git a/src/test/util/net.h b/src/test/util/net.h index ec6b4e6e8833f..749b5b33644a3 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -166,6 +166,8 @@ class StaticContentsSock : public Sock return 0; } + bool IsSelectable() const override { return true; } + bool Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred = nullptr) const override diff --git a/src/util/sock.cpp b/src/util/sock.cpp index fed50444a973e..c312a88276ce4 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -117,11 +117,12 @@ int Sock::GetSockName(sockaddr* name, socklen_t* name_len) const return getsockname(m_socket, name, name_len); } -bool IsSelectableSocket(const SOCKET& s) { +bool Sock::IsSelectable() const +{ #if defined(USE_POLL) || defined(WIN32) return true; #else - return (s < FD_SETSIZE); + return m_socket < FD_SETSIZE; #endif } @@ -193,10 +194,10 @@ bool Sock::WaitMany(std::chrono::milliseconds timeout, EventsPerSock& events_per SOCKET socket_max{0}; for (const auto& [sock, events] : events_per_sock) { - const auto& s = sock->m_socket; - if (!IsSelectableSocket(s)) { + if (!sock->IsSelectable()) { return false; } + const auto& s = sock->m_socket; if (events.requested & RECV) { FD_SET(s, &recv); } diff --git a/src/util/sock.h b/src/util/sock.h index 9ec53ec91d17b..ab9c6737c6517 100644 --- a/src/util/sock.h +++ b/src/util/sock.h @@ -133,6 +133,12 @@ class Sock */ [[nodiscard]] virtual int GetSockName(sockaddr* name, socklen_t* name_len) const; + /** + * Check if the underlying socket can be used for `select(2)` (or the `Wait()` method). + * @return true if selectable + */ + [[nodiscard]] virtual bool IsSelectable() const; + using Event = uint8_t; /** @@ -267,8 +273,6 @@ class Sock void Close(); }; -bool IsSelectableSocket(const SOCKET& s); - /** Return readable error string for a network error code */ std::string NetworkErrorString(int err); From 29f66f76826056f53d971ac734b7ed49b39848d3 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Tue, 5 Jul 2022 12:20:42 +0200 Subject: [PATCH 0025/1293] moveonly: move SetSocketNonBlocking() from netbase to util/sock To be converted to a method of the `Sock` class. --- src/netbase.cpp | 15 --------------- src/netbase.h | 2 -- src/util/sock.cpp | 15 +++++++++++++++ src/util/sock.h | 3 +++ 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/netbase.cpp b/src/netbase.cpp index 22cf4415c80b6..b13d6b1ae0bd8 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -717,21 +717,6 @@ bool LookupSubNet(const std::string& subnet_str, CSubNet& subnet_out) return false; } -bool SetSocketNonBlocking(const SOCKET& hSocket) -{ -#ifdef WIN32 - u_long nOne = 1; - if (ioctlsocket(hSocket, FIONBIO, &nOne) == SOCKET_ERROR) { -#else - int fFlags = fcntl(hSocket, F_GETFL, 0); - if (fcntl(hSocket, F_SETFL, fFlags | O_NONBLOCK) == SOCKET_ERROR) { -#endif - return false; - } - - return true; -} - void InterruptSocks5(bool interrupt) { interruptSocks5Recv = interrupt; diff --git a/src/netbase.h b/src/netbase.h index fadc8b418e685..f7816f5d1de71 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -221,8 +221,6 @@ bool ConnectSocketDirectly(const CService &addrConnect, const Sock& sock, int nT */ bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_t port, const Sock& sock, int nTimeout, bool& outProxyConnectionFailed); -/** Enable non-blocking mode for a socket */ -bool SetSocketNonBlocking(const SOCKET& hSocket); void InterruptSocks5(bool interrupt); /** diff --git a/src/util/sock.cpp b/src/util/sock.cpp index c312a88276ce4..b2b5133629c8f 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -117,6 +117,21 @@ int Sock::GetSockName(sockaddr* name, socklen_t* name_len) const return getsockname(m_socket, name, name_len); } +bool SetSocketNonBlocking(const SOCKET& hSocket) +{ +#ifdef WIN32 + u_long nOne = 1; + if (ioctlsocket(hSocket, FIONBIO, &nOne) == SOCKET_ERROR) { +#else + int fFlags = fcntl(hSocket, F_GETFL, 0); + if (fcntl(hSocket, F_SETFL, fFlags | O_NONBLOCK) == SOCKET_ERROR) { +#endif + return false; + } + + return true; +} + bool Sock::IsSelectable() const { #if defined(USE_POLL) || defined(WIN32) diff --git a/src/util/sock.h b/src/util/sock.h index ab9c6737c6517..b97977da95bf5 100644 --- a/src/util/sock.h +++ b/src/util/sock.h @@ -273,6 +273,9 @@ class Sock void Close(); }; +/** Enable non-blocking mode for a socket */ +bool SetSocketNonBlocking(const SOCKET& hSocket); + /** Return readable error string for a network error code */ std::string NetworkErrorString(int err); From b527b549504672704a61f70d2565b9489aaaba91 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Mon, 19 Apr 2021 11:35:09 +0200 Subject: [PATCH 0026/1293] net: convert standalone SetSocketNonBlocking() to Sock::SetNonBlocking() This further encapsulates syscalls inside the `Sock` class. Co-authored-by: practicalswift --- src/netbase.cpp | 5 ++--- src/test/fuzz/util.cpp | 13 +++++++++++++ src/test/fuzz/util.h | 2 ++ src/test/util/net.h | 2 ++ src/util/sock.cpp | 18 +++++++++++------- src/util/sock.h | 9 ++++++--- 6 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/netbase.cpp b/src/netbase.cpp index b13d6b1ae0bd8..d504107d91c64 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -304,8 +304,7 @@ enum class IntrRecvError { * read. * * @see This function can be interrupted by calling InterruptSocks5(bool). - * Sockets can be made non-blocking with SetSocketNonBlocking(const - * SOCKET&). + * Sockets can be made non-blocking with Sock::SetNonBlocking(). */ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, const Sock& sock) { @@ -525,7 +524,7 @@ std::unique_ptr CreateSockTCP(const CService& address_family) } // Set the non-blocking option on the socket. - if (!SetSocketNonBlocking(sock->Get())) { + if (!sock->SetNonBlocking()) { LogPrintf("Error setting socket to non-blocking: %s\n", NetworkErrorString(WSAGetLastError())); return nullptr; } diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index 1d1985667126d..0faece58546e9 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -254,6 +254,19 @@ int FuzzedSock::GetSockName(sockaddr* name, socklen_t* name_len) const return 0; } +bool FuzzedSock::SetNonBlocking() const +{ + constexpr std::array setnonblocking_errnos{ + EBADF, + EPERM, + }; + if (m_fuzzed_data_provider.ConsumeBool()) { + SetFuzzedErrNo(m_fuzzed_data_provider, setnonblocking_errnos); + return false; + } + return true; +} + bool FuzzedSock::IsSelectable() const { return m_selectable; diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 92ad08ec59aed..c5ae78da67400 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -80,6 +80,8 @@ class FuzzedSock : public Sock int GetSockName(sockaddr* name, socklen_t* name_len) const override; + bool SetNonBlocking() const override; + bool IsSelectable() const override; bool Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred = nullptr) const override; diff --git a/src/test/util/net.h b/src/test/util/net.h index 749b5b33644a3..814a4ff9c85c0 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -166,6 +166,8 @@ class StaticContentsSock : public Sock return 0; } + bool SetNonBlocking() const override { return true; } + bool IsSelectable() const override { return true; } bool Wait(std::chrono::milliseconds timeout, diff --git a/src/util/sock.cpp b/src/util/sock.cpp index b2b5133629c8f..84ac2759faaab 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -117,18 +117,22 @@ int Sock::GetSockName(sockaddr* name, socklen_t* name_len) const return getsockname(m_socket, name, name_len); } -bool SetSocketNonBlocking(const SOCKET& hSocket) +bool Sock::SetNonBlocking() const { #ifdef WIN32 - u_long nOne = 1; - if (ioctlsocket(hSocket, FIONBIO, &nOne) == SOCKET_ERROR) { + u_long on{1}; + if (ioctlsocket(m_socket, FIONBIO, &on) == SOCKET_ERROR) { + return false; + } #else - int fFlags = fcntl(hSocket, F_GETFL, 0); - if (fcntl(hSocket, F_SETFL, fFlags | O_NONBLOCK) == SOCKET_ERROR) { -#endif + const int flags{fcntl(m_socket, F_GETFL, 0)}; + if (flags == SOCKET_ERROR) { return false; } - + if (fcntl(m_socket, F_SETFL, flags | O_NONBLOCK) == SOCKET_ERROR) { + return false; + } +#endif return true; } diff --git a/src/util/sock.h b/src/util/sock.h index b97977da95bf5..791228490417b 100644 --- a/src/util/sock.h +++ b/src/util/sock.h @@ -133,6 +133,12 @@ class Sock */ [[nodiscard]] virtual int GetSockName(sockaddr* name, socklen_t* name_len) const; + /** + * Set the non-blocking option on the socket. + * @return true if set successfully + */ + [[nodiscard]] virtual bool SetNonBlocking() const; + /** * Check if the underlying socket can be used for `select(2)` (or the `Wait()` method). * @return true if selectable @@ -273,9 +279,6 @@ class Sock void Close(); }; -/** Enable non-blocking mode for a socket */ -bool SetSocketNonBlocking(const SOCKET& hSocket); - /** Return readable error string for a network error code */ std::string NetworkErrorString(int err); From 6d58117a31a88eec3f0a103f9d1fc26cf0b48348 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sun, 2 Jan 2022 11:10:25 +0200 Subject: [PATCH 0027/1293] build: Build minisketch test in `make check`, not in `make` --- ci/test/06_script_b.sh | 2 ++ src/Makefile.am | 1 + src/Makefile.minisketch.include | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index b60c9b6d30a77..9b8f3a02445f8 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -9,12 +9,14 @@ export LC_ALL=C.UTF-8 if [[ $HOST = *-mingw32 ]]; then # Generate all binaries, so that they can be wrapped CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 + CI_EXEC make "$MAKEJOBS" -C src minisketch/test.exe VERBOSE=1 CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" fi if [ -n "$QEMU_USER_CMD" ]; then # Generate all binaries, so that they can be wrapped CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 + CI_EXEC make "$MAKEJOBS" -C src minisketch/test VERBOSE=1 CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" fi diff --git a/src/Makefile.am b/src/Makefile.am index 23bc1800953a2..3b49e0cb01be4 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -20,6 +20,7 @@ noinst_LTLIBRARIES = bin_PROGRAMS = noinst_PROGRAMS = +check_PROGRAMS = TESTS = BENCHMARKS = diff --git a/src/Makefile.minisketch.include b/src/Makefile.minisketch.include index b337f483498ed..1363bec34eac2 100644 --- a/src/Makefile.minisketch.include +++ b/src/Makefile.minisketch.include @@ -31,7 +31,7 @@ if ENABLE_TESTS if !ENABLE_FUZZ MINISKETCH_TEST = minisketch/test TESTS += $(MINISKETCH_TEST) -noinst_PROGRAMS += $(MINISKETCH_TEST) +check_PROGRAMS += $(MINISKETCH_TEST) minisketch_test_SOURCES = $(MINISKETCH_TEST_SOURCES_INT) minisketch_test_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS) From fa3ea81c3e7d962715788ab5525958a532c51414 Mon Sep 17 00:00:00 2001 From: MacroFake Date: Mon, 30 May 2022 16:10:32 +0200 Subject: [PATCH 0028/1293] refactor: Add LIFETIMEBOUND / -Wdangling-gsl to Assert() --- src/test/util_tests.cpp | 5 +++++ src/util/check.h | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index fda56ccff7819..5766fff92d2a1 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -122,6 +122,11 @@ BOOST_AUTO_TEST_CASE(util_check) // Check nested Asserts BOOST_CHECK_EQUAL(Assert((Assert(x).test() ? 3 : 0)), 3); + + // Check -Wdangling-gsl does not trigger when copying the int. (It would + // trigger on "const int&") + const int nine{*Assert(std::optional{9})}; + BOOST_CHECK_EQUAL(9, nine); } BOOST_AUTO_TEST_CASE(util_criticalsection) diff --git a/src/util/check.h b/src/util/check.h index aca957925a282..49f07de9dd640 100644 --- a/src/util/check.h +++ b/src/util/check.h @@ -9,6 +9,7 @@ #include #endif +#include #include #include @@ -24,7 +25,7 @@ class NonFatalCheckError : public std::runtime_error /** Helper for CHECK_NONFATAL() */ template -T&& inline_check_non_fatal(T&& val, const char* file, int line, const char* func, const char* assertion) +T&& inline_check_non_fatal(LIFETIMEBOUND T&& val, const char* file, int line, const char* func, const char* assertion) { if (!(val)) { throw NonFatalCheckError( @@ -56,7 +57,7 @@ void assertion_fail(const char* file, int line, const char* func, const char* as /** Helper for Assert()/Assume() */ template -T&& inline_assertion_check(T&& val, [[maybe_unused]] const char* file, [[maybe_unused]] int line, [[maybe_unused]] const char* func, [[maybe_unused]] const char* assertion) +T&& inline_assertion_check(LIFETIMEBOUND T&& val, [[maybe_unused]] const char* file, [[maybe_unused]] int line, [[maybe_unused]] const char* func, [[maybe_unused]] const char* assertion) { if constexpr (IS_ASSERT #ifdef ABORT_ON_FAILED_ASSUME From 599ff5adfc7e1227c6d97d861d0715aee57611dd Mon Sep 17 00:00:00 2001 From: furszy Date: Thu, 30 Jun 2022 20:51:40 -0300 Subject: [PATCH 0029/1293] wallet: avoid double TopUp() calls on descriptor wallets Move TopUp() responsibility from the wallet class to each scriptpubkeyman. So each spkm can decide to call it or not after perform the basic checks for the new destination request. Reason: We were calling it twice in the following flows for descriptor wallets: A) CWallet::GetNewDestination: 1) Calls spk_man->TopUp() 2) Calls spk_man->GetNewDestination() --> which, after the basic script checks, calls TopUp() again. B) CWallet::GetReservedDestination: 1) Calls spk_man->TopUp() 2) Calls spk_man->GetReservedDestination() --> which calls to GetNewDestination (which calls to TopUp again). --- src/wallet/scriptpubkeyman.cpp | 6 ++++++ src/wallet/wallet.cpp | 6 +----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 7ff017775e07f..496dfd81a72f3 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -28,6 +28,9 @@ util::Result LegacyScriptPubKeyMan::GetNewDestination(const Outp } assert(type != OutputType::BECH32M); + // Fill-up keypool if needed + TopUp(); + LOCK(cs_KeyStore); // Generate a new key that is added to wallet @@ -304,6 +307,9 @@ util::Result LegacyScriptPubKeyMan::GetReservedDestination(const return util::Error{_("Error: Keypool ran out, please call keypoolrefill first")}; } + // Fill-up keypool if needed + TopUp(); + if (!ReserveKeyFromKeyPool(index, keypool, internal)) { return util::Error{_("Error: Keypool ran out, please call keypoolrefill first")}; } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index de1078e646453..117a747c9767f 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2346,7 +2346,6 @@ util::Result CWallet::GetNewDestination(const OutputType type, c return util::Error{strprintf(_("Error: No %s addresses available."), FormatOutputType(type))}; } - spk_man->TopUp(); auto op_dest = spk_man->GetNewDestination(type); if (op_dest) { SetAddressBook(*op_dest, label, "receive"); @@ -2440,10 +2439,7 @@ util::Result ReserveDestination::GetReservedDestination(bool int return util::Error{strprintf(_("Error: No %s addresses available."), FormatOutputType(type))}; } - if (nIndex == -1) - { - m_spk_man->TopUp(); - + if (nIndex == -1) { CKeyPool keypool; auto op_address = m_spk_man->GetReservedDestination(type, internal, nIndex, keypool); if (!op_address) return op_address; From 76b982a4a5328c1357dbc5361317f682db160876 Mon Sep 17 00:00:00 2001 From: furszy Date: Sat, 4 Jun 2022 14:03:25 -0300 Subject: [PATCH 0030/1293] wallet: remove unused `nAccountingEntryNumber` field --- src/wallet/wallet.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index cf33ea21f2aff..b6bcca401536d 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -398,7 +398,6 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati TxItems wtxOrdered; int64_t nOrderPosNext GUARDED_BY(cs_wallet) = 0; - uint64_t nAccountingEntryNumber = 0; std::map m_address_book GUARDED_BY(cs_wallet); const CAddressBookData* FindAddressBookEntry(const CTxDestination&, bool allow_change = false) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); From bfb9b94ebefdb95ac7656836975b3d5afc428744 Mon Sep 17 00:00:00 2001 From: furszy Date: Fri, 1 Jul 2022 16:56:51 -0300 Subject: [PATCH 0031/1293] wallet: remove duplicate descriptor type check in GetNewDestination --- src/wallet/scriptpubkeyman.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 496dfd81a72f3..daf122e0e1d41 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -1676,7 +1676,7 @@ util::Result DescriptorScriptPubKeyMan::GetNewDestination(const std::optional desc_addr_type = m_wallet_descriptor.descriptor->GetOutputType(); assert(desc_addr_type); if (type != *desc_addr_type) { - throw std::runtime_error(std::string(__func__) + ": Types are inconsistent"); + throw std::runtime_error(std::string(__func__) + ": Types are inconsistent. Stored type does not match type of newly generated address"); } TopUp(); @@ -1694,11 +1694,8 @@ util::Result DescriptorScriptPubKeyMan::GetNewDestination(const } CTxDestination dest; - std::optional out_script_type = m_wallet_descriptor.descriptor->GetOutputType(); - if (out_script_type && out_script_type == type) { - ExtractDestination(scripts_temp[0], dest); - } else { - throw std::runtime_error(std::string(__func__) + ": Types are inconsistent. Stored type does not match type of newly generated address"); + if (!ExtractDestination(scripts_temp[0], dest)) { + return util::Error{_("Error: Cannot extract destination from the generated scriptpubkey")}; // shouldn't happen } m_wallet_descriptor.next_index++; WalletBatch(m_storage.GetDatabase()).WriteDescriptor(GetID(), m_wallet_descriptor); From fdb8dc8a5a10c39927dc4b8ddc5e0038a633c50e Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Sun, 14 Aug 2022 15:58:47 -0400 Subject: [PATCH 0032/1293] gui: Show watchonly balance only for Legacy wallets Descriptor wallets do not have a watchonly balance as wallets are designated watchonly or not. Thus we should not be displaying the empty watchonly balance for descriptor wallets. --- src/qt/sendcoinsdialog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index bd44d127818b9..b9623d2041019 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -703,7 +703,7 @@ void SendCoinsDialog::setBalance(const interfaces::WalletBalances& balances) CAmount balance = balances.balance; if (model->wallet().hasExternalSigner()) { ui->labelBalanceName->setText(tr("External balance:")); - } else if (model->wallet().privateKeysDisabled()) { + } else if (model->wallet().isLegacy() && model->wallet().privateKeysDisabled()) { balance = balances.watch_only_balance; ui->labelBalanceName->setText(tr("Watch-only balance:")); } From 0652dc53b291bd295caff4093ec2854fd4b34645 Mon Sep 17 00:00:00 2001 From: Jeremy Rubin Date: Tue, 16 Aug 2022 12:09:27 -0700 Subject: [PATCH 0033/1293] [BugFix]: Do not allow deserializing PSBT with empty PSBT_OUT_TAP_TREE --- src/psbt.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/psbt.h b/src/psbt.h index d5c67802c7e48..b8c35c99707fd 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -862,6 +862,9 @@ struct PSBTOutput std::vector tree_v; s >> tree_v; SpanReader s_tree(s.GetType(), s.GetVersion(), tree_v); + if (s_tree.empty()) { + throw std::ios_base::failure("Output Taproot tree must not be empty"); + } while (!s_tree.empty()) { uint8_t depth; uint8_t leaf_ver; From 91497031cbd74a0665b7fc31eb6b73bfb7bd0d40 Mon Sep 17 00:00:00 2001 From: brunoerg Date: Thu, 16 Jun 2022 19:13:18 -0300 Subject: [PATCH 0034/1293] rest: add `/deploymentinfo` --- src/rest.cpp | 44 ++++++++++++++++++++++++++++++++++++++++++ src/rpc/blockchain.cpp | 2 +- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/rest.cpp b/src/rest.cpp index 85815a9c8baae..8fdc3ef38d969 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -590,6 +590,48 @@ static bool rest_chaininfo(const std::any& context, HTTPRequest* req, const std: } } + +RPCHelpMan getdeploymentinfo(); + +static bool rest_deploymentinfo(const std::any& context, HTTPRequest* req, const std::string& str_uri_part) +{ + if (!CheckWarmup(req)) return false; + + std::string hash_str; + const RESTResponseFormat rf = ParseDataFormat(hash_str, str_uri_part); + + switch (rf) { + case RESTResponseFormat::JSON: { + JSONRPCRequest jsonRequest; + jsonRequest.context = context; + jsonRequest.params = UniValue(UniValue::VARR); + + if (!hash_str.empty()) { + uint256 hash; + if (!ParseHashStr(hash_str, hash)) { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hash_str); + } + + const ChainstateManager* chainman = GetChainman(context, req); + if (!chainman) return false; + if (!WITH_LOCK(::cs_main, return chainman->m_blockman.LookupBlockIndex(ParseHashV(hash_str, "blockhash")))) { + return RESTERR(req, HTTP_BAD_REQUEST, "Block not found"); + } + + jsonRequest.params.pushKV("blockhash", hash_str); + } + + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, getdeploymentinfo().HandleRequest(jsonRequest).write() + "\n"); + return true; + } + default: { + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); + } + } + +} + static bool rest_mempool(const std::any& context, HTTPRequest* req, const std::string& str_uri_part) { if (!CheckWarmup(req)) @@ -935,6 +977,8 @@ static const struct { {"/rest/mempool/", rest_mempool}, {"/rest/headers/", rest_headers}, {"/rest/getutxos", rest_getutxos}, + {"/rest/deploymentinfo/", rest_deploymentinfo}, + {"/rest/deploymentinfo", rest_deploymentinfo}, {"/rest/blockhashbyheight/", rest_blockhash_by_height}, }; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 8f116a05efc39..ad7961a270479 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1307,7 +1307,7 @@ UniValue DeploymentInfo(const CBlockIndex* blockindex, const ChainstateManager& } } // anon namespace -static RPCHelpMan getdeploymentinfo() +RPCHelpMan getdeploymentinfo() { return RPCHelpMan{"getdeploymentinfo", "Returns an object containing various state info regarding deployments of consensus changes.", From 3e44bee08eb93e086179b92007649d47652aa439 Mon Sep 17 00:00:00 2001 From: brunoerg Date: Thu, 16 Jun 2022 19:13:37 -0300 Subject: [PATCH 0035/1293] test: add coverage for `/rest/deploymentinfo` --- test/functional/interface_rest.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index f36bbda3afd0b..eec1c8fffbf6a 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -383,6 +383,17 @@ def run_test(self): assert_equal(self.test_rest_request(f"/headers/{bb_hash}", query_params={"count": 1}), self.test_rest_request(f"/headers/1/{bb_hash}")) assert_equal(self.test_rest_request(f"/blockfilterheaders/basic/{bb_hash}", query_params={"count": 1}), self.test_rest_request(f"/blockfilterheaders/basic/5/{bb_hash}")) + self.log.info("Test the /deploymentinfo URI") + + deployment_info = self.nodes[0].getdeploymentinfo() + assert_equal(deployment_info, self.test_rest_request('/deploymentinfo')) + + non_existing_blockhash = '42759cde25462784395a337460bde75f58e73d3f08bd31fdc3507cbac856a2c4' + resp = self.test_rest_request(f'/deploymentinfo/{non_existing_blockhash}', ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), "Block not found") + + resp = self.test_rest_request(f"/deploymentinfo/{INVALID_PARAM}", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), f"Invalid hash: {INVALID_PARAM}") if __name__ == '__main__': RESTTest().main() From 5c960200242d237f2cf74309b8fd29e8162682ed Mon Sep 17 00:00:00 2001 From: brunoerg Date: Fri, 17 Jun 2022 17:56:42 -0300 Subject: [PATCH 0036/1293] doc: add `/deploymentinfo` in REST-interface --- doc/REST-interface.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 4b46f29153105..e86f93702b4b2 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -84,6 +84,15 @@ Returns various state info regarding block chain processing. Only supports JSON as output format. Refer to the `getblockchaininfo` RPC help for details. +#### Deployment info +`GET /rest/deploymentinfo.json` +`GET /rest/deploymentinfo/.json` + +Returns an object containing various state info regarding deployments of +consensus changes at the current chain tip, or at if provided. +Only supports JSON as output format. +Refer to the `getdeploymentinfo` RPC help for details. + #### Query UTXO set `GET /rest/getutxos//-/-/.../-.` From a8250e30f16f2919ea5aa122b2880b076bd398a3 Mon Sep 17 00:00:00 2001 From: brunoerg Date: Thu, 7 Jul 2022 18:10:24 -0300 Subject: [PATCH 0037/1293] doc: add release note about `/rest/deploymentinfo` --- doc/release-notes-25412.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/release-notes-25412.md diff --git a/doc/release-notes-25412.md b/doc/release-notes-25412.md new file mode 100644 index 0000000000000..b11fe73d450aa --- /dev/null +++ b/doc/release-notes-25412.md @@ -0,0 +1,5 @@ +New REST endpoint +----------------- + +- A new `/rest/deploymentinfo` endpoint has been added for fetching various + state info regarding deployments of consensus changes. (#25412) From cc8dff5f8ff865cb7b3cbd8410e254de5dc86d19 Mon Sep 17 00:00:00 2001 From: fanquake Date: Fri, 19 Aug 2022 09:44:28 +0100 Subject: [PATCH 0038/1293] depends: Boost 1.80.0 https://www.boost.org/users/history/version_1_80_0.html --- depends/packages/boost.mk | 4 ++-- doc/dependencies.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index 563848c39889d..e3625d97f5661 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -1,8 +1,8 @@ package=boost -$(package)_version=1.77.0 +$(package)_version=1.80.0 $(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/$($(package)_version)/source/ $(package)_file_name=boost_$(subst .,_,$($(package)_version)).tar.bz2 -$(package)_sha256_hash=fc9f85fc030e233142908241af7a846e60630aa7388de9a5fafb1f3a26840854 +$(package)_sha256_hash=1e19565d82e43bc59209a168f5ac899d3ba471d55c7610c677d4ccf2c9c500c0 define $(package)_stage_cmds mkdir -p $($(package)_staging_prefix_dir)/include && \ diff --git a/doc/dependencies.md b/doc/dependencies.md index 8b8259ab0a79e..c7ad5abf2051e 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -17,7 +17,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | Dependency | Releases | Version used | Minimum required | Runtime | | --- | --- | --- | --- | --- | -| [Boost](../depends/packages/boost.mk) | [link](https://www.boost.org/users/download/) | [1.77.0](https://github.com/bitcoin/bitcoin/pull/24383) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | +| [Boost](../depends/packages/boost.mk) | [link](https://www.boost.org/users/download/) | [1.80.0](https://github.com/bitcoin/bitcoin/pull/25873) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | | [libevent](../depends/packages/libevent.mk) | [link](https://github.com/libevent/libevent/releases) | [2.1.12-stable](https://github.com/bitcoin/bitcoin/pull/21991) | [2.1.8](https://github.com/bitcoin/bitcoin/pull/24681) | No | | glibc | [link](https://www.gnu.org/software/libc/) | N/A | [2.18](https://github.com/bitcoin/bitcoin/pull/23511) | Yes | | Linux Kernel | [link](https://www.kernel.org/) | N/A | 3.2.0 | Yes | From 9cbfe40d8af8567682284890c080b0c3cf434490 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Fri, 29 Jul 2022 13:23:29 +0200 Subject: [PATCH 0039/1293] net: remove useless call to IsReachable() from CConnman::Bind() `CConnman::Bind()` is called without `BF_EXPLICIT` only when passed either `0.0.0.0` or `::`. For those addresses `IsReachable()` is always true (regardless of the `-onlynet=` setting!), meaning that the `if` condition never evaluates to true. `IsReachable()` is always true for the "any" IPv4 and IPv6 addresses because `CNetAddr::GetNetwork()` returns `NET_UNROUTABLE` instead of `NET_IPV4` or `NET_IPV6` and the network `NET_UNROUTABLE` is always considered reachable. It follows that `BF_EXPLICIT` is unnecessary, remove it too. --- src/net.cpp | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index c4aaac4986a6d..f3e38872cb13c 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -91,13 +91,12 @@ static constexpr auto FEELER_SLEEP_WINDOW{1s}; /** Used to pass flags to the Bind() function */ enum BindFlags { BF_NONE = 0, - BF_EXPLICIT = (1U << 0), - BF_REPORT_ERROR = (1U << 1), + BF_REPORT_ERROR = (1U << 0), /** * Do not call AddLocal() for our special addresses, e.g., for incoming * Tor connections, to prevent gossiping them over the network. */ - BF_DONT_ADVERTISE = (1U << 2), + BF_DONT_ADVERTISE = (1U << 1), }; // The set of sockets cannot be modified while waiting @@ -2204,9 +2203,6 @@ bool CConnman::Bind(const CService& addr_, unsigned int flags, NetPermissionFlag { const CService addr{MaybeFlipIPv6toCJDNS(addr_)}; - if (!(flags & BF_EXPLICIT) && !IsReachable(addr)) { - return false; - } bilingual_str strError; if (!BindListenPort(addr, strError, permissions)) { if ((flags & BF_REPORT_ERROR) && m_client_interface) { @@ -2226,13 +2222,13 @@ bool CConnman::InitBinds(const Options& options) { bool fBound = false; for (const auto& addrBind : options.vBinds) { - fBound |= Bind(addrBind, (BF_EXPLICIT | BF_REPORT_ERROR), NetPermissionFlags::None); + fBound |= Bind(addrBind, BF_REPORT_ERROR, NetPermissionFlags::None); } for (const auto& addrBind : options.vWhiteBinds) { - fBound |= Bind(addrBind.m_service, (BF_EXPLICIT | BF_REPORT_ERROR), addrBind.m_flags); + fBound |= Bind(addrBind.m_service, BF_REPORT_ERROR, addrBind.m_flags); } for (const auto& addr_bind : options.onion_binds) { - fBound |= Bind(addr_bind, BF_EXPLICIT | BF_DONT_ADVERTISE, NetPermissionFlags::None); + fBound |= Bind(addr_bind, BF_DONT_ADVERTISE, NetPermissionFlags::None); } if (options.bind_on_any) { struct in_addr inaddr_any; From 65471008e0a4d65e301a5b934928d38c80a21f2c Mon Sep 17 00:00:00 2001 From: fanquake Date: Fri, 15 Jul 2022 11:07:08 +0100 Subject: [PATCH 0040/1293] depends: libnatpmp 07004b97cf691774efebe70404cf22201e4d330d This pulls in two changes I've upstreamed: Support for pkg-config: https://github.com/miniupnp/libnatpmp/pull/19 Suppressing a deprecation warning: https://github.com/miniupnp/libnatpmp/pull/28 Somewhat related to #22644. --- depends/packages/libnatpmp.mk | 4 ++-- doc/dependencies.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/depends/packages/libnatpmp.mk b/depends/packages/libnatpmp.mk index cdcf8c0bf2c18..2eddc76d9cf4e 100644 --- a/depends/packages/libnatpmp.mk +++ b/depends/packages/libnatpmp.mk @@ -1,8 +1,8 @@ package=libnatpmp -$(package)_version=4536032ae32268a45c073a4d5e91bbab4534773a +$(package)_version=07004b97cf691774efebe70404cf22201e4d330d $(package)_download_path=https://github.com/miniupnp/libnatpmp/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=543b460aab26acf91e11d15e17d8798f845304199eea2d76c2f444ec749c5383 +$(package)_sha256_hash=9321953ceb39d07c25463e266e50d0ae7b64676bb3a986d932b18881ed94f1fb define $(package)_set_vars $(package)_build_opts=CC="$($(package)_cc)" diff --git a/doc/dependencies.md b/doc/dependencies.md index 8b8259ab0a79e..ec019f003ac0a 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -35,7 +35,7 @@ You can find installation instructions in the `build-*.md` file for your platfor ### Networking | Dependency | Releases | Version used | Minimum required | Runtime | | --- | --- | --- | --- | --- | -| [libnatpmp](../depends/packages/libnatpmp.mk) | [link](https://github.com/miniupnp/libnatpmp/) | commit [4536032...](https://github.com/bitcoin/bitcoin/pull/21209) | | No | +| [libnatpmp](../depends/packages/libnatpmp.mk) | [link](https://github.com/miniupnp/libnatpmp/) | commit [07004b9...](https://github.com/bitcoin/bitcoin/pull/25917) | | No | | [MiniUPnPc](../depends/packages/miniupnpc.mk) | [link](https://miniupnp.tuxfamily.org/) | [2.2.2](https://github.com/bitcoin/bitcoin/pull/20421) | 1.9 | No | ### Notifications From 1b77db265317a6470d0914b520f04eb64b3c0942 Mon Sep 17 00:00:00 2001 From: w0xlt Date: Sat, 27 Aug 2022 03:09:38 -0300 Subject: [PATCH 0041/1293] test: add `ismine` test for descriptor scriptpubkeyman --- src/wallet/test/ismine_tests.cpp | 297 +++++++++++++++++++++++++++++-- 1 file changed, 281 insertions(+), 16 deletions(-) diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index dd5cd0af46f35..126fba99284a6 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include #include