diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 24a12d34d2..a61d66dbd7 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -517,6 +517,7 @@ public: std::optional FetchBlock(NodeId peer_id, const CBlockIndex& block_index) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) const override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); + std::vector GetOrphanTransactions() override EXCLUSIVE_LOCKS_REQUIRED(!m_tx_download_mutex); PeerManagerInfo GetInfo() const override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); void SendPings() override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); void RelayTransaction(const uint256& txid, const uint256& wtxid) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); @@ -1934,6 +1935,12 @@ bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) c return true; } +std::vector PeerManagerImpl::GetOrphanTransactions() +{ + LOCK(m_tx_download_mutex); + return m_orphanage.GetOrphanTransactions(); +} + PeerManagerInfo PeerManagerImpl::GetInfo() const { return PeerManagerInfo{ diff --git a/src/net_processing.h b/src/net_processing.h index a413db98e8..b9028b0c69 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -7,6 +7,7 @@ #define BITCOIN_NET_PROCESSING_H #include +#include #include #include @@ -93,6 +94,8 @@ public: /** Get statistics from node state */ virtual bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) const = 0; + virtual std::vector GetOrphanTransactions() = 0; + /** Get peer manager info. */ virtual PeerManagerInfo GetInfo() const = 0; diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index e51f1eb582..2bbe18e4f6 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -274,6 +274,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getmempoolinfo", 0, "with_fee_histogram" }, { "getrawmempool", 0, "verbose" }, { "getrawmempool", 1, "mempool_sequence" }, + { "getorphantxs", 0, "verbosity" }, { "estimatesmartfee", 0, "conf_target" }, { "estimaterawfee", 0, "conf_target" }, { "estimaterawfee", 1, "threshold" }, diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index f96c5fb8a4..6aa2910ebd 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -8,8 +8,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -29,6 +31,7 @@ #include #include #include +#include #include #include @@ -1050,6 +1053,107 @@ static RPCHelpMan savemempool() }; } +static std::vector OrphanDescription() +{ + return { + RPCResult{RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, + RPCResult{RPCResult::Type::STR_HEX, "wtxid", "The transaction witness hash in hex"}, + RPCResult{RPCResult::Type::NUM, "bytes", "The serialized transaction size in bytes"}, + RPCResult{RPCResult::Type::NUM, "vsize", "The virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted. CAUTION: Since orphan transactions are missing input data, this can be incorrect!"}, + RPCResult{RPCResult::Type::NUM, "weight", "The transaction weight as defined in BIP 141."}, + RPCResult{RPCResult::Type::NUM_TIME, "entry", "The entry time into the orphanage expressed in " + UNIX_EPOCH_TIME}, + RPCResult{RPCResult::Type::NUM_TIME, "expiration", "The orphan expiration time expressed in " + UNIX_EPOCH_TIME}, + RPCResult{RPCResult::Type::ARR, "from", "", + { + RPCResult{RPCResult::Type::NUM, "peer_id", "Peer ID"}, + }}, + }; +} + +static UniValue OrphanToJSON(const TxOrphanage::OrphanTxBase& orphan) +{ + UniValue o(UniValue::VOBJ); + o.pushKV("txid", orphan.tx->GetHash().ToString()); + o.pushKV("wtxid", orphan.tx->GetWitnessHash().ToString()); + o.pushKV("bytes", orphan.tx->GetTotalSize()); + o.pushKV("vsize", GetVirtualTransactionSize(*orphan.tx)); + o.pushKV("weight", GetTransactionWeight(*orphan.tx)); + o.pushKV("entry", int64_t{TicksSinceEpoch(orphan.nTimeExpire - ORPHAN_TX_EXPIRE_TIME)}); + o.pushKV("expiration", int64_t{TicksSinceEpoch(orphan.nTimeExpire)}); + UniValue from(UniValue::VARR); + from.push_back(orphan.fromPeer); // only one fromPeer for now + o.pushKV("from", from); + return o; +} + +static RPCHelpMan getorphantxs() +{ + return RPCHelpMan{"getorphantxs", + "\nShows transactions in the tx orphanage.\n" + "\nEXPERIMENTAL warning: this call may be changed in future releases.\n", + { + {"verbosity", RPCArg::Type::NUM, RPCArg::Default{0}, "0 for an array of txids (may contain duplicates), 1 for an array of objects with tx details, and 2 for details from (1) and tx hex", + RPCArgOptions{.skip_type_check = true}}, + }, + { + RPCResult{"for verbose = 0", + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, + }}, + RPCResult{"for verbose = 1", + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::OBJ, "", "", OrphanDescription()}, + }}, + RPCResult{"for verbose = 2", + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::OBJ, "", "", + Cat>( + OrphanDescription(), + {{RPCResult::Type::STR_HEX, "hex", "The serialized, hex-encoded transaction data"}} + ) + }, + }}, + }, + RPCExamples{ + HelpExampleCli("getorphantxs", "2") + + HelpExampleRpc("getorphantxs", "2") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + const NodeContext& node = EnsureAnyNodeContext(request.context); + PeerManager& peerman = EnsurePeerman(node); + std::vector orphanage = peerman.GetOrphanTransactions(); + + int verbosity{ParseVerbosity(request.params[0], /*default_verbosity=*/0, /*allow_bool*/false)}; + + UniValue ret(UniValue::VARR); + + if (verbosity == 0) { + for (auto const& orphan : orphanage) { + ret.push_back(orphan.tx->GetHash().ToString()); + } + } else if (verbosity == 1) { + for (auto const& orphan : orphanage) { + ret.push_back(OrphanToJSON(orphan)); + } + } else if (verbosity == 2) { + for (auto const& orphan : orphanage) { + UniValue o{OrphanToJSON(orphan)}; + o.pushKV("hex", EncodeHexTx(*orphan.tx)); + ret.push_back(o); + } + } else { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid verbosity value " + ToString(verbosity)); + } + + return ret; + }, + }; +} + static RPCHelpMan submitpackage() { return RPCHelpMan{"submitpackage", @@ -1266,6 +1370,7 @@ void RegisterMempoolRPCCommands(CRPCTable& t) {"blockchain", &importmempool}, {"blockchain", &savemempool}, {"blockchain", &maxmempool}, + {"hidden", &getorphantxs}, {"rawtransactions", &submitpackage}, {"rawtransactions", &listmempooltransactions}, }; diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 8ec89d5020..a9f927e6c6 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -80,6 +80,21 @@ void RPCTypeCheckObj(const UniValue& o, } } +int ParseVerbosity(const UniValue& arg, int default_verbosity, bool allow_bool) +{ + if (!arg.isNull()) { + if (arg.isBool()) { + if (!allow_bool) { + throw JSONRPCError(RPC_TYPE_ERROR, "Verbosity was boolean but only integer allowed"); + } + return arg.get_bool(); // true = 1 + } else { + return arg.getInt(); + } + } + return default_verbosity; +} + CAmount AmountFromValue(const UniValue& value, int decimals) { if (!value.isNum() && !value.isStr()) diff --git a/src/rpc/util.h b/src/rpc/util.h index d93f562bd5..51a5b5d888 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -101,6 +101,17 @@ uint256 ParseHashO(const UniValue& o, std::string_view strKey); std::vector ParseHexV(const UniValue& v, std::string_view name); std::vector ParseHexO(const UniValue& o, std::string_view strKey); +/** + * Parses verbosity from provided UniValue. + * + * @param[in] arg The verbosity argument as an int (0, 1, 2,...) or bool if allow_bool is set to true + * @param[in] default_verbosity The value to return if verbosity argument is null + * @param[in] allow_bool If true, allows arg to be a bool and parses it + * @returns An integer describing the verbosity level (e.g. 0, 1, 2, etc.) + * @throws JSONRPCError if allow_bool is false but arg provided is boolean + */ +int ParseVerbosity(const UniValue& arg, int default_verbosity, bool allow_bool); + /** * Validate and return a CAmount from a UniValue number or string. * diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index fb0b3b56ea..8b55bf4484 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -147,6 +147,7 @@ const std::vector RPC_COMMANDS_SAFE_FOR_FUZZING{ "getnetworkhashps", "getnetworkinfo", "getnodeaddresses", + "getorphantxs", "getpeerinfo", "getprioritisedtransactions", "getrawaddrman", diff --git a/src/txorphanage.cpp b/src/txorphanage.cpp index faab208333..bc027f8890 100644 --- a/src/txorphanage.cpp +++ b/src/txorphanage.cpp @@ -33,7 +33,7 @@ bool TxOrphanage::AddTx(const CTransactionRef& tx, NodeId peer) return false; } - auto ret = m_orphans.emplace(wtxid, OrphanTx{tx, peer, Now() + ORPHAN_TX_EXPIRE_TIME, m_orphan_list.size()}); + auto ret = m_orphans.emplace(wtxid, OrphanTx{{tx, peer, Now() + ORPHAN_TX_EXPIRE_TIME}, m_orphan_list.size()}); assert(ret.second); m_orphan_list.push_back(ret.first); for (const CTxIn& txin : tx->vin) { @@ -277,3 +277,13 @@ std::vector> TxOrphanage::GetChildrenFromDiff } return children_found; } + +std::vector TxOrphanage::GetOrphanTransactions() const +{ + std::vector ret; + ret.reserve(m_orphans.size()); + for (auto const& o : m_orphans) { + ret.push_back({o.second.tx, o.second.fromPeer, o.second.nTimeExpire}); + } + return ret; +} diff --git a/src/txorphanage.h b/src/txorphanage.h index 2c53d1d40f..5501d10922 100644 --- a/src/txorphanage.h +++ b/src/txorphanage.h @@ -72,11 +72,17 @@ public: return m_orphans.size(); } -protected: - struct OrphanTx { + /** Allows providing orphan information externally */ + struct OrphanTxBase { CTransactionRef tx; NodeId fromPeer; NodeSeconds nTimeExpire; + }; + + std::vector GetOrphanTransactions() const; + +protected: + struct OrphanTx : public OrphanTxBase { size_t list_pos; }; diff --git a/test/functional/p2p_orphan_handling.py b/test/functional/p2p_orphan_handling.py index 22600bf8a4..9ec23e9d6e 100755 --- a/test/functional/p2p_orphan_handling.py +++ b/test/functional/p2p_orphan_handling.py @@ -5,6 +5,7 @@ import time +from test_framework.mempool_util import tx_in_orphanage from test_framework.messages import ( CInv, CTxInWitness, @@ -41,6 +42,8 @@ from test_framework.wallet import ( # for one peer and y seconds for another, use specific values instead. TXREQUEST_TIME_SKIP = NONPREF_PEER_TX_DELAY + TXID_RELAY_DELAY + OVERLOADED_PEER_TX_DELAY + 1 +DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100 + def cleanup(func): # Time to fastfoward (using setmocktime) in between subtests to ensure they do not interfere with # one another, in seconds. Equal to 12 hours, which is enough to expire anything that may exist @@ -566,6 +569,47 @@ class OrphanHandlingTest(BitcoinTestFramework): assert tx_child["txid"] in node_mempool assert_equal(node.getmempoolentry(tx_child["txid"])["wtxid"], tx_child["wtxid"]) + @cleanup + def test_max_orphan_amount(self): + self.log.info("Check that we never exceed our storage limits for orphans") + + node = self.nodes[0] + self.generate(self.wallet, 1) + peer_1 = node.add_p2p_connection(P2PInterface()) + + self.log.info("Check that orphanage is empty on start of test") + assert len(node.getorphantxs()) == 0 + + self.log.info("Filling up orphanage with " + str(DEFAULT_MAX_ORPHAN_TRANSACTIONS) + "(DEFAULT_MAX_ORPHAN_TRANSACTIONS) orphans") + orphans = [] + parent_orphans = [] + for _ in range(DEFAULT_MAX_ORPHAN_TRANSACTIONS): + tx_parent_1 = self.wallet.create_self_transfer() + tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"]) + parent_orphans.append(tx_parent_1["tx"]) + orphans.append(tx_child_1["tx"]) + peer_1.send_message(msg_tx(tx_child_1["tx"])) + + peer_1.sync_with_ping() + orphanage = node.getorphantxs() + assert_equal(len(orphanage), DEFAULT_MAX_ORPHAN_TRANSACTIONS) + + for orphan in orphans: + assert tx_in_orphanage(node, orphan) + + self.log.info("Check that we do not add more than the max orphan amount") + tx_parent_1 = self.wallet.create_self_transfer() + tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"]) + peer_1.send_and_ping(msg_tx(tx_child_1["tx"])) + parent_orphans.append(tx_parent_1["tx"]) + orphanage = node.getorphantxs() + assert_equal(len(orphanage), DEFAULT_MAX_ORPHAN_TRANSACTIONS) + + self.log.info("Clearing the orphanage") + for index, parent_orphan in enumerate(parent_orphans): + peer_1.send_and_ping(msg_tx(parent_orphan)) + assert_equal(len(node.getorphantxs()),0) + def run_test(self): self.nodes[0].setmocktime(int(time.time())) @@ -582,6 +626,7 @@ class OrphanHandlingTest(BitcoinTestFramework): self.test_same_txid_orphan() self.test_same_txid_orphan_of_orphan() self.test_orphan_txid_inv() + self.test_max_orphan_amount() if __name__ == '__main__': diff --git a/test/functional/rpc_orphans.py b/test/functional/rpc_orphans.py new file mode 100755 index 0000000000..4871166a39 --- /dev/null +++ b/test/functional/rpc_orphans.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Tests for orphan related RPCs.""" + +import time + +from test_framework.mempool_util import ( + ORPHAN_TX_EXPIRE_TIME, + tx_in_orphanage, +) +from test_framework.messages import msg_tx +from test_framework.p2p import P2PInterface +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.wallet import MiniWallet + + +class OrphanRPCsTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.test_orphan_activity() + self.test_orphan_details() + self.test_misc() + + def test_orphan_activity(self): + self.log.info("Check that orphaned transactions are returned with getorphantxs") + node = self.nodes[0] + + self.log.info("Create two 1P1C packages, but only broadcast the children") + tx_parent_1 = self.wallet.create_self_transfer() + tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"]) + tx_parent_2 = self.wallet.create_self_transfer() + tx_child_2 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_2["new_utxo"]) + peer = node.add_p2p_connection(P2PInterface()) + peer.send_and_ping(msg_tx(tx_child_1["tx"])) + peer.send_and_ping(msg_tx(tx_child_2["tx"])) + + self.log.info("Check that neither parent is in the mempool") + assert_equal(node.getmempoolinfo()["size"], 0) + + orphanage = node.getorphantxs(verbosity=0) + self.log.info("Check the size of the orphanage") + assert_equal(len(orphanage), 2) + self.log.info("Check that undefined verbosity is disallowed") + assert_raises_rpc_error(-8, "Invalid verbosity value -1", node.getorphantxs, verbosity=-1) + assert_raises_rpc_error(-8, "Invalid verbosity value 3", node.getorphantxs, verbosity=3) + self.log.info("Check that both children are in the orphanage") + assert tx_in_orphanage(node, tx_child_1["tx"]) + assert tx_in_orphanage(node, tx_child_2["tx"]) + + self.log.info("Broadcast parent 1") + peer.send_and_ping(msg_tx(tx_parent_1["tx"])) + self.log.info("Check that parent 1 and child 1 are in the mempool") + raw_mempool = node.getrawmempool() + assert_equal(len(raw_mempool), 2) + assert tx_parent_1["txid"] in raw_mempool + assert tx_child_1["txid"] in raw_mempool + + self.log.info("Check that orphanage only contains child 2") + orphanage = node.getorphantxs() + assert_equal(len(orphanage), 1) + assert tx_in_orphanage(node, tx_child_2["tx"]) + + peer.send_and_ping(msg_tx(tx_parent_2["tx"])) + self.log.info("Check that all parents and children are now in the mempool") + raw_mempool = node.getrawmempool() + assert_equal(len(raw_mempool), 4) + assert tx_parent_1["txid"] in raw_mempool + assert tx_child_1["txid"] in raw_mempool + assert tx_parent_2["txid"] in raw_mempool + assert tx_child_2["txid"] in raw_mempool + self.log.info("Check that the orphanage is empty") + assert_equal(len(node.getorphantxs()), 0) + + self.log.info("Confirm the transactions (clears mempool)") + self.generate(node, 1) + assert_equal(node.getmempoolinfo()["size"], 0) + + def test_orphan_details(self): + self.log.info("Check the transaction details returned from getorphantxs") + node = self.nodes[0] + + self.log.info("Create two orphans, from different peers") + tx_parent_1 = self.wallet.create_self_transfer() + tx_child_1 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_1["new_utxo"]) + tx_parent_2 = self.wallet.create_self_transfer() + tx_child_2 = self.wallet.create_self_transfer(utxo_to_spend=tx_parent_2["new_utxo"]) + peer_1 = node.add_p2p_connection(P2PInterface()) + peer_2 = node.add_p2p_connection(P2PInterface()) + entry_time = int(time.time()) + node.setmocktime(entry_time) + peer_1.send_and_ping(msg_tx(tx_child_1["tx"])) + peer_2.send_and_ping(msg_tx(tx_child_2["tx"])) + + orphanage = node.getorphantxs(verbosity=2) + assert tx_in_orphanage(node, tx_child_1["tx"]) + assert tx_in_orphanage(node, tx_child_2["tx"]) + + self.log.info("Check that orphan 1 and 2 were from different peers") + assert orphanage[0]["from"][0] != orphanage[1]["from"][0] + + self.log.info("Unorphan child 2") + peer_2.send_and_ping(msg_tx(tx_parent_2["tx"])) + assert not tx_in_orphanage(node, tx_child_2["tx"]) + + self.log.info("Checking orphan details") + orphanage = node.getorphantxs(verbosity=1) + assert_equal(len(node.getorphantxs()), 1) + orphan_1 = orphanage[0] + self.orphan_details_match(orphan_1, tx_child_1, verbosity=1) + self.log.info("Checking orphan entry/expiration times") + assert_equal(orphan_1["entry"], entry_time) + assert_equal(orphan_1["expiration"], entry_time + ORPHAN_TX_EXPIRE_TIME) + + self.log.info("Checking orphan details (verbosity 2)") + orphanage = node.getorphantxs(verbosity=2) + orphan_1 = orphanage[0] + self.orphan_details_match(orphan_1, tx_child_1, verbosity=2) + + def orphan_details_match(self, orphan, tx, verbosity): + self.log.info("Check txid/wtxid of orphan") + assert_equal(orphan["txid"], tx["txid"]) + assert_equal(orphan["wtxid"], tx["wtxid"]) + + self.log.info("Check the sizes of orphan") + assert_equal(orphan["bytes"], len(tx["tx"].serialize())) + assert_equal(orphan["vsize"], tx["tx"].get_vsize()) + assert_equal(orphan["weight"], tx["tx"].get_weight()) + + if verbosity == 2: + self.log.info("Check the transaction hex of orphan") + assert_equal(orphan["hex"], tx["hex"]) + + def test_misc(self): + node = self.nodes[0] + assert_raises_rpc_error(-3, "Verbosity was boolean but only integer allowed", node.getorphantxs, verbosity=True) + assert_raises_rpc_error(-3, "Verbosity was boolean but only integer allowed", node.getorphantxs, verbosity=False) + help_output = node.help() + self.log.info("Check that getorphantxs is a hidden RPC") + assert "getorphantxs" not in help_output + assert "unknown command: getorphantxs" not in node.help("getorphantxs") + + +if __name__ == '__main__': + OrphanRPCsTest(__file__).main() diff --git a/test/functional/test_framework/mempool_util.py b/test/functional/test_framework/mempool_util.py index 148cc935ed..67343ae485 100644 --- a/test/functional/test_framework/mempool_util.py +++ b/test/functional/test_framework/mempool_util.py @@ -8,6 +8,7 @@ from decimal import Decimal from .blocktools import ( COINBASE_MATURITY, ) +from .messages import CTransaction from .util import ( assert_equal, assert_greater_than, @@ -18,6 +19,8 @@ from .wallet import ( MiniWallet, ) +ORPHAN_TX_EXPIRE_TIME = 1200 + def fill_mempool(test_framework, node): """Fill mempool until eviction. @@ -79,3 +82,8 @@ def fill_mempool(test_framework, node): test_framework.log.debug("Check that mempoolminfee is larger than minrelaytxfee") assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) assert_greater_than(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) + +def tx_in_orphanage(node, tx: CTransaction) -> bool: + """Returns true if the transaction is in the orphanage.""" + found = [o for o in node.getorphantxs(verbosity=1) if o["txid"] == tx.rehash() and o["wtxid"] == tx.getwtxid()] + return len(found) == 1 diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 9da2bac6ce..abafafc4d2 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -157,6 +157,7 @@ BASE_SCRIPTS = [ 'wallet_importmulti.py --legacy-wallet', 'mempool_limit.py', 'rpc_txoutproof.py', + 'rpc_orphans.py', 'wallet_listreceivedby.py --legacy-wallet', 'wallet_listreceivedby.py --descriptors', 'wallet_abandonconflict.py --legacy-wallet',