From c0c7cb57d6fefe36e8fe2071f000d90003ed774f Mon Sep 17 00:00:00 2001 From: niftynei Date: Mon, 4 Dec 2023 21:47:36 -0600 Subject: [PATCH 1/2] rpc: add new `listmempooltransactions` Allow bitcoin-core clients to get more interesting information out of the mempool more easily, including by filtering for 'latest txs'. Eventually would be a good interface for grabbing all mempool transaction data, including information about evicted transactions (which would be useful for avoiding situations where removed transactions contained data important for bitcoin applications). This would require the addition of a mempool tx log, however the `mempool_sequence` id plus this call and the use of a ring buffer(?) would allow for clients to get complete scans of anything a bitcoin core node had seen in its mempool. That's future work. For now we start with the interface over existing information. Github-Pull: #29016 Rebased-From: cb466f4451e105028d41d81535e1c90c7259d4ad --- src/Makefile.am | 1 + src/rpc/client.cpp | 2 + src/rpc/mempool.cpp | 97 ++++++++++++++++++++++++++++++++++++++ src/rpc/mempool.h | 3 ++ src/rpc/rawtransaction.cpp | 3 +- src/rpc/rawtransaction.h | 14 ++++++ src/test/fuzz/rpc.cpp | 1 + 7 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 src/rpc/rawtransaction.h diff --git a/src/Makefile.am b/src/Makefile.am index 1ccb5332c4..54503f788d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -262,6 +262,7 @@ BITCOIN_CORE_H = \ rpc/mempool.h \ rpc/mining.h \ rpc/protocol.h \ + rpc/rawtransaction.h \ rpc/rawtransaction_util.h \ rpc/register.h \ rpc/request.h \ diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index ae0bfac9d1..483c3ddbd6 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -316,6 +316,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "stop", 0, "wait" }, { "addnode", 2, "v2transport" }, { "addconnection", 2, "v2transport" }, + { "listmempooltransactions", 0, "start_sequence"}, + { "listmempooltransactions", 1, "verbose"}, }; // clang-format on diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index b97c4a5e2f..cb113ab728 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include #include @@ -413,6 +415,100 @@ static RPCHelpMan maxmempool() }; } +static RPCHelpMan listmempooltransactions() +{ + return RPCHelpMan{"listmempooltransactions", + "\nReturns all transactions in the mempool. Can be filtered by mempool_sequence\n" + "\nAllows for syncing with current mempool entries via polling (not zmq).", + { + {"start_sequence", RPCArg::Type::NUM, RPCArg::Default{0}, "The mempool_sequence to start the results to. Defaults to 0 (zero, all transactions)."}, + {"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "True for a json object, false for array of transaction ids"}, + }, + { + RPCResult{"for verbose = false", + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::NUM, "mempool_sequence", "The current max mempool sequence value."}, + {RPCResult::Type::ARR, "txs", "", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::NUM, "entry_sequence", "The mempool sequence value for this transaction entry."}, + {RPCResult::Type::STR_HEX, "txid", "The transaction id"}, + }}, + }}, + }}, + RPCResult{"for verbose = true", + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::NUM, "mempool_sequence", "The current max mempool sequence value."}, + {RPCResult::Type::ARR, "txs", "", + { + {RPCResult::Type::OBJ, "", "", + { + Cat>( + { + {RPCResult::Type::NUM, "entry_sequence", "The mempool sequence value for this transaction entry."}, + }, + DecodeTxDoc(/*txid_field_doc=*/"The transaction id of the mempool transaction")), + }}, + }}, + }}, + }, + RPCExamples{ + HelpExampleCli("listmempooltransactions", "true") + + HelpExampleRpc("listmempooltransactions", "true") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + uint64_t start_mempool_sequence = 0; + if (!request.params[0].isNull()) { + start_mempool_sequence = request.params[0].getInt(); + } + + bool fVerbose = false; + if (!request.params[1].isNull()) + fVerbose = request.params[1].get_bool(); + + return MempoolTxsToJSON(EnsureAnyMemPool(request.context), fVerbose, start_mempool_sequence); + }, + }; +} + +UniValue MempoolTxsToJSON(const CTxMemPool& pool, bool verbose, uint64_t sequence_start) +{ + uint64_t mempool_sequence; + + LOCK(pool.cs); + mempool_sequence = pool.GetSequence(); + + UniValue o(UniValue::VOBJ); + o.pushKV("mempool_sequence", mempool_sequence); + + UniValue a(UniValue::VARR); + for (const CTxMemPoolEntry& e : pool.mapTx) { + UniValue txentry(UniValue::VOBJ); + + // We skip anything not requested. + if (e.GetSequence() < sequence_start) + continue; + + txentry.pushKV("entry_sequence", e.GetSequence()); + + if (verbose) { + // We could also calculate fees etc for this transaction, but yolo. + TxToUniv(e.GetTx(), /*block_hash=*/uint256::ZERO, /*entry=*/txentry, /*include_hex=*/false); + } else { + txentry.pushKV("txid", e.GetTx().GetHash().ToString()); + } + + a.push_back(txentry); + } + + o.pushKV("txs", a); + return o; +} + static RPCHelpMan getrawmempool() { return RPCHelpMan{"getrawmempool", @@ -1068,6 +1164,7 @@ void RegisterMempoolRPCCommands(CRPCTable& t) {"blockchain", &savemempool}, {"blockchain", &maxmempool}, {"rawtransactions", &submitpackage}, + {"rawtransactions", &listmempooltransactions}, }; for (const auto& c : commands) { t.appendCommand(c.name, &c); diff --git a/src/rpc/mempool.h b/src/rpc/mempool.h index 229d7d52dd..d0e4f9ca06 100644 --- a/src/rpc/mempool.h +++ b/src/rpc/mempool.h @@ -14,4 +14,7 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool); /** Mempool to JSON */ UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose = false, bool include_mempool_sequence = false); +/** Mempool Txs to JSON */ +UniValue MempoolTxsToJSON(const CTxMemPool& pool, bool verbose = false, uint64_t sequence_start = 0); + #endif // BITCOIN_RPC_MEMPOOL_H diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 21bc0e52f1..d3681a7056 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -93,7 +94,7 @@ static std::vector ScriptPubKeyDoc() { }; } -static std::vector DecodeTxDoc(const std::string& txid_field_doc) +std::vector DecodeTxDoc(const std::string& txid_field_doc) { return { {RPCResult::Type::STR_HEX, "txid", txid_field_doc}, diff --git a/src/rpc/rawtransaction.h b/src/rpc/rawtransaction.h new file mode 100644 index 0000000000..9b5f746223 --- /dev/null +++ b/src/rpc/rawtransaction.h @@ -0,0 +1,14 @@ +// Copyright (c) 2017-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_RPC_RAWTRANSACTION_H +#define BITCOIN_RPC_RAWTRANSACTION_H + +#include + +struct RPCResult; + +std::vector DecodeTxDoc(const std::string& txid_field_doc); + +#endif // BITCOIN_RPC_RAWTRANSACTION_H diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 2447ae3662..68c1205b4c 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -156,6 +156,7 @@ const std::vector RPC_COMMANDS_SAFE_FOR_FUZZING{ "invalidateblock", "joinpsbts", "listbanned", + "listmempooltransactions", "listprunelocks", "logging", "maxmempool", From a434ab0528131989ff8505a58ce620bbdaf89d42 Mon Sep 17 00:00:00 2001 From: niftynei Date: Mon, 4 Dec 2023 21:50:47 -0600 Subject: [PATCH 2/2] rest: add `listmempooltransactions` to the REST API Make the `listmempooltransactions` rpc available via the REST api Github-Pull: #29016 Rebased-From: 07008477b8174d00627c5fe75859bc1f85602c69 --- src/rest.cpp | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/rest.cpp b/src/rest.cpp index c42bc8e40c..0a90ead1f9 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -643,6 +643,45 @@ static bool rest_deploymentinfo(const std::any& context, HTTPRequest* req, const } +static bool rest_mempool_transactions(const std::any& context, HTTPRequest* req, const std::string& str_uri_part) +{ + if (!CheckWarmup(req)) + return false; + + std::string param; + const RESTResponseFormat rf = ParseDataFormat(param, str_uri_part); + if (param != "contents" && param != "info") { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/mempool/transactions/.json"); + } + + const CTxMemPool* mempool = GetMemPool(context, req); + if (!mempool) return false; + + switch (rf) { + case RESTResponseFormat::JSON: { + std::string str_json; + std::string raw_sequence_start; + const bool verbose = param == "contents"; + + try { + raw_sequence_start = req->GetQueryParameter("sequence_start").value_or("0"); + } catch (const std::runtime_error& e) { + return RESTERR(req, HTTP_BAD_REQUEST, e.what()); + } + + const auto sequence_start{ToIntegral(raw_sequence_start)}; + str_json = MempoolTxsToJSON(*mempool, verbose, sequence_start.value()).write() + "\n"; + + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, str_json); + return true; + } + default: { + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); + } + } +} + static bool rest_mempool(const std::any& context, HTTPRequest* req, const std::string& str_uri_part) { if (!CheckWarmup(req)) @@ -1014,6 +1053,7 @@ static const struct { {"/rest/blockfilterheaders/", rest_filter_header}, {"/rest/chaininfo", rest_chaininfo}, {"/rest/mempool/", rest_mempool}, + {"/rest/mempool/transactions", rest_mempool_transactions}, {"/rest/headers/", rest_headers}, {"/rest/getutxos", rest_getutxos}, {"/rest/deploymentinfo/", rest_deploymentinfo},