Merge 15836 via fee_histogram+pr15836_api

This commit is contained in:
Luke Dashjr 2025-03-05 03:27:08 +00:00
commit b891a04599
7 changed files with 362 additions and 10 deletions

View File

@ -33,6 +33,7 @@
#include <validation.h>
#include <any>
#include <optional>
#include <vector>
#include <univalue.h>
@ -650,8 +651,8 @@ static bool rest_mempool(const std::any& context, HTTPRequest* req, const std::s
std::string param;
const RESTResponseFormat rf = ParseDataFormat(param, str_uri_part);
if (param != "contents" && param != "info") {
return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/mempool/<info|contents>.json");
if (param != "contents" && param != "info" && param != "info/with_fee_histogram") {
return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/mempool/<info|info/with_fee_histogram|contents>.json");
}
const CTxMemPool* mempool = GetMemPool(context, req);
@ -685,8 +686,10 @@ static bool rest_mempool(const std::any& context, HTTPRequest* req, const std::s
return RESTERR(req, HTTP_BAD_REQUEST, "Verbose results cannot contain mempool sequence values. (hint: set \"verbose=false\")");
}
str_json = MempoolToJSON(*mempool, verbose, mempool_sequence).write() + "\n";
} else if (param == "info/with_fee_histogram") {
str_json = MempoolInfoToJSON(*mempool, MempoolInfoToJSON_const_histogram_floors).write() + "\n";
} else {
str_json = MempoolInfoToJSON(*mempool).write() + "\n";
str_json = MempoolInfoToJSON(*mempool, std::nullopt).write() + "\n";
}
req->WriteHeader("Content-Type", "application/json");

View File

@ -250,6 +250,8 @@ static const CRPCConvertParam vRPCConvertParams[] =
{ "getblockstats", 1, "stats" },
{ "pruneblockchain", 0, "height" },
{ "keypoolrefill", 0, "newsize" },
{ "getmempoolinfo", 0, "fee_histogram" },
{ "getmempoolinfo", 0, "with_fee_histogram" },
{ "getrawmempool", 0, "verbose" },
{ "getrawmempool", 1, "mempool_sequence" },
{ "estimatesmartfee", 0, "conf_target" },

View File

@ -15,6 +15,7 @@
#include <policy/rbf.h>
#include <policy/settings.h>
#include <primitives/transaction.h>
#include <rpc/mempool.h>
#include <rpc/server.h>
#include <rpc/server_util.h>
#include <rpc/util.h>
@ -25,6 +26,7 @@
#include <util/strencodings.h>
#include <util/time.h>
#include <optional>
#include <utility>
using node::DumpMempool;
@ -661,7 +663,7 @@ static RPCHelpMan gettxspendingprevout()
};
}
UniValue MempoolInfoToJSON(const CTxMemPool& pool)
UniValue MempoolInfoToJSON(const CTxMemPool& pool, const std::optional<MempoolHistogramFeeRates>& histogram_floors)
{
// Make sure this call is atomic in the pool.
LOCK(pool.cs);
@ -677,14 +679,75 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool)
ret.pushKV("incrementalrelayfee", ValueFromAmount(pool.m_opts.incremental_relay_feerate.GetFeePerK()));
ret.pushKV("unbroadcastcount", uint64_t{pool.GetUnbroadcastTxs().size()});
ret.pushKV("fullrbf", pool.m_opts.full_rbf);
if (histogram_floors) {
const MempoolHistogramFeeRates& floors{histogram_floors.value()};
std::vector<uint64_t> sizes(floors.size(), 0);
std::vector<uint64_t> count(floors.size(), 0);
std::vector<CAmount> fees(floors.size(), 0);
for (const CTxMemPoolEntry& e : pool.mapTx) {
const CAmount fee{e.GetFee()};
const uint32_t size{uint32_t(e.GetTxSize())};
const CAmount afees{e.GetModFeesWithAncestors()}, dfees{e.GetModFeesWithDescendants()};
const uint32_t asize{uint32_t(e.GetSizeWithAncestors())}, dsize{uint32_t(e.GetSizeWithDescendants())};
// Do not use CFeeRate here, since it rounds up, and this should be rounding down
const CAmount fpb{fee / size}; // Fee rate per byte
const CAmount afpb{afees / asize}; // Fee rate per byte including ancestors
const CAmount dfpb{dfees / dsize}; // Fee rate per byte including descendants
// Fee rate per byte including ancestors & descendants
// (fee/size are included in both, so subtracted to avoid double-counting)
const CAmount tfpb{(afees + dfees - fee) / (asize + dsize - size)};
const CAmount fee_rate{std::max(std::min(dfpb, tfpb), std::min(fpb, afpb))};
// Distribute fee rates
for (size_t i = floors.size(); i > 0;) {
--i;
if (fee_rate >= floors[i]) {
sizes[i] += size;
++count[i];
fees[i] += fee;
break;
}
}
}
// Track total amount of available fees in fee rate groups
CAmount total_fees = 0;
UniValue info(UniValue::VOBJ);
for (size_t i = 0; i < floors.size(); ++i) {
UniValue info_sub(UniValue::VOBJ);
info_sub.pushKV("sizes", sizes[i]);
info_sub.pushKV("count", count.at(i));
info_sub.pushKV("fees", fees.at(i));
info_sub.pushKV("from_feerate", floors[i]);
info_sub.pushKV("to_feerate", i == floors.size() - 1 ? std::numeric_limits<int64_t>::max() : floors[i + 1]);
total_fees += fees.at(i);
info.pushKV(ToString(floors[i]), info_sub);
}
info.pushKV("total_fees", total_fees);
ret.pushKV("fee_histogram", info);
}
return ret;
}
static RPCHelpMan getmempoolinfo()
{
return RPCHelpMan{"getmempoolinfo",
"Returns details on the active state of the TX memory pool.",
{},
"Returns details on the active state of the TX memory pool.\n",
{
{"fee_histogram|with_fee_histogram", {RPCArg::Type::ARR, RPCArg::Type::BOOL}, RPCArg::Optional::OMITTED, "Fee statistics grouped by fee rate ranges",
{
{"fee_rate", RPCArg::Type::NUM, RPCArg::Optional::NO, "Fee rate (in " + CURRENCY_ATOM + "/vB) to group the fees by"},
},
},
},
RPCResult{
RPCResult::Type::OBJ, "", "",
{
@ -699,14 +762,57 @@ static RPCHelpMan getmempoolinfo()
{RPCResult::Type::NUM, "incrementalrelayfee", "minimum fee rate increment for mempool limiting or replacement in " + CURRENCY_UNIT + "/kvB"},
{RPCResult::Type::NUM, "unbroadcastcount", "Current number of transactions that haven't passed initial broadcast yet"},
{RPCResult::Type::BOOL, "fullrbf", "True if the mempool accepts RBF without replaceability signaling inspection"},
{RPCResult::Type::OBJ_DYN, "fee_histogram", /*optional=*/true, "",
{
{RPCResult::Type::OBJ, "<fee_rate_group>", "Fee rate group named by its lower bound (in " + CURRENCY_ATOM + "/vB), identical to the \"from_feerate\" field below",
{
{RPCResult::Type::NUM, "sizes", "Cumulative size of all transactions in the fee rate group (in vBytes)"},
{RPCResult::Type::NUM, "count", "Number of transactions in the fee rate group"},
{RPCResult::Type::NUM, "fees", "Cumulative fees of all transactions in the fee rate group (in " + CURRENCY_ATOM + ")"},
{RPCResult::Type::NUM, "from_feerate", "Group contains transactions with fee rates equal or greater than this value (in " + CURRENCY_ATOM + "/vB)"},
{RPCResult::Type::NUM, "to_feerate", /*optional=*/true, "Group contains transactions with fee rates equal or less than this value (in " + CURRENCY_ATOM + "/vB)"},
}},
{RPCResult::Type::ELISION, "", ""},
{RPCResult::Type::NUM, "total_fees", "Total available fees in mempool (in " + CURRENCY_ATOM + ")"},
}, /*skip_type_check=*/ true},
}},
RPCExamples{
HelpExampleCli("getmempoolinfo", "")
+ HelpExampleRpc("getmempoolinfo", "")
HelpExampleCli("getmempoolinfo", "") +
HelpExampleCli("getmempoolinfo", R"("[0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 17, 20, 25, 30, 40, 50, 60, 70, 80, 100, 120, 140, 170, 200]")") +
HelpExampleRpc("getmempoolinfo", "") +
HelpExampleRpc("getmempoolinfo", R"([0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 17, 20, 25, 30, 40, 50, 60, 70, 80, 100, 120, 140, 170, 200])")
},
[&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue
{
return MempoolInfoToJSON(EnsureAnyMemPool(request.context));
MempoolHistogramFeeRates histogram_floors;
std::optional<MempoolHistogramFeeRates> histogram_floors_opt = std::nullopt;
if (request.params[0].isBool()) {
if (request.params[0].isTrue()) {
histogram_floors_opt = MempoolInfoToJSON_const_histogram_floors;
}
} else if (!request.params[0].isNull()) {
const UniValue histogram_floors_univalue = request.params[0].get_array();
if (histogram_floors_univalue.empty()) {
throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid number of parameters");
}
for (size_t i = 0; i < histogram_floors_univalue.size(); ++i) {
int64_t value = histogram_floors_univalue[i].getInt<int64_t>();
if (value < 0) {
throw JSONRPCError(RPC_INVALID_PARAMETER, "Non-negative values are expected");
} else if (i > 0 && histogram_floors.back() >= value) {
throw JSONRPCError(RPC_INVALID_PARAMETER, "Strictly increasing values are expected");
}
histogram_floors.push_back(value);
}
histogram_floors_opt = std::optional<MempoolHistogramFeeRates>(std::move(histogram_floors));
}
return MempoolInfoToJSON(EnsureAnyMemPool(request.context), histogram_floors_opt);
},
};
}

View File

@ -5,11 +5,27 @@
#ifndef BITCOIN_RPC_MEMPOOL_H
#define BITCOIN_RPC_MEMPOOL_H
#include <consensus/amount.h>
#include <optional>
#include <vector>
class CTxMemPool;
class UniValue;
typedef std::vector<CAmount> MempoolHistogramFeeRates;
/* TODO: define log scale formular for dynamically creating the
* feelimits but with the property of not constantly changing
* (and thus screw up client implementations) */
static const MempoolHistogramFeeRates MempoolInfoToJSON_const_histogram_floors{
1, 2, 3, 4, 5, 6, 7, 8, 10,
12, 14, 17, 20, 25, 30, 40, 50, 60, 70, 80, 100,
120, 140, 170, 200, 250, 300, 400, 500, 600, 700, 800, 1000,
1200, 1400, 1700, 2000, 2500, 3000, 4000, 5000, 6000, 7000, 8000, 10000};
/** Mempool information to JSON */
UniValue MempoolInfoToJSON(const CTxMemPool& pool);
UniValue MempoolInfoToJSON(const CTxMemPool& pool, const std::optional<MempoolHistogramFeeRates>& histogram_floors);
/** Mempool to JSON */
UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose = false, bool include_mempool_sequence = false);

View File

@ -345,6 +345,9 @@ class RESTTest (BitcoinTestFramework):
for obj in [json_obj, mempool_info]:
obj.pop("unbroadcastcount")
assert_equal(json_obj, mempool_info)
json_obj = self.test_rest_request("/mempool/info/with_fee_histogram")
mempool_info = self.nodes[0].getmempoolinfo(with_fee_histogram=True)
assert_equal(json_obj, mempool_info)
# Check that there are our submitted transactions in the TX memory pool
json_obj = self.test_rest_request("/mempool/contents")

View File

@ -0,0 +1,221 @@
#!/usr/bin/env python3
# Copyright (c) 2023 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test mempool fee histogram."""
from decimal import Decimal
from test_framework.blocktools import COINBASE_MATURITY
from test_framework.messages import (
COIN,
)
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import (
assert_equal,
assert_greater_than,
assert_greater_than_or_equal,
)
def get_actual_fee_rate(fee_in_satoshis, vsize):
# NOTE: Must round down, unlike ceildiv/get_fee
fee_rate = fee_in_satoshis // vsize
return str(fee_rate)
def get_tx_details(node, txid):
info = node.gettransaction(txid=txid)
info.update(node.getrawtransaction(txid=txid, verbose=True))
info['fee'] = int(-info['fee'] * COIN) # convert to satoshis
info['feerate'] = get_actual_fee_rate(info['fee'], info['vsize'])
return info
class MempoolFeeHistogramTest(BitcoinTestFramework):
def add_options(self, parser):
self.add_wallet_options(parser)
def set_test_params(self):
self.setup_clean_chain = True
self.num_nodes = 1
def skip_test_if_missing_module(self):
self.skip_if_no_wallet()
def run_test(self):
node = self.nodes[0]
self.generate(self.nodes[0], COINBASE_MATURITY + 2, sync_fun=self.no_op)
# We have two UTXOs (utxo_1 and utxo_2) and we create three changeless transactions:
# - tx1 (5 sat/vB): spending utxo_1
# - tx2 (14 sat/vB): spending output from tx1
# - tx3 (6 sat/vB): spending utxo_2 and the output from tx2
self.log.info("Test getmempoolinfo does not return fee histogram by default")
assert ("fee_histogram" not in node.getmempoolinfo())
self.log.info("Test getmempoolinfo returns empty fee histogram when mempool is empty")
info = node.getmempoolinfo([1, 2, 3])
(non_empty_groups, empty_groups, total_fees) = self.histogram_stats(info['fee_histogram'])
assert_equal(0, non_empty_groups)
assert_equal(3, empty_groups)
assert_equal(0, total_fees)
for i in ['1', '2', '3']:
assert_equal(0, info['fee_histogram'][i]['sizes'])
assert_equal(0, info['fee_histogram'][i]['count'])
assert_equal(0, info['fee_histogram'][i]['fees'])
assert_equal(int(i), info['fee_histogram'][i]['from_feerate'])
self.log.info("Test that we have two spendable UTXOs and lock the second one")
utxos = node.listunspent()
assert_equal(2, len(utxos))
node.lockunspent(False, [{"txid": utxos[1]["txid"], "vout": utxos[1]["vout"]}])
self.log.info("Send tx1 transaction with 5 sat/vB fee rate")
tx1_txid = node.sendtoaddress(address=node.getnewaddress(), amount=Decimal("50.0"), fee_rate=5, subtractfeefromamount=True)
tx1_info = get_tx_details(node, tx1_txid)
self.log.info(f"Test fee rate histogram when mempool contains 1 transaction (tx1: {tx1_info['feerate']} sat/vB)")
info = node.getmempoolinfo([1, 3, 5, 10])
(non_empty_groups, empty_groups, total_fees) = self.histogram_stats(info['fee_histogram'])
assert_equal(1, non_empty_groups)
assert_equal(3, empty_groups)
assert_equal(1, info['fee_histogram'][tx1_info['feerate']]['count'])
assert_equal(total_fees, info['fee_histogram']['total_fees'])
assert_equal(0, info['fee_histogram']['1']['sizes'])
assert_equal(0, info['fee_histogram']['1']['count'])
assert_equal(0, info['fee_histogram']['1']['fees'])
assert_equal(1, info['fee_histogram']['1']['from_feerate'])
assert_equal(0, info['fee_histogram']['3']['sizes'])
assert_equal(0, info['fee_histogram']['3']['count'])
assert_equal(0, info['fee_histogram']['3']['fees'])
assert_equal(3, info['fee_histogram']['3']['from_feerate'])
assert_equal(tx1_info['vsize'], info['fee_histogram']['5']['sizes'])
assert_equal(1, info['fee_histogram']['5']['count'])
assert_equal(tx1_info['fee'], info['fee_histogram']['5']['fees'])
assert_equal(5, info['fee_histogram']['5']['from_feerate'])
assert_equal(0, info['fee_histogram']['10']['sizes'])
assert_equal(0, info['fee_histogram']['10']['count'])
assert_equal(0, info['fee_histogram']['10']['fees'])
assert_equal(10, info['fee_histogram']['10']['from_feerate'])
self.log.info("Send tx2 transaction with 14 sat/vB fee rate (spends tx1 UTXO)")
tx2_txid = node.sendtoaddress(address=node.getnewaddress(), amount=Decimal("25.0"), fee_rate=14, subtractfeefromamount=True)
tx2_info = get_tx_details(node, tx2_txid)
self.log.info(f"Test fee rate histogram when mempool contains 2 transactions (tx1: {tx1_info['feerate']} sat/vB, tx2: {tx2_info['feerate']} sat/vB)")
info = node.getmempoolinfo([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])
# Verify that both tx1 and tx2 are reported in 8 sat/vB fee rate group
(non_empty_groups, empty_groups, total_fees) = self.histogram_stats(info['fee_histogram'])
tx1p2_feerate = get_actual_fee_rate(tx1_info['fee'] + tx2_info['fee'], tx1_info['vsize'] + tx2_info['vsize'])
assert_equal(1, non_empty_groups)
assert_equal(14, empty_groups)
assert_equal(2, info['fee_histogram'][tx1p2_feerate]['count'])
assert_equal(total_fees, info['fee_histogram']['total_fees'])
# Unlock the second UTXO which we locked
node.lockunspent(True, [{"txid": utxos[1]["txid"], "vout": utxos[1]["vout"]}])
self.log.info("Send tx3 transaction with 6 sat/vB fee rate (spends all available UTXOs)")
tx3_txid = node.sendtoaddress(address=node.getnewaddress(), amount=Decimal("99.9"), fee_rate=6, subtractfeefromamount=True)
tx3_info = get_tx_details(node, tx3_txid)
self.log.info(f"Test fee rate histogram when mempool contains 3 transactions (tx1: {tx1_info['feerate']} sat/vB, tx2: {tx2_info['feerate']} sat/vB, tx3: {tx3_info['feerate']} sat/vB)")
info = node.getmempoolinfo([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])
# Verify that each of 6, 8 and 9 sat/vB fee rate groups contain one transaction
# tx1 should be grouped with tx2 + tx3 (descendants)
# tx2 should be grouped with tx1 (ancestors only)
# tx3 should be alone
expected_histogram = dict(
tuple(
(str(n), {
'from_feerate': n,
'to_feerate': n + 1,
'count': 0,
'fees': 0,
'sizes': 0,
}) for n in range(1, 16)
) + (
('total_fees', tx1_info['fee'] + tx2_info['fee'] + tx3_info['fee']),
)
)
expected_frg = expected_histogram
expected_frg['15']['to_feerate'] = 9223372036854775807
tx1p2p3_feerate = get_actual_fee_rate(expected_histogram['total_fees'], tx1_info['vsize'] + tx2_info['vsize'] + tx3_info['vsize'])
def inc_expected(feerate, txinfo):
this_frg = expected_frg[feerate]
this_frg['count'] += 1
this_frg['fees'] += txinfo['fee']
this_frg['sizes'] += txinfo['vsize']
inc_expected(tx1p2p3_feerate, tx1_info)
inc_expected(tx1p2_feerate, tx2_info)
inc_expected(tx3_info['feerate'], tx3_info)
assert_equal(expected_histogram, info['fee_histogram'])
self.log.info("Test fee rate histogram with default groups")
info = node.getmempoolinfo(with_fee_histogram=True)
# Verify that the 6 sat/vB fee rate group has one transaction, and the 8-9 sat/vB fee rate group has two
for collapse_n in (9, 11, 13, 15):
for field in ('count', 'sizes', 'fees'):
expected_frg[str(collapse_n - 1)][field] += expected_frg[str(collapse_n)][field]
expected_frg[str(collapse_n - 1)]['to_feerate'] += 1
del expected_frg[str(collapse_n)]
expected_frg['14']['to_feerate'] += 1 # 16 is also skipped
for new_n in (17, 20, 25) + tuple(range(30, 90, 10)) + (100, 120, 140, 170, 200, 250) + tuple(range(300, 900, 100)) + (1000, 1200, 1400, 1700, 2000, 2500) + tuple(range(3000, 9000, 1000)) + (10000,):
frinfo = info['fee_histogram'][str(new_n)]
assert frinfo['to_feerate'] > frinfo['from_feerate']
del frinfo['to_feerate']
assert_equal(frinfo, {
'from_feerate': new_n,
'count': 0,
'fees': 0,
'sizes': 0,
})
del info['fee_histogram'][str(new_n)]
assert_equal(expected_histogram, info['fee_histogram'])
self.log.info("Test getmempoolinfo(with_fee_histogram=False) does not return fee histogram")
assert('fee_histogram' not in node.getmempoolinfo(with_fee_histogram=False))
def histogram_stats(self, histogram):
total_fees = 0
empty_count = 0
non_empty_count = 0
for key, bin in histogram.items():
if key == 'total_fees':
continue
assert_equal(int(key), bin['from_feerate'])
if bin['fees'] > 0:
assert_greater_than(bin['count'], 0)
else:
assert_equal(bin['count'], 0)
assert_greater_than_or_equal(bin['fees'], 0)
assert_greater_than_or_equal(bin['sizes'], 0)
if bin['to_feerate'] is not None:
assert_greater_than_or_equal(bin['to_feerate'], bin['from_feerate'])
for next_key in sorted((*(int(a) for a in histogram.keys() if a != 'total_fees'), 0x7fffffffffffffff)):
if int(next_key) <= int(key):
continue
assert_equal(bin['to_feerate'], int(next_key))
break
total_fees += bin['fees']
if bin['count'] == 0:
empty_count += 1
else:
non_empty_count += 1
return (non_empty_count, empty_count, total_fees)
if __name__ == '__main__':
MempoolFeeHistogramTest(__file__).main()

View File

@ -304,6 +304,7 @@ BASE_SCRIPTS = [
'p2p_initial_headers_sync.py',
'feature_nulldummy.py',
'mempool_accept.py',
'mempool_fee_histogram.py',
'mempool_expiry.py',
'wallet_import_with_label.py --legacy-wallet',
'wallet_importdescriptors.py --descriptors',