Merge dustdynamic-27+knots

This commit is contained in:
Luke Dashjr 2024-08-01 00:06:00 +00:00
commit f50f599fb4
10 changed files with 234 additions and 2 deletions

View File

@ -651,7 +651,11 @@ void SetupServerArgs(ArgsManager& argsman)
argsman.AddArg("-acceptnonstdtxn", strprintf("Relay and mine \"non-standard\" transactions (default: %u)", DEFAULT_ACCEPT_NON_STD_TXN), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
argsman.AddArg("-incrementalrelayfee=<amt>", strprintf("Fee rate (in %s/kvB) used to define cost of relay, used for mempool limiting and replacement policy. (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_INCREMENTAL_RELAY_FEE)), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::NODE_RELAY);
argsman.AddArg("-dustrelayfee=<amt>", strprintf("Fee rate (in %s/kvB) used to define dust, the value of an output such that it will cost more than its value in fees at this fee rate to spend it. (default: %s)", CURRENCY_UNIT, FormatMoney(DUST_RELAY_TX_FEE)), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::NODE_RELAY);
argsman.AddArg("-dustrelayfee=<amt>", strprintf("Fee rate (in %s/kvB) used to define dust, the value of an output such that it will cost more than its value in fees at this fee rate to spend it. (default: %s)", CURRENCY_UNIT, FormatMoney(DUST_RELAY_TX_FEE)), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
argsman.AddArg("-dustdynamic=off|[<multiplier>*]target:<blocks>|[<multiplier>*]mempool:<kvB>",
strprintf("Automatically raise dustrelayfee based on either the expected fee to be mined within <blocks> blocks, or to be within the best <kvB> kvB of this node's mempool. If unspecified, multiplier is %s. (default: %s)",
DEFAULT_DUST_RELAY_MULTIPLIER,
DEFAULT_DUST_DYNAMIC), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
argsman.AddArg("-acceptstalefeeestimates", strprintf("Read fee estimates even if they are stale (%sdefault: %u) fee estimates are considered stale if they are %s hours old", "regtest only; ", DEFAULT_ACCEPT_STALE_FEE_ESTIMATES, Ticks<std::chrono::hours>(MAX_FILE_AGE)), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
argsman.AddArg("-bytespersigop", strprintf("Equivalent bytes per sigop in transactions for relay and mining (default: %u)", DEFAULT_BYTES_PER_SIGOP), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
argsman.AddArg("-bytespersigopstrict", strprintf("Minimum bytes per sigop in transactions we relay and mine (default: %u)", DEFAULT_BYTES_PER_SIGOP_STRICT), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
@ -1582,6 +1586,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
assert(!node.chainman);
CTxMemPool::Options mempool_opts{
.estimator = node.fee_estimator.get(),
.scheduler = &*node.scheduler,
.check_ratio = chainparams.DefaultConsistencyChecks() ? 1 : 0,
};
auto result{ApplyArgsManOptions(args, chainparams, mempool_opts)};

View File

@ -13,6 +13,9 @@
#include <cstdint>
#include <optional>
class CBlockPolicyEstimator;
class CScheduler;
enum class RBFPolicy { Never, OptIn, Always };
enum class TRUCPolicy { Reject, Accept, Enforce };
@ -40,6 +43,9 @@ namespace kernel {
* Most of the time, this struct should be referenced as CTxMemPool::Options.
*/
struct MemPoolOptions {
/* Used to estimate appropriate transaction fees. */
CBlockPolicyEstimator* estimator{nullptr};
CScheduler* scheduler;
/* The ratio used to determine how often sanity checks will run. */
int check_ratio{0};
int64_t max_size_bytes{DEFAULT_MAX_MEMPOOL_SIZE_MB * 1'000'000};
@ -48,6 +54,10 @@ struct MemPoolOptions {
/** A fee rate smaller than this is considered zero fee (for relaying, mining and transaction creation) */
CFeeRate min_relay_feerate{DEFAULT_MIN_RELAY_TX_FEE};
CFeeRate dust_relay_feerate{DUST_RELAY_TX_FEE};
/** Negative for a target number of blocks, positive for target kB into current mempool. */
int32_t dust_relay_target;
/** Multiplier for dustdynamic assignments, in thousandths. */
int dust_relay_multiplier{DEFAULT_DUST_RELAY_MULTIPLIER};
/**
* A data carrying output is an unspendable output containing data. The script
* type is designated as TxoutType::NULL_DATA.

View File

@ -13,14 +13,20 @@
#include <logging.h>
#include <node/interface_ui.h>
#include <policy/feerate.h>
#include <policy/fees.h>
#include <policy/policy.h>
#include <tinyformat.h>
#include <util/error.h>
#include <util/moneystr.h>
#include <util/result.h>
#include <util/strencodings.h>
#include <util/translation.h>
#include <chrono>
#include <cstdint>
#include <memory>
#include <string_view>
#include <utility>
using kernel::MemPoolLimits;
using kernel::MemPoolOptions;
@ -38,6 +44,51 @@ void ApplyArgsManOptions(const ArgsManager& argsman, MemPoolLimits& mempool_limi
}
}
util::Result<std::pair<int32_t, int>> ParseDustDynamicOpt(std::string_view optstr, const unsigned int max_fee_estimate_blocks)
{
if (optstr == "0" || optstr == "off") {
return std::pair<int32_t, int>(0, DEFAULT_DUST_RELAY_MULTIPLIER);
}
int multiplier{DEFAULT_DUST_RELAY_MULTIPLIER};
if (auto pos = optstr.find('*'); pos != optstr.npos) {
int64_t parsed;
if ((!ParseFixedPoint(optstr.substr(0, pos), 3, &parsed)) || parsed > std::numeric_limits<int>::max() || parsed < 1) {
return util::Error{_("failed to parse multiplier")};
}
multiplier = parsed;
optstr.remove_prefix(pos + 1);
}
if (optstr.rfind("target:", 0) == 0) {
if (!max_fee_estimate_blocks) {
return util::Error{_("\"target\" mode requires fee estimator (disabled)")};
}
const auto val = ToIntegral<uint16_t>(optstr.substr(7));
if (!val) {
return util::Error{_("failed to parse target block count")};
}
if (*val < 2) {
return util::Error{_("target must be at least 2 blocks")};
}
if (*val > max_fee_estimate_blocks) {
return util::Error{strprintf(_("target can only be at most %s blocks"), max_fee_estimate_blocks)};
}
return std::pair<int32_t, int>(-*val, multiplier);
} else if (optstr.rfind("mempool:", 0) == 0) {
const auto val = ToIntegral<int32_t>(optstr.substr(8));
if (!val) {
return util::Error{_("failed to parse mempool position")};
}
if (*val < 1) {
return util::Error{_("mempool position must be at least 1 kB")};
}
return std::pair<int32_t, int>(*val, multiplier);
} else {
return util::Error{strprintf(_("\"%s\""), optstr)};
}
}
util::Result<void> ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, MemPoolOptions& mempool_opts)
{
mempool_opts.check_ratio = argsman.GetIntArg("-checkmempool", mempool_opts.check_ratio);
@ -78,6 +129,16 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& argsman, const CChainP
return util::Error{AmountErrMsg("dustrelayfee", argsman.GetArg("-dustrelayfee", ""))};
}
}
if (argsman.IsArgSet("-dustdynamic")) {
const auto optstr = argsman.GetArg("-dustdynamic", DEFAULT_DUST_DYNAMIC);
const auto max_fee_estimate_blocks = mempool_opts.estimator ? mempool_opts.estimator->HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE) : (unsigned int)0;
const auto parsed = ParseDustDynamicOpt(optstr, max_fee_estimate_blocks);
if (!parsed) {
return util::Error{strprintf(_("Invalid mode for dustdynamic: %s"), util::ErrorString(parsed))};
}
mempool_opts.dust_relay_target = parsed->first;
mempool_opts.dust_relay_multiplier = parsed->second;
}
mempool_opts.permit_bare_pubkey = argsman.GetBoolArg("-permitbarepubkey", DEFAULT_PERMIT_BAREPUBKEY);

View File

@ -7,6 +7,10 @@
#include <util/result.h>
#include <cstdint>
#include <string_view>
#include <utility>
class ArgsManager;
class CChainParams;
struct bilingual_str;
@ -14,6 +18,8 @@ namespace kernel {
struct MemPoolOptions;
};
[[nodiscard]] util::Result<std::pair<int32_t, int>> ParseDustDynamicOpt(std::string_view optstr, unsigned int max_fee_estimate_blocks);
/**
* Overlay the options set in \p argsman on top of corresponding members in \p mempool_opts.
* Returns an error if one was encountered.

View File

@ -76,6 +76,7 @@ public:
std::string SatsToString() const;
friend CFeeRate operator*(const CFeeRate& f, int a) { return CFeeRate(a * f.nSatoshisPerK); }
friend CFeeRate operator*(int a, const CFeeRate& f) { return CFeeRate(a * f.nSatoshisPerK); }
friend CFeeRate operator/(const CFeeRate& f, int a) { return CFeeRate(f.nSatoshisPerK / a); }
SERIALIZE_METHODS(CFeeRate, obj) { READWRITE(obj.nSatoshisPerK); }
};

View File

@ -64,6 +64,8 @@ static constexpr unsigned int MAX_STANDARD_SCRIPTSIG_SIZE{1650};
* only increase the dust limit after prior releases were already not creating
* outputs below the new threshold */
static constexpr unsigned int DUST_RELAY_TX_FEE{3000};
static const std::string DEFAULT_DUST_DYNAMIC{"off"};
static const int DEFAULT_DUST_RELAY_MULTIPLIER{3'000};
static const std::string DEFAULT_SPKREUSE{"allow"};
/** Default for -minrelaytxfee, minimum relay fee for transactions */
static constexpr unsigned int DEFAULT_MIN_RELAY_TX_FEE{1000};

View File

@ -884,6 +884,22 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool, const std::optional<MempoolHi
ret.pushKV("mempoolminfee", ValueFromAmount(std::max(pool.GetMinFee(), pool.m_min_relay_feerate).GetFeePerK()));
ret.pushKV("minrelaytxfee", ValueFromAmount(pool.m_min_relay_feerate.GetFeePerK()));
ret.pushKV("incrementalrelayfee", ValueFromAmount(pool.m_incremental_relay_feerate.GetFeePerK()));
ret.pushKV("dustrelayfee", ValueFromAmount(pool.m_dust_relay_feerate.GetFeePerK()));
ret.pushKV("dustrelayfeefloor", ValueFromAmount(pool.m_dust_relay_feerate_floor.GetFeePerK()));
if (pool.m_dust_relay_target == 0) {
ret.pushKV("dustdynamic", "off");
} else {
std::string multiplier_str = strprintf("%u", pool.m_dust_relay_multiplier / 1000);
if (pool.m_dust_relay_multiplier % 1000) {
multiplier_str += strprintf(".%03u", pool.m_dust_relay_multiplier % 1000);
while (multiplier_str.back() == '0') multiplier_str.pop_back();
}
if (pool.m_dust_relay_target < 0) {
ret.pushKV("dustdynamic", strprintf("%s*target:%u", multiplier_str, -pool.m_dust_relay_target));
} else { // pool.m_dust_relay_target > 0
ret.pushKV("dustdynamic", strprintf("%s*mempool:%u", multiplier_str, pool.m_dust_relay_target));
}
}
ret.pushKV("unbroadcastcount", uint64_t{pool.GetUnbroadcastTxs().size()});
ret.pushKV("fullrbf", (pool.m_rbf_policy == RBFPolicy::Always));
switch (pool.m_rbf_policy) {
@ -977,6 +993,9 @@ static RPCHelpMan getmempoolinfo()
{RPCResult::Type::STR_AMOUNT, "mempoolminfee", "Minimum fee rate in " + CURRENCY_UNIT + "/kvB for tx to be accepted. Is the maximum of minrelaytxfee and minimum mempool fee"},
{RPCResult::Type::STR_AMOUNT, "minrelaytxfee", "Current minimum relay fee for transactions"},
{RPCResult::Type::NUM, "incrementalrelayfee", "minimum fee rate increment for mempool limiting or replacement in " + CURRENCY_UNIT + "/kvB"},
{RPCResult::Type::NUM, "dustrelayfee", "Current fee rate used to define dust, the value of an output so small it will cost more to spend than its value, in " + CURRENCY_UNIT + "/kvB"},
{RPCResult::Type::NUM, "dustrelayfeefloor", "Minimum fee rate used to define dust in " + CURRENCY_UNIT + "/kvB"},
{RPCResult::Type::STR, "dustdynamic", "Method for automatic adjustments to dustrelayfee (one of: off, target:<blocks>, or mempool:<kB>)"},
{RPCResult::Type::NUM, "unbroadcastcount", "Current number of transactions that haven't passed initial broadcast yet"},
{RPCResult::Type::BOOL, "fullrbf", "True if the mempool accepts RBF without replaceability signaling inspection"},
{RPCResult::Type::STR, "rbf_policy", "Policy used for replacing conflicting transactions by fee (one of: never, optin, always)"},

View File

@ -14,10 +14,12 @@
#include <crypto/ripemd160.h>
#include <logging.h>
#include <policy/coin_age_priority.h>
#include <policy/fees.h>
#include <policy/policy.h>
#include <policy/settings.h>
#include <random.h>
#include <reverse_iterator.h>
#include <scheduler.h>
#include <script/script.h>
#include <util/check.h>
#include <util/moneystr.h>
@ -406,11 +408,16 @@ void CTxMemPoolEntry::UpdateAncestorState(int32_t modifySize, CAmount modifyFee,
CTxMemPool::CTxMemPool(const Options& opts)
: m_check_ratio{opts.check_ratio},
minerPolicyEstimator{opts.estimator},
scheduler{opts.scheduler},
m_max_size_bytes{opts.max_size_bytes},
m_expiry{opts.expiry},
m_incremental_relay_feerate{opts.incremental_relay_feerate},
m_min_relay_feerate{opts.min_relay_feerate},
m_dust_relay_feerate{opts.dust_relay_feerate},
m_dust_relay_feerate_floor{opts.dust_relay_feerate},
m_dust_relay_target{opts.dust_relay_target},
m_dust_relay_multiplier{opts.dust_relay_multiplier},
m_permit_bare_pubkey{opts.permit_bare_pubkey},
m_permit_bare_multisig{opts.permit_bare_multisig},
m_max_datacarrier_bytes{opts.max_datacarrier_bytes},
@ -420,6 +427,12 @@ CTxMemPool::CTxMemPool(const Options& opts)
m_persist_v1_dat{opts.persist_v1_dat},
m_limits{opts.limits}
{
Assert(scheduler || !m_dust_relay_target);
if (scheduler) {
scheduler->scheduleEvery([this]{
UpdateDynamicDustFeerate();
}, DYNAMIC_DUST_FEERATE_UPDATE_INTERVAL);
}
}
bool CTxMemPool::isSpent(const COutPoint& outpoint) const
@ -688,6 +701,36 @@ void CTxMemPool::removeForBlock(const std::vector<CTransactionRef>& vtx, unsigne
blockSinceLastRollingFeeBump = true;
}
void CTxMemPool::UpdateDynamicDustFeerate()
{
CFeeRate est_feerate{0};
if (m_dust_relay_target < 0 && minerPolicyEstimator) {
static constexpr double target_success_threshold{0.8};
est_feerate = minerPolicyEstimator->estimateRawFee(-m_dust_relay_target, target_success_threshold, FeeEstimateHorizon::LONG_HALFLIFE, nullptr);
} else if (m_dust_relay_target > 0) {
auto bytes_remaining = int64_t{m_dust_relay_target} * 1'000;
LOCK(cs);
for (auto mi = mapTx.get<ancestor_score>().begin(); mi != mapTx.get<ancestor_score>().end(); ++mi) {
bytes_remaining -= mi->GetTxSize();
if (bytes_remaining <= 0) {
est_feerate = CFeeRate(mi->GetFee(), mi->GetTxSize());
break;
}
}
}
est_feerate = (est_feerate * m_dust_relay_multiplier) / 1'000;
if (est_feerate < m_dust_relay_feerate_floor) {
est_feerate = m_dust_relay_feerate_floor;
}
if (m_dust_relay_feerate != est_feerate) {
LogPrint(BCLog::MEMPOOL, "Updating dust feerate to %s\n", est_feerate.ToString(FeeEstimateMode::SAT_VB));
m_dust_relay_feerate = est_feerate;
}
}
void CTxMemPool::check(const CCoinsViewCache& active_coins_tip, int64_t spendheight) const
{
if (m_check_ratio == 0) return;

View File

@ -43,6 +43,8 @@
class CChain;
class CScript;
static constexpr std::chrono::minutes DYNAMIC_DUST_FEERATE_UPDATE_INTERVAL{15};
/** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */
static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF;
@ -210,6 +212,8 @@ struct entry_time {};
struct ancestor_score {};
struct index_by_wtxid {};
class CBlockPolicyEstimator;
/**
* Information about a mempool transaction.
*/
@ -309,6 +313,8 @@ class CTxMemPool
protected:
const int m_check_ratio; //!< Value n means that 1 times in n we check.
std::atomic<unsigned int> nTransactionsUpdated{0}; //!< Used by getblocktemplate to trigger CreateNewBlock() invocation
CBlockPolicyEstimator* const minerPolicyEstimator;
CScheduler* scheduler;
uint64_t totalTxSize GUARDED_BY(cs){0}; //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141.
CAmount m_total_fee GUARDED_BY(cs){0}; //!< sum of all mempool tx's fees (NOT modified fee)
@ -449,7 +455,10 @@ public:
const std::chrono::seconds m_expiry;
const CFeeRate m_incremental_relay_feerate;
const CFeeRate m_min_relay_feerate;
const CFeeRate m_dust_relay_feerate;
CFeeRate m_dust_relay_feerate;
CFeeRate m_dust_relay_feerate_floor;
int32_t m_dust_relay_target;
int m_dust_relay_multiplier;
const bool m_permit_bare_pubkey;
const bool m_permit_bare_multisig;
const std::optional<unsigned> m_max_datacarrier_bytes;
@ -513,6 +522,8 @@ public:
*/
void UpdateDependentPriorities(const CTransaction &tx, unsigned int nBlockHeight, bool addToChain);
void UpdateDynamicDustFeerate();
/** Affect CreateNewBlock prioritisation of transactions */
void PrioritiseTransaction(const uint256& hash, double dPriorityDelta, const CAmount& nFeeDelta);
void PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta) { PrioritiseTransaction(hash, 0., nFeeDelta); }

View File

@ -27,6 +27,7 @@ from test_framework.wallet import MiniWallet
MAX_FILE_AGE = 60
SECONDS_PER_HOUR = 60 * 60
target_success_threshold = 0.8
def small_txpuzzle_randfee(
wallet, from_node, conflist, unconflist, amount, min_fee, fee_increment, batch_reqs
@ -150,6 +151,19 @@ def make_tx(wallet, utxo, feerate):
)
def get_feerate_into_mempool(node, kB):
mempool_entries = list(node.getrawmempool(verbose=True).values())
for entry in mempool_entries:
entry['feerate_BTC/vB'] = entry['fees']['modified'] / entry['vsize']
mempool_entries.sort(key=lambda entry: entry['feerate_BTC/vB'], reverse=True)
bytes_remaining = kB * 1000
for entry in mempool_entries:
bytes_remaining -= entry['vsize']
if bytes_remaining <= 0:
return satoshi_round(entry['feerate_BTC/vB'] * 1000)
raise AssertionError('Entire mempool is smaller than %s kB' % (kB,))
class EstimateFeeTest(BitcoinTestFramework):
def set_test_params(self):
self.num_nodes = 3
@ -249,6 +263,63 @@ class EstimateFeeTest(BitcoinTestFramework):
self.log.info("Final estimates after emptying mempools")
check_estimates(self.nodes[1], self.fees_per_kb)
def test_feerate_dustrelayfee_common(self, node, multiplier, dust_mode, desc, expected_base):
dust_parameter = f"-dustdynamic={dust_mode}".replace('=3*', '=')
self.log.info(f"Test dust limit setting {dust_parameter} (fee estimation for {desc})")
self.restart_node(0, extra_args=[dust_parameter])
assert_equal(node.getmempoolinfo()['dustdynamic'], dust_mode)
with node.wait_for_debug_log([b'Updating dust feerate']):
node.mockscheduler(SECONDS_PER_HOUR)
mempool_info = node.getmempoolinfo()
assert_equal(mempool_info['dustrelayfee'], satoshi_round(expected_base() * multiplier))
assert mempool_info['dustrelayfee'] > mempool_info['dustrelayfeefloor']
def test_feerate_dustrelayfee_target(self, node, multiplier, dustfee_target):
dust_mode = f"{multiplier}*target:{dustfee_target}"
self.test_feerate_dustrelayfee_common(node, multiplier, dust_mode, f'{dustfee_target} blocks', lambda: node.estimaterawfee(dustfee_target, target_success_threshold)['long']['feerate'])
def test_feerate_dustrelayfee_mempool(self, node, multiplier, dustfee_kB):
dust_mode = f"{multiplier}*mempool:{dustfee_kB}"
self.test_feerate_dustrelayfee_common(node, multiplier, dust_mode, f'{dustfee_kB} kB into mempool', lambda: get_feerate_into_mempool(node, dustfee_kB))
def test_feerate_dustrelayfee(self):
node = self.nodes[0]
# test dustdynamic=target:<blocks>
for dustfee_target in (2, 8, 1008):
for multiplier in (Decimal('0.5'), 1, 3, Decimal('3.3'), 10, Decimal('10.001')):
self.test_feerate_dustrelayfee_target(node, multiplier, dustfee_target)
# Fill mempool up
mempool_size = 0
batch_sendtx_reqs = []
min_fee = Decimal("0.00001")
while mempool_size < 52000:
(tx_bytes, fee) = small_txpuzzle_randfee(
self.wallet,
self.nodes[0],
self.confutxo,
self.memutxo,
Decimal("0.005"),
min_fee,
min_fee,
batch_sendtx_reqs,
)
mempool_size += tx_bytes
node.batch(batch_sendtx_reqs)
# test dustdynamic=mempool:<kB>
for dustfee_kB in (1, 10, 50):
for multiplier in (Decimal('0.5'), 1, 3, Decimal('3.3'), 10, Decimal('10.001')):
self.test_feerate_dustrelayfee_mempool(node, multiplier, dustfee_kB)
# Restore nodes to a normal state, wiping the mempool
self.stop_node(0)
(self.nodes[0].chain_path / 'mempool.dat').unlink()
self.start_node(0)
self.connect_nodes(1, 0)
self.connect_nodes(0, 2)
def test_feerate_mempoolminfee(self):
high_val = 3 * self.nodes[1].estimatesmartfee(1)["feerate"]
self.restart_node(1, extra_args=[f"-minrelaytxfee={high_val}", '-rest'])
@ -428,6 +499,8 @@ class EstimateFeeTest(BitcoinTestFramework):
self.log.info("Test fee_estimates.dat is flushed periodically")
self.test_estimate_dat_is_flushed_periodically()
self.test_feerate_dustrelayfee()
# check that the effective feerate is greater than or equal to the mempoolminfee even for high mempoolminfee
self.log.info(
"Test fee rate estimation after restarting node with high MempoolMinFee"