Merge dustdynamic-28+knots

src/init.cpp

@@ -671,7 +671,11 @@ void SetupServerArgs(ArgsManager& argsman)
 
     argsman.AddArg("-acceptnonstdtxn", strprintf("Relay and mine \"non-standard\" transactions (default: %u)", DEFAULT_ACCEPT_NON_STD_TXN), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
     argsman.AddArg("-incrementalrelayfee=<amt>", strprintf("Fee rate (in %s/kvB) used to define cost of relay, used for mempool limiting and replacement policy. (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_INCREMENTAL_RELAY_FEE)), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::NODE_RELAY);
-    argsman.AddArg("-dustrelayfee=<amt>", strprintf("Fee rate (in %s/kvB) used to define dust, the value of an output such that it will cost more than its value in fees at this fee rate to spend it. (default: %s)", CURRENCY_UNIT, FormatMoney(DUST_RELAY_TX_FEE)), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::NODE_RELAY);
+    argsman.AddArg("-dustrelayfee=<amt>", strprintf("Fee rate (in %s/kvB) used to define dust, the value of an output such that it will cost more than its value in fees at this fee rate to spend it. (default: %s)", CURRENCY_UNIT, FormatMoney(DUST_RELAY_TX_FEE)), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
+    argsman.AddArg("-dustdynamic=off|[<multiplier>*]target:<blocks>|[<multiplier>*]mempool:<kvB>",
+                   strprintf("Automatically raise dustrelayfee based on either the expected fee to be mined within <blocks> blocks, or to be within the best <kvB> kvB of this node's mempool. If unspecified, multiplier is %s. (default: %s)",
+                             DEFAULT_DUST_RELAY_MULTIPLIER / 1000.,
+                             DEFAULT_DUST_DYNAMIC), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
     argsman.AddArg("-acceptstalefeeestimates", strprintf("Read fee estimates even if they are stale (%sdefault: %u) fee estimates are considered stale if they are %s hours old", "regtest only; ", DEFAULT_ACCEPT_STALE_FEE_ESTIMATES, Ticks<std::chrono::hours>(MAX_FILE_AGE)), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
     argsman.AddArg("-bytespersigop", strprintf("Equivalent bytes per sigop in transactions for relay and mining (default: %u)", DEFAULT_BYTES_PER_SIGOP), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
     argsman.AddArg("-bytespersigopstrict", strprintf("Minimum bytes per sigop in transactions we relay and mine (default: %u)", DEFAULT_BYTES_PER_SIGOP_STRICT), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY);
@@ -1639,6 +1643,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
     assert(!node.chainman);
 
     CTxMemPool::Options mempool_opts{
+        .estimator = node.fee_estimator.get(),
+        .scheduler = &*node.scheduler,
         .check_ratio = chainparams.DefaultConsistencyChecks() ? 1 : 0,
         .signals = &validation_signals,
     };

src/kernel/mempool_options.h

@@ -13,6 +13,8 @@
 #include <cstdint>
 #include <optional>
 
+class CBlockPolicyEstimator;
+class CScheduler;
 class ValidationSignals;
 
 enum class RBFPolicy { Never, OptIn, Always };
@@ -42,6 +44,9 @@ namespace kernel {
  * Most of the time, this struct should be referenced as CTxMemPool::Options.
  */
 struct MemPoolOptions {
+    /* Used to estimate appropriate transaction fees. */
+    CBlockPolicyEstimator* estimator{nullptr};
+    CScheduler* scheduler{nullptr};
     /* The ratio used to determine how often sanity checks will run.  */
     int check_ratio{0};
     int64_t max_size_bytes{DEFAULT_MAX_MEMPOOL_SIZE_MB * 1'000'000};
@@ -50,6 +55,11 @@ struct MemPoolOptions {
     /** A fee rate smaller than this is considered zero fee (for relaying, mining and transaction creation) */
     CFeeRate min_relay_feerate{DEFAULT_MIN_RELAY_TX_FEE};
     CFeeRate dust_relay_feerate{DUST_RELAY_TX_FEE};
+    CFeeRate dust_relay_feerate_floor{DUST_RELAY_TX_FEE};
+    /** Negative for a target number of blocks, positive for target kB into current mempool. */
+    int32_t dust_relay_target{0};
+    /** Multiplier for dustdynamic assignments, in thousandths. */
+    int dust_relay_multiplier{DEFAULT_DUST_RELAY_MULTIPLIER};
     /**
      * A data carrying output is an unspendable output containing data. The script
      * type is designated as TxoutType::NULL_DATA.

src/node/mempool_args.cpp

@@ -14,14 +14,20 @@
 #include <logging.h>
 #include <node/interface_ui.h>
 #include <policy/feerate.h>
+#include <policy/fees.h>
 #include <policy/policy.h>
 #include <tinyformat.h>
 #include <util/moneystr.h>
+#include <util/result.h>
+#include <util/strencodings.h>
 #include <util/string.h>
 #include <util/translation.h>
 
 #include <chrono>
+#include <cstdint>
 #include <memory>
+#include <string_view>
+#include <utility>
 
 using common::AmountErrMsg;
 using kernel::MemPoolLimits;
@@ -40,6 +46,51 @@ void ApplyArgsManOptions(const ArgsManager& argsman, MemPoolLimits& mempool_limi
 }
 }
 
+util::Result<std::pair<int32_t, int>> ParseDustDynamicOpt(std::string_view optstr, const unsigned int max_fee_estimate_blocks)
+{
+    if (optstr == "0" || optstr == "off") {
+        return std::pair<int32_t, int>(0, DEFAULT_DUST_RELAY_MULTIPLIER);
+    }
+
+    int multiplier{DEFAULT_DUST_RELAY_MULTIPLIER};
+    if (auto pos = optstr.find('*'); pos != optstr.npos) {
+        int64_t parsed;
+        if ((!ParseFixedPoint(optstr.substr(0, pos), 3, &parsed)) || parsed > std::numeric_limits<int>::max() || parsed < 1) {
+            return util::Error{_("failed to parse multiplier")};
+        }
+        multiplier = parsed;
+        optstr.remove_prefix(pos + 1);
+    }
+
+    if (optstr.rfind("target:", 0) == 0) {
+        if (!max_fee_estimate_blocks) {
+            return util::Error{_("\"target\" mode requires fee estimator (disabled)")};
+        }
+        const auto val = ToIntegral<uint16_t>(optstr.substr(7));
+        if (!val) {
+            return util::Error{_("failed to parse target block count")};
+        }
+        if (*val < 2) {
+            return util::Error{_("target must be at least 2 blocks")};
+        }
+        if (*val > max_fee_estimate_blocks) {
+            return util::Error{strprintf(_("target can only be at most %s blocks"), max_fee_estimate_blocks)};
+        }
+        return std::pair<int32_t, int>(-*val, multiplier);
+    } else if (optstr.rfind("mempool:", 0) == 0) {
+        const auto val = ToIntegral<int32_t>(optstr.substr(8));
+        if (!val) {
+            return util::Error{_("failed to parse mempool position")};
+        }
+        if (*val < 1) {
+            return util::Error{_("mempool position must be at least 1 kB")};
+        }
+        return std::pair<int32_t, int>(*val, multiplier);
+    } else {
+        return util::Error{strprintf(_("\"%s\""), optstr)};
+    }
+}
+
 util::Result<void> ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, MemPoolOptions& mempool_opts)
 {
     mempool_opts.check_ratio = argsman.GetIntArg("-checkmempool", mempool_opts.check_ratio);
@@ -80,6 +131,16 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& argsman, const CChainP
             return util::Error{AmountErrMsg("dustrelayfee", argsman.GetArg("-dustrelayfee", ""))};
         }
     }
+    if (argsman.IsArgSet("-dustdynamic")) {
+        const auto optstr = argsman.GetArg("-dustdynamic", DEFAULT_DUST_DYNAMIC);
+        const auto max_fee_estimate_blocks = mempool_opts.estimator ? mempool_opts.estimator->HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE) : (unsigned int)0;
+        const auto parsed = ParseDustDynamicOpt(optstr, max_fee_estimate_blocks);
+        if (!parsed) {
+            return util::Error{strprintf(_("Invalid mode for dustdynamic: %s"), util::ErrorString(parsed))};
+        }
+        mempool_opts.dust_relay_target = parsed->first;
+        mempool_opts.dust_relay_multiplier = parsed->second;
+    }
 
     mempool_opts.permit_bare_pubkey = argsman.GetBoolArg("-permitbarepubkey", DEFAULT_PERMIT_BAREPUBKEY);
 

src/node/mempool_args.h

@@ -7,6 +7,10 @@
 
 #include <util/result.h>
 
+#include <cstdint>
+#include <string_view>
+#include <utility>
+
 class ArgsManager;
 class CChainParams;
 struct bilingual_str;
@@ -14,6 +18,8 @@ namespace kernel {
 struct MemPoolOptions;
 };
 
+[[nodiscard]] util::Result<std::pair<int32_t, int>> ParseDustDynamicOpt(std::string_view optstr, unsigned int max_fee_estimate_blocks);
+
 /**
  * Overlay the options set in \p argsman on top of corresponding members in \p mempool_opts.
  * Returns an error if one was encountered.

src/policy/feerate.h

@@ -74,6 +74,7 @@ public:
     std::string SatsToString() const;
     friend CFeeRate operator*(const CFeeRate& f, int a) { return CFeeRate(a * f.nSatoshisPerK); }
     friend CFeeRate operator*(int a, const CFeeRate& f) { return CFeeRate(a * f.nSatoshisPerK); }
+    friend CFeeRate operator/(const CFeeRate& f, int a) { return CFeeRate(f.nSatoshisPerK / a); }
 
     SERIALIZE_METHODS(CFeeRate, obj) { READWRITE(obj.nSatoshisPerK); }
 };

src/policy/policy.h

@@ -67,6 +67,8 @@ static constexpr unsigned int MAX_STANDARD_SCRIPTSIG_SIZE{1650};
  * only increase the dust limit after prior releases were already not creating
  * outputs below the new threshold */
 static constexpr unsigned int DUST_RELAY_TX_FEE{3000};
+static const std::string DEFAULT_DUST_DYNAMIC{"off"};
+static const int DEFAULT_DUST_RELAY_MULTIPLIER{3'000};
 static const std::string DEFAULT_SPKREUSE{"allow"};
 /** Default for -minrelaytxfee, minimum relay fee for transactions */
 static constexpr unsigned int DEFAULT_MIN_RELAY_TX_FEE{1000};

src/rpc/mempool.cpp

@@ -891,6 +891,22 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool, const std::optional<MempoolHi
     ret.pushKV("mempoolminfee", ValueFromAmount(std::max(pool.GetMinFee(), pool.m_opts.min_relay_feerate).GetFeePerK()));
     ret.pushKV("minrelaytxfee", ValueFromAmount(pool.m_opts.min_relay_feerate.GetFeePerK()));
     ret.pushKV("incrementalrelayfee", ValueFromAmount(pool.m_opts.incremental_relay_feerate.GetFeePerK()));
+    ret.pushKV("dustrelayfee", ValueFromAmount(pool.m_opts.dust_relay_feerate.GetFeePerK()));
+    ret.pushKV("dustrelayfeefloor", ValueFromAmount(pool.m_opts.dust_relay_feerate_floor.GetFeePerK()));
+    if (pool.m_opts.dust_relay_target == 0) {
+        ret.pushKV("dustdynamic", "off");
+    } else {
+        std::string multiplier_str = strprintf("%u", pool.m_opts.dust_relay_multiplier / 1000);
+        if (pool.m_opts.dust_relay_multiplier % 1000) {
+            multiplier_str += strprintf(".%03u", pool.m_opts.dust_relay_multiplier % 1000);
+            while (multiplier_str.back() == '0') multiplier_str.pop_back();
+        }
+        if (pool.m_opts.dust_relay_target < 0) {
+            ret.pushKV("dustdynamic", strprintf("%s*target:%u", multiplier_str, -pool.m_opts.dust_relay_target));
+        } else { // pool.m_opts.dust_relay_target > 0
+            ret.pushKV("dustdynamic", strprintf("%s*mempool:%u", multiplier_str, pool.m_opts.dust_relay_target));
+        }
+    }
     ret.pushKV("unbroadcastcount", uint64_t{pool.GetUnbroadcastTxs().size()});
     ret.pushKV("fullrbf", (pool.m_opts.rbf_policy == RBFPolicy::Always));
     switch (pool.m_opts.rbf_policy) {
@@ -984,6 +1000,9 @@ static RPCHelpMan getmempoolinfo()
                 {RPCResult::Type::STR_AMOUNT, "mempoolminfee", "Minimum fee rate in " + CURRENCY_UNIT + "/kvB for tx to be accepted. Is the maximum of minrelaytxfee and minimum mempool fee"},
                 {RPCResult::Type::STR_AMOUNT, "minrelaytxfee", "Current minimum relay fee for transactions"},
                 {RPCResult::Type::NUM, "incrementalrelayfee", "minimum fee rate increment for mempool limiting or replacement in " + CURRENCY_UNIT + "/kvB"},
+                {RPCResult::Type::NUM, "dustrelayfee", "Current fee rate used to define dust, the value of an output so small it will cost more to spend than its value, in " + CURRENCY_UNIT + "/kvB"},
+                {RPCResult::Type::NUM, "dustrelayfeefloor", "Minimum fee rate used to define dust in " + CURRENCY_UNIT + "/kvB"},
+                {RPCResult::Type::STR, "dustdynamic", "Method for automatic adjustments to dustrelayfee (one of: off, target:<blocks>, or mempool:<kB>)"},
                 {RPCResult::Type::NUM, "unbroadcastcount", "Current number of transactions that haven't passed initial broadcast yet"},
                 {RPCResult::Type::BOOL, "fullrbf", "True if the mempool accepts RBF without replaceability signaling inspection"},
                 {RPCResult::Type::STR, "rbf_policy", "Policy used for replacing conflicting transactions by fee (one of: never, optin, always)"},

src/txmempool.cpp

@@ -14,9 +14,11 @@
 #include <crypto/ripemd160.h>
 #include <logging.h>
 #include <policy/coin_age_priority.h>
+#include <policy/fees.h>
 #include <policy/policy.h>
 #include <policy/settings.h>
 #include <random.h>
+#include <scheduler.h>
 #include <tinyformat.h>
 #include <script/script.h>
 #include <util/check.h>
@@ -421,6 +423,13 @@ static CTxMemPool::Options&& Flatten(CTxMemPool::Options&& opts, bilingual_str&
 CTxMemPool::CTxMemPool(Options opts, bilingual_str& error)
     : m_opts{Flatten(std::move(opts), error)}
 {
+    Assert(m_opts.scheduler || !m_opts.dust_relay_target);
+    m_opts.dust_relay_feerate_floor = m_opts.dust_relay_feerate;
+    if (m_opts.scheduler) {
+        m_opts.scheduler->scheduleEvery([this]{
+            UpdateDynamicDustFeerate();
+        }, DYNAMIC_DUST_FEERATE_UPDATE_INTERVAL);
+    }
 }
 
 bool CTxMemPool::isSpent(const COutPoint& outpoint) const
@@ -691,6 +700,36 @@ void CTxMemPool::removeForBlock(const std::vector<CTransactionRef>& vtx, unsigne
     blockSinceLastRollingFeeBump = true;
 }
 
+void CTxMemPool::UpdateDynamicDustFeerate()
+{
+    CFeeRate est_feerate{0};
+    if (m_opts.dust_relay_target < 0 && m_opts.estimator) {
+        static constexpr double target_success_threshold{0.8};
+        est_feerate = m_opts.estimator->estimateRawFee(-m_opts.dust_relay_target, target_success_threshold, FeeEstimateHorizon::LONG_HALFLIFE, nullptr);
+    } else if (m_opts.dust_relay_target > 0) {
+        auto bytes_remaining = int64_t{m_opts.dust_relay_target} * 1'000;
+        LOCK(cs);
+        for (auto mi = mapTx.get<ancestor_score>().begin(); mi != mapTx.get<ancestor_score>().end(); ++mi) {
+            bytes_remaining -= mi->GetTxSize();
+            if (bytes_remaining <= 0) {
+                est_feerate = CFeeRate(mi->GetFee(), mi->GetTxSize());
+                break;
+            }
+        }
+    }
+
+    est_feerate = (est_feerate * m_opts.dust_relay_multiplier) / 1'000;
+
+    if (est_feerate < m_opts.dust_relay_feerate_floor) {
+        est_feerate = m_opts.dust_relay_feerate_floor;
+    }
+
+    if (m_opts.dust_relay_feerate != est_feerate) {
+        LogPrint(BCLog::MEMPOOL, "Updating dust feerate to %s\n", est_feerate.ToString(FeeEstimateMode::SAT_VB));
+        m_opts.dust_relay_feerate = est_feerate;
+    }
+}
+
 void CTxMemPool::check(const CCoinsViewCache& active_coins_tip, int64_t spendheight) const
 {
     if (m_opts.check_ratio == 0) return;

src/txmempool.h

@@ -47,6 +47,8 @@ class ValidationSignals;
 
 struct bilingual_str;
 
+static constexpr std::chrono::minutes DYNAMIC_DUST_FEERATE_UPDATE_INTERVAL{15};
+
 /** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */
 static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF;
 
@@ -214,6 +216,8 @@ struct entry_time {};
 struct ancestor_score {};
 struct index_by_wtxid {};
 
+class CBlockPolicyEstimator;
+
 /**
  * Information about a mempool transaction.
  */
@@ -502,6 +506,8 @@ public:
      */
     void UpdateDependentPriorities(const CTransaction &tx, unsigned int nBlockHeight, bool addToChain);
 
+    void UpdateDynamicDustFeerate();
+
     /** Affect CreateNewBlock prioritisation of transactions */
     void PrioritiseTransaction(const uint256& hash, double dPriorityDelta, const CAmount& nFeeDelta);
     void PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta) { PrioritiseTransaction(hash, 0., nFeeDelta); }

test/functional/feature_fee_estimation.py

@@ -27,6 +27,7 @@ from test_framework.wallet import MiniWallet
 
 MAX_FILE_AGE = 60
 SECONDS_PER_HOUR = 60 * 60
+target_success_threshold = 0.8
 
 def small_txpuzzle_randfee(
     wallet, from_node, conflist, unconflist, amount, min_fee, fee_increment, batch_reqs
@@ -161,6 +162,19 @@ def check_fee_estimates_btw_modes(node, expected_conservative, expected_economic
     assert_equal(fee_est_default, rest_getfee(node.url, 'unset', 1)['feerate'])
 
 
+def get_feerate_into_mempool(node, kB):
+    mempool_entries = list(node.getrawmempool(verbose=True).values())
+    for entry in mempool_entries:
+        entry['feerate_BTC/vB'] = entry['fees']['modified'] / entry['vsize']
+    mempool_entries.sort(key=lambda entry: entry['feerate_BTC/vB'], reverse=True)
+    bytes_remaining = kB * 1000
+    for entry in mempool_entries:
+        bytes_remaining -= entry['vsize']
+        if bytes_remaining <= 0:
+            return satoshi_round(entry['feerate_BTC/vB'] * 1000)
+    raise AssertionError('Entire mempool is smaller than %s kB' % (kB,))
+
+
 class EstimateFeeTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 3
@@ -261,6 +275,63 @@ class EstimateFeeTest(BitcoinTestFramework):
         self.log.info("Final estimates after emptying mempools")
         check_estimates(self.nodes[1], self.fees_per_kb)
 
+    def test_feerate_dustrelayfee_common(self, node, multiplier, dust_mode, desc, expected_base):
+        dust_parameter = f"-dustdynamic={dust_mode}".replace('=3*', '=')
+        self.log.info(f"Test dust limit setting {dust_parameter} (fee estimation for {desc})")
+        self.restart_node(0, extra_args=[dust_parameter])
+        assert_equal(node.getmempoolinfo()['dustdynamic'], dust_mode)
+        with node.busy_wait_for_debug_log([b'Updating dust feerate']):
+            node.mockscheduler(SECONDS_PER_HOUR)
+        mempool_info = node.getmempoolinfo()
+        assert_equal(mempool_info['dustrelayfee'], satoshi_round(expected_base() * multiplier))
+        assert mempool_info['dustrelayfee'] > mempool_info['dustrelayfeefloor']
+
+    def test_feerate_dustrelayfee_target(self, node, multiplier, dustfee_target):
+        dust_mode = f"{multiplier}*target:{dustfee_target}"
+        self.test_feerate_dustrelayfee_common(node, multiplier, dust_mode, f'{dustfee_target} blocks', lambda: node.estimaterawfee(dustfee_target, target_success_threshold)['long']['feerate'])
+
+    def test_feerate_dustrelayfee_mempool(self, node, multiplier, dustfee_kB):
+        dust_mode = f"{multiplier}*mempool:{dustfee_kB}"
+        self.test_feerate_dustrelayfee_common(node, multiplier, dust_mode, f'{dustfee_kB} kB into mempool', lambda: get_feerate_into_mempool(node, dustfee_kB))
+
+    def test_feerate_dustrelayfee(self):
+        node = self.nodes[0]
+
+        # test dustdynamic=target:<blocks>
+        for dustfee_target in (2, 8, 1008):
+            for multiplier in (Decimal('0.5'), 1, 3, Decimal('3.3'), 10, Decimal('10.001')):
+                self.test_feerate_dustrelayfee_target(node, multiplier, dustfee_target)
+
+        # Fill mempool up
+        mempool_size = 0
+        batch_sendtx_reqs = []
+        min_fee = Decimal("0.00001")
+        while mempool_size < 52000:
+            (tx_bytes, fee) = small_txpuzzle_randfee(
+                self.wallet,
+                self.nodes[0],
+                self.confutxo,
+                self.memutxo,
+                Decimal("0.005"),
+                min_fee,
+                min_fee,
+                batch_sendtx_reqs,
+            )
+            mempool_size += tx_bytes
+        node.batch(batch_sendtx_reqs)
+
+        # test dustdynamic=mempool:<kB>
+        for dustfee_kB in (1, 10, 50):
+            for multiplier in (Decimal('0.5'), 1, 3, Decimal('3.3'), 10, Decimal('10.001')):
+                self.test_feerate_dustrelayfee_mempool(node, multiplier, dustfee_kB)
+
+        # Restore nodes to a normal state, wiping the mempool
+        self.stop_node(0)
+        (self.nodes[0].chain_path / 'mempool.dat').unlink()
+        self.start_node(0)
+        self.connect_nodes(1, 0)
+        self.connect_nodes(0, 2)
+
     def test_feerate_mempoolminfee(self):
         high_val = 3 * self.nodes[1].estimatesmartfee(1)["feerate"]
         self.restart_node(1, extra_args=[f"-minrelaytxfee={high_val}", '-rest'])
@@ -475,6 +546,8 @@ class EstimateFeeTest(BitcoinTestFramework):
         self.log.info("Test fee_estimates.dat is flushed periodically")
         self.test_estimate_dat_is_flushed_periodically()
 
+        self.test_feerate_dustrelayfee()
+
         # check that the effective feerate is greater than or equal to the mempoolminfee even for high mempoolminfee
         self.log.info(
             "Test fee rate estimation after restarting node with high MempoolMinFee"