Merge 8501 via old_stats_rpc-26

src/Makefile.am

@@ -280,6 +280,7 @@ BITCOIN_CORE_H = \
   script/solver.h \
   shutdown.h \
   signet.h \
+  stats/stats.h \
   streams.h \
   support/allocators/pool.h \
   support/allocators/secure.h \
@@ -466,6 +467,8 @@ libbitcoin_node_a_SOURCES = \
   script/sigcache.cpp \
   shutdown.cpp \
   signet.cpp \
+  stats/rpc_stats.cpp \
+  stats/stats.cpp \
   timedata.cpp \
   torcontrol.cpp \
   txdb.cpp \

src/Makefile.test.include

@@ -169,6 +169,9 @@ BITCOIN_TESTS =\
   test/versionbits_tests.cpp \
   test/xoroshiro128plusplus_tests.cpp
 
+BITCOIN_TESTS += \
+  stats/test/stats_tests.cpp
+
 if ENABLE_WALLET
 BITCOIN_TESTS += \
   wallet/test/feebumper_tests.cpp \

src/common/args.cpp

@@ -652,6 +652,9 @@ std::string ArgsManager::GetHelpMessage() const
             case OptionsCategory::REGISTER_COMMANDS:
                 usage += HelpMessageGroup("Register Commands:");
                 break;
+            case OptionsCategory::STATS:
+                usage += HelpMessageGroup("Statistic options:");
+                break;
             default:
                 break;
         }

src/common/args.h

@@ -63,6 +63,7 @@ enum class OptionsCategory {
     GUI,
     COMMANDS,
     REGISTER_COMMANDS,
+    STATS,
 
     HIDDEN // Always the last option to avoid printing these in the help
 };

src/init.cpp

@@ -67,6 +67,7 @@
 #include <scheduler.h>
 #include <script/sigcache.h>
 #include <shutdown.h>
+#include <stats/stats.h>
 #include <sync.h>
 #include <timedata.h>
 #include <torcontrol.h>
@@ -650,6 +651,8 @@ void SetupServerArgs(ArgsManager& argsman)
     hidden_args.emplace_back("-daemonwait");
 #endif
 
+    CStats::AddStatsOptions();
+
     // Add the hidden options
     argsman.AddHiddenArgs(hidden_args);
 }
@@ -1034,6 +1037,8 @@ bool AppInitParameterInteraction(const ArgsManager& args)
         }
     }
 
+    if (!CStats::parameterInteraction()) return false;
+
     return true;
 }
 

src/rpc/register.h

@@ -20,6 +20,7 @@ void RegisterRawTransactionRPCCommands(CRPCTable &tableRPC);
 void RegisterSignMessageRPCCommands(CRPCTable&);
 void RegisterSignerRPCCommands(CRPCTable &tableRPC);
 void RegisterTxoutProofRPCCommands(CRPCTable&);
+void RegisterStatsRPCCommands(CRPCTable&);
 
 static inline void RegisterAllCoreRPCCommands(CRPCTable &t)
 {
@@ -36,6 +37,7 @@ static inline void RegisterAllCoreRPCCommands(CRPCTable &t)
     RegisterSignerRPCCommands(t);
 #endif // ENABLE_EXTERNAL_SIGNER
     RegisterTxoutProofRPCCommands(t);
+    RegisterStatsRPCCommands(t);
 }
 
 #endif // BITCOIN_RPC_REGISTER_H

src/stats/rpc_stats.cpp

@@ -0,0 +1,79 @@
+// Copyright (c) 2016 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <rpc/server.h>
+#include <rpc/util.h>
+#include <stats/stats.h>
+#include <util/strencodings.h>
+
+#include <stdint.h>
+
+#include <univalue.h>
+
+static RPCHelpMan getmempoolstats()
+{
+    return RPCHelpMan{"getmempoolstats",
+                "\nReturns the collected mempool statistics (non-linear non-interpolated samples).\n",
+                {},
+                RPCResult{
+                    RPCResult::Type::OBJ, "", "",
+                    {
+                        {RPCResult::Type::NUM_TIME, "time_from", "Timestamp, first sample"},
+                        {RPCResult::Type::NUM_TIME, "time_to", "Timestamp, last sample"},
+                        {RPCResult::Type::ARR, "samples", "",
+                        {
+                            {RPCResult::Type::ARR_FIXED, "", "",
+                            {
+                                {RPCResult::Type::NUM, "", "Sample time in seconds (relative to other sample times only)"},
+                                {RPCResult::Type::NUM, "", "Number of transactions in the memory pool"},
+                                {RPCResult::Type::NUM, "", "Memory usage by memory pool"},
+                                {RPCResult::Type::NUM, "", "Minimum fee per kB"},
+                            }},
+                        }},
+                    }},
+                RPCExamples{
+                    HelpExampleCli("getmempoolstats", "")
+            + HelpExampleRpc("getmempoolstats", "")
+                },
+        [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue
+{
+    // get stats from the core stats model
+    uint64_t timeFrom = 0;
+    uint64_t timeTo = 0;
+    mempoolSamples_t samples = CStats::DefaultStats()->mempoolGetValuesInRange(timeFrom, timeTo);
+
+    // use "flat" json encoding for performance reasons
+    UniValue samplesObj(UniValue::VARR);
+    for (struct CStatsMempoolSample& sample : samples) {
+        UniValue singleSample(UniValue::VARR);
+        singleSample.push_back(UniValue((uint64_t)sample.m_time_delta));
+        singleSample.push_back(UniValue(sample.m_tx_count));
+        singleSample.push_back(UniValue(sample.m_dyn_mem_usage));
+        singleSample.push_back(UniValue(sample.m_min_fee_per_k));
+        samplesObj.push_back(singleSample);
+    }
+
+    UniValue result(UniValue::VOBJ);
+    result.pushKV("time_from", timeFrom);
+    result.pushKV("time_to", timeTo);
+    result.pushKV("samples", samplesObj);
+
+    return result;
+},
+    };
+}
+
+void RegisterStatsRPCCommands(CRPCTable& t)
+{
+// clang-format off
+static const CRPCCommand commands[] =
+{ //  category              actor (function)
+  //  --------------------- ------------------------
+    { "stats",              &getmempoolstats,                    },
+};
+// clang-format on
+    for (const auto& c : commands) {
+        t.appendCommand(c.name, &c);
+    }
+}

src/stats/stats.cpp

@@ -0,0 +1,144 @@
+// Copyright (c) 2016 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <stats/stats.h>
+
+#include <common/args.h>
+#include <memusage.h>
+#include <util/time.h>
+
+#include <cmath>
+
+static const uint32_t SAMPLE_MIN_DELTA_IN_SEC = 2;
+static const int CLEANUP_SAMPLES_THRESHOLD = 100;
+size_t CStats::maxStatsMemory = 0;
+const size_t CStats::DEFAULT_MAX_STATS_MEMORY = 10 * 1024 * 1024; //10 MB
+const bool CStats::DEFAULT_STATISTICS_ENABLED = false;
+std::atomic<bool> CStats::m_stats_enabled(false); //disable stats by default
+
+CStats* CStats::m_shared_instance{nullptr};
+
+CStats* CStats::DefaultStats()
+{
+    if (!m_shared_instance)
+        m_shared_instance = new CStats();
+
+    return m_shared_instance;
+}
+
+void CStats::addMempoolSample(int64_t txcount, int64_t dynUsage, int64_t currentMinRelayFee)
+{
+    if (!m_stats_enabled)
+        return;
+
+    uint64_t now = GetTime();
+    {
+        LOCK(cs_stats);
+
+        // set the mempool stats start time if this is the first sample
+        if (m_mempool_stats.m_start_time == 0)
+            m_mempool_stats.m_start_time = now;
+
+        // ensure the minimum time delta between samples
+        if (m_mempool_stats.m_samples.size() && m_mempool_stats.m_start_time + m_mempool_stats.m_samples.back().m_time_delta + SAMPLE_MIN_DELTA_IN_SEC >= now) {
+            return;
+        }
+
+        // calculate the current time delta and add a sample
+        uint32_t timeDelta = now - m_mempool_stats.m_start_time; //truncate to uint32_t should be sufficient
+        m_mempool_stats.m_samples.push_back({timeDelta, txcount, dynUsage, currentMinRelayFee});
+        m_mempool_stats.m_cleanup_counter++;
+
+        // check if we should cleanup the container
+        if (m_mempool_stats.m_cleanup_counter >= CLEANUP_SAMPLES_THRESHOLD) {
+            //check memory usage
+            if (memusage::DynamicUsage(m_mempool_stats.m_samples) > maxStatsMemory && m_mempool_stats.m_samples.size() > 1) {
+                // only shrink if the vector.capacity() is > the target for performance reasons
+                m_mempool_stats.m_samples.shrink_to_fit();
+                const size_t memUsage = memusage::DynamicUsage(m_mempool_stats.m_samples);
+                // calculate the amount of samples we need to remove
+                size_t itemsToRemove = (memUsage - maxStatsMemory + sizeof(m_mempool_stats.m_samples[0]) - 1) / sizeof(m_mempool_stats.m_samples[0]);
+
+                // sanity check; always keep the most recent sample we just added
+                if (m_mempool_stats.m_samples.size() <= itemsToRemove) {
+                    itemsToRemove = m_mempool_stats.m_samples.size() - 1;
+                }
+                m_mempool_stats.m_samples.erase(m_mempool_stats.m_samples.begin(), m_mempool_stats.m_samples.begin() + itemsToRemove);
+            }
+            // shrink vector
+            m_mempool_stats.m_samples.shrink_to_fit();
+            m_mempool_stats.m_cleanup_counter = 0;
+        }
+
+        // fire signal
+        MempoolStatsDidChange();
+    }
+}
+
+mempoolSamples_t CStats::mempoolGetValuesInRange(uint64_t& fromTime, uint64_t& toTime)
+{
+    if (!m_stats_enabled)
+        return mempoolSamples_t();
+
+    LOCK(cs_stats);
+
+    // if empty, return directly
+    if (!m_mempool_stats.m_samples.size())
+        return m_mempool_stats.m_samples;
+
+
+    if (!(fromTime == 0 && toTime == 0) && (fromTime > m_mempool_stats.m_start_time + m_mempool_stats.m_samples.front().m_time_delta || toTime < m_mempool_stats.m_start_time + m_mempool_stats.m_samples.back().m_time_delta)) {
+        mempoolSamples_t::iterator fromSample = m_mempool_stats.m_samples.begin();
+        mempoolSamples_t::iterator toSample = std::prev(m_mempool_stats.m_samples.end());
+
+        // create subset of samples
+        bool fromSet = false;
+        for (mempoolSamples_t::iterator it = m_mempool_stats.m_samples.begin(); it != m_mempool_stats.m_samples.end(); ++it) {
+            if (m_mempool_stats.m_start_time + (*it).m_time_delta >= fromTime && !fromSet) {
+                fromSample = it;
+                fromSet = true;
+            }
+            else if (m_mempool_stats.m_start_time + (*it).m_time_delta > toTime) {
+                toSample = std::prev(it);
+                break;
+            }
+        }
+
+        mempoolSamples_t subset(fromSample, toSample + 1);
+
+        // set the fromTime and toTime pass-by-ref parameters
+        fromTime = m_mempool_stats.m_start_time + (*fromSample).m_time_delta;
+        toTime = m_mempool_stats.m_start_time + (*toSample).m_time_delta;
+
+        // return subset
+        return subset;
+    }
+
+    // return all available samples
+    fromTime = m_mempool_stats.m_start_time + m_mempool_stats.m_samples.front().m_time_delta;
+    toTime = m_mempool_stats.m_start_time + m_mempool_stats.m_samples.back().m_time_delta;
+    return m_mempool_stats.m_samples;
+}
+
+void CStats::setMaxMemoryUsageTarget(size_t maxMem)
+{
+    m_stats_enabled = (maxMem > 0);
+
+    LOCK(cs_stats);
+    maxStatsMemory = maxMem;
+}
+
+void CStats::AddStatsOptions()
+{
+    gArgs.AddArg("-statsenable", strprintf("Enable statistics (default: %u)", DEFAULT_STATISTICS_ENABLED), ArgsManager::ALLOW_ANY, OptionsCategory::STATS);
+    gArgs.AddArg("-statsmaxmemorytarget=<n>", strprintf("Set the memory limit target for statistics in bytes (default: %u)", DEFAULT_MAX_STATS_MEMORY), ArgsManager::ALLOW_ANY, OptionsCategory::STATS);
+}
+
+bool CStats::parameterInteraction()
+{
+    if (gArgs.GetBoolArg("-statsenable", DEFAULT_STATISTICS_ENABLED))
+        DefaultStats()->setMaxMemoryUsageTarget(gArgs.GetIntArg("-statsmaxmemorytarget", DEFAULT_MAX_STATS_MEMORY));
+
+    return true;
+}

src/stats/stats.h

@@ -0,0 +1,77 @@
+// Copyright (c) 2016 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_STATS_STATS_H
+#define BITCOIN_STATS_STATS_H
+
+#include <sync.h>
+
+#include <atomic>
+#include <stdlib.h>
+#include <vector>
+
+#include <boost/signals2/signal.hpp>
+
+struct CStatsMempoolSample {
+    uint32_t m_time_delta;    //use 32bit time delta to save memory
+    int64_t m_tx_count;       //transaction count
+    int64_t m_dyn_mem_usage;  //dynamic mempool usage
+    int64_t m_min_fee_per_k;  //min fee per Kb
+};
+
+typedef std::vector<struct CStatsMempoolSample> mempoolSamples_t;
+
+// simple mempool stats container
+class CStatsMempool
+{
+public:
+    uint64_t m_start_time;      //start time of the container
+    mempoolSamples_t m_samples;
+    uint64_t m_cleanup_counter; //internal counter to trogger cleanups
+
+    CStatsMempool()
+    {
+        m_start_time = 0;
+        m_cleanup_counter = 0;
+    }
+};
+
+// Class that manages various types of statistics and its memory consumption
+class CStats
+{
+private:
+    static size_t maxStatsMemory;                  //maximum amount of memory to use for the stats
+
+    static CStats* m_shared_instance;
+    mutable RecursiveMutex cs_stats;
+
+    CStatsMempool m_mempool_stats; //mempool stats container
+
+public:
+    static const size_t DEFAULT_MAX_STATS_MEMORY; //default maximum of memory to use
+    static const bool DEFAULT_STATISTICS_ENABLED; //default value for enabling statistics
+
+    static std::atomic<bool> m_stats_enabled; //if enabled, stats will be collected
+    static CStats* DefaultStats(); //shared instance
+
+    /* signals */
+    boost::signals2::signal<void(void)> MempoolStatsDidChange; //mempool stats update signal
+
+    /* add a mempool stats sample */
+    void addMempoolSample(int64_t txcount, int64_t dynUsage, int64_t currentMinRelayFee);
+
+    /* get all mempool samples in range */
+    mempoolSamples_t mempoolGetValuesInRange(uint64_t& fromTime, uint64_t& toTime);
+
+    /* set the target for the maximum memory consumption (in bytes) */
+    void setMaxMemoryUsageTarget(size_t maxMem);
+
+    /* register the statistics module help strings */
+    static void AddStatsOptions();
+
+    /* access the parameters and map it to the internal model */
+    static bool parameterInteraction();
+};
+
+#endif // BITCOIN_STATS_STATS_H

src/stats/test/stats_tests.cpp

@@ -0,0 +1,76 @@
+// Copyright (c) 2016 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <stats/stats.h>
+
+#include <test/util/setup_common.h>
+#include <util/time.h>
+
+#include <boost/test/unit_test.hpp>
+
+BOOST_FIXTURE_TEST_SUITE(stats_tests, BasicTestingSetup)
+
+BOOST_AUTO_TEST_CASE(stats)
+{
+    CStats::DefaultStats()->setMaxMemoryUsageTarget(CStats::DEFAULT_MAX_STATS_MEMORY);
+
+    uint64_t start = GetTime();
+    SetMockTime(start);
+
+    CStats::DefaultStats()->addMempoolSample(0, 1, 1);
+    SetMockTime(start + 1);
+    CStats::DefaultStats()->addMempoolSample(0, 2, 2); //1second should be to short
+    SetMockTime(start + 5);
+    CStats::DefaultStats()->addMempoolSample(3, 4, 3);
+
+    uint64_t queryFromTime = start;
+    uint64_t queryToTime = start + 3600;
+    mempoolSamples_t samples = CStats::DefaultStats()->mempoolGetValuesInRange(queryFromTime, queryToTime);
+
+    BOOST_CHECK_EQUAL(samples[0].m_time_delta, 0U);
+    BOOST_CHECK_EQUAL(samples[1].m_time_delta, 5U);
+    BOOST_CHECK_EQUAL(samples[1].m_tx_count, 3);
+    BOOST_CHECK_EQUAL(samples[1].m_dyn_mem_usage, 4);
+
+    // check retrieving a subset of the available samples
+    queryFromTime = start;
+    queryToTime = start;
+    samples = CStats::DefaultStats()->mempoolGetValuesInRange(queryFromTime, queryToTime);
+    BOOST_CHECK_EQUAL(samples.size(), 1U);
+
+    // add some samples
+    for (int i = 0; i < 10000; i++) {
+        SetMockTime(start + 10 + i * 5);
+        CStats::DefaultStats()->addMempoolSample(i, i + 1, i + 2);
+    }
+
+    queryFromTime = start + 3600;
+    queryToTime = start + 3600;
+    samples = CStats::DefaultStats()->mempoolGetValuesInRange(queryFromTime, queryToTime);
+    BOOST_CHECK_EQUAL(samples.size(), 1U); //get a single sample
+
+    queryFromTime = start;
+    queryToTime = start + 3600;
+    samples = CStats::DefaultStats()->mempoolGetValuesInRange(queryFromTime, queryToTime);
+    BOOST_CHECK(samples.size() >= 3600 / 5);
+
+    // reduce max memory and add 100 samples to ensure it triggers the cleanup
+    CStats::DefaultStats()->setMaxMemoryUsageTarget(10 * 1024);
+    for (int i = 10000; i < 10100; i++) {
+        SetMockTime(start + 10 + i * 5);
+        CStats::DefaultStats()->addMempoolSample(i, i + 1, i + 2);
+    }
+
+    queryFromTime = start;
+    queryToTime = start + 100;
+    samples = CStats::DefaultStats()->mempoolGetValuesInRange(queryFromTime, queryToTime);
+    BOOST_CHECK_EQUAL(samples.size(), 1U);
+
+    queryFromTime = 0; // no range limits
+    queryToTime = 0;   // no range  limits
+    samples = CStats::DefaultStats()->mempoolGetValuesInRange(queryFromTime, queryToTime);
+    BOOST_CHECK_EQUAL(samples.size() < 1000U, true);
+}
+
+BOOST_AUTO_TEST_SUITE_END()

src/test/fuzz/rpc.cpp

@@ -137,6 +137,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{
     "getmempoolancestors",
     "getmempooldescendants",
     "getmempoolentry",
+    "getmempoolstats",
     "getmempoolinfo",
     "getmininginfo",
     "getnettotals",

src/validation.cpp

@@ -43,6 +43,7 @@
 #include <script/script.h>
 #include <script/sigcache.h>
 #include <signet.h>
+#include <stats/stats.h>
 #include <tinyformat.h>
 #include <txdb.h>
 #include <txmempool.h>
@@ -1220,6 +1221,8 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef
     AssertLockHeld(cs_main);
     LOCK(m_pool.cs); // mempool "read lock" (held through GetMainSignals().TransactionAddedToMempool())
 
+    const CFeeRate mempool_min_fee_rate = m_pool.GetMinFee();
+
     Workspace ws(ptx);
 
     if (!PreChecks(args, ws)) return MempoolAcceptResult::Failure(ws.m_state);
@@ -1244,6 +1247,9 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef
 
     GetMainSignals().TransactionAddedToMempool(ptx, m_pool.GetAndIncrementSequence());
 
+    // update mempool stats cache
+    CStats::DefaultStats()->addMempoolSample(m_pool.size(), m_pool.DynamicMemoryUsage(), mempool_min_fee_rate.GetFeePerK());
+
     return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees,
                                         effective_feerate, single_wtxid);
 }
@@ -2813,6 +2819,12 @@ bool Chainstate::DisconnectTip(BlockValidationState& state, DisconnectedBlockTra
     // Let wallets know transactions went from 1-confirmed to
     // 0-confirmed or conflicted:
     GetMainSignals().BlockDisconnected(pblock, pindexDelete);
+
+    if (m_mempool) {
+        // add mempool stats sample
+        CStats::DefaultStats()->addMempoolSample(m_mempool->size(), m_mempool->DynamicMemoryUsage(), m_mempool->GetMinFee().GetFeePerK());
+    }
+
     return true;
 }
 
@@ -2938,6 +2950,11 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew,
     m_chain.SetTip(*pindexNew);
     UpdateTip(pindexNew);
 
+    if (m_mempool) {
+        // add mempool stats sample
+        CStats::DefaultStats()->addMempoolSample(m_mempool->size(), m_mempool->DynamicMemoryUsage(), m_mempool->GetMinFee().GetFeePerK());
+    }
+
     const auto time_6{SteadyClock::now()};
     time_post_connect += time_6 - time_5;
     time_total += time_6 - time_1;

test/functional/rpc_help.py

@@ -103,7 +103,7 @@ class HelpRpcTest(BitcoinTestFramework):
         # command titles
         titles = [line[3:-3] for line in node.help().splitlines() if line.startswith('==')]
 
-        components = ['Blockchain', 'Control', 'Mining', 'Network', 'Rawtransactions', 'Util']
+        components = ['Blockchain', 'Control', 'Mining', 'Network', 'Rawtransactions', 'Stats', 'Util']
 
         if self.is_wallet_compiled():
             components.append('Wallet')