Merge 8501 via old_stats_rpc-25

src/Makefile.am

@@ -265,6 +265,7 @@ BITCOIN_CORE_H = \
   script/standard.h \
   shutdown.h \
   signet.h \
+  stats/stats.h \
   streams.h \
   support/allocators/secure.h \
   support/allocators/zeroafterfree.h \
@@ -442,6 +443,8 @@ libbitcoin_node_a_SOURCES = \
   script/sigcache.cpp \
   shutdown.cpp \
   signet.cpp \
+  stats/rpc_stats.cpp \
+  stats/stats.cpp \
   timedata.cpp \
   torcontrol.cpp \
   txdb.cpp \

src/Makefile.test.include

@@ -166,6 +166,9 @@ BITCOIN_TESTS =\
   test/versionbits_tests.cpp \
   test/xoroshiro128plusplus_tests.cpp
 
+BITCOIN_TESTS += \
+  stats/test/stats_tests.cpp
+
 if ENABLE_WALLET
 BITCOIN_TESTS += \
   wallet/test/feebumper_tests.cpp \

src/init.cpp

@@ -62,6 +62,7 @@
 #include <script/sigcache.h>
 #include <script/standard.h>
 #include <shutdown.h>
+#include <stats/stats.h>
 #include <sync.h>
 #include <timedata.h>
 #include <torcontrol.h>
@@ -633,6 +634,8 @@ void SetupServerArgs(ArgsManager& argsman)
     argsman.AddArg("-sandbox=<mode>", "Use the experimental syscall sandbox in the specified mode (-sandbox=log-and-abort or -sandbox=abort). Allow only expected syscalls to be used by bitcoind. Note that this is an experimental new feature that may cause bitcoind to exit or crash unexpectedly: use with caution. In the \"log-and-abort\" mode the invocation of an unexpected syscall results in a debug handler being invoked which will log the incident and terminate the program (without executing the unexpected syscall). In the \"abort\" mode the invocation of an unexpected syscall results in the entire process being killed immediately by the kernel without executing the unexpected syscall.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
 #endif // USE_SYSCALL_SANDBOX
 
+    CStats::AddStatsOptions();
+
     // Add the hidden options
     argsman.AddHiddenArgs(hidden_args);
 }
@@ -1049,6 +1052,8 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb
         }
     }
 
+    if (!CStats::parameterInteraction()) return false;
+
     return true;
 }
 

src/rpc/register.h

@@ -20,6 +20,7 @@ void RegisterRawTransactionRPCCommands(CRPCTable &tableRPC);
 void RegisterSignMessageRPCCommands(CRPCTable&);
 void RegisterSignerRPCCommands(CRPCTable &tableRPC);
 void RegisterTxoutProofRPCCommands(CRPCTable&);
+void RegisterStatsRPCCommands(CRPCTable&);
 
 static inline void RegisterAllCoreRPCCommands(CRPCTable &t)
 {
@@ -36,6 +37,7 @@ static inline void RegisterAllCoreRPCCommands(CRPCTable &t)
     RegisterSignerRPCCommands(t);
 #endif // ENABLE_EXTERNAL_SIGNER
     RegisterTxoutProofRPCCommands(t);
+    RegisterStatsRPCCommands(t);
 }
 
 #endif // BITCOIN_RPC_REGISTER_H

src/stats/rpc_stats.cpp

@@ -0,0 +1,80 @@
+// Copyright (c) 2016 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <rpc/server.h>
+#include <rpc/util.h>
+#include <stats/stats.h>
+#include <util/system.h>
+#include <util/strencodings.h>
+
+#include <stdint.h>
+
+#include <univalue.h>
+
+static RPCHelpMan getmempoolstats()
+{
+    return RPCHelpMan{"getmempoolstats",
+                "\nReturns the collected mempool statistics (non-linear non-interpolated samples).\n",
+                {},
+                RPCResult{
+                    RPCResult::Type::OBJ, "", "",
+                    {
+                        {RPCResult::Type::NUM_TIME, "time_from", "Timestamp, first sample"},
+                        {RPCResult::Type::NUM_TIME, "time_to", "Timestamp, last sample"},
+                        {RPCResult::Type::ARR, "samples", "",
+                        {
+                            {RPCResult::Type::ARR_FIXED, "", "",
+                            {
+                                {RPCResult::Type::NUM, "", "Sample time in seconds (relative to other sample times only)"},
+                                {RPCResult::Type::NUM, "", "Number of transactions in the memory pool"},
+                                {RPCResult::Type::NUM, "", "Memory usage by memory pool"},
+                                {RPCResult::Type::NUM, "", "Minimum fee per kB"},
+                            }},
+                        }},
+                    }},
+                RPCExamples{
+                    HelpExampleCli("getmempoolstats", "")
+            + HelpExampleRpc("getmempoolstats", "")
+                },
+        [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue
+{
+    // get stats from the core stats model
+    uint64_t timeFrom = 0;
+    uint64_t timeTo = 0;
+    mempoolSamples_t samples = CStats::DefaultStats()->mempoolGetValuesInRange(timeFrom, timeTo);
+
+    // use "flat" json encoding for performance reasons
+    UniValue samplesObj(UniValue::VARR);
+    for (struct CStatsMempoolSample& sample : samples) {
+        UniValue singleSample(UniValue::VARR);
+        singleSample.push_back(UniValue((uint64_t)sample.m_time_delta));
+        singleSample.push_back(UniValue(sample.m_tx_count));
+        singleSample.push_back(UniValue(sample.m_dyn_mem_usage));
+        singleSample.push_back(UniValue(sample.m_min_fee_per_k));
+        samplesObj.push_back(singleSample);
+    }
+
+    UniValue result(UniValue::VOBJ);
+    result.pushKV("time_from", timeFrom);
+    result.pushKV("time_to", timeTo);
+    result.pushKV("samples", samplesObj);
+
+    return result;
+},
+    };
+}
+
+void RegisterStatsRPCCommands(CRPCTable& t)
+{
+// clang-format off
+static const CRPCCommand commands[] =
+{ //  category              actor (function)
+  //  --------------------- ------------------------
+    { "stats",              &getmempoolstats,                    },
+};
+// clang-format on
+    for (const auto& c : commands) {
+        t.appendCommand(c.name, &c);
+    }
+}

src/stats/stats.cpp

@@ -0,0 +1,145 @@
+// Copyright (c) 2016 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <stats/stats.h>
+
+#include <memusage.h>
+#include <util/time.h>
+
+#include <util/system.h>
+
+#include <cmath>
+
+static const uint32_t SAMPLE_MIN_DELTA_IN_SEC = 2;
+static const int CLEANUP_SAMPLES_THRESHOLD = 100;
+size_t CStats::maxStatsMemory = 0;
+const size_t CStats::DEFAULT_MAX_STATS_MEMORY = 10 * 1024 * 1024; //10 MB
+const bool CStats::DEFAULT_STATISTICS_ENABLED = false;
+std::atomic<bool> CStats::m_stats_enabled(false); //disable stats by default
+
+CStats* CStats::m_shared_instance{nullptr};
+
+CStats* CStats::DefaultStats()
+{
+    if (!m_shared_instance)
+        m_shared_instance = new CStats();
+
+    return m_shared_instance;
+}
+
+void CStats::addMempoolSample(int64_t txcount, int64_t dynUsage, int64_t currentMinRelayFee)
+{
+    if (!m_stats_enabled)
+        return;
+
+    uint64_t now = GetTime();
+    {
+        LOCK(cs_stats);
+
+        // set the mempool stats start time if this is the first sample
+        if (m_mempool_stats.m_start_time == 0)
+            m_mempool_stats.m_start_time = now;
+
+        // ensure the minimum time delta between samples
+        if (m_mempool_stats.m_samples.size() && m_mempool_stats.m_start_time + m_mempool_stats.m_samples.back().m_time_delta + SAMPLE_MIN_DELTA_IN_SEC >= now) {
+            return;
+        }
+
+        // calculate the current time delta and add a sample
+        uint32_t timeDelta = now - m_mempool_stats.m_start_time; //truncate to uint32_t should be sufficient
+        m_mempool_stats.m_samples.push_back({timeDelta, txcount, dynUsage, currentMinRelayFee});
+        m_mempool_stats.m_cleanup_counter++;
+
+        // check if we should cleanup the container
+        if (m_mempool_stats.m_cleanup_counter >= CLEANUP_SAMPLES_THRESHOLD) {
+            //check memory usage
+            if (memusage::DynamicUsage(m_mempool_stats.m_samples) > maxStatsMemory && m_mempool_stats.m_samples.size() > 1) {
+                // only shrink if the vector.capacity() is > the target for performance reasons
+                m_mempool_stats.m_samples.shrink_to_fit();
+                const size_t memUsage = memusage::DynamicUsage(m_mempool_stats.m_samples);
+                // calculate the amount of samples we need to remove
+                size_t itemsToRemove = (memUsage - maxStatsMemory + sizeof(m_mempool_stats.m_samples[0]) - 1) / sizeof(m_mempool_stats.m_samples[0]);
+
+                // sanity check; always keep the most recent sample we just added
+                if (m_mempool_stats.m_samples.size() <= itemsToRemove) {
+                    itemsToRemove = m_mempool_stats.m_samples.size() - 1;
+                }
+                m_mempool_stats.m_samples.erase(m_mempool_stats.m_samples.begin(), m_mempool_stats.m_samples.begin() + itemsToRemove);
+            }
+            // shrink vector
+            m_mempool_stats.m_samples.shrink_to_fit();
+            m_mempool_stats.m_cleanup_counter = 0;
+        }
+
+        // fire signal
+        MempoolStatsDidChange();
+    }
+}
+
+mempoolSamples_t CStats::mempoolGetValuesInRange(uint64_t& fromTime, uint64_t& toTime)
+{
+    if (!m_stats_enabled)
+        return mempoolSamples_t();
+
+    LOCK(cs_stats);
+
+    // if empty, return directly
+    if (!m_mempool_stats.m_samples.size())
+        return m_mempool_stats.m_samples;
+
+
+    if (!(fromTime == 0 && toTime == 0) && (fromTime > m_mempool_stats.m_start_time + m_mempool_stats.m_samples.front().m_time_delta || toTime < m_mempool_stats.m_start_time + m_mempool_stats.m_samples.back().m_time_delta)) {
+        mempoolSamples_t::iterator fromSample = m_mempool_stats.m_samples.begin();
+        mempoolSamples_t::iterator toSample = std::prev(m_mempool_stats.m_samples.end());
+
+        // create subset of samples
+        bool fromSet = false;
+        for (mempoolSamples_t::iterator it = m_mempool_stats.m_samples.begin(); it != m_mempool_stats.m_samples.end(); ++it) {
+            if (m_mempool_stats.m_start_time + (*it).m_time_delta >= fromTime && !fromSet) {
+                fromSample = it;
+                fromSet = true;
+            }
+            else if (m_mempool_stats.m_start_time + (*it).m_time_delta > toTime) {
+                toSample = std::prev(it);
+                break;
+            }
+        }
+
+        mempoolSamples_t subset(fromSample, toSample + 1);
+
+        // set the fromTime and toTime pass-by-ref parameters
+        fromTime = m_mempool_stats.m_start_time + (*fromSample).m_time_delta;
+        toTime = m_mempool_stats.m_start_time + (*toSample).m_time_delta;
+
+        // return subset
+        return subset;
+    }
+
+    // return all available samples
+    fromTime = m_mempool_stats.m_start_time + m_mempool_stats.m_samples.front().m_time_delta;
+    toTime = m_mempool_stats.m_start_time + m_mempool_stats.m_samples.back().m_time_delta;
+    return m_mempool_stats.m_samples;
+}
+
+void CStats::setMaxMemoryUsageTarget(size_t maxMem)
+{
+    m_stats_enabled = (maxMem > 0);
+
+    LOCK(cs_stats);
+    maxStatsMemory = maxMem;
+}
+
+void CStats::AddStatsOptions()
+{
+    gArgs.AddArg("-statsenable", strprintf("Enable statistics (default: %u)", DEFAULT_STATISTICS_ENABLED), ArgsManager::ALLOW_ANY, OptionsCategory::STATS);
+    gArgs.AddArg("-statsmaxmemorytarget=<n>", strprintf("Set the memory limit target for statistics in bytes (default: %u)", DEFAULT_MAX_STATS_MEMORY), ArgsManager::ALLOW_ANY, OptionsCategory::STATS);
+}
+
+bool CStats::parameterInteraction()
+{
+    if (gArgs.GetBoolArg("-statsenable", DEFAULT_STATISTICS_ENABLED))
+        DefaultStats()->setMaxMemoryUsageTarget(gArgs.GetIntArg("-statsmaxmemorytarget", DEFAULT_MAX_STATS_MEMORY));
+
+    return true;
+}

src/stats/stats.h

@@ -0,0 +1,77 @@
+// Copyright (c) 2016 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_STATS_STATS_H
+#define BITCOIN_STATS_STATS_H
+
+#include <sync.h>
+
+#include <atomic>
+#include <stdlib.h>
+#include <vector>
+
+#include <boost/signals2/signal.hpp>
+
+struct CStatsMempoolSample {
+    uint32_t m_time_delta;    //use 32bit time delta to save memory
+    int64_t m_tx_count;       //transaction count
+    int64_t m_dyn_mem_usage;  //dynamic mempool usage
+    int64_t m_min_fee_per_k;  //min fee per Kb
+};
+
+typedef std::vector<struct CStatsMempoolSample> mempoolSamples_t;
+
+// simple mempool stats container
+class CStatsMempool
+{
+public:
+    uint64_t m_start_time;      //start time of the container
+    mempoolSamples_t m_samples;
+    uint64_t m_cleanup_counter; //internal counter to trogger cleanups
+
+    CStatsMempool()
+    {
+        m_start_time = 0;
+        m_cleanup_counter = 0;
+    }
+};
+
+// Class that manages various types of statistics and its memory consumption
+class CStats
+{
+private:
+    static size_t maxStatsMemory;                  //maximum amount of memory to use for the stats
+
+    static CStats* m_shared_instance;
+    mutable RecursiveMutex cs_stats;
+
+    CStatsMempool m_mempool_stats; //mempool stats container
+
+public:
+    static const size_t DEFAULT_MAX_STATS_MEMORY; //default maximum of memory to use
+    static const bool DEFAULT_STATISTICS_ENABLED; //default value for enabling statistics
+
+    static std::atomic<bool> m_stats_enabled; //if enabled, stats will be collected
+    static CStats* DefaultStats(); //shared instance
+
+    /* signals */
+    boost::signals2::signal<void(void)> MempoolStatsDidChange; //mempool stats update signal
+
+    /* add a mempool stats sample */
+    void addMempoolSample(int64_t txcount, int64_t dynUsage, int64_t currentMinRelayFee);
+
+    /* get all mempool samples in range */
+    mempoolSamples_t mempoolGetValuesInRange(uint64_t& fromTime, uint64_t& toTime);
+
+    /* set the target for the maximum memory consumption (in bytes) */
+    void setMaxMemoryUsageTarget(size_t maxMem);
+
+    /* register the statistics module help strings */
+    static void AddStatsOptions();
+
+    /* access the parameters and map it to the internal model */
+    static bool parameterInteraction();
+};
+
+#endif // BITCOIN_STATS_STATS_H

src/stats/test/stats_tests.cpp

@@ -0,0 +1,76 @@
+// Copyright (c) 2016 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <stats/stats.h>
+
+#include <test/util/setup_common.h>
+#include <util/system.h>
+
+#include <boost/test/unit_test.hpp>
+
+BOOST_FIXTURE_TEST_SUITE(stats_tests, BasicTestingSetup)
+
+BOOST_AUTO_TEST_CASE(stats)
+{
+    CStats::DefaultStats()->setMaxMemoryUsageTarget(CStats::DEFAULT_MAX_STATS_MEMORY);
+
+    uint64_t start = GetTime();
+    SetMockTime(start);
+
+    CStats::DefaultStats()->addMempoolSample(0, 1, 1);
+    SetMockTime(start + 1);
+    CStats::DefaultStats()->addMempoolSample(0, 2, 2); //1second should be to short
+    SetMockTime(start + 5);
+    CStats::DefaultStats()->addMempoolSample(3, 4, 3);
+
+    uint64_t queryFromTime = start;
+    uint64_t queryToTime = start + 3600;
+    mempoolSamples_t samples = CStats::DefaultStats()->mempoolGetValuesInRange(queryFromTime, queryToTime);
+
+    BOOST_CHECK_EQUAL(samples[0].m_time_delta, 0U);
+    BOOST_CHECK_EQUAL(samples[1].m_time_delta, 5U);
+    BOOST_CHECK_EQUAL(samples[1].m_tx_count, 3);
+    BOOST_CHECK_EQUAL(samples[1].m_dyn_mem_usage, 4);
+
+    // check retrieving a subset of the available samples
+    queryFromTime = start;
+    queryToTime = start;
+    samples = CStats::DefaultStats()->mempoolGetValuesInRange(queryFromTime, queryToTime);
+    BOOST_CHECK_EQUAL(samples.size(), 1U);
+
+    // add some samples
+    for (int i = 0; i < 10000; i++) {
+        SetMockTime(start + 10 + i * 5);
+        CStats::DefaultStats()->addMempoolSample(i, i + 1, i + 2);
+    }
+
+    queryFromTime = start + 3600;
+    queryToTime = start + 3600;
+    samples = CStats::DefaultStats()->mempoolGetValuesInRange(queryFromTime, queryToTime);
+    BOOST_CHECK_EQUAL(samples.size(), 1U); //get a single sample
+
+    queryFromTime = start;
+    queryToTime = start + 3600;
+    samples = CStats::DefaultStats()->mempoolGetValuesInRange(queryFromTime, queryToTime);
+    BOOST_CHECK(samples.size() >= 3600 / 5);
+
+    // reduce max memory and add 100 samples to ensure it triggers the cleanup
+    CStats::DefaultStats()->setMaxMemoryUsageTarget(10 * 1024);
+    for (int i = 10000; i < 10100; i++) {
+        SetMockTime(start + 10 + i * 5);
+        CStats::DefaultStats()->addMempoolSample(i, i + 1, i + 2);
+    }
+
+    queryFromTime = start;
+    queryToTime = start + 100;
+    samples = CStats::DefaultStats()->mempoolGetValuesInRange(queryFromTime, queryToTime);
+    BOOST_CHECK_EQUAL(samples.size(), 1U);
+
+    queryFromTime = 0; // no range limits
+    queryToTime = 0;   // no range  limits
+    samples = CStats::DefaultStats()->mempoolGetValuesInRange(queryFromTime, queryToTime);
+    BOOST_CHECK_EQUAL(samples.size() < 1000U, true);
+}
+
+BOOST_AUTO_TEST_SUITE_END()

src/test/fuzz/rpc.cpp

@@ -134,6 +134,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{
     "getmempoolancestors",
     "getmempooldescendants",
     "getmempoolentry",
+    "getmempoolstats",
     "gettxspendingprevout",
     "getmempoolinfo",
     "getmininginfo",

src/util/system.cpp

@@ -682,6 +682,9 @@ std::string ArgsManager::GetHelpMessage() const
             case OptionsCategory::REGISTER_COMMANDS:
                 usage += HelpMessageGroup("Register Commands:");
                 break;
+            case OptionsCategory::STATS:
+                usage += HelpMessageGroup("Statistic options:");
+                break;
             default:
                 break;
         }

src/util/system.h

@@ -85,6 +85,7 @@ enum class OptionsCategory {
     GUI,
     COMMANDS,
     REGISTER_COMMANDS,
+    STATS,
 
     HIDDEN // Always the last option to avoid printing these in the help
 };

src/validation.cpp

@@ -39,6 +39,7 @@
 #include <script/sigcache.h>
 #include <shutdown.h>
 #include <signet.h>
+#include <stats/stats.h>
 #include <tinyformat.h>
 #include <txdb.h>
 #include <txmempool.h>
@@ -1199,6 +1200,8 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef
     AssertLockHeld(cs_main);
     LOCK(m_pool.cs); // mempool "read lock" (held through GetMainSignals().TransactionAddedToMempool())
 
+    const CFeeRate mempool_min_fee_rate = m_pool.GetMinFee();
+
     Workspace ws(ptx);
 
     if (!PreChecks(args, ws)) return MempoolAcceptResult::Failure(ws.m_state);
@@ -1223,6 +1226,9 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef
 
     GetMainSignals().TransactionAddedToMempool(ptx, m_pool.GetAndIncrementSequence());
 
+    // update mempool stats cache
+    CStats::DefaultStats()->addMempoolSample(m_pool.size(), m_pool.DynamicMemoryUsage(), mempool_min_fee_rate.GetFeePerK());
+
     return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees,
                                         effective_feerate, single_wtxid);
 }
@@ -2741,6 +2747,12 @@ bool Chainstate::DisconnectTip(BlockValidationState& state, DisconnectedBlockTra
     // Let wallets know transactions went from 1-confirmed to
     // 0-confirmed or conflicted:
     GetMainSignals().BlockDisconnected(pblock, pindexDelete);
+
+    if (m_mempool) {
+        // add mempool stats sample
+        CStats::DefaultStats()->addMempoolSample(m_mempool->size(), m_mempool->DynamicMemoryUsage(), m_mempool->GetMinFee().GetFeePerK());
+    }
+
     return true;
 }
 
@@ -2868,6 +2880,11 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew,
     m_chain.SetTip(*pindexNew);
     UpdateTip(pindexNew);
 
+    if (m_mempool) {
+        // add mempool stats sample
+        CStats::DefaultStats()->addMempoolSample(m_mempool->size(), m_mempool->DynamicMemoryUsage(), m_mempool->GetMinFee().GetFeePerK());
+    }
+
     const auto time_6{SteadyClock::now()};
     time_post_connect += time_6 - time_5;
     time_total += time_6 - time_1;

test/functional/rpc_help.py

@@ -103,7 +103,7 @@ class HelpRpcTest(BitcoinTestFramework):
         # command titles
         titles = [line[3:-3] for line in node.help().splitlines() if line.startswith('==')]
 
-        components = ['Blockchain', 'Control', 'Mining', 'Network', 'Rawtransactions', 'Util']
+        components = ['Blockchain', 'Control', 'Mining', 'Network', 'Rawtransactions', 'Stats', 'Util']
 
         if self.is_wallet_compiled():
             components.append('Wallet')