Add mempool statistics collector

src/Makefile.am

@@ -278,6 +278,7 @@ BITCOIN_CORE_H = \
   script/signingprovider.h \
   script/solver.h \
   signet.h \
+  stats/stats.h \
   streams.h \
   support/allocators/pool.h \
   support/allocators/secure.h \
@@ -467,6 +468,7 @@ libbitcoin_node_a_SOURCES = \
   rpc/txoutproof.cpp \
   script/sigcache.cpp \
   signet.cpp \
+  stats/stats.cpp \
   torcontrol.cpp \
   txdb.cpp \
   txmempool.cpp \

src/Makefile.test.include

@@ -177,6 +177,9 @@ BITCOIN_TESTS =\
   test/validationinterface_tests.cpp \
   test/versionbits_tests.cpp
 
+BITCOIN_TESTS += \
+  stats/test/stats_tests.cpp
+
 if ENABLE_WALLET
 BITCOIN_TESTS += \
   wallet/test/feebumper_tests.cpp \

src/stats/stats.cpp

@@ -0,0 +1,118 @@
+// Copyright (c) 2016 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <stats/stats.h>
+
+#include <common/args.h>
+#include <memusage.h>
+#include <util/time.h>
+
+#include <cmath>
+
+static const uint32_t SAMPLE_MIN_DELTA_IN_SEC = 2;
+static const int CLEANUP_SAMPLES_THRESHOLD = 100;
+static const size_t MAX_MEMORY_STATS = 10 * 1024 * 1024; //10 MB
+std::atomic<bool> CStats::m_stats_enabled(false); //disable stats by default
+
+CStats* CStats::m_shared_instance{nullptr};
+
+CStats* CStats::DefaultStats()
+{
+    if (!m_shared_instance)
+        m_shared_instance = new CStats();
+
+    return m_shared_instance;
+}
+
+void CStats::addMempoolSample(int64_t txcount, int64_t dynUsage, int64_t currentMinRelayFee)
+{
+    if (!m_stats_enabled)
+        return;
+
+    uint64_t now = GetTime();
+    {
+        LOCK(cs_stats);
+
+        // set the mempool stats start time if this is the first sample
+        if (m_mempool_stats.m_start_time == 0)
+            m_mempool_stats.m_start_time = now;
+
+        // ensure the minimum time delta between samples
+        if (m_mempool_stats.m_samples.size() && m_mempool_stats.m_samples.back().m_time_delta + SAMPLE_MIN_DELTA_IN_SEC >= now - m_mempool_stats.m_start_time)
+            return;
+
+        // calculate the current time delta and add a sample
+        uint32_t timeDelta = now - m_mempool_stats.m_start_time; //truncate to uint32_t should be sufficient
+        m_mempool_stats.m_samples.push_back({timeDelta, txcount, dynUsage, currentMinRelayFee});
+        m_mempool_stats.m_cleanup_counter++;
+
+        // check if we should cleanup the container
+        if (m_mempool_stats.m_cleanup_counter >= CLEANUP_SAMPLES_THRESHOLD) {
+            //check memory usage
+            int32_t memDelta = memusage::DynamicUsage(m_mempool_stats.m_samples) - MAX_MEMORY_STATS;
+            if (memDelta > 0 && m_mempool_stats.m_samples.size()) {
+                // only shrink if the vector.capacity() is > the target for performance reasons
+                m_mempool_stats.m_samples.shrink_to_fit();
+                int32_t memUsage = memusage::DynamicUsage(m_mempool_stats.m_samples);
+                // calculate the amount of samples we need to remove
+                size_t itemsToRemove = ceil((memUsage - MAX_MEMORY_STATS) / sizeof(m_mempool_stats.m_samples[0]));
+
+                // make sure the vector contains more items then we'd like to remove
+                if (m_mempool_stats.m_samples.size() > itemsToRemove)
+                    m_mempool_stats.m_samples.erase(m_mempool_stats.m_samples.begin(), m_mempool_stats.m_samples.begin() + itemsToRemove);
+            }
+            // shrink vector
+            m_mempool_stats.m_samples.shrink_to_fit();
+            m_mempool_stats.m_cleanup_counter = 0;
+        }
+
+        // fire signal
+        MempoolStatsDidChange();
+    }
+}
+
+mempoolSamples_t CStats::mempoolGetValuesInRange(uint64_t& fromTime, uint64_t& toTime)
+{
+    if (!m_stats_enabled)
+        return mempoolSamples_t();
+
+    LOCK(cs_stats);
+
+    // if empty, return directly
+    if (!m_mempool_stats.m_samples.size())
+        return m_mempool_stats.m_samples;
+
+
+    if (!(fromTime == 0 && toTime == 0) && (fromTime > m_mempool_stats.m_start_time + m_mempool_stats.m_samples.front().m_time_delta || toTime < m_mempool_stats.m_start_time + m_mempool_stats.m_samples.back().m_time_delta)) {
+        mempoolSamples_t::iterator fromSample = m_mempool_stats.m_samples.begin();
+        mempoolSamples_t::iterator toSample = std::prev(m_mempool_stats.m_samples.end());
+
+        // create subset of samples
+        bool fromSet = false;
+        for (mempoolSamples_t::iterator it = m_mempool_stats.m_samples.begin(); it != m_mempool_stats.m_samples.end(); ++it) {
+            if (m_mempool_stats.m_start_time + (*it).m_time_delta >= fromTime && !fromSet) {
+                fromSample = it;
+                fromSet = true;
+            }
+            else if (m_mempool_stats.m_start_time + (*it).m_time_delta > toTime) {
+                toSample = std::prev(it);
+                break;
+            }
+        }
+
+        mempoolSamples_t subset(fromSample, toSample + 1);
+
+        // set the fromTime and toTime pass-by-ref parameters
+        fromTime = m_mempool_stats.m_start_time + (*fromSample).m_time_delta;
+        toTime = m_mempool_stats.m_start_time + (*toSample).m_time_delta;
+
+        // return subset
+        return subset;
+    }
+
+    // return all available samples
+    fromTime = m_mempool_stats.m_start_time + m_mempool_stats.m_samples.front().m_time_delta;
+    toTime = m_mempool_stats.m_start_time + m_mempool_stats.m_samples.back().m_time_delta;
+    return m_mempool_stats.m_samples;
+}

src/stats/stats.h

@@ -0,0 +1,63 @@
+// Copyright (c) 2016 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_STATS_STATS_H
+#define BITCOIN_STATS_STATS_H
+
+#include <sync.h>
+
+#include <atomic>
+#include <stdlib.h>
+#include <vector>
+
+#include <boost/signals2/signal.hpp>
+
+struct CStatsMempoolSample {
+    uint32_t m_time_delta;    //use 32bit time delta to save memory
+    int64_t m_tx_count;       //transaction count
+    int64_t m_dyn_mem_usage;  //dynamic mempool usage
+    int64_t m_min_fee_per_k;  //min fee per Kb
+};
+
+typedef std::vector<struct CStatsMempoolSample> mempoolSamples_t;
+
+// simple mempool stats container
+class CStatsMempool
+{
+public:
+    uint64_t m_start_time;      //start time of the container
+    mempoolSamples_t m_samples;
+    uint64_t m_cleanup_counter; //internal counter to trogger cleanups
+
+    CStatsMempool()
+    {
+        m_start_time = 0;
+        m_cleanup_counter = 0;
+    }
+};
+
+// Class that manages various types of statistics and its memory consumption
+class CStats
+{
+private:
+    static CStats* m_shared_instance;
+    mutable RecursiveMutex cs_stats;
+
+    CStatsMempool m_mempool_stats; //mempool stats container
+
+public:
+    static std::atomic<bool> m_stats_enabled; //if enabled, stats will be collected
+    static CStats* DefaultStats(); //shared instance
+
+    /* signals */
+    boost::signals2::signal<void(void)> MempoolStatsDidChange; //mempool stats update signal
+
+    /* add a mempool stats sample */
+    void addMempoolSample(int64_t txcount, int64_t dynUsage, int64_t currentMinRelayFee);
+
+    /* get all mempool samples in range */
+    mempoolSamples_t mempoolGetValuesInRange(uint64_t& fromTime, uint64_t& toTime);
+};
+
+#endif // BITCOIN_STATS_STATS_H

src/stats/test/stats_tests.cpp

@@ -0,0 +1,59 @@
+// Copyright (c) 2016 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <stats/stats.h>
+
+#include <test/util/setup_common.h>
+#include <util/time.h>
+
+#include <boost/test/unit_test.hpp>
+
+BOOST_FIXTURE_TEST_SUITE(stats_tests, BasicTestingSetup)
+
+BOOST_AUTO_TEST_CASE(stats)
+{
+    CStats::m_stats_enabled = true;
+
+    uint64_t start = GetTime();
+    SetMockTime(start);
+
+    CStats::DefaultStats()->addMempoolSample(0, 1, 1);
+    SetMockTime(start + 1);
+    CStats::DefaultStats()->addMempoolSample(0, 2, 2); //1second should be to short
+    SetMockTime(start + 5);
+    CStats::DefaultStats()->addMempoolSample(3, 4, 3);
+
+    uint64_t queryFromTime = start;
+    uint64_t queryToTime = start + 3600;
+    mempoolSamples_t samples = CStats::DefaultStats()->mempoolGetValuesInRange(queryFromTime, queryToTime);
+
+    BOOST_CHECK_EQUAL(samples[0].m_time_delta, 0U);
+    BOOST_CHECK_EQUAL(samples[1].m_time_delta, 5U);
+    BOOST_CHECK_EQUAL(samples[1].m_tx_count, 3);
+    BOOST_CHECK_EQUAL(samples[1].m_dyn_mem_usage, 4);
+
+    // check retrieving a subset of the available samples
+    queryFromTime = start;
+    queryToTime = start;
+    samples = CStats::DefaultStats()->mempoolGetValuesInRange(queryFromTime, queryToTime);
+    BOOST_CHECK_EQUAL(samples.size(), 1U);
+
+    // add some samples
+    for (int i = 0; i < 10000; i++) {
+        SetMockTime(start + 10 + i * 5);
+        CStats::DefaultStats()->addMempoolSample(i, i + 1, i + 2);
+    }
+
+    queryFromTime = start + 3600;
+    queryToTime = start + 3600;
+    samples = CStats::DefaultStats()->mempoolGetValuesInRange(queryFromTime, queryToTime);
+    BOOST_CHECK_EQUAL(samples.size(), 1U); //get a single sample
+
+    queryFromTime = start;
+    queryToTime = start + 3600;
+    samples = CStats::DefaultStats()->mempoolGetValuesInRange(queryFromTime, queryToTime);
+    BOOST_CHECK(samples.size() >= 3600 / 5);
+}
+
+BOOST_AUTO_TEST_SUITE_END()

src/validation.cpp

@@ -43,6 +43,7 @@
 #include <script/script.h>
 #include <script/sigcache.h>
 #include <signet.h>
+#include <stats/stats.h>
 #include <tinyformat.h>
 #include <txdb.h>
 #include <txmempool.h>
@@ -1437,6 +1438,8 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef
     AssertLockHeld(cs_main);
     LOCK(m_pool.cs); // mempool "read lock" (held through m_pool.m_opts.signals->TransactionAddedToMempool())
 
+    const CFeeRate mempool_min_fee_rate = m_pool.GetMinFee();
+
     Workspace ws(ptx);
     const std::vector<Wtxid> single_wtxid{ws.m_ptx->GetWitnessHash()};
 
@@ -1499,6 +1502,9 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef
                  ws.m_vsize - static_cast<int>(m_subpackage.m_conflicting_size));
     }
 
+    // update mempool stats cache
+    CStats::DefaultStats()->addMempoolSample(m_pool.size(), m_pool.DynamicMemoryUsage(), mempool_min_fee_rate.GetFeePerK());
+
     return MempoolAcceptResult::Success(std::move(m_subpackage.m_replaced_transactions), ws.m_vsize, ws.m_base_fees,
                                         effective_feerate, single_wtxid);
 }
@@ -3085,6 +3091,12 @@ bool Chainstate::DisconnectTip(BlockValidationState& state, DisconnectedBlockTra
     if (m_chainman.m_options.signals) {
         m_chainman.m_options.signals->BlockDisconnected(pblock, pindexDelete);
     }
+
+    if (m_mempool) {
+        // add mempool stats sample
+        CStats::DefaultStats()->addMempoolSample(m_mempool->size(), m_mempool->DynamicMemoryUsage(), m_mempool->GetMinFee().GetFeePerK());
+    }
+
     return true;
 }
 
@@ -3208,6 +3220,11 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew,
     m_chain.SetTip(*pindexNew);
     UpdateTip(pindexNew);
 
+    if (m_mempool) {
+        // add mempool stats sample
+        CStats::DefaultStats()->addMempoolSample(m_mempool->size(), m_mempool->DynamicMemoryUsage(), m_mempool->GetMinFee().GetFeePerK());
+    }
+
     const auto time_6{SteadyClock::now()};
     m_chainman.time_post_connect += time_6 - time_5;
     m_chainman.time_total += time_6 - time_1;