diff --git a/configure.ac b/configure.ac index d2329d5e6b..153c45b564 100644 --- a/configure.ac +++ b/configure.ac @@ -940,6 +940,21 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [ AC_MSG_RESULT([no])] ) +AC_MSG_CHECKING(for compatible sysinfo call) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], + [[ + struct sysinfo info; + int rv = sysinfo(&info); + unsigned long test = info.freeram + info.bufferram + info.mem_unit; + ]])], + [ + AC_MSG_RESULT(yes); + AC_DEFINE(HAVE_LINUX_SYSINFO, 1, [Define this symbol if you have a Linux-compatible sysinfo call]) + ],[ + AC_MSG_RESULT(no) + ] +) + dnl Check for posix_fallocate AC_MSG_CHECKING([for posix_fallocate]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ diff --git a/src/common/system.cpp b/src/common/system.cpp index 6d04c8a7bc..d7dd357f8c 100644 --- a/src/common/system.cpp +++ b/src/common/system.cpp @@ -22,6 +22,10 @@ #include #endif +#ifdef HAVE_LINUX_SYSINFO +#include +#endif + #include #include #include @@ -110,3 +114,39 @@ int64_t GetStartupTime() { return nStartupTime; } + +size_t g_low_memory_threshold = 10 * 1024 * 1024 /* 10 MB */; + +bool SystemNeedsMemoryReleased() +{ + if (g_low_memory_threshold <= 0) { + // Intentionally bypass other metrics when disabled entirely + return false; + } +#ifdef WIN32 + MEMORYSTATUSEX mem_status; + mem_status.dwLength = sizeof(mem_status); + if (GlobalMemoryStatusEx(&mem_status)) { + if (mem_status.dwMemoryLoad >= 99 || + mem_status.ullAvailPhys < g_low_memory_threshold || + mem_status.ullAvailVirtual < g_low_memory_threshold) { + LogPrintf("%s: YES: %s%% memory load; %s available physical memory; %s available virtual memory\n", __func__, int(mem_status.dwMemoryLoad), size_t(mem_status.ullAvailPhys), size_t(mem_status.ullAvailVirtual)); + return true; + } + } +#endif +#ifdef HAVE_LINUX_SYSINFO + struct sysinfo sys_info; + if (!sysinfo(&sys_info)) { + // Explicitly 64-bit in case of 32-bit userspace on 64-bit kernel + const uint64_t free_ram = uint64_t(sys_info.freeram) * sys_info.mem_unit; + const uint64_t buffer_ram = uint64_t(sys_info.bufferram) * sys_info.mem_unit; + if (free_ram + buffer_ram < g_low_memory_threshold) { + LogPrintf("%s: YES: %s free RAM + %s buffer RAM\n", __func__, free_ram, buffer_ram); + return true; + } + } +#endif + // NOTE: sysconf(_SC_AVPHYS_PAGES) doesn't account for caches on at least Linux, so not safe to use here + return false; +} diff --git a/src/common/system.h b/src/common/system.h index d9115d3b33..f572d716df 100644 --- a/src/common/system.h +++ b/src/common/system.h @@ -23,6 +23,10 @@ std::string ShellEscape(const std::string& arg); void runCommand(const std::string& strCommand); #endif +extern size_t g_low_memory_threshold; + +bool SystemNeedsMemoryReleased(); + /** * Return the number of cores available on the current system. * @note This does count virtual cores, such as those provided by HyperThreading. diff --git a/src/init.cpp b/src/init.cpp index 44883b4b62..2e835a3381 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -500,6 +500,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-includeconf=", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-allowignoredconf", strprintf("For backwards compatibility, treat an unused %s file in the datadir as a warning, not an error.", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-loadblock=", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-lowmem=", strprintf("If system available memory falls below MiB, flush caches (0 to disable, default: %s)", g_low_memory_threshold / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-maxmempool=", strprintf("Keep the transaction memory pool below megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE_MB), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-maxorphantx=", strprintf("Keep at most unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-mempoolexpiry=", strprintf("Do not keep transactions in the mempool longer than hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY_HOURS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -1554,6 +1555,13 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) bool do_reindex{args.GetBoolArg("-reindex", false)}; const bool do_reindex_chainstate{args.GetBoolArg("-reindex-chainstate", false)}; + if (gArgs.IsArgSet("-lowmem")) { + g_low_memory_threshold = gArgs.GetIntArg("-lowmem", 0 /* not used */) * 1024 * 1024; + } + if (g_low_memory_threshold > 0) { + LogPrintf("* Flushing caches if available system memory drops below %s MiB\n", g_low_memory_threshold / 1024 / 1024); + } + for (bool fLoaded = false; !fLoaded && !ShutdownRequested(node);) { bilingual_str mempool_error; node.mempool = std::make_unique(mempool_opts, mempool_error); diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 1c02066047..b0ae2df674 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. // #include +#include #include #include #include @@ -25,6 +26,8 @@ BOOST_FIXTURE_TEST_SUITE(validation_chainstate_tests, ChainTestingSetup) //! BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) { + g_low_memory_threshold = 0; // disable to get deterministic flushing + ChainstateManager& manager = *Assert(m_node.chainman); CTxMemPool& mempool = *Assert(m_node.mempool); Chainstate& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool)); diff --git a/src/validation.cpp b/src/validation.cpp index 1a8cc4de10..d338301093 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -2839,8 +2840,15 @@ bool Chainstate::FlushStateToDisk( } // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing). bool fCacheLarge = mode == FlushStateMode::PERIODIC && cache_state >= CoinsCacheSizeState::LARGE; - // The cache is over the limit, we have to write now. - bool fCacheCritical = mode == FlushStateMode::IF_NEEDED && cache_state >= CoinsCacheSizeState::CRITICAL; + bool fCacheCritical = false; + if (mode == FlushStateMode::IF_NEEDED) { + if (cache_state >= CoinsCacheSizeState::CRITICAL) { + // The cache is over the limit, we have to write now. + fCacheCritical = true; + } else if (SystemNeedsMemoryReleased()) { + fCacheCritical = true; + } + } // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. bool fPeriodicWrite = mode == FlushStateMode::PERIODIC && nNow > m_last_write + DATABASE_WRITE_INTERVAL; // It's been very long since we flushed the cache. Do this infrequently, to optimize cache usage. diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 12d5a3b3eb..4e67c85a0a 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -819,6 +820,9 @@ BOOST_FIXTURE_TEST_CASE(wallet_descriptor_test, BasicTestingSetup) //! rescanning where new transactions in new blocks could be lost. BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) { + // FIXME: this test fails for some reason if there's a flush + g_low_memory_threshold = 0; + m_args.ForceSetArg("-unsafesqlitesync", "1"); // Create new wallet with known key and unload it. WalletContext context;