Try to use posix_fadvise with CBufferedFile

This primarily affects blocks when bitcoin is launched with -reindex, as
that causes the block files to be loaded as CBufferedFile objects one at
a time as the reindex progresses.

Co-Authored-By: Luke Dashjr <luke-jr+git@utopios.org>

src/streams.h

@@ -9,6 +9,7 @@
 #include <serialize.h>
 #include <span.h>
 #include <support/allocators/zeroafterfree.h>
+#include <util/fs_helpers.h>
 #include <util/overflow.h>
 
 #include <algorithm>
@@ -621,13 +622,20 @@ public:
     {
         if (nRewindIn >= nBufSize)
             throw std::ios_base::failure("Rewind limit must be less than buffer size");
+        AdviseSequential(m_src.Get());
     }
 
     int GetVersion() const { return m_src.GetVersion(); }
 
     ~BufferedFile() { fclose(); }
 
-    int fclose() { return m_src.fclose(); }
+    int fclose()
+    {
+        if (auto rel{m_src.release()}) {
+            return CloseAndUncache(rel);
+        }
+        return 0;
+    }
 
     //! check whether we're at the end of the source file
     bool eof() const {

src/util/fs_helpers.cpp

@@ -225,7 +225,8 @@ void AllocateFileRange(FILE* file, unsigned int offset, unsigned int length)
     ftruncate(fileno(file), static_cast<off_t>(offset) + length);
 #else
 #if defined(HAVE_POSIX_FALLOCATE)
-    // Version using posix_fallocate
+    // Use posix_fallocate to advise the kernel how much data we have to write,
+    // if this system supports it.
     off_t nEndPos = (off_t)offset + length;
     if (0 == posix_fallocate(fileno(file), 0, nEndPos)) return;
 #endif
@@ -245,6 +246,46 @@ void AllocateFileRange(FILE* file, unsigned int offset, unsigned int length)
 #endif
 }
 
+FILE* AdviseSequential(FILE *file) {
+#if _POSIX_C_SOURCE >= 200112L
+    // Since this whole thing is advisory anyway, we can ignore any errors
+    // encountered up to and including the posix_fadvise call. However, we must
+    // rewind the file to the appropriate position if we've changed the seek
+    // offset.
+    if (file == nullptr) {
+        return nullptr;
+    }
+    const int fd = fileno(file);
+    if (fd == -1) {
+        return file;
+    }
+    const off_t start = lseek(fd, 0, SEEK_CUR);
+    if (start == -1) {
+        return file;
+    }
+    posix_fadvise(fd, start, 0, POSIX_FADV_WILLNEED);
+    posix_fadvise(fd, start, 0, POSIX_FADV_SEQUENTIAL);
+#endif
+    return file;
+}
+
+int CloseAndUncache(FILE *file) {
+#if _POSIX_C_SOURCE >= 200112L
+    // Ignore any errors up to and including the posix_fadvise call since it's
+    // advisory.
+    if (file != nullptr) {
+        const int fd = fileno(file);
+        if (fd != -1) {
+            const off_t end = lseek(fd, 0, SEEK_END);
+            if (end != (off_t)-1) {
+                posix_fadvise(fd, 0, end, POSIX_FADV_DONTNEED);
+            }
+        }
+    }
+#endif
+    return std::fclose(file);
+}
+
 #ifdef WIN32
 fs::path GetSpecialFolderPath(int nFolder, bool fCreate)
 {

src/util/fs_helpers.h

@@ -48,6 +48,15 @@ bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes = 0);
  */
 std::streampos GetFileSize(const char* path, std::streamsize max = std::numeric_limits<std::streamsize>::max());
 
+//! Return the original FILE* unchanged. On systems that support it,
+//! also advise the OS that the file will be accessed sequentially.
+FILE* AdviseSequential(FILE*);
+
+//! Close a file and return the result of fclose(). On systems that
+//! support it, advise the OS to remove the file contents from the page
+//! cache (which can help on memory-constrained systems).
+int CloseAndUncache(FILE*);
+
 /** Release all directory locks. This is used for unit testing only, at runtime
  * the global destructor will take care of the locks.
  */