Merge 15421 via tor_subprocess-25+knots

configure.ac

@@ -331,11 +331,20 @@ AC_ARG_ENABLE([werror],
     [enable_werror=$enableval],
     [enable_werror=no])
 
+AC_ARG_WITH([boost-process],
+    [AS_HELP_STRING([--without-boost-process],[disable features that require Boost::Process (default is yes if usable)])],
+    [use_boost_process=$withval],
+    [use_boost_process=auto])
+
 AC_ARG_ENABLE([external-signer],
     [AS_HELP_STRING([--enable-external-signer],[compile external signer support (default is auto, requires Boost::Process)])],
     [use_external_signer=$enableval],
     [use_external_signer=auto])
 
+if test "$use_external_signer" = "yes" && test "$use_boost_process" = "no"; then
+  AC_MSG_ERROR([External signing (enabled explicitly) requires Boost::Process (disabled explicitly)])
+fi
+
 AC_ARG_ENABLE([lto],
     [AS_HELP_STRING([--enable-lto],[build using LTO (default is no)])],
     [enable_lto=$enableval],
@@ -1501,6 +1510,7 @@ if test "$enable_fuzz" = "yes"; then
   bitcoin_enable_qt_dbus=no
   use_bench=no
   use_tests=no
+  use_boost_process=no
   use_external_signer=no
   use_upnp=no
   use_natpmp=no
@@ -1697,7 +1707,7 @@ if test "$use_boost" = "yes"; then
   fi
 fi
 
-if test "$use_external_signer" != "no"; then
+if test "$use_boost_process" != "no"; then
   AC_MSG_CHECKING([whether Boost.Process can be used])
   TEMP_CXXFLAGS="$CXXFLAGS"
   dnl Boost 1.78 requires the following workaround.
@@ -1750,16 +1760,30 @@ if test "$use_external_signer" != "no"; then
   CXXFLAGS="$TEMP_CXXFLAGS"
   AC_MSG_RESULT([$have_boost_process])
   if test "$have_boost_process" = "yes"; then
-    use_external_signer="yes"
+    use_boost_process=yes
-    AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [1], [Define if external signer support is enabled])
+    AC_DEFINE([HAVE_BOOST_PROCESS], [1], [Define if Boost::Process is available])
     AC_DEFINE([BOOST_PROCESS_USE_STD_FS], [1], [Defined to avoid Boost::Process trying to use Boost Filesystem])
   else
-    if test "$use_external_signer" = "yes"; then
+    if test "$use_boost_process" = "yes"; then
-      AC_MSG_ERROR([External signing is not supported for this Boost version])
+      AC_MSG_ERROR([Boost::Process is not supported for this Boost version. Use --without-boost-process])
     fi
-    use_external_signer="no";
+    use_boost_process=no
   fi
 fi
+
+AC_MSG_CHECKING([if external signer support should be enabled])
+if test "$use_external_signer" = "yes"; then
+  if test "$use_boost_process" = "no"; then
+    AC_MSG_RESULT([yes])
+    AC_MSG_ERROR([External signing is not supported for this Boost version])
+  fi
+elif test "$use_external_signer" != "no"; then
+  use_external_signer="$use_boost_process"
+fi
+AC_MSG_RESULT([$use_external_signer])
+if test "$use_external_signer" = "yes"; then
+  AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [1], [Define if external signer support is enabled])
+fi
 AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "$use_external_signer" = "yes"])
 
 dnl Do not compile with syscall sandbox support when compiling under the sanitizers.
@@ -2238,6 +2262,7 @@ esac
 
 echo
 echo "Options used to compile and link:"
+echo "  boost process   = $use_boost_process"
 echo "  external signer = $use_external_signer"
 echo "  multiprocess    = $build_multiprocess"
 echo "  with experimental syscall sandbox support = $use_syscall_sandbox"

src/common/run_command.h

@@ -14,7 +14,7 @@
 
 #include <string>
 
-#if defined(ENABLE_EXTERNAL_SIGNER) && defined(BOOST_POSIX_API)
+#if defined(HAVE_BOOST_PROCESS) && defined(BOOST_POSIX_API)
 #include <fcntl.h>
 #ifdef FD_CLOEXEC
 #include <unistd.h>
@@ -38,11 +38,11 @@
 #pragma GCC diagnostic pop
 #endif
 #endif // FD_CLOEXEC
-#endif // ENABLE_EXTERNAL_SIGNER && BOOST_POSIX_API
+#endif // HAVE_BOOST_PROCESS && BOOST_POSIX_API
 
 class UniValue;
 
-#if defined(ENABLE_EXTERNAL_SIGNER) && defined(BOOST_POSIX_API) && defined(FD_CLOEXEC)
+#if defined(HAVE_BOOST_PROCESS) && defined(BOOST_POSIX_API) && defined(FD_CLOEXEC)
 /**
  * Ensure a boost::process::child has its non-std fds all closed when exec
  * is called.

src/init.cpp

@@ -508,6 +508,11 @@ void SetupServerArgs(ArgsManager& argsman)
     argsman.AddArg("-timeout=<n>", strprintf("Specify socket connection timeout in milliseconds. If an initial attempt to connect is unsuccessful after this amount of time, drop it (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION);
     argsman.AddArg("-peertimeout=<n>", strprintf("Specify a p2p connection timeout delay in seconds. After connecting to a peer, wait this amount of time before considering disconnection based on inactivity (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION);
     argsman.AddArg("-torcontrol=<ip>:<port>", strprintf("Tor control host and port to use if onion listening enabled (default: %s). If no port is specified, the default port of %i will be used.", DEFAULT_TOR_CONTROL, DEFAULT_TOR_CONTROL_PORT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION);
+#ifdef HAVE_BOOST_PROCESS
+    argsman.AddArg("-torexecute=<command>", strprintf("Tor command to use if not already running (default: %s)", DEFAULT_TOR_EXECUTE), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION);
+#else
+    hidden_args.emplace_back("-torexecute=<command>");
+#endif
     argsman.AddArg("-torpassword=<pass>", "Tor control port password (default: empty)", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::CONNECTION);
 #ifdef USE_UPNP
 #if 0

src/net.cpp

@@ -303,7 +303,8 @@ bool AddLocal(const CService& addr_, int nScore)
     if (!fDiscover && nScore < LOCAL_MANUAL)
         return false;
 
-    if (!IsReachable(addr))
+    // IPv4 and IPv6 cannot be connected to unless their networks are reachable, but Tor is not necessarily bidirectional
+    if (!(IsReachable(addr) || addr.IsTor()))
         return false;
 
     LogPrintf("AddLocal(%s,%i)\n", addr.ToStringAddrPort(), nScore);

src/torcontrol.cpp

@@ -12,6 +12,7 @@
 #include <net.h>
 #include <netaddress.h>
 #include <netbase.h>
+#include <util/fs.h>
 #include <util/readwritefile.h>
 #include <util/strencodings.h>
 #include <util/syscall_sandbox.h>
@@ -19,11 +20,34 @@
 #include <util/thread.h>
 #include <util/time.h>
 
+#include <chrono>
 #include <deque>
+#include <fstream>
 #include <functional>
 #include <set>
 #include <vector>
 
+#ifdef HAVE_BOOST_PROCESS
+// Boost 1.77 requires the following workaround.
+// See: https://github.com/boostorg/process/issues/213
+#include <algorithm>
+#if defined(WIN32) && !defined(__kernel_entry)
+// Boost 1.71-1.77 requires the following workaround for compatibility with mingw-w64 compiler.
+// See: https://github.com/bitcoin/bitcoin/pull/22348
+#define __kernel_entry
+#endif
+#if defined(__GNUC__)
+// Boost 1.78 requires the following workaround.
+// See: https://github.com/boostorg/process/issues/235
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wnarrowing"
+#endif
+#include <boost/process.hpp>
+#if defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+#endif
+
 #include <event2/buffer.h>
 #include <event2/bufferevent.h>
 #include <event2/event.h>
@@ -32,6 +56,7 @@
 
 /** Default control ip and port */
 const std::string DEFAULT_TOR_CONTROL = "127.0.0.1:" + ToString(DEFAULT_TOR_CONTROL_PORT);
+const std::string DEFAULT_TOR_EXECUTE = "tor";
 /** Tor cookie size (from control-spec.txt) */
 static const int TOR_COOKIE_SIZE = 32;
 /** Size of client/server nonce for SAFECOOKIE */
@@ -64,6 +89,10 @@ TorControlConnection::~TorControlConnection()
         bufferevent_free(b_conn);
 }
 
+void TorControlConnection::IgnoreReplyHandler(TorControlConnection &a, const TorControlReply &b)
+{
+}
+
 void TorControlConnection::readcb(struct bufferevent *bev, void *ctx)
 {
     TorControlConnection *self = static_cast<TorControlConnection*>(ctx);
@@ -302,18 +331,20 @@ std::map<std::string,std::string> ParseTorReplyMapping(const std::string &s)
     return mapping;
 }
 
-TorController::TorController(struct event_base* _base, const std::string& tor_control_center, const CService& target):
+TorController::TorController(struct event_base* _base, const std::string& tor_control_center, const CService& target, const std::string& execute):
     base(_base),
-    m_tor_control_center(tor_control_center), conn(base), reconnect(true), reconnect_timeout(RECONNECT_TIMEOUT_START),
+    m_connect_tor_control_center(tor_control_center), conn(base), reconnect(true), reconnect_timeout(RECONNECT_TIMEOUT_START),
+    m_execute(execute),
     m_target(target)
 {
     reconnect_ev = event_new(base, -1, 0, reconnect_cb, this);
     if (!reconnect_ev)
         LogPrintf("tor: Failed to create event for reconnection: out of memory?\n");
     // Start connection attempts immediately
-    if (!conn.Connect(m_tor_control_center, std::bind(&TorController::connected_cb, this, std::placeholders::_1),
+    m_current_tor_control_center = tor_control_center;
+    if (!conn.Connect(m_current_tor_control_center, std::bind(&TorController::connected_cb, this, std::placeholders::_1),
          std::bind(&TorController::disconnected_cb, this, std::placeholders::_1) )) {
-        LogPrintf("tor: Initiating connection to Tor control port %s failed\n", m_tor_control_center);
+        LogPrintf("tor: Initiating connection to Tor control port %s failed\n", m_current_tor_control_center);
     }
     // Read service private key if cached
     std::pair<bool,std::string> pkf = ReadBinaryFile(GetPrivateKeyFile());
@@ -332,6 +363,12 @@ TorController::~TorController()
     if (service.IsValid()) {
         RemoveLocal(service);
     }
+#ifdef HAVE_BOOST_PROCESS
+    if (m_process) {
+        conn.Command("SIGNAL SHUTDOWN");
+        delete m_process;
+    }
+#endif
 }
 
 void TorController::get_socks_cb(TorControlConnection& _conn, const TorControlReply& reply)
@@ -441,9 +478,20 @@ void TorController::auth_cb(TorControlConnection& _conn, const TorControlReply&
     if (reply.code == 250) {
         LogPrint(BCLog::TOR, "Authentication successful\n");
 
+#ifdef HAVE_BOOST_PROCESS
+        if (m_process) {
+            _conn.Command("TAKEOWNERSHIP");
+        }
+#endif
+
         // Now that we know Tor is running setup the proxy for onion addresses
         // if -onion isn't set to something else.
-        if (gArgs.GetArg("-onion", "") == "") {
+        // NOTE: Our own private Tor doesn't do SOCKS, so don't configure it
+        if (gArgs.GetArg("-onion", "") == ""
+#ifdef HAVE_BOOST_PROCESS
+            && !m_process
+#endif
+        ) {
             _conn.Command("GETINFO net/listeners/socks", std::bind(&TorController::get_socks_cb, this, std::placeholders::_1, std::placeholders::_2));
         }
 
@@ -600,22 +648,95 @@ void TorController::protocolinfo_cb(TorControlConnection& _conn, const TorContro
 
 void TorController::connected_cb(TorControlConnection& _conn)
 {
+    m_try_exec = false;
     reconnect_timeout = RECONNECT_TIMEOUT_START;
     // First send a PROTOCOLINFO command to figure out what authentication is expected
     if (!_conn.Command("PROTOCOLINFO 1", std::bind(&TorController::protocolinfo_cb, this, std::placeholders::_1, std::placeholders::_2)))
         LogPrintf("tor: Error sending initial protocolinfo command\n");
 }
 
+std::string TorController::LaunchTor()
+{
+#ifdef HAVE_BOOST_PROCESS
+    fs::path tor_datadir = gArgs.GetDataDirNet() / "tor";
+    const fs::path controlport_env_filepath = tor_datadir / "controlport.env";
+    fs::remove(controlport_env_filepath);  // may throw exceptions
+
+    if (m_process) {
+        m_process->terminate();
+        delete m_process;
+        m_process = nullptr;
+    }
+
+    boost::process::opstream in;
+    try {
+        m_process = new boost::process::child(m_execute + " -f -", boost::process::std_in < in
+#ifdef HAVE_BPE_CLOSE_EXCESS_FDS
+            , bpe_close_excess_fds()
+#endif
+        );
+    } catch (...) {
+        LogPrint(BCLog::TOR, "tor: Failed to execute Tor process\n");
+        throw;
+    }
+    in << "SOCKSPort 0" << std::endl;
+    in << "DataDirectory " << fs::PathToString(tor_datadir) << std::endl;
+    in << "ControlPort auto" << std::endl;
+    in << "ControlPortWriteToFile " << fs::PathToString(controlport_env_filepath) << std::endl;
+    in << "CookieAuthentication 1" << std::endl;
+    in.pipe().close();
+
+    while (!fs::exists(controlport_env_filepath)) {
+        if (!m_process->running()) {
+            LogPrint(BCLog::TOR, "tor: Tor process died before making control port file\n");
+            throw std::runtime_error("tor process died");
+        }
+        std::this_thread::sleep_for(std::chrono::seconds(1));
+    }
+
+    std::ifstream controlport_file(controlport_env_filepath);
+    std::string portline;
+    controlport_file >> portline;
+    if (portline.compare(0, 5, "PORT=")) {
+        LogPrint(BCLog::TOR, "tor: Unrecognized control port line in file\n");
+        m_process->terminate();
+        delete m_process;
+        m_process = nullptr;
+        throw std::runtime_error("port line unrecognized");
+    }
+
+    return portline.substr(5);
+#else
+    throw std::runtime_error("not supported");
+#endif
+}
+
 void TorController::disconnected_cb(TorControlConnection& _conn)
 {
     // Stop advertising service when disconnected
     if (service.IsValid())
         RemoveLocal(service);
     service = CService();
+
+#ifdef HAVE_BOOST_PROCESS
+    if (m_try_exec && !m_execute.empty()) {
+        LogPrint(BCLog::TOR, "tor: Not connected to Tor control port %s, trying to launch via %s\n", m_current_tor_control_center, m_execute);
+        try {
+            m_current_tor_control_center = LaunchTor();
+            Reconnect();
+            return;
+        } catch (...) {
+            // fall through to normal reconnect logic
+        }
+    }
+#endif
+
     if (!reconnect)
         return;
 
-    LogPrint(BCLog::TOR, "Not connected to Tor control port %s, trying to reconnect\n", m_tor_control_center);
+    LogPrint(BCLog::TOR, "Not connected to Tor control port %s, trying to reconnect in %s seconds\n", m_current_tor_control_center, reconnect_timeout);
+    m_current_tor_control_center = m_connect_tor_control_center;
+    m_try_exec = true;  // if this fails
 
     // Single-shot timer for reconnect. Use exponential backoff.
     struct timeval time = MillisToTimeval(int64_t(reconnect_timeout * 1000.0));
@@ -629,9 +750,9 @@ void TorController::Reconnect()
     /* Try to reconnect and reestablish if we get booted - for example, Tor
      * may be restarting.
      */
-    if (!conn.Connect(m_tor_control_center, std::bind(&TorController::connected_cb, this, std::placeholders::_1),
+    if (!conn.Connect(m_current_tor_control_center, std::bind(&TorController::connected_cb, this, std::placeholders::_1),
          std::bind(&TorController::disconnected_cb, this, std::placeholders::_1) )) {
-        LogPrintf("tor: Re-initiating connection to Tor control port %s failed\n", m_tor_control_center);
+        LogPrintf("tor: Re-initiating connection to Tor control port %s failed\n", m_current_tor_control_center);
     }
 }
 
@@ -653,7 +774,17 @@ static std::thread torControlThread;
 static void TorControlThread(CService onion_service_target)
 {
     SetSyscallSandboxPolicy(SyscallSandboxPolicy::TOR_CONTROL);
-    TorController ctrl(gBase, gArgs.GetArg("-torcontrol", DEFAULT_TOR_CONTROL), onion_service_target);
+#ifdef HAVE_BOOST_PROCESS
+    std::string execute_command = gArgs.GetArg("-torexecute", DEFAULT_TOR_EXECUTE);
+    if (execute_command == "1") {
+        execute_command = DEFAULT_TOR_EXECUTE;
+    } else if (execute_command == "0") {
+        execute_command.clear();
+    }
+#else
+    const std::string execute_command;
+#endif
+    TorController ctrl(gBase, gArgs.GetArg("-torcontrol", DEFAULT_TOR_CONTROL), onion_service_target, execute_command);
 
     event_base_dispatch(gBase);
 }

src/torcontrol.h

@@ -8,9 +8,31 @@
 #ifndef BITCOIN_TORCONTROL_H
 #define BITCOIN_TORCONTROL_H
 
+#if defined(HAVE_CONFIG_H)
+#include <config/bitcoin-config.h>
+#endif
+
 #include <netaddress.h>
 #include <util/fs.h>
 
+#ifdef HAVE_BOOST_PROCESS
+#if defined(WIN32) && !defined(__kernel_entry)
+// A workaround for boost 1.71 incompatibility with mingw-w64 compiler.
+// For details see https://github.com/bitcoin/bitcoin/pull/22348.
+#define __kernel_entry
+#endif
+#if defined(__GNUC__)
+// Boost 1.78 requires the following workaround.
+// See: https://github.com/boostorg/process/issues/235
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wnarrowing"
+#endif
+#include <boost/process.hpp>
+#if defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+#endif
+
 #include <boost/signals2/signal.hpp>
 
 #include <event2/bufferevent.h>
@@ -26,6 +48,7 @@ class CService;
 
 constexpr int DEFAULT_TOR_CONTROL_PORT = 9051;
 extern const std::string DEFAULT_TOR_CONTROL;
+extern const std::string DEFAULT_TOR_EXECUTE;
 static const bool DEFAULT_LISTEN_ONION = true;
 
 void StartTorControl(CService onion_service_target);
@@ -58,6 +81,7 @@ class TorControlConnection
 public:
     typedef std::function<void(TorControlConnection&)> ConnectionCB;
     typedef std::function<void(TorControlConnection &,const TorControlReply &)> ReplyHandlerCB;
+    static void IgnoreReplyHandler(TorControlConnection &, const TorControlReply &);
 
     /** Create a new TorControlConnection.
      */
@@ -82,7 +106,7 @@ public:
      * A trailing CRLF is automatically added.
      * Return true on success.
      */
-    bool Command(const std::string &cmd, const ReplyHandlerCB& reply_handler);
+    bool Command(const std::string &cmd, const ReplyHandlerCB& reply_handler = IgnoreReplyHandler);
 
     /** Response handlers for async replies */
     boost::signals2::signal<void(TorControlConnection &,const TorControlReply &)> async_handler;
@@ -113,7 +137,7 @@ private:
 class TorController
 {
 public:
-    TorController(struct event_base* base, const std::string& tor_control_center, const CService& target);
+    TorController(struct event_base* base, const std::string& tor_control_center, const CService& target, const std::string& execute);
     TorController() : conn{nullptr} {
         // Used for testing only.
     }
@@ -126,13 +150,19 @@ public:
     void Reconnect();
 private:
     struct event_base* base;
-    const std::string m_tor_control_center;
+    const std::string m_connect_tor_control_center;
+    std::string m_current_tor_control_center;
     TorControlConnection conn;
     std::string private_key;
     std::string service_id;
+    bool m_try_exec{true};
     bool reconnect;
     struct event *reconnect_ev = nullptr;
     float reconnect_timeout;
+    std::string m_execute{DEFAULT_TOR_EXECUTE};
+#ifdef HAVE_BOOST_PROCESS
+    boost::process::child *m_process{nullptr};
+#endif
     CService service;
     const CService m_target;
     /** Cookie for SAFECOOKIE auth */
@@ -158,6 +188,8 @@ public:
 
     /** Callback for reconnect timer */
     static void reconnect_cb(evutil_socket_t fd, short what, void *arg);
+
+    std::string LaunchTor();
 };
 
 #endif // BITCOIN_TORCONTROL_H