Merge 24202 via rpc_dumptxoutset_hr

src/rpc/blockchain.cpp

@@ -2598,11 +2598,32 @@ static RPCHelpMan getblockfilter()
  */
 static RPCHelpMan dumptxoutset()
 {
+    static const std::vector<std::pair<std::string, coinascii_cb_t>> ascii_types{
+        {"txid",         [](const COutPoint& k, const Coin& c) { return k.hash.GetHex(); }},
+        {"vout",         [](const COutPoint& k, const Coin& c) { return ToString(static_cast<int32_t>(k.n)); }},
+        {"value",        [](const COutPoint& k, const Coin& c) { return ToString(c.out.nValue); }},
+        {"coinbase",     [](const COutPoint& k, const Coin& c) { return ToString(c.fCoinBase); }},
+        {"height",       [](const COutPoint& k, const Coin& c) { return ToString(static_cast<uint32_t>(c.nHeight)); }},
+        {"scriptPubKey", [](const COutPoint& k, const Coin& c) { return HexStr(c.out.scriptPubKey); }},
+        // add any other desired items here
+    };
+
+    std::vector<RPCArg> ascii_args;
+    std::transform(std::begin(ascii_types), std::end(ascii_types), std::back_inserter(ascii_args),
+            [](const std::pair<std::string, coinascii_cb_t>& t) { return RPCArg{t.first, RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Info to write for a given UTXO"}; });
+
     return RPCHelpMan{
         "dumptxoutset",
-        "Write the serialized UTXO set to a file.",
+        "Write the UTXO set to a file.",
         {
             {"path", RPCArg::Type::STR, RPCArg::Optional::NO, "Path to the output file. If relative, will be prefixed by datadir."},
+            {"format", RPCArg::Type::ARR, RPCArg::DefaultHint{"compact serialized format"},
+                                        "If no argument is provided, a compact binary serialized format is used; otherwise only requested items "
+                                        "available below are written in ASCII format (if an empty array is provided, all items are written in ASCII).",
+                                        ascii_args,
+                                        RPCArgOptions{.oneline_description="format"}},
+            {"show_header", RPCArg::Type::BOOL, RPCArg::Default{true}, "Whether to include the header line in non-serialized (ASCII) mode"},
+            {"separator", RPCArg::Type::STR, RPCArg::Default{","}, "Field separator to use in non-serialized (ASCII) mode"},
         },
         RPCResult{
             RPCResult::Type::OBJ, "", "",
@@ -2616,10 +2637,33 @@ static RPCHelpMan dumptxoutset()
                 }
         },
         RPCExamples{
-            HelpExampleCli("dumptxoutset", "utxo.dat")
+            HelpExampleCli("dumptxoutset", "utxo.dat") +
+            HelpExampleCli("dumptxoutset", "utxo.dat '[]'") +
+            HelpExampleCli("dumptxoutset", "utxo.dat '[\"txid\", \"vout\"]' false ':'")
         },
         [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue
 {
+    // handle optional ASCII parameters
+    const bool is_human_readable = !request.params[1].isNull();
+    const bool show_header = request.params[2].isNull() || request.params[2].get_bool();
+    const auto separator = request.params[3].isNull() ? MakeByteSpan(",").first(1) : MakeByteSpan(request.params[3].get_str());
+    std::vector<std::pair<std::string, coinascii_cb_t>> requested;
+    if (is_human_readable) {
+        const auto& arr = request.params[1].get_array();
+        const std::unordered_map<std::string, coinascii_cb_t> ascii_map(std::begin(ascii_types), std::end(ascii_types));
+        for (size_t i = 0; i < arr.size(); ++i) {
+            const auto it = ascii_map.find(arr[i].get_str());
+            if (it == std::end(ascii_map))
+                throw JSONRPCError(RPC_INVALID_PARAMETER, "unable to find item '"+arr[i].get_str()+"'");
+
+            requested.emplace_back(*it);
+        }
+
+        // if nothing was found, shows everything by default
+        if (requested.size() == 0)
+            requested = ascii_types;
+    }
+
     const ArgsManager& args{EnsureAnyArgsman(request.context)};
     const fs::path path = fsbridge::AbsPathJoin(args.GetDataDirNet(), fs::u8path(request.params[0].get_str()));
     // Write to a temporary path and then move into `path` on completion
@@ -2633,7 +2677,7 @@ static RPCHelpMan dumptxoutset()
             "move it out of the way first");
     }
 
-    FILE* file{fsbridge::fopen(temppath, "wb")};
+    FILE* file{fsbridge::fopen(temppath, !is_human_readable ? "wb" : "w")};
     AutoFile afile{file};
     if (afile.IsNull()) {
         throw JSONRPCError(
@@ -2643,6 +2687,8 @@ static RPCHelpMan dumptxoutset()
 
     NodeContext& node = EnsureAnyNodeContext(request.context);
     UniValue result = CreateUTXOSnapshot(
+        is_human_readable,
+        show_header, separator, requested,
         node, node.chainman->ActiveChainstate(), afile, path, temppath);
     fs::rename(temppath, path);
 
@@ -2653,6 +2699,10 @@ static RPCHelpMan dumptxoutset()
 }
 
 UniValue CreateUTXOSnapshot(
+    const bool is_human_readable,
+    const bool show_header,
+    const Span<const std::byte>& separator,
+    const std::vector<std::pair<std::string, coinascii_cb_t>>& requested,
     NodeContext& node,
     Chainstate& chainstate,
     AutoFile& afile,
@@ -2663,6 +2713,9 @@ UniValue CreateUTXOSnapshot(
     std::optional<CCoinsStats> maybe_stats;
     const CBlockIndex* tip;
 
+    // used when human readable format is requested
+    const auto line_separator = MakeByteSpan("\n").first(1);
+
     {
         // We need to lock cs_main to ensure that the coinsdb isn't written to
         // between (i) flushing coins cache to disk (coinsdb), (ii) getting stats
@@ -2693,9 +2746,20 @@ UniValue CreateUTXOSnapshot(
         tip->nHeight, tip->GetBlockHash().ToString(),
         fs::PathToString(path), fs::PathToString(temppath)));
 
+    if (!is_human_readable) {
     SnapshotMetadata metadata{tip->GetBlockHash(), maybe_stats->coins_count};
 
     afile << metadata;
+    } else if (show_header) {
+        afile.write(MakeByteSpan("#(blockhash " + tip->GetBlockHash().ToString() + " ) "));
+        for (auto it = std::begin(requested); it != std::end(requested); ++it) {
+            if (it != std::begin(requested)) {
+                afile.write(separator);
+            }
+            afile.write(MakeByteSpan(it->first));
+        }
+        afile.write(line_separator);
+    }
 
     COutPoint key;
     Coin coin;
@@ -2705,8 +2769,17 @@ UniValue CreateUTXOSnapshot(
         if (iter % 5000 == 0) node.rpc_interruption_point();
         ++iter;
         if (pcursor->GetKey(key) && pcursor->GetValue(coin)) {
+            if (!is_human_readable) {
             afile << key;
             afile << coin;
+            } else {
+                for (auto it = std::begin(requested); it != std::end(requested); ++it) {
+                    if (it != std::begin(requested))
+                        afile.write(separator);
+                    afile.write(MakeByteSpan(it->second(key, coin)));
+                }
+                afile.write(line_separator);
+            }
         }
 
         pcursor->Next();

src/rpc/blockchain.h

@@ -5,8 +5,10 @@
 #ifndef BITCOIN_RPC_BLOCKCHAIN_H
 #define BITCOIN_RPC_BLOCKCHAIN_H
 
+#include <coins.h>
 #include <consensus/amount.h>
 #include <core_io.h>
+#include <span.h>
 #include <streams.h>
 #include <sync.h>
 #include <util/fs.h>
@@ -25,6 +27,7 @@ struct NodeContext;
 } // namespace node
 
 static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5;
+using coinascii_cb_t = std::function<std::string(const COutPoint&, const Coin&)>;
 
 /**
  * Get the difficulty of the net wrt to the given block index.
@@ -51,6 +54,10 @@ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES],
  * @return a UniValue map containing metadata about the snapshot.
  */
 UniValue CreateUTXOSnapshot(
+    const bool is_human_readable,
+    const bool show_header,
+    const Span<const std::byte>& separator,
+    const std::vector<std::pair<std::string, coinascii_cb_t>>& requested,
     node::NodeContext& node,
     Chainstate& chainstate,
     AutoFile& afile,

src/rpc/client.cpp

@@ -93,6 +93,8 @@ static const CRPCConvertParam vRPCConvertParams[] =
     { "scanblocks", 5, "options" },
     { "scanblocks", 5, "filter_false_positives" },
     { "scantxoutset", 1, "scanobjects" },
+    { "dumptxoutset", 1, "format" },
+    { "dumptxoutset", 2, "show_header" },
     { "addmultisigaddress", 0, "nrequired" },
     { "addmultisigaddress", 1, "keys" },
     { "createmultisig", 0, "nrequired" },

src/test/util/chainstate.h

@@ -48,6 +48,7 @@ CreateAndActivateUTXOSnapshot(
     AutoFile auto_outfile{outfile};
 
     UniValue result = CreateUTXOSnapshot(
+        false, false, Span<std::byte>(), {},
         node, node.chainman->ActiveChainstate(), auto_outfile, snapshot_path, snapshot_path);
     LogPrintf(
         "Wrote UTXO snapshot to %s: %s\n", fs::PathToString(snapshot_path.make_preferred()), result.write());

test/functional/rpc_dumptxoutset.py

@@ -12,6 +12,8 @@ from test_framework.util import (
     assert_raises_rpc_error,
     sha256sum_file,
 )
+import hashlib
+import os
 
 
 class DumptxoutsetTest(BitcoinTestFramework):
@@ -19,16 +21,30 @@ class DumptxoutsetTest(BitcoinTestFramework):
         self.setup_clean_chain = True
         self.num_nodes = 1
 
-    def run_test(self):
-        """Test a trivial usage of the dumptxoutset RPC command."""
-        node = self.nodes[0]
-        mocktime = node.getblockheader(node.getblockhash(0))['time'] + 1
-        node.setmocktime(mocktime)
-        self.generate(node, COINBASE_MATURITY)
+    @staticmethod
+    def check_output_file(path, is_human_readable, expected_digest):
+        with open(str(path), 'rb') as f:
+            content = f.read()
 
-        FILENAME = 'txoutset.dat'
-        out = node.dumptxoutset(FILENAME)
-        expected_path = node.datadir_path / self.chain / FILENAME
+            if is_human_readable:
+                # Normalise platform EOL to \n, while making sure any stray \n becomes a literal backslash+n to avoid a false positive
+                # This ensures the platform EOL and only the platform EOL produces the expected hash
+                linesep = os.linesep.encode('utf8')
+                content = b'\n'.join(line.replace(b'\n', b'\\n') for line in content.split(linesep))
+
+            digest = hashlib.sha256(content).hexdigest()
+            # UTXO snapshot hash should be deterministic based on mocked time.
+            assert_equal(digest, expected_digest)
+
+    def test_dump_file(self, testname, params, expected_digest):
+        node = self.nodes[0]
+
+        self.log.info(testname)
+        filename = testname + '_txoutset.dat'
+        is_human_readable = not params.get('format') is None
+
+        out = node.dumptxoutset(path=filename, **params)
+        expected_path = node.datadir_path / self.chain / filename
 
         assert expected_path.is_file()
 
@@ -40,22 +56,56 @@ class DumptxoutsetTest(BitcoinTestFramework):
             out['base_hash'],
             '09abf0e7b510f61ca6cf33bab104e9ee99b3528b371d27a2d4b39abb800fba7e')
 
-        # UTXO snapshot hash should be deterministic based on mocked time.
-        assert_equal(
-            sha256sum_file(str(expected_path)).hex(),
-            'b1bacb602eacf5fbc9a7c2ef6eeb0d229c04e98bdf0c2ea5929012cd0eae3830')
+        self.check_output_file(expected_path, is_human_readable, expected_digest)
 
         assert_equal(
             out['txoutset_hash'], 'a0b7baa3bf5ccbd3279728f230d7ca0c44a76e9923fca8f32dbfd08d65ea496a')
         assert_equal(out['nchaintx'], 101)
 
-        # Specifying a path to an existing or invalid file will fail.
+        self.log.info("Test that a path to an existing or invalid file will fail")
         assert_raises_rpc_error(
-            -8, '{} already exists'.format(FILENAME),  node.dumptxoutset, FILENAME)
+            -8, '{} already exists'.format(filename),  node.dumptxoutset, filename)
         invalid_path = node.datadir_path / "invalid" / "path"
         assert_raises_rpc_error(
             -8, "Couldn't open file {}.incomplete for writing".format(invalid_path), node.dumptxoutset, invalid_path)
 
+        if params.get('format') == ():
+            with open(expected_path, 'r', encoding='utf-8') as f:
+                content = f.readlines()
+                sep = params.get('separator', ',')
+                if params.get('show_header', True):
+                    assert_equal(content.pop(0).rstrip(),
+                        "#(blockhash 09abf0e7b510f61ca6cf33bab104e9ee99b3528b371d27a2d4b39abb800fba7e ) txid{s}vout{s}value{s}coinbase{s}height{s}scriptPubKey".format(s=sep))
+                assert_equal(content[0].rstrip(),
+                    "b9edce02689692b1cdc3440d03011486a27c46b966248b922cc6e4315e900708{s}0{s}5000000000{s}1{s}78{s}76a9142b4569203694fc997e13f2c0a1383b9e16c77a0d88ac".format(s=sep))
+
+    def run_test(self):
+        """Test a trivial usage of the dumptxoutset RPC command."""
+        node = self.nodes[0]
+        mocktime = node.getblockheader(node.getblockhash(0))['time'] + 1
+        node.setmocktime(mocktime)
+        self.generate(node, COINBASE_MATURITY)
+
+        self.test_dump_file('no_option',           {},
+                            'b1bacb602eacf5fbc9a7c2ef6eeb0d229c04e98bdf0c2ea5929012cd0eae3830')
+        self.test_dump_file('all_data',            {'format': ()},
+                            '50d7bf3ecca8c5daf648aca884b91496386d8269ef001ff95a1db4381d399bfb')
+        self.test_dump_file('partial_data_1',      {'format': ('txid',)},
+                            'f9966db510b46d865a9412da88d17ac2c05c6bfe612ffc7c1b004aec1b508c5c')
+        self.test_dump_file('partial_data_order',  {'format': ('height', 'vout')},
+                            '0ef7e361fde77f5c9f3667b1d8ce4351ec8dc81826937da0dab5631e2aedc5fe')
+        self.test_dump_file('partial_data_double', {'format': ('scriptPubKey', 'scriptPubKey')},
+                            '8bd128d326b971ea37bd28c016aae506e29d23dac578edd849636a8ab2ee31a8')
+        self.test_dump_file('no_header',           {'format': (), 'show_header': False},
+                            'af1f38ee1d1b8bbdc117ab7e8353910dab5ab45f18be27aa4fa7d96ccc96a050')
+        self.test_dump_file('separator',           {'format': (), 'separator': ':'},
+                            '5bee81096e400d1b3bf02de432e0fd4af8f4d9244907dc1c857ec329c5ce4490')
+        self.test_dump_file('all_options',         {'format': (), 'show_header': False, 'separator': ':'},
+                            '5c52c2a9bdb23946eb0f6d088f25ed8f5d9ebc3a3512182287975f1041cdedb4')
+
+        # Other failing tests
+        assert_raises_rpc_error(
+            -8, 'unable to find item \'sample\'',  node.dumptxoutset, path='xxx', format=['sample'])
 
 if __name__ == '__main__':
     DumptxoutsetTest().main()