From c86d2d1cb30fdaef8eedd4a6eed94b8d1dfec006 Mon Sep 17 00:00:00 2001 From: w0xlt <94266259+w0xlt@users.noreply.github.com> Date: Tue, 8 Feb 2022 22:41:35 -0300 Subject: [PATCH] rpc: allow dumptxoutset to dump human-readable data Co-authored-by: Shashwat Vangani Co-authored-by: Luke Dashjr --- src/rpc/blockchain.cpp | 95 ++++++++++++++++++++++++++++++++++---- src/rpc/blockchain.h | 7 +++ src/rpc/client.cpp | 2 + src/test/util/chainstate.h | 1 + 4 files changed, 95 insertions(+), 10 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index b449444aff..6bce983d7c 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -2647,11 +2648,32 @@ static RPCHelpMan getblockfilter() */ static RPCHelpMan dumptxoutset() { + static const std::vector> ascii_types{ + {"txid", [](const COutPoint& k, const Coin& c) { return k.hash.GetHex(); }}, + {"vout", [](const COutPoint& k, const Coin& c) { return util::ToString(static_cast(k.n)); }}, + {"value", [](const COutPoint& k, const Coin& c) { return util::ToString(c.out.nValue); }}, + {"coinbase", [](const COutPoint& k, const Coin& c) { return util::ToString(c.fCoinBase); }}, + {"height", [](const COutPoint& k, const Coin& c) { return util::ToString(static_cast(c.nHeight)); }}, + {"scriptPubKey", [](const COutPoint& k, const Coin& c) { return HexStr(c.out.scriptPubKey); }}, + // add any other desired items here + }; + + std::vector ascii_args; + std::transform(std::begin(ascii_types), std::end(ascii_types), std::back_inserter(ascii_args), + [](const std::pair& t) { return RPCArg{t.first, RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Info to write for a given UTXO"}; }); + return RPCHelpMan{ "dumptxoutset", - "Write the serialized UTXO set to a file.", + "Write the UTXO set to a file.", { {"path", RPCArg::Type::STR, RPCArg::Optional::NO, "Path to the output file. If relative, will be prefixed by datadir."}, + {"format", RPCArg::Type::ARR, RPCArg::DefaultHint{"compact serialized format"}, + "If no argument is provided, a compact binary serialized format is used; otherwise only requested items " + "available below are written in ASCII format (if an empty array is provided, all items are written in ASCII).", + ascii_args, + RPCArgOptions{.oneline_description="format"}}, + {"show_header", RPCArg::Type::BOOL, RPCArg::Default{true}, "Whether to include the header line in non-serialized (ASCII) mode"}, + {"separator", RPCArg::Type::STR, RPCArg::Default{","}, "Field separator to use in non-serialized (ASCII) mode"}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -2665,10 +2687,33 @@ static RPCHelpMan dumptxoutset() } }, RPCExamples{ - HelpExampleCli("dumptxoutset", "utxo.dat") + HelpExampleCli("dumptxoutset", "utxo.dat") + + HelpExampleCli("dumptxoutset", "utxo.dat '[]'") + + HelpExampleCli("dumptxoutset", "utxo.dat '[\"txid\", \"vout\"]' false ':'") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + // handle optional ASCII parameters + const bool is_human_readable = !request.params[1].isNull(); + const bool show_header = request.params[2].isNull() || request.params[2].get_bool(); + const auto separator = request.params[3].isNull() ? MakeByteSpan(",").first(1) : MakeByteSpan(request.params[3].get_str()); + std::vector> requested; + if (is_human_readable) { + const auto& arr = request.params[1].get_array(); + const std::unordered_map ascii_map(std::begin(ascii_types), std::end(ascii_types)); + for (size_t i = 0; i < arr.size(); ++i) { + const auto it = ascii_map.find(arr[i].get_str()); + if (it == std::end(ascii_map)) + throw JSONRPCError(RPC_INVALID_PARAMETER, "unable to find item '"+arr[i].get_str()+"'"); + + requested.emplace_back(*it); + } + + // if nothing was found, shows everything by default + if (requested.size() == 0) + requested = ascii_types; + } + const ArgsManager& args{EnsureAnyArgsman(request.context)}; const fs::path path = fsbridge::AbsPathJoin(args.GetDataDirNet(), fs::u8path(request.params[0].get_str())); // Write to a temporary path and then move into `path` on completion @@ -2682,7 +2727,7 @@ static RPCHelpMan dumptxoutset() "move it out of the way first"); } - FILE* file{fsbridge::fopen(temppath, "wb")}; + FILE* file{fsbridge::fopen(temppath, !is_human_readable ? "wb" : "w")}; AutoFile afile{file}; if (afile.IsNull()) { throw JSONRPCError( @@ -2692,6 +2737,8 @@ static RPCHelpMan dumptxoutset() NodeContext& node = EnsureAnyNodeContext(request.context); UniValue result = CreateUTXOSnapshot( + is_human_readable, + show_header, separator, requested, node, node.chainman->ActiveChainstate(), afile, path, temppath); fs::rename(temppath, path); @@ -2702,6 +2749,10 @@ static RPCHelpMan dumptxoutset() } UniValue CreateUTXOSnapshot( + const bool is_human_readable, + const bool show_header, + const Span& separator, + const std::vector>& requested, NodeContext& node, Chainstate& chainstate, AutoFile& afile, @@ -2712,6 +2763,9 @@ UniValue CreateUTXOSnapshot( std::optional maybe_stats; const CBlockIndex* tip; + // used when human readable format is requested + const auto line_separator = MakeByteSpan("\n").first(1); + { // We need to lock cs_main to ensure that the coinsdb isn't written to // between (i) flushing coins cache to disk (coinsdb), (ii) getting stats @@ -2742,9 +2796,20 @@ UniValue CreateUTXOSnapshot( tip->nHeight, tip->GetBlockHash().ToString(), fs::PathToString(path), fs::PathToString(temppath))); - SnapshotMetadata metadata{chainstate.m_chainman.GetParams().MessageStart(), tip->GetBlockHash(), maybe_stats->coins_count}; + if (!is_human_readable) { + SnapshotMetadata metadata{chainstate.m_chainman.GetParams().MessageStart(), tip->GetBlockHash(), maybe_stats->coins_count}; - afile << metadata; + afile << metadata; + } else if (show_header) { + afile.write(MakeByteSpan("#(blockhash " + tip->GetBlockHash().ToString() + " ) ")); + for (auto it = std::begin(requested); it != std::end(requested); ++it) { + if (it != std::begin(requested)) { + afile.write(separator); + } + afile.write(MakeByteSpan(it->first)); + } + afile.write(line_separator); + } COutPoint key; Txid last_hash; @@ -2776,12 +2841,22 @@ UniValue CreateUTXOSnapshot( if (iter % 5000 == 0) node.rpc_interruption_point(); ++iter; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { - if (key.hash != last_hash) { - write_coins_to_file(afile, last_hash, coins, written_coins_count); - last_hash = key.hash; - coins.clear(); + if (!is_human_readable) { + if (key.hash != last_hash) { + write_coins_to_file(afile, last_hash, coins, written_coins_count); + last_hash = key.hash; + coins.clear(); + } + coins.emplace_back(key.n, coin); + } else { + for (auto it = std::begin(requested); it != std::end(requested); ++it) { + if (it != std::begin(requested)) + afile.write(separator); + afile.write(MakeByteSpan(it->second(key, coin))); + } + afile.write(line_separator); + ++written_coins_count; } - coins.emplace_back(key.n, coin); } pcursor->Next(); } diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index f6a7fe236c..2ea02ed482 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -5,8 +5,10 @@ #ifndef BITCOIN_RPC_BLOCKCHAIN_H #define BITCOIN_RPC_BLOCKCHAIN_H +#include #include #include +#include #include #include #include @@ -26,6 +28,7 @@ struct NodeContext; } // namespace node static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5; +using coinascii_cb_t = std::function; /** * Get the difficulty of the net wrt to the given block index. @@ -52,6 +55,10 @@ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], * @return a UniValue map containing metadata about the snapshot. */ UniValue CreateUTXOSnapshot( + const bool is_human_readable, + const bool show_header, + const Span& separator, + const std::vector>& requested, node::NodeContext& node, Chainstate& chainstate, AutoFile& afile, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index b866fa484b..1b9fbc9e5a 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -93,6 +93,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "scanblocks", 5, "options" }, { "scanblocks", 5, "filter_false_positives" }, { "scantxoutset", 1, "scanobjects" }, + { "dumptxoutset", 1, "format" }, + { "dumptxoutset", 2, "show_header" }, { "addmultisigaddress", 0, "nrequired" }, { "addmultisigaddress", 1, "keys" }, { "createmultisig", 0, "nrequired" }, diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index e604b44c16..52e20c711d 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -48,6 +48,7 @@ CreateAndActivateUTXOSnapshot( AutoFile auto_outfile{outfile}; UniValue result = CreateUTXOSnapshot( + false, false, Span(), {}, node, node.chainman->ActiveChainstate(), auto_outfile, snapshot_path, snapshot_path); LogPrintf( "Wrote UTXO snapshot to %s: %s\n", fs::PathToString(snapshot_path.make_preferred()), result.write());