From 2ee1991c7d95367ee5ad7c1dbfa62231f761eae2 Mon Sep 17 00:00:00 2001 From: Roman Zeyde Date: Tue, 11 Aug 2020 17:57:17 +0300 Subject: [PATCH] RPC: Add getblocklocations call This RPC allows the client to retrieve the file system locations of the confirmed blocks and their undo data, to allow building efficient indexes outside of Bitcoin Core. An example usage is described here: https://github.com/romanz/electrs/issues/308 By using the new RPC, it is possible to build an address-based index taking ~24GB and a txindex taking ~6GB (as of Dec. 2020). --- src/rpc/blockchain.cpp | 61 ++++++++++++++++++ src/rpc/client.cpp | 1 + test/functional/rpc_getblocklocations.py | 78 ++++++++++++++++++++++++ test/functional/test_runner.py | 1 + 4 files changed, 141 insertions(+) create mode 100755 test/functional/rpc_getblocklocations.py diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 7b84747a3f..2a734d55d9 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2869,6 +2869,66 @@ return RPCHelpMan{ }; } +static RPCHelpMan getblocklocations() +{ + return RPCHelpMan{"getblocklocations", + "\nEXPERIMENTAL warning: this call may be removed or changed in future releases.\n" + "\nReturns a JSON for the file system location of 'blockhash' block and undo data.\n" + "\nIt is possible to return also the locations of previous blocks, by specifying 'nblocks' > 1.\n", + { + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The block hash"}, + {"nblocks", RPCArg::Type::NUM, RPCArg::Optional::NO, "Maximum number locations to return (up to genesis block)"}, + }, + { + RPCResult{ + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::NUM, "file", "blk*.dat/rev*.dat file index"}, + {RPCResult::Type::NUM, "data", "block data file offset"}, + {RPCResult::Type::NUM, "undo", "undo data file offset (if exists)"}, + {RPCResult::Type::STR_HEX, "prev", "previous block hash"}, + } + }, + }, + RPCExamples{ + HelpExampleCli("getblocklocation", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\" 10") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + + ChainstateManager& chainman = EnsureAnyChainman(request.context); + if (chainman.m_blockman.IsPruneMode()) { + throw JSONRPCError(RPC_MISC_ERROR, "Block locations are not available in prune mode"); + } + + uint256 hash(ParseHashV(request.params[0], "blockhash")); + size_t nblocks = request.params[1].getInt(); + + const CBlockIndex* pblockindex = WITH_LOCK(cs_main, return chainman.m_blockman.LookupBlockIndex(hash)); + if (!pblockindex) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + + UniValue result(UniValue::VARR); + do { + UniValue location(UniValue::VOBJ); + location.pushKV("file", (uint64_t)pblockindex->nFile); + location.pushKV("data", (uint64_t)pblockindex->nDataPos); + if (pblockindex->nUndoPos) { + location.pushKV("undo", (uint64_t)pblockindex->nUndoPos); + } + if (pblockindex->pprev) { + location.pushKV("prev", pblockindex->pprev->GetBlockHash().GetHex()); + } else { + location.pushKV("prev", uint256().GetHex()); + } + result.push_back(location); + pblockindex = pblockindex->pprev; + } while (result.size() < nblocks && pblockindex); + return result; +}, + }; +} + void RegisterBlockchainRPCCommands(CRPCTable& t) { @@ -2902,6 +2962,7 @@ void RegisterBlockchainRPCCommands(CRPCTable& t) {"hidden", &waitforblock}, {"hidden", &waitforblockheight}, {"hidden", &syncwithvalidationinterfacequeue}, + {"hidden", &getblocklocations}, }; for (const auto& c : commands) { t.appendCommand(c.name, &c); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 49820f25a3..d629a533ad 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -109,6 +109,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getblock", 1, "verbosity" }, { "getblock", 1, "verbose" }, { "getblockheader", 1, "verbose" }, + { "getblocklocations", 1, "nblocks" }, { "getchaintxstats", 0, "nblocks" }, { "gettransaction", 1, "include_watchonly" }, { "gettransaction", 2, "verbose" }, diff --git a/test/functional/rpc_getblocklocations.py b/test/functional/rpc_getblocklocations.py new file mode 100755 index 0000000000..5623b00c6d --- /dev/null +++ b/test/functional/rpc_getblocklocations.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the getblocklocations rpc call.""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import (assert_equal, assert_raises_rpc_error) +from test_framework.messages import ser_vector + + +class GetblocklocationsTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + """Test a trivial usage of the getblocklocations RPC command.""" + node = self.nodes[0] + test_block_count = 7 + self.generate(node, test_block_count) + + NULL_HASH = '0000000000000000000000000000000000000000000000000000000000000000' + + block_hashes = [node.getblockhash(height) for height in range(test_block_count)] + block_hashes.reverse() + + block_locations = {} + def check_consistency(tip, a): + for o in a: + if tip in block_locations: + assert_equal(block_locations[tip], o) + else: + block_locations[tip] = o + tip = o['prev'] + + # Get blocks' locations using several batch sizes + last_locations = None + for batch_size in range(1, 10): + locations = [] + tip = block_hashes[0] + while tip != NULL_HASH: + locations.extend(node.getblocklocations(tip, batch_size)) + check_consistency(block_hashes[0], locations) + tip = locations[-1]['prev'] + if last_locations: assert_equal(last_locations, locations) + last_locations = locations + + # Read blocks' data from the file system + blocks_dir = node.chain_path / 'blocks' + with (blocks_dir / 'blk00000.dat').open('rb') as blkfile: + for block_hash in block_hashes: + location = block_locations[block_hash] + block_bytes = bytes.fromhex(node.getblock(block_hash, 0)) + assert_file_contains(blkfile, location['data'], block_bytes) + + + empty_undo = ser_vector([]) # empty blocks = no transactions to undo + with (blocks_dir / 'rev00000.dat').open('rb') as revfile: + for block_hash in block_hashes[:-1]: # skip genesis block (has no undo) + location = block_locations[block_hash] + assert_file_contains(revfile, location['undo'], empty_undo) + + # Fail getting unknown block + unknown_block_hash = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' + assert_raises_rpc_error(-5, 'Block not found', node.getblocklocations, unknown_block_hash, 3) + + # Fail in pruned mode + self.restart_node(0, ['-prune=1']) + tip = block_hashes[0] + assert_raises_rpc_error(-1, 'Block locations are not available in prune mode', node.getblocklocations, tip, 3) + + +def assert_file_contains(fileobj, offset, data): + fileobj.seek(offset) + assert_equal(fileobj.read(len(data)), data) + +if __name__ == '__main__': + GetblocklocationsTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index fbf48a0e4d..353164b1f8 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -316,6 +316,7 @@ BASE_SCRIPTS = [ 'wallet_fallbackfee.py --legacy-wallet', 'wallet_fallbackfee.py --descriptors', 'rpc_dumptxoutset.py', + 'rpc_getblocklocations.py', 'feature_minchainwork.py', 'rpc_estimatefee.py', 'rpc_getblockstats.py',