From 620e79b72f6dd6eb0bd2798c4da893081d8e7f39 Mon Sep 17 00:00:00 2001 From: Roman Zeyde Date: Tue, 11 Aug 2020 17:57:17 +0300 Subject: [PATCH 1/6] RPC: Add getblocklocations call This RPC allows the client to retrieve the file system locations of the confirmed blocks and their undo data, to allow building efficient indexes outside of Bitcoin Core. An example usage is described here: https://github.com/romanz/electrs/issues/308 By using the new RPC, it is possible to build an address-based index taking ~24GB and a txindex taking ~6GB (as of Dec. 2020). --- src/rpc/blockchain.cpp | 61 +++++++++++++++++ src/rpc/client.cpp | 1 + test/functional/rpc_getblocklocations.py | 83 ++++++++++++++++++++++++ test/functional/test_runner.py | 1 + 4 files changed, 146 insertions(+) create mode 100755 test/functional/rpc_getblocklocations.py diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 7b84747a3f..2a734d55d9 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2869,6 +2869,66 @@ return RPCHelpMan{ }; } +static RPCHelpMan getblocklocations() +{ + return RPCHelpMan{"getblocklocations", + "\nEXPERIMENTAL warning: this call may be removed or changed in future releases.\n" + "\nReturns a JSON for the file system location of 'blockhash' block and undo data.\n" + "\nIt is possible to return also the locations of previous blocks, by specifying 'nblocks' > 1.\n", + { + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The block hash"}, + {"nblocks", RPCArg::Type::NUM, RPCArg::Optional::NO, "Maximum number locations to return (up to genesis block)"}, + }, + { + RPCResult{ + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::NUM, "file", "blk*.dat/rev*.dat file index"}, + {RPCResult::Type::NUM, "data", "block data file offset"}, + {RPCResult::Type::NUM, "undo", "undo data file offset (if exists)"}, + {RPCResult::Type::STR_HEX, "prev", "previous block hash"}, + } + }, + }, + RPCExamples{ + HelpExampleCli("getblocklocation", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\" 10") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + + ChainstateManager& chainman = EnsureAnyChainman(request.context); + if (chainman.m_blockman.IsPruneMode()) { + throw JSONRPCError(RPC_MISC_ERROR, "Block locations are not available in prune mode"); + } + + uint256 hash(ParseHashV(request.params[0], "blockhash")); + size_t nblocks = request.params[1].getInt(); + + const CBlockIndex* pblockindex = WITH_LOCK(cs_main, return chainman.m_blockman.LookupBlockIndex(hash)); + if (!pblockindex) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + + UniValue result(UniValue::VARR); + do { + UniValue location(UniValue::VOBJ); + location.pushKV("file", (uint64_t)pblockindex->nFile); + location.pushKV("data", (uint64_t)pblockindex->nDataPos); + if (pblockindex->nUndoPos) { + location.pushKV("undo", (uint64_t)pblockindex->nUndoPos); + } + if (pblockindex->pprev) { + location.pushKV("prev", pblockindex->pprev->GetBlockHash().GetHex()); + } else { + location.pushKV("prev", uint256().GetHex()); + } + result.push_back(location); + pblockindex = pblockindex->pprev; + } while (result.size() < nblocks && pblockindex); + return result; +}, + }; +} + void RegisterBlockchainRPCCommands(CRPCTable& t) { @@ -2902,6 +2962,7 @@ void RegisterBlockchainRPCCommands(CRPCTable& t) {"hidden", &waitforblock}, {"hidden", &waitforblockheight}, {"hidden", &syncwithvalidationinterfacequeue}, + {"hidden", &getblocklocations}, }; for (const auto& c : commands) { t.appendCommand(c.name, &c); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 49820f25a3..d629a533ad 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -109,6 +109,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getblock", 1, "verbosity" }, { "getblock", 1, "verbose" }, { "getblockheader", 1, "verbose" }, + { "getblocklocations", 1, "nblocks" }, { "getchaintxstats", 0, "nblocks" }, { "gettransaction", 1, "include_watchonly" }, { "gettransaction", 2, "verbose" }, diff --git a/test/functional/rpc_getblocklocations.py b/test/functional/rpc_getblocklocations.py new file mode 100755 index 0000000000..a9b24fdb86 --- /dev/null +++ b/test/functional/rpc_getblocklocations.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the getblocklocations rpc call.""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import (assert_equal, assert_raises_rpc_error) +from test_framework.messages import ser_vector + + +class GetblocklocationsTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [ + [ + "-blocksxor=0", + ], + ] + + def run_test(self): + """Test a trivial usage of the getblocklocations RPC command.""" + node = self.nodes[0] + test_block_count = 7 + self.generate(node, test_block_count) + + NULL_HASH = '0000000000000000000000000000000000000000000000000000000000000000' + + block_hashes = [node.getblockhash(height) for height in range(test_block_count)] + block_hashes.reverse() + + block_locations = {} + def check_consistency(tip, a): + for o in a: + if tip in block_locations: + assert_equal(block_locations[tip], o) + else: + block_locations[tip] = o + tip = o['prev'] + + # Get blocks' locations using several batch sizes + last_locations = None + for batch_size in range(1, 10): + locations = [] + tip = block_hashes[0] + while tip != NULL_HASH: + locations.extend(node.getblocklocations(tip, batch_size)) + check_consistency(block_hashes[0], locations) + tip = locations[-1]['prev'] + if last_locations: assert_equal(last_locations, locations) + last_locations = locations + + # Read blocks' data from the file system + blocks_dir = node.chain_path / 'blocks' + with (blocks_dir / 'blk00000.dat').open('rb') as blkfile: + for block_hash in block_hashes: + location = block_locations[block_hash] + block_bytes = bytes.fromhex(node.getblock(block_hash, 0)) + assert_file_contains(blkfile, location['data'], block_bytes) + + + empty_undo = ser_vector([]) # empty blocks = no transactions to undo + with (blocks_dir / 'rev00000.dat').open('rb') as revfile: + for block_hash in block_hashes[:-1]: # skip genesis block (has no undo) + location = block_locations[block_hash] + assert_file_contains(revfile, location['undo'], empty_undo) + + # Fail getting unknown block + unknown_block_hash = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' + assert_raises_rpc_error(-5, 'Block not found', node.getblocklocations, unknown_block_hash, 3) + + # Fail in pruned mode + self.restart_node(0, ['-prune=1']) + tip = block_hashes[0] + assert_raises_rpc_error(-1, 'Block locations are not available in prune mode', node.getblocklocations, tip, 3) + + +def assert_file_contains(fileobj, offset, data): + fileobj.seek(offset) + assert_equal(fileobj.read(len(data)), data) + +if __name__ == '__main__': + GetblocklocationsTest(__file__).main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index fbf48a0e4d..353164b1f8 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -316,6 +316,7 @@ BASE_SCRIPTS = [ 'wallet_fallbackfee.py --legacy-wallet', 'wallet_fallbackfee.py --descriptors', 'rpc_dumptxoutset.py', + 'rpc_getblocklocations.py', 'feature_minchainwork.py', 'rpc_estimatefee.py', 'rpc_getblockstats.py', From 91c9e1463915e734ba3a87b77e28b28378e46756 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Tue, 17 May 2022 08:13:17 +0000 Subject: [PATCH 2/6] Bugfix: RPC/blockchain: Hold cs_main for CBlockIndex access as needed to avoid races in getblocklocations --- src/rpc/blockchain.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 2a734d55d9..af450f2956 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2910,11 +2910,19 @@ static RPCHelpMan getblocklocations() UniValue result(UniValue::VARR); do { + int64_t file_num; + uint64_t data_pos, undo_pos; + { + LOCK(::cs_main); + file_num = pblockindex->nFile; + data_pos = pblockindex->nDataPos; + undo_pos = pblockindex->nUndoPos; + } UniValue location(UniValue::VOBJ); - location.pushKV("file", (uint64_t)pblockindex->nFile); - location.pushKV("data", (uint64_t)pblockindex->nDataPos); - if (pblockindex->nUndoPos) { - location.pushKV("undo", (uint64_t)pblockindex->nUndoPos); + location.pushKV("file", file_num); + location.pushKV("data", data_pos); + if (undo_pos) { + location.pushKV("undo", undo_pos); } if (pblockindex->pprev) { location.pushKV("prev", pblockindex->pprev->GetBlockHash().GetHex()); From 32c2b6e32693dd185b2298e72c26e8e50dc51c2d Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Wed, 16 Nov 2022 03:41:37 +0000 Subject: [PATCH 3/6] RPC/Blockchain: Bugfix: Correct getblocklocations example RPC method name --- src/rpc/blockchain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index af450f2956..f1eba9722a 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2891,7 +2891,7 @@ static RPCHelpMan getblocklocations() }, }, RPCExamples{ - HelpExampleCli("getblocklocation", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\" 10") + HelpExampleCli("getblocklocations", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\" 10") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { From e56f5e725544336ce2d47ff7c30abfe88d1a70c7 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Thu, 27 Jul 2023 00:29:33 +0000 Subject: [PATCH 4/6] Bugfix: RPC/Blockchain: Correct getblocklocations return type (array of objects, not array with named keys) --- src/rpc/blockchain.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f1eba9722a..e8b9f496d4 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2883,10 +2883,13 @@ static RPCHelpMan getblocklocations() RPCResult{ RPCResult::Type::ARR, "", "", { - {RPCResult::Type::NUM, "file", "blk*.dat/rev*.dat file index"}, - {RPCResult::Type::NUM, "data", "block data file offset"}, - {RPCResult::Type::NUM, "undo", "undo data file offset (if exists)"}, - {RPCResult::Type::STR_HEX, "prev", "previous block hash"}, + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::NUM, "file", "blk*.dat/rev*.dat file index"}, + {RPCResult::Type::NUM, "data", "block data file offset"}, + {RPCResult::Type::NUM, "undo", /*optional=*/true, "undo data file offset (if exists)"}, + {RPCResult::Type::STR_HEX, "prev", "previous block hash"}, + }}, } }, }, From e1a10af807d6252951920c4f6de4648734ec2f88 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Tue, 10 Oct 2023 00:29:48 +0000 Subject: [PATCH 5/6] Bugfix: QA/fuzz: Add getblocklocations to RPC_COMMANDS_SAFE_FOR_FUZZING --- src/test/fuzz/rpc.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 270cab58e2..143f9ba955 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -120,6 +120,7 @@ const std::vector RPC_COMMANDS_SAFE_FOR_FUZZING{ "getblockfrompeer", // when no peers are connected, no p2p message is sent "getblockhash", "getblockheader", + "getblocklocations", "getblockstats", "getblocktemplate", "getchaintips", From 099f47c89f9a9d4e8d0705c969ba6a73b27116da Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Sun, 16 Feb 2025 16:12:56 +0000 Subject: [PATCH 6/6] QA: rpc_getblocklocations: Support for testing with blocksxor enabled --- test/functional/rpc_getblocklocations.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/test/functional/rpc_getblocklocations.py b/test/functional/rpc_getblocklocations.py index a9b24fdb86..bafa53a794 100755 --- a/test/functional/rpc_getblocklocations.py +++ b/test/functional/rpc_getblocklocations.py @@ -4,7 +4,11 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the getblocklocations rpc call.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (assert_equal, assert_raises_rpc_error) +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + util_xor, +) from test_framework.messages import ser_vector @@ -12,11 +16,6 @@ class GetblocklocationsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 - self.extra_args = [ - [ - "-blocksxor=0", - ], - ] def run_test(self): """Test a trivial usage of the getblocklocations RPC command.""" @@ -50,20 +49,22 @@ class GetblocklocationsTest(BitcoinTestFramework): if last_locations: assert_equal(last_locations, locations) last_locations = locations + xor_key = node.read_xor_key() + # Read blocks' data from the file system blocks_dir = node.chain_path / 'blocks' with (blocks_dir / 'blk00000.dat').open('rb') as blkfile: for block_hash in block_hashes: location = block_locations[block_hash] block_bytes = bytes.fromhex(node.getblock(block_hash, 0)) - assert_file_contains(blkfile, location['data'], block_bytes) + assert_file_contains(blkfile, location['data'], block_bytes, xor_key) empty_undo = ser_vector([]) # empty blocks = no transactions to undo with (blocks_dir / 'rev00000.dat').open('rb') as revfile: for block_hash in block_hashes[:-1]: # skip genesis block (has no undo) location = block_locations[block_hash] - assert_file_contains(revfile, location['undo'], empty_undo) + assert_file_contains(revfile, location['undo'], empty_undo, xor_key) # Fail getting unknown block unknown_block_hash = '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' @@ -75,9 +76,11 @@ class GetblocklocationsTest(BitcoinTestFramework): assert_raises_rpc_error(-1, 'Block locations are not available in prune mode', node.getblocklocations, tip, 3) -def assert_file_contains(fileobj, offset, data): +def assert_file_contains(fileobj, offset, data, xor_key): fileobj.seek(offset) - assert_equal(fileobj.read(len(data)), data) + read_data = fileobj.read(len(data)) + read_data = util_xor(read_data, xor_key, offset=offset) + assert_equal(read_data, data) if __name__ == '__main__': GetblocklocationsTest(__file__).main()