#3605 Add both reindex options & FAQ entry (#3623)

fixes #3605
This commit is contained in:
/rootzoll 2023-01-25 19:07:13 +01:00 committed by GitHub
parent 0012aea921
commit d731823758
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 88 additions and 61 deletions

24
FAQ.md
View File

@ -302,6 +302,17 @@ You can also put an empty file just called `hdmi` (without any ending) onto the
## Debug
### How do I generate a Debug Report?
If your RaspiBlitz is not working correctly and you like to get help from the community, it's good to provide more debug information, so others can better diagnose your problem - please follow the following steps to generate a debug report:
- SSH into your raspiblitz as admin user with your password A
- If you see the menu - use CTRL+C to get to the terminal
- To generate debug report run: `debug`, optionally create a link with `debug -l`
- Then copy all output beginning with `*** RASPIBLITZ LOGS ***` and share this
*PLEASE NOTICE: It's possible that these logs can contain private information (like IPs, node IDs, ...) - just share publicly what you feel OK with.*
### I have the full blockchain on another storage. How do I copy it to the RaspiBlitz?
Copying a already synced blockchain from another storage (e.g. your Laptop or external hard drive) can be a quick way to get the RaspiBlitz started or replacing a corrupted blockchain with a fresh one. Also that way you have synced and verified the blockchain yourself, and are not trusting the RaspiBlitz Torrent downloads (Don't trust, verify).
@ -318,16 +329,13 @@ If everything described above is in order, start the setup of the new RaspiBlitz
Once you finished all the transfers, the Raspiblitz will make a quick-check on the data - but that will not guarantee that everything in detail was OK with the transfer. Check further FAQ answers if you get stuck or see a final sync with a value below 90%.
### How do I generate a Debug Report?
### Bitcoind tells me to reindex - how can I do this?
If your RaspiBlitz is not working correctly and you like to get help from the community, it's good to provide more debug information, so others can better diagnose your problem - please follow the following steps to generate a debug report:
To find/access information fast in large data sets like the Bitcoin blockhain indexes are needed. Those indexes can get corrupted on your HDD/SSD and to repair them they need to be rebuild - re-indexed. Bitcoind has two different options to do this - a fast way called "reindex-chainstate" (which just rebuilds the UTXO set from the blocks as you have them) and the slow but complete way called just "reindex" that would even recheck all your block data - see for details here: https://bitcoin.stackexchange.com/questions/60709/when-should-i-use-reindex-chainstate-and-when-reindex
So if you read in your debug logs of bitcoind that you should "reindex" you can try first just to do a fast "reindex-chainstate" and if that didnt worked a slow and full "reindex".
- SSH into your raspiblitz as admin user with your password A
- If you see the menu - use CTRL+C to get to the terminal
- To generate debug report run: `debug`, optionally create a link with `debug -l`
- Then copy all output beginning with `*** RASPIBLITZ LOGS ***` and share this
*PLEASE NOTICE: It's possible that these logs can contain private information (like IPs, node IDs, ...) - just share publicly what you feel OK with.*
See the raspiblitz script `./config.scripts/network.reindex.sh` or the REAPIR menu to start these processes.
### Why is my "final sync" taking so long?

View File

@ -353,7 +353,7 @@ else
appInfoLine=""
# Electrum Server - electrs
if [ "${ElectRS}" = "on" ]; then
if [ "${ElectRS}" == "on" ]; then
error=""
source <(sudo /home/admin/config.scripts/bonus.electrs.sh status-sync 2>/dev/null)
if [ ${#infoSync} -gt 0 ]; then
@ -361,15 +361,10 @@ else
fi
fi
# BTC RPC EXPLORER
if [ "${BTCRPCexplorer}" = "on" ]; then
error=""
source <(sudo /home/admin/config.scripts/bonus.btc-rpc-explorer.sh status 2>/dev/null)
if [ ${#error} -gt 0 ]; then
appInfoLine="ERROR BTC-RPC-Explorer: ${error} (try restart)"
elif [ "${isIndexed}" = "0" ]; then
appInfoLine="BTC-RPC-Explorer: ${indexInfo}"
fi
# Transaction Index
source <(/home/admin/config.scripts/network.txindex.sh status)
if [ "${txindex}" == "1" ] && [ "${isIndexed}" != "1" ]; then
appInfoLine="Transaction Index: ${indexInfo}"
fi
if [ ${#appInfoLine} -gt 0 ]; then

View File

@ -65,6 +65,9 @@ RaspiBlitz image to your SD card.
" 12 40
}
# get status of txindex
source <(/home/admin/config.scripts/network.txindex.sh status)
OPTIONS=()
#OPTIONS+=(HARDWARE "Run Hardwaretest")
OPTIONS+=(SOFTWARE "Run Softwaretest (DebugReport)")
@ -76,8 +79,13 @@ if [ "${lightning}" == "cl" ] || [ "${cl}" == "on" ]; then
fi
OPTIONS+=(MIGRATION "Migrate Blitz Data to new Hardware")
OPTIONS+=(COPY-SOURCE "Copy Blockchain Source Modus")
OPTIONS+=(REINDEX "Redindex Bitcoin Blockchain")
OPTIONS+=(DELETE-INDEX "Delete Bitcoin Transaction-Index")
if [ "${txindex}" == "1" ]; then
OPTIONS+=(DELETE-INDEX "Reindex Bitcoin Transaction-Index")
elif [ "${indexByteSize}" != "0" ]; then
OPTIONS+=(DELETE-INDEX "Delete Bitcoin Transaction-Index")
fi
OPTIONS+=(REINDEX-UTXO "Redindex Just Bitcoin Chainstate (Fast)")
OPTIONS+=(REINDEX-FULL "Redindex Full Bitcoin Blockchain (Slow)")
OPTIONS+=(RESET-CHAIN "Delete Blockchain & Re-Download")
OPTIONS+=(RESET-HDD "Delete HDD Data but keep Blockchain")
OPTIONS+=(RESET-ALL "Delete HDD completely to start fresh")
@ -157,8 +165,12 @@ case $CHOICE in
/home/admin/config.scripts/network.txindex.sh delete
exit 0;
;;
REINDEX)
/home/admin/config.scripts/network.reindex.sh reindex main
REINDEX-UTXO)
/home/admin/config.scripts/network.reindex.sh reindex-chainstate mainnet
exit 0;
;;
REINDEX-FULL)
/home/admin/config.scripts/network.reindex.sh reindex mainnet
exit 0;
;;
COPY-SOURCE)

View File

@ -98,20 +98,21 @@ if [ "$1" = "status-sync" ] || [ "$1" = "status" ]; then
echo "serviceRunning=${serviceRunning}"
if [ ${serviceRunning} -eq 1 ]; then
# Experimental try to get sync Info
syncedToBlock=$(sudo journalctl -u electrs --no-pager -n2000 | grep "height=" | tail -n1| cut -d= -f3)
blockchainHeight=$(sudo -u bitcoin ${network}-cli getblockchaininfo 2>/dev/null | jq -r '.headers' | sed 's/[^0-9]*//g')
lastBlockchainHeight=$(($blockchainHeight -1))
syncProgress=0
if [ "${syncedToBlock}" != "" ] && [ "${blockchainHeight}" != "" ] && [ "${blockchainHeight}" != "0" ]; then
syncProgress="$(echo "$syncedToBlock" "$blockchainHeight" | awk '{printf "%.2f", $1 / $2 * 100}')"
fi
echo "syncProgress=${syncProgress}%"
if [ "${syncedToBlock}" = "${blockchainHeight}" ] || [ "${syncedToBlock}" = "${lastBlockchainHeight}" ]; then
echo "tipSynced=1"
else
echo "tipSynced=0"
fi
# Experimental try to get sync Info (electrs debug info would need more details)
#source <(/home/admin/_cache.sh get btc_mainnet_blocks_headers)
#blockchainHeight="${btc_mainnet_blocks_headers}"
#lastBlockchainHeight=$(($blockchainHeight -1))
#syncedToBlock=$(sudo journalctl -u electrs --no-pager -n2000 | grep "height=" | tail -n1| cut -d= -f3)
#syncProgress=0
#if [ "${syncedToBlock}" != "" ] && [ "${blockchainHeight}" != "" ] && [ "${blockchainHeight}" != "0" ]; then
# syncProgress="$(echo "$syncedToBlock" "$blockchainHeight" | awk '{printf "%.2f", $1 / $2 * 100}')"
#fi
#echo "syncProgress=${syncProgress}%"
#if [ "${syncedToBlock}" = "${blockchainHeight}" ] || [ "${syncedToBlock}" = "${lastBlockchainHeight}" ]; then
# echo "tipSynced=1"
#else
# echo "tipSynced=0"
#fi
# check if initial sync was done, by setting a file as once electrs is the first time responding on port 50001
electrumResponding=$(echo '{"jsonrpc":"2.0","method":"server.ping","params":[],"id":"electrs-check"}' | netcat -w 2 127.0.0.1 50001 | grep -c "result")
@ -135,7 +136,7 @@ if [ "$1" = "status-sync" ] || [ "$1" = "status" ]; then
fi
else
echo "tipSynced=0"
# echo "tipSynced=0"
echo "initialSynced=0"
echo "electrumResponding=0"
echo "infoSync='Not running - check: sudo journalctl -u electrs'"

View File

@ -4,28 +4,29 @@
if [ $# -eq 0 ] || [ "$1" = "-h" ] || [ "$1" = "-help" ]; then
echo "script to run re-index if the blockchain - blocks will not be deleted but re-indexed"
echo "will trigger reboot after started and progress can be monitored thru normal sync status"
echo "network.reindex.sh reindex [mainnet|testnet|signet] --> use to start re-index chain"
echo "There are two ways to re-index - for details see: https://bitcoin.stackexchange.com/a/60711"
echo "network.reindex.sh reindex [mainnet|testnet|signet] --> re-index chain & repair corrupt blocks"
echo "network.reindex.sh reindex-chainstate [mainnet|testnet|signet] --> only re-build UTXO set (fast)"
exit 1
fi
source /mnt/hdd/raspiblitz.conf
###################
# START
###################
if [ "$1" = "reindex" ]; then
if [ "$1" = "reindex" ] || [ "$1" = "reindex-chainstate" ]; then
action="$1"
# network prefixes
if [ "$2" = "mainnet" ]; then
echo "# network.reindex.sh reindex --> mainnet"
echo "# network.reindex.sh ${action} --> mainnet"
prefix=""
netparam=""
elif [ "$2" = "testnet" ]; then
echo "# network.reindex.sh reindex --> testnet"
echo "# network.reindex.sh ${action} --> testnet"
prefix="t"
netparam="-testnet "
elif [ "$2" = "signet" ]; then
echo "# network.reindex.sh reindex --> signet"
echo "# network.reindex.sh ${action} --> signet"
prefix="s"
netparam="-signet "
else
@ -38,8 +39,8 @@ if [ "$1" = "reindex" ]; then
sudo systemctl stop ${prefix}${network}d
# starting reindex
echo "# starting ${network} service with -reindex flag"
sudo -u bitcoin /usr/local/bin/${network}d ${netparam}-daemon -reindex -conf=/mnt/hdd/${network}/${network}.conf -datadir=/mnt/hdd/${network} 1>&2
echo "# starting ${network} service with -${action} flag"
sudo -u bitcoin /usr/local/bin/${network}d ${netparam}-daemon -blockfilterindex=0 -${action} -conf=/mnt/hdd/${network}/${network}.conf -datadir=/mnt/hdd/${network} 1>&2
echo "# waiting 10 secs"
sleep 10
echo "# going into reboot - reindex process can be monitored like normal blockchain sync status"

View File

@ -30,36 +30,47 @@ if [ "$1" = "status" ]; then
echo "##### STATUS TXINDEX"
indexByteSize=$(sudo du -s /mnt/hdd/bitcoin/indexes/txindex 2>/dev/null | cut -f1)
if [ "${indexByteSize}" == "" ]; then
indexByteSize=0
fi
echo "txindex=${txindex}"
echo "indexByteSize=${indexByteSize}"
if [ ${txindex} -eq 0 ]; then
exit 0
fi
# try to gather if still indexing
source <(/home/admin/_cache.sh get btc_mainnet_blocks_headers)
blockchainHeight="${btc_mainnet_blocks_headers}"
indexedToBlock=$(sudo tail -n 200 /mnt/hdd/${network}${pathAdd}/debug.log | grep "Syncing txindex with block chain from height" | tail -n 1 | cut -d " " -f 9 | sed 's/[^0-9]*//g')
blockchainHeight=$(sudo -u bitcoin ${network}-cli getblockchaininfo 2>/dev/null | jq -r '.blocks' | sed 's/[^0-9]*//g')
indexFinished=$(sudo tail -n 200 /mnt/hdd/${network}${pathAdd}/debug.log | grep -c "txindex is enabled at height")
echo "indexedToBlock=${indexedToBlock}"
echo "blockchainHeight=${blockchainHeight}"
echo "indexFinished=${indexFinished}"
if [ ${#indexedToBlock} -eq 0 ] || [ ${indexFinished} -gt 0 ] || [ "${indexedToBlock}" = "${blockchainHeight}" ]; then
echo "isIndexed=1"
indexedToBlock=$blockchainHeight
indexFinished=1
indexInfo="OK"
else
echo "isIndexed=0"
if [ ${#indexedToBlock} -gt 0 ] && [ ${#blockchainHeight} -gt 0 ]; then
progressPercent=$(printf %.2f $(echo "${indexedToBlock}/${blockchainHeight}*100" | bc -l))
indexInfo="Indexing is at ${progressPercent}% (please wait)"
indexInfo="Building ${progressPercent}% (please wait)"
else
indexInfo="Indexing is running (please wait)"
indexInfo="Building (please wait)"
fi
echo "indexInfo='${indexInfo}'"
fi
fi
echo "indexFinished=${indexFinished}"
echo "indexedToBlock=${indexedToBlock}"
echo "blockchainHeight=${blockchainHeight}"
exit 0
fi
###################
# switch on
###################
@ -84,27 +95,26 @@ if [ "$1" = "1" ] || [ "$1" = "on" ]; then
fi
fi
###################
# switch off
###################
if [ "$1" = "0" ] || [ "$1" = "off" ]; then
echo "# changing config ..."
sudo sed -i "s/^txindex=.*/txindex=0/g" /mnt/hdd/${network}/${network}.conf
echo "# deinstalling apps needing txindex ..."
sudo -u admin /home/admin/config.scripts/bonus.btc-rpc-explorer.sh off
echo "# restarting bitcoind ..."
sudo systemctl restart ${network}d
exit 0
fi
###################
# delete (and make sure all using apps are deinstalled)
# on version update check all bonus scripts that this network.txindex.sh on
###################
if [ "$1" = "delete" ]; then
echo "# deinstalling apps needing txindex ..."
sudo -u admin /home/admin/config.scripts/bonus.btc-rpc-explorer.sh off
echo "# changing config ..."
echo "# stopping bitcoind ..."
sudo systemctl stop ${network}d
sudo sed -i "s/^txindex=.*/txindex=0/g" /mnt/hdd/${network}/${network}.conf
echo "# deleting tx index ..."
sudo rm -r /mnt/hdd/${network}/indexes/txindex
echo "# restarting bitcoind ..."