Added Reindex Chainstate Action (#27)

* Create reindex_chainstate.sh

* Update Dockerfile

* Update main.rs for reindex_chainstate

* Update manifest.yaml

* Update main.rs

* Update main.rs

* Update action.ts for coinstatsindex

* Update setConfig.ts for coinstatsindex

* Update manifest.yaml for coinstats index

* Update getConfig.ts for coinstatsindex

* Update bitcoin.conf.template for coinstats index

* Update getConfig.ts to make v2transport protocol default

* Update reindex.sh naming to 'Knots'

* Update reindex_chainstate.sh for 'Knots' naming

* Update main.rs to fix issue

* Update main.rs dumb error

* Update main.rs to keep progress to next halving and sync summary

* Update main.rs to simplify

* Update main.rs Simplify + reduce complex math for longevity

Dockerfile

@@ -83,6 +83,7 @@ COPY --from=bitcoin-core /opt /opt
 COPY ./manager/target/${ARCH}-unknown-linux-musl/release/bitcoind-manager \
      ./docker_entrypoint.sh \
      ./actions/reindex.sh \
+     ./actions/reindex_chainstate.sh \
      ./actions/prioritise-transaction.sh \
      ./check-rpc.sh \
      ./check-synced.sh \

actions/reindex.sh

@@ -5,14 +5,14 @@ set -e
 touch /root/.bitcoin/requires.reindex
 action_result_running="    {
     \"version\": \"0\",
-    \"message\": \"Bitcoin Core restarting in reindex mode\",
+    \"message\": \"Bitcoin Knots restarting in reindex mode\",
     \"value\": null,
     \"copyable\": false,
     \"qr\": false
 }"
 action_result_stopped="    {
     \"version\": \"0\",
-    \"message\": \"Bitcoin Core will reindex the next time the service is started\",
+    \"message\": \"Bitcoin Knots will reindex the next time the service is started\",
     \"value\": null,
     \"copyable\": false,
     \"qr\": false

actions/reindex_chainstate.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+
+set -e
+
+action_result_running="    {
+    \"version\": \"0\",
+    \"message\": \"Bitcoin Knots restarting in reindex chainstate mode\",
+    \"value\": null,
+    \"copyable\": false,
+    \"qr\": false
+}"
+action_result_stopped="    {
+    \"version\": \"0\",
+    \"message\": \"Bitcoin Knots will reindex the chainstate the next time the service is started\",
+    \"value\": null,
+    \"copyable\": false,
+    \"qr\": false
+}"
+action_result_pruned="    {
+    \"version\": \"0\",
+    \"message\": \"Bitcoin Knots does not allow reindex-chainstate for pruned nodes. If the Chainstate is corrupted on a pruned node the entire blockchain will need to be re-downloaded from genesis with the 'Reindex Blockchain' action\",
+    \"value\": null,
+    \"copyable\": false,
+    \"qr\": false
+}"
+
+pruned=$(yq e '.advanced.pruning.mode' /root/.bitcoin/start9/config.yaml)
+
+if [ "$pruned" != "disabled" ]; then
+  echo $action_result_pruned
+else
+  touch /root/.bitcoin/requires.reindex_chainstate
+  bitcoin-cli -rpcconnect=bitcoind.embassy stop >/dev/null 2>/dev/null && echo $action_result_running || echo $action_result_stopped
+fi

assets/compat/bitcoin.conf.template

@@ -124,6 +124,11 @@ zmqpubsequence=tcp://0.0.0.0:28333
 txindex=1
 }}
 
+## COINSTATSINDEX
+{{#IF coinstatsindex
+coinstatsindex=1
+}}
+
 ## DATACARRIER
 {{#IF blkconstr.datacarrier
 datacarrier=1
@@ -219,4 +224,4 @@ blockmaxsize={{advanced.templateconstruction.blockmaxsize}}
 
 {{#IF advanced.templateconstruction.blockmaxweight
 blockmaxweight={{advanced.templateconstruction.blockmaxweight}}
-}}
+}}

manager/src/main.rs

@@ -33,11 +33,8 @@ pub struct ChainInfo {
 
 #[derive(Clone, Debug, serde::Deserialize)]
 pub struct NetworkInfo {
-    #[allow(dead_code)]
     connections: usize,
-    #[allow(dead_code)]
     connections_in: usize,
-    #[allow(dead_code)]
     connections_out: usize,
 }
 
@@ -115,50 +112,6 @@ pub struct Stat {
 
 fn sidecar(config: &Mapping, addr: &str) -> Result<(), Box<dyn Error>> {
     let mut stats = LinearMap::new();
-    // Node uptime with years included
-    let uptime_info = std::process::Command::new("bitcoin-cli")
-        .arg("-conf=/root/.bitcoin/bitcoin.conf")
-        .arg("uptime")
-        .output()?;
-
-    if uptime_info.status.success() {
-        let uptime_seconds = String::from_utf8_lossy(&uptime_info.stdout)
-            .trim()
-            .parse::<u64>()
-            .unwrap_or(0);
-
-        let years = uptime_seconds / 31_536_000;
-        let days = (uptime_seconds % 31_536_000) / 86400;
-        let hours = (uptime_seconds % 86400) / 3600;
-        let minutes = (uptime_seconds % 3600) / 60;
-
-        let uptime_formatted = if years > 0 {
-            format!("{} years, {} days, {} hours, {} minutes", years, days, hours, minutes)
-        } else if days > 0 {
-            format!("{} days, {} hours, {} minutes", days, hours, minutes)
-        } else if hours > 0 {
-            format!("{} hours, {} minutes", hours, minutes)
-        } else {
-            format!("{} minutes", minutes)
-        };
-
-        stats.insert(
-            Cow::from("Node Uptime"),
-            Stat {
-                value_type: "string",
-                value: uptime_formatted,
-                description: Some(Cow::from("Total time the Bitcoin node has been running")),
-                copyable: false,
-                qr: false,
-                masked: false,
-            },
-        );
-    } else {
-        eprintln!(
-            "Error retrieving uptime info: {}",
-            std::str::from_utf8(&uptime_info.stderr).unwrap_or("UNKNOWN ERROR")
-        );
-    }
     if let (Some(user), Some(pass)) = (
         config
             .get(&Value::String("rpc".to_owned()))
@@ -215,132 +168,24 @@ fn sidecar(config: &Mapping, addr: &str) -> Result<(), Box<dyn Error>> {
             },
         );
     }
-    
-    // Calculate total supply and progress to the next halving
-    let blockchain_info = std::process::Command::new("bitcoin-cli")
-        .arg("-conf=/root/.bitcoin/bitcoin.conf")
-        .arg("getblockchaininfo")
-        .output()?;
-
-    if blockchain_info.status.success() {
-        let blockchain_data: serde_json::Value = serde_json::from_slice(&blockchain_info.stdout)?;
-        let current_height = blockchain_data["blocks"].as_u64().unwrap_or(0);
-
-        // Calculate total supply based on height and halving intervals
-        let halving_interval = 210_000;
-        let mut subsidy = 50.0; // Initial subsidy in BTC
-        let mut total_supply = 0.0;
-        let mut remaining_height = current_height;
-
-        while remaining_height > 0 {
-            let blocks_in_this_epoch = remaining_height.min(halving_interval);
-            total_supply += blocks_in_this_epoch as f64 * subsidy;
-            remaining_height = remaining_height.saturating_sub(halving_interval);
-            subsidy /= 2.0;
-        }
-
-        stats.insert(
-            Cow::from("Total Bitcoin Supply"),
-            Stat {
-                value_type: "string",
-                value: format!("{:.8} BTC", total_supply),
-                description: Some(Cow::from("Current total supply of Bitcoin based on issuance schedule")),
-                copyable: false,
-                qr: false,
-                masked: false,
-            },
-        );
-
-        // Progress to Next Halving calculation
-        let last_halving_block = (current_height / halving_interval) * halving_interval;
-        let blocks_since_last_halving = current_height - last_halving_block;
-        let progress_to_next_halving = (blocks_since_last_halving as f64 / halving_interval as f64) * 100.0;
-
-        stats.insert(
-            Cow::from("Progress to Next Halving"),
-            Stat {
-                value_type: "string",
-                value: format!("{:.2}%", progress_to_next_halving),
-                description: Some(Cow::from("Percentage of blocks completed until the next Bitcoin halving")),
-                copyable: false,
-                qr: false,
-                masked: false,
-            },
-        );
-    }
-
-// New section to fetch and display a simplified mempool summary
-let mempool_info = std::process::Command::new("bitcoin-cli")
-    .arg("-conf=/root/.bitcoin/bitcoin.conf")
-    .arg("getmempoolinfo")
-    .output()?;
-
-if mempool_info.status.success() {
-    let mempool_data: serde_json::Value = serde_json::from_slice(&mempool_info.stdout)?;
-
-    let max_mempool = mempool_data["maxmempool"].as_u64().unwrap_or(0) as f64 / 1024_f64.powf(2.0); // Convert to MB
-    let mempool_usage = mempool_data["usage"].as_u64().unwrap_or(0) as f64 / 1024_f64.powf(2.0); // Convert to MB
-    let mempool_percent = if max_mempool > 0.0 {
-        (mempool_usage / max_mempool) * 100.0
-    } else {
-        0.0
-    };
-    let tx_count = mempool_data["size"].as_u64().unwrap_or(0); // Number of transactions
-
-    // Insert the simplified mempool summary
-    stats.insert(
-        Cow::from("Mempool Summary"),
-        Stat {
-            value_type: "string",
-            value: format!(
-                "{:.2}/{:.2} MB ({:.2}%), {} Transactions",
-                mempool_usage, max_mempool, mempool_percent, tx_count
-            ),
-            description: Some(Cow::from("Summary of current mempool usage and transaction count")),
-            copyable: false,
-            qr: false,
-            masked: false,
-        },
-    );
-} else {
-    eprintln!(
-        "Error retrieving mempool info: {}",
-        std::str::from_utf8(&mempool_info.stderr).unwrap_or("UNKNOWN ERROR")
-    );
-}
-
-    // Existing code for blockchain and network info retrieval continues here...
     let info_res = std::process::Command::new("bitcoin-cli")
         .arg("-conf=/root/.bitcoin/bitcoin.conf")
         .arg("getblockchaininfo")
         .output()?;
     if info_res.status.success() {
         let info: ChainInfo = serde_json::from_slice(&info_res.stdout)?;
-        // Consolidated block height and sync progress information
-        // Extract values for current synced blocks, total headers, and sync progress percentage
-        let current_blocks = info.blocks;
-        let total_headers = info.headers;
-        let sync_progress = if current_blocks < total_headers {
-            100.0 * info.verificationprogress
-        } else {
-            100.0
-        };
-
-        // Insert the simplified blockchain sync summary
         stats.insert(
             Cow::from("Blockchain Sync Summary"),
             Stat {
                 value_type: "string",
-                value: format!(
-                    "{}/{} ({:.2}%)",
-                    current_blocks, total_headers, sync_progress
-                ),
-                description: Some(Cow::from("Current synced block height out of total headers and sync progress")),
+                value: format!("{} / {} ({:.2}%)", info.blocks, info.headers, 100.0 * info.verificationprogress),
+                description: Some(Cow::from("Current synced block height / actual network block height and sync percentage")),
                 copyable: false,
                 qr: false,
                 masked: false,
             },
-        );   
+        );
+
         for (sf_name, sf_data) in info.softforks {
             let sf_name_pretty = sf_name.to_title_case();
             let status_desc = Some(Cow::from(format!(
@@ -545,9 +390,8 @@ if mempool_info.status.success() {
     )?;
     Ok(())
 }
-    
 
-fn inner_main(reindex: bool) -> Result<(), Box<dyn Error>> {
+fn inner_main(reindex: bool, reindex_chainstate: bool) -> Result<(), Box<dyn Error>> {
     while !Path::new("/root/.bitcoin/start9/config.yaml").exists() {
         std::thread::sleep(std::time::Duration::from_secs(1));
     }
@@ -585,6 +429,18 @@ fn inner_main(reindex: bool) -> Result<(), Box<dyn Error>> {
     }
     if reindex {
         btc_args.push("-reindex".to_owned());
+        match fs::remove_file("/root/.bitcoin/requires.reindex") {
+            Ok(()) => (),
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => (),
+            a => a?,
+        }
+    } else if reindex_chainstate {
+        btc_args.push("-reindex-chainstate".to_owned());
+        match fs::remove_file("/root/.bitcoin/requires.reindex_chainstate") {
+            Ok(()) => (),
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => (),
+            a => a?,
+        }
     }
 
     std::io::copy(
@@ -599,13 +455,6 @@ fn inner_main(reindex: bool) -> Result<(), Box<dyn Error>> {
     let mut child = std::process::Command::new("bitcoind")
         .args(btc_args)
         .spawn()?;
-    if reindex {
-        match fs::remove_file("/root/.bitcoin/requires.reindex") {
-            Ok(()) => (),
-            Err(e) if e.kind() == std::io::ErrorKind::NotFound => (),
-            a => a?,
-        }
-    }
     let raw_child = child.id();
     *CHILD_PID.lock().unwrap() = Some(raw_child);
     let pruned = {
@@ -663,6 +512,7 @@ fn inner_main(reindex: bool) -> Result<(), Box<dyn Error>> {
 fn main() -> Result<(), Box<dyn Error>> {
     env_logger::Builder::from_env(Env::default().default_filter_or("warn")).init();
     let reindex = Path::new("/root/.bitcoin/requires.reindex").exists();
+    let reindex_chainstate = Path::new("/root/.bitcoin/requires.reindex_chainstate").exists();
     ctrlc::set_handler(move || {
         if let Some(raw_child) = *CHILD_PID.lock().unwrap() {
             use nix::{
@@ -674,7 +524,7 @@ fn main() -> Result<(), Box<dyn Error>> {
             std::process::exit(143)
         }
     })?;
-    inner_main(reindex)
+    inner_main(reindex, reindex_chainstate)
 }
 
 fn human_readable_timestamp(unix_time: u64) -> String {

manifest.yaml

@@ -146,6 +146,22 @@ actions:
       mounts:
         main: /root/.bitcoin
       io-format: json
+  reindex-chainstate:
+    name: "Reindex Chainstate"
+    description: "Rebuilds the chainstate database using existing block index data; as the block index is not rebuilt, 'reindex_chainstate' should be strictly faster than 'reindex'. This action should only be used in the case of chainstate corruption; if the blocks stored on disk are corrupted, the 'reindex' action will need to be run instead."
+    warning: While faster than 'Reindex', 'Reindex Chainstate' can still take several days or more to complete. Pruned nodes do not allow 'reindex-chainstate'; if you are running a pruned node and suspect chainstate corruption the 'reindex' action (requiring redownloading the entire Blockchain) should be run instead.
+    allowed-statuses:
+      - running
+      - stopped
+    implementation:
+      type: docker
+      image: main
+      system: false
+      entrypoint: reindex_chainstate.sh
+      args: []
+      mounts:
+        main: /root/.bitcoin
+      io-format: json
   delete-txindex:
     name: "Delete Transaction Index"
     description: "Deletes the Transaction Index (txindex) in case it gets corrupted."
@@ -154,6 +170,14 @@ actions:
       - stopped
     implementation:
       type: script
+  delete-coinstatsindex:
+    name: "Delete Coinstats Index"
+    description: "Deletes the Coinstats Index (coinstatsindex) in case it gets corrupted."
+    warning: The Coinstats Index will be rebuilt once Bitcoin Core is started again, unless you deactivate it in the config settings. Please don't do this unless instructed to by Start9 support staff.
+    allowed-statuses:
+      - stopped
+    implementation:
+      type: script
   delete-peers:
     name: "Delete Peer List"
     description: "Deletes the Peer List (peers.dat) in case it gets corrupted."

scripts/services/action.ts

@@ -57,4 +57,32 @@ export const action = {
       },
     };
   },
+  async "delete-coinstatsindex"(
+    effect: T.Effects,
+    _input?: T.Config,
+  ): Promise<T.ResultType<T.ActionResult>> {
+    const coinstatsinfoLocation = {
+      path: "indexes/coinstats",
+      volumeId: "main",
+    };
+    if (await util.exists(effect, coinstatsinfoLocation) === false) {
+      return {
+        result: {
+          copyable: false,
+          message: "coinstatsindex doesn't exist",
+          version: "0",
+          qr: false,
+        },
+      };
+    }
+    await effect.removeDir(coinstatsinfoLocation);
+    return {
+      result: {
+        copyable: false,
+        message: "Deleted coinstatsindex",
+        version: "0",
+        qr: false,
+      },
+    };
+  },
 };

scripts/services/getConfig.ts

@@ -329,6 +329,12 @@ export const getConfig: T.ExpectedExports.getConfig = async (effects) => {
       description: "Enable the Transaction Index (txindex)",
       default: allowUnpruned,
     },
+    coinstatsindex: {
+      type: "boolean",
+      name: "Coinstats Index",
+      description: "Enabling Coinstats Index reduces the time for the gettxoutsetinfo RPC to complete at the cost of using additional disk space",
+      default: false,
+    },
     wallet: {
       type: "object",
       name: "Wallet",
@@ -410,7 +416,7 @@ export const getConfig: T.ExpectedExports.getConfig = async (effects) => {
               type: "boolean",
               name: "Use V2 P2P Transport Protocol",
               description: "Enable or disable the use of BIP324 V2 P2P transport protocol.",
-              default: false,
+              default: true,
             },
             addnode: {
               name: "Add Nodes",

scripts/services/setConfig.ts

@@ -1,36 +1,39 @@
-import {
-  matches,
-  compat,
-  types,
-  YAML
-} from "../dependencies.ts";
-const { number, shape, boolean } = matches;
+import { matches, types, YAML } from "../dependencies.ts";
+const { number } = matches;
 
 export const setConfig: types.ExpectedExports.setConfig = async (
   effects: types.Effects,
   // deno-lint-ignore no-explicit-any
-  newConfig: any,
+  newConfig: any
 ) => {
-  if (!(newConfig?.rpc?.enable || !(newConfig.advanced?.pruning?.mode === "manual"))) {
+  if (newConfig.advanced.pruning.mode === "manual" && !newConfig.rpc.enable) {
     return {
-      error: "RPC must be enabled for manual.",
+      error: "RPC must be enabled for manual pruning.",
     };
   }
-  if (
-    !(!newConfig.txindex || (newConfig.advanced?.pruning?.mode === "disabled"))
-  ) {
+
+  if (newConfig.txindex && newConfig.advanced.pruning.mode !== "disabled") {
     return {
       error: "Txindex not allowed on pruned nodes.",
     };
   }
-  // true, false only fail case
+
   if (
-    !(!newConfig.advanced.blockfilters.peerblockfilters ||
-      (newConfig.advanced.blockfilters.blockfilterindex))
+    newConfig.coinstatsindex &&
+    newConfig.advanced.pruning.mode !== "disabled"
+  ) {
+    return {
+      error: "Coinstats index not allowed on pruned nodes.",
+    };
+  }
+
+  if (
+    newConfig.advanced.blockfilters.peerblockfilters &&
+    !newConfig.advanced.blockfilters.blockfilterindex
   ) {
     return {
       error:
-        "'Compute Compact Block Filters' must be enabled if 'Serve Compact Block Filters to Peers' is enabled.",
+        '"Compute Compact Block Filters" must be enabled if "Serve Compact Block Filters to Peers" is enabled.',
     };
   }
 
@@ -41,10 +44,12 @@ export const setConfig: types.ExpectedExports.setConfig = async (
 
   // config-set.sh
 
-  const oldConfig = await effects.readFile({
-    path: "start9/config.yaml",
-    volumeId: "main",
-  }).catch(() => null);
+  const oldConfig = await effects
+    .readFile({
+      path: "start9/config.yaml",
+      volumeId: "main",
+    })
+    .catch(() => null);
   if (oldConfig) {
     await effects.writeFile({
       path: "start9/config-old.yaml",
@@ -56,7 +61,7 @@ export const setConfig: types.ExpectedExports.setConfig = async (
     let oldPruningSize = 0;
     if (oldPruningTl !== "disabled") {
       oldPruningSize = number.unsafeCast(
-        oldConfigParsed?.advanced?.pruning?.size,
+        oldConfigParsed?.advanced?.pruning?.size
       );
     }
     const newPruningTl = newConfig.advanced.pruning.mode;
@@ -67,7 +72,8 @@ export const setConfig: types.ExpectedExports.setConfig = async (
     if (oldPruningTl == "disabled" || !oldPruningTl) {
       effects.debug("No reindex required");
     } else if (
-      oldPruningTl === newPruningTl && oldPruningSize >= newPruningSize
+      oldPruningTl === newPruningTl &&
+      oldPruningSize >= newPruningSize
     ) {
       effects.debug("No reindex required");
     } else {
@@ -81,7 +87,7 @@ export const setConfig: types.ExpectedExports.setConfig = async (
   } else {
     effects.debug("No reindex required");
   }
-  
+
   await effects.writeFile({
     path: "start9/config.yaml",
     toWrite: YAML.stringify(newConfig),