Merge 30860 via bashcomp_bcli_generate-28

Makefile.am

@@ -39,6 +39,7 @@ DIST_CONTRIB = \
 	       $(top_srcdir)/test/sanitizer_suppressions/lsan \
 	       $(top_srcdir)/test/sanitizer_suppressions/tsan \
 	       $(top_srcdir)/test/sanitizer_suppressions/ubsan \
+	       $(top_srcdir)/contrib/completions \
 	       $(top_srcdir)/contrib/linearize/linearize-data.py \
 	       $(top_srcdir)/contrib/linearize/linearize-hashes.py \
 	       $(top_srcdir)/contrib/signet/miner

src/rpc/node.cpp

@@ -269,6 +269,32 @@ static RPCHelpMan logging()
     };
 }
 
+static RPCHelpMan format()
+{
+    return RPCHelpMan{"format",
+        "\nFormat data we have about an RPC command in the format specified\n",
+        {
+            {"command", RPCArg::Type::STR, RPCArg::Optional::NO, "Command to query"},
+            {"output", RPCArg::Type::STR, RPCArg::Optional::NO, "Output format. Accepted values: args_cli"},
+        },
+        RPCResult{RPCResult::Type::STR, "data", "Formated data about command"},
+        RPCExamples{""},
+        [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue
+{
+    const std::string command = request.params[0].get_str();
+    JSONRPCRequest jreq(request);
+    jreq.mode = JSONRPCRequest::GET_HELP;
+
+    try {
+        tableRPC.execute(command, jreq);
+    } catch(const UniValue& e) {
+        return e["message"];
+    }
+    return NullUniValue;
+},
+    };
+}
+
 static RPCHelpMan echo(const std::string& name)
 {
     return RPCHelpMan{name,
@@ -408,6 +434,7 @@ void RegisterNodeRPCCommands(CRPCTable& t)
         {"util", &getindexinfo},
         {"hidden", &setmocktime},
         {"hidden", &mockscheduler},
+        {"hidden", &format},
         {"hidden", &echo},
         {"hidden", &echojson},
         {"hidden", &echoipc},

src/rpc/server.cpp

@@ -11,6 +11,7 @@
 #include <common/system.h>
 #include <logging.h>
 #include <node/context.h>
+#include <rpc/request.h>
 #include <rpc/server_util.h>
 #include <rpc/util.h>
 #include <sync.h>
@@ -498,7 +499,7 @@ static bool ExecuteCommands(const std::vector<const CRPCCommand*>& commands, con
     return false;
 }
 
-UniValue CRPCTable::execute(const JSONRPCRequest &request) const
+UniValue CRPCTable::execute(const std::string method, const JSONRPCRequest &request) const
 {
     // Return immediately if in warmup
     {
@@ -508,7 +509,7 @@ UniValue CRPCTable::execute(const JSONRPCRequest &request) const
     }
 
     // Find method
-    auto it = mapCommands.find(request.strMethod);
+    auto it = mapCommands.find(method);
     if (it != mapCommands.end()) {
         UniValue result;
         if (ExecuteCommands(it->second, request, result)) {
@@ -518,6 +519,11 @@ UniValue CRPCTable::execute(const JSONRPCRequest &request) const
     throw JSONRPCError(RPC_METHOD_NOT_FOUND, "Method not found");
 }
 
+UniValue CRPCTable::execute(const JSONRPCRequest &request) const
+{
+    return this->execute(request.strMethod, request);
+}
+
 static bool ExecuteCommand(const CRPCCommand& command, const JSONRPCRequest& request, UniValue& result, bool last_handler)
 {
     try {

src/rpc/server.h

@@ -138,13 +138,22 @@ public:
     std::string help(const std::string& name, const JSONRPCRequest& helpreq) const;
 
     /**
-     * Execute a method.
+     * Execute the request.strMethod.
      * @param request The JSONRPCRequest to execute
      * @returns Result of the call.
      * @throws an exception (UniValue) when an error happens.
      */
     UniValue execute(const JSONRPCRequest &request) const;
 
+    /**
+     * Execute a method.
+     * @param method The method to execute
+     * @param request The JSONRPCRequest to execute
+     * @returns Result of the call.
+     * @throws an exception (UniValue) when an error happens.
+     */
+    UniValue execute(const std::string method, const JSONRPCRequest &request) const;
+
     /**
     * Returns a list of registered commands
     * @returns List of registered commands.

src/rpc/util.cpp

@@ -663,7 +663,11 @@ UniValue RPCHelpMan::HandleRequest(const JSONRPCRequest& request) const
      * the user is asking for help information, and throw help when appropriate.
      */
     if (request.mode == JSONRPCRequest::GET_HELP || !IsValidNumArgs(request.params.size())) {
-        throw std::runtime_error(ToString());
+        std::string help_format = "default";
+        if (request.strMethod == "format") {
+            help_format = request.params[1].get_str();
+        }
+        throw std::runtime_error(ToString(help_format));
     }
     UniValue arg_mismatch{UniValue::VOBJ};
     for (size_t i{0}; i < m_args.size(); ++i) {
@@ -850,6 +854,34 @@ std::string RPCHelpMan::ToString() const
     return ret;
 }
 
+std::string RPCHelpMan::ToStringArgsCli() const
+{
+    std::string res;
+    for (const auto& arg : m_args) {
+        const bool is_file = ToLower(arg.m_description).find("file") != std::string::npos;
+        res += arg.m_names + ":" + (is_file ? "file" : arg.ToTypeString()) + ",";
+    }
+
+    if (res.size() > 0) {
+        res.pop_back();
+    }
+
+    return res;
+}
+
+std::string RPCHelpMan::ToString(const std::string& format) const
+{
+    if (format == "default") {
+        return this->ToString();
+    }
+
+    if (format == "args_cli") {
+        return this->ToStringArgsCli();
+    }
+
+    throw std::runtime_error("unrecogonized help format");
+}
+
 UniValue RPCHelpMan::GetArgMap() const
 {
     UniValue arr{UniValue::VARR};
@@ -957,6 +989,32 @@ bool RPCArg::IsOptional() const
     }
 }
 
+std::string RPCArg::ToTypeString() const
+{
+    switch (m_type) {
+    case Type::STR_HEX:
+    case Type::STR:
+        return "string";
+    case Type::NUM:
+        return "numeric";
+    case Type::AMOUNT:
+        return "numeric or string";
+    case Type::RANGE:
+        return "numeric or array";
+    case Type::BOOL:
+        return "boolean";
+    case Type::OBJ:
+    case Type::OBJ_NAMED_PARAMS:
+    case Type::OBJ_USER_KEYS:
+        return "json object";
+    case Type::ARR:
+        return"json array";
+    } // no default case, so the compiler can warn about missing cases
+
+    //gcc and msvc might complain we don't return anything even if we handle all cases
+    throw std::runtime_error("unknown argument type");
+}
+
 std::string RPCArg::ToDescriptionString(bool is_named_arg) const
 {
     std::string ret;
@@ -964,39 +1022,7 @@ std::string RPCArg::ToDescriptionString(bool is_named_arg) const
     if (m_opts.type_str.size() != 0) {
         ret += m_opts.type_str.at(1);
     } else {
-        switch (m_type) {
+        ret += this->ToTypeString();
-        case Type::STR_HEX:
-        case Type::STR: {
-            ret += "string";
-            break;
-        }
-        case Type::NUM: {
-            ret += "numeric";
-            break;
-        }
-        case Type::AMOUNT: {
-            ret += "numeric or string";
-            break;
-        }
-        case Type::RANGE: {
-            ret += "numeric or array";
-            break;
-        }
-        case Type::BOOL: {
-            ret += "boolean";
-            break;
-        }
-        case Type::OBJ:
-        case Type::OBJ_NAMED_PARAMS:
-        case Type::OBJ_USER_KEYS: {
-            ret += "json object";
-            break;
-        }
-        case Type::ARR: {
-            ret += "json array";
-            break;
-        }
-        } // no default case, so the compiler can warn about missing cases
     }
     if (m_fallback.index() == 1) {
         ret += ", optional, default=" + std::get<RPCArg::DefaultHint>(m_fallback);

src/rpc/util.h

@@ -302,6 +302,10 @@ struct RPCArg {
      * Set oneline to get the oneline representation (less whitespace)
      */
     std::string ToStringObj(bool oneline) const;
+    /**
+     * Return the type as a string
+     */
+    std::string ToTypeString() const;
     /**
      * Return the description string, including the argument type and whether
      * the argument is required.
@@ -503,6 +507,8 @@ public:
         }
     }
     std::string ToString() const;
+    std::string ToStringArgsCli() const;
+    std::string ToString(const std::string& format) const;
     /** Return the named args that need to be converted from string to another JSON type */
     UniValue GetArgMap() const;
     /** If the supplied number of args is neither too small nor too high */

src/test/fuzz/rpc.cpp

@@ -112,6 +112,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{
     "estimaterawfee",
     "estimatesmartfee",
     "finalizepsbt",
+    "format",
     "generate",
     "generateblock",
     "getaddednodeinfo",

test/functional/data/completion/bitcoin-cli.footer.bash-completion

@@ -0,0 +1,47 @@
+
+    case "$cur" in
+        -conf=*)
+            cur="${cur#*=}"
+            _filedir
+            return 0
+            ;;
+        -datadir=*)
+            cur="${cur#*=}"
+            _filedir -d
+            return 0
+            ;;
+        -*=*)	# prevent nonsense completions
+            return 0
+            ;;
+        *)
+            local helpopts commands
+
+            # only parse -help if senseful
+            if [[ -z "$cur" || "$cur" =~ ^- ]]; then
+                helpopts=$($bitcoin_cli -help 2>&1 | awk '$1 ~ /^-/ { sub(/=.*/, "="); print $1 }' )
+            fi
+
+            # only parse help if senseful
+            if [[ -z "$cur" || "$cur" =~ ^[a-z] ]]; then
+                commands=$(_bitcoin_rpc help 2>/dev/null | awk '$1 ~ /^[a-z]/ { print $1; }')
+            fi
+
+            COMPREPLY=( $( compgen -W "$helpopts $commands" -- "$cur" ) )
+
+            # Prevent space if an argument is desired
+            if [[ $COMPREPLY == *= ]]; then
+                compopt -o nospace
+            fi
+            return 0
+            ;;
+    esac
+} &&
+complete -F _bitcoin_cli bitcoin-cli
+
+# Local variables:
+# mode: shell-script
+# sh-basic-offset: 4
+# sh-indent-comment: t
+# indent-tabs-mode: nil
+# End:
+# ex: ts=4 sw=4 et filetype=sh

test/functional/data/completion/bitcoin-cli.header.bash-completion

@@ -0,0 +1,29 @@
+# Copyright (c) 2012-2024 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+# call $bitcoin-cli for RPC
+_bitcoin_rpc() {
+    # determine already specified args necessary for RPC
+    local rpcargs=()
+    for i in ${COMP_LINE}; do
+        case "$i" in
+            -conf=*|-datadir=*|-regtest|-rpc*|-testnet|-testnet4)
+                rpcargs=( "${rpcargs[@]}" "$i" )
+                ;;
+        esac
+    done
+    $bitcoin_cli "${rpcargs[@]}" "$@"
+}
+
+_bitcoin_cli() {
+    local cur prev words=() cword
+    local bitcoin_cli
+
+    # save and use original argument to invoke bitcoin-cli for -help, help and RPC
+    # as bitcoin-cli might not be in $PATH
+    bitcoin_cli="$1"
+
+    COMPREPLY=()
+    _get_comp_words_by_ref -n = cur prev words cword
+

test/functional/test_runner.py

@@ -186,6 +186,7 @@ BASE_SCRIPTS = [
     'feature_bind_extra.py',
     'mempool_resurrect.py',
     'wallet_txn_doublespend.py --mineblock',
+    'tool_cli_bash_completion.py',
     'tool_wallet.py --legacy-wallet',
     'tool_wallet.py --legacy-wallet --bdbro',
     'tool_wallet.py --legacy-wallet --bdbro --swap-bdb-endian',

test/functional/tool_cli_bash_completion.py

@@ -0,0 +1,282 @@
+#!/usr/bin/env python3
+
+from os import path
+from collections import defaultdict
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal
+
+
+# bash cli completion file header
+COMPLETION_HEADER = """# Dynamic bash programmable completion for bitcoin-cli(1)
+#     DO NOT EDIT THIS FILE BY HAND -- THIS WILL FAIL THE FUNCTIONAL TEST tool_cli_completion
+# This file is auto-generated by the functional test tool_cli_completion.
+# If you want to modify this file, modify test/functional/tool_cli_completion.py and re-autogenerate
+# this file via the --overwrite test flag.
+
+"""
+
+# option types which are limited to certain values
+TYPED_OPTIONS = [
+        ["estimate_mode", {"UNSET", "ECONOMICAL", "CONSERVATIVE"}],
+        ["sighashtype", {"ALL", "NONE", "SINGLE", "ALL|ANYONECANPAY",
+                         "NONE|ANYONECANPAY", "SINGLE|ANYONECANPAY"}]
+]
+
+
+class PossibleArgs():
+    """ Helper class to store options associated to a command. """
+    def __init__(self, command):
+        self.command = command
+        self.arguments = {}
+
+    def set_args(self, position, values):
+        """ Set the position-th positional argument as having values as possible values. """
+        if position in self.arguments:
+            raise AssertionError(f"The positional parameter at position {position} is already defined for command '{self.command}'")
+
+        self.arguments[position] = values
+        return self
+
+    def set_bool_args(self, position):
+        return self.set_args(position, {"true", "false"})
+
+    def set_file_args(self, position):
+        # We consider an empty string as a file value for the sake of simplicity (don't
+        # have to create an extra level of indirection).
+        return self.set_args(position, {""})
+
+    def set_unknown_args(self, position):
+        return self.set_args(position, {})
+
+    def set_typed_option(self, position, arg_name):
+        """ Checks if arg_name is a typed option; if it is, sets it and return True. """
+        for option_type in TYPED_OPTIONS:
+            if arg_name == option_type[0]:
+                self.set_args(position, option_type[1])
+                return True
+        return False
+
+    def has_option(self, position):
+        return position in self.arguments and len(self.arguments[position]) > 0
+
+    def get_num_args(self):
+        """ Return the max number of positional argument the option accepts. """
+        pos = list(self.arguments.keys())
+        if len(pos) == 0:
+            return 0
+
+        return max(pos)
+
+    def generate_autocomplete(self, pos):
+        """ Generate the autocomplete file line relevent to the given position pos. """
+        if len(self.arguments[pos]) == 0:
+            raise AssertionError(f"generating undefined arg id {pos} ({self.arguments})")
+
+        # handle special file case
+        if len(self.arguments[pos]) == 1 and len(next(iter(self.arguments[pos]))) == 0:
+            return "_filedir"
+
+        # a set order is undefined, so we order args alphabetically
+        args = list(self.arguments[pos])
+        args.sort()
+
+        return "COMPREPLY=( $( compgen -W \"" + ' '.join(args) + "\" -- \"$cur\" ) )"
+
+# commands where the option type can only be difficultly derived from the help message
+SPECIAL_OPTIONS = [
+        PossibleArgs("addnode").set_args(2, {"add", "remove", "onetry"}),
+        PossibleArgs("setban").set_args(2, {"add", "remove"}),
+]
+
+
+def generate_start_complete(cword):
+    """ Generate the start of an autocomplete block (beware of indentation). """
+    if cword > 1:
+        return f"""    if ((cword > {cword})); then
+        case ${{words[cword-{cword}]}} in"""
+
+    return "    case \"$prev\" in"
+
+
+def generate_end_complete(cword):
+    """ Generate the end of an autocomplete block. """
+    if cword > 1:
+        return f"\n{' ' * 8}esac\n{' ' * 4}fi\n\n"
+
+    return f"\n{' ' * 4}esac\n"
+
+
+class CliCompletionTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 1
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_cli()
+        # self.skip_if_no_wallet()
+        self.skip_if_no_bitcoind_zmq()
+
+    def add_options(self, parser):
+        parser.add_argument(
+                '--header',
+                help='Static header part of the bash completion file',
+        )
+
+        parser.add_argument(
+                '--footer',
+                help='Static footer part of the bash completion file',
+        )
+
+        parser.add_argument(
+                '--completion',
+                help='Location of the current bash completion file',
+        )
+
+        parser.add_argument(
+                '--overwrite',
+                default=False,
+                action='store_true',
+                help='Force the test to overwrite the file pointer to by the --completion'
+                     'to the newly generated completion file',
+        )
+    def parse_single_helper(self, option):
+        """ Complete the arguments of option via the RPC format command. """
+
+        res = self.nodes[0].format(command=option.command, output='args_cli')
+        if len(res) == 0:
+            return option
+
+        if res.count('\n') > 1:
+            raise AssertionError(
+                f"command {option.command} doesn't support format RPC. Should it be a hidden command? "
+                f"Please call RPCHelpMan::Check when adding a new non-hidden command. Returned: {res}"
+            )
+
+        for idx, argument in enumerate(res.split(",")):
+            elems = argument.split(":")
+
+            if option.set_typed_option(idx+1, elems[0]):
+                continue
+
+            if elems[1] == "boolean":
+                option.set_bool_args(idx+1)
+                continue
+
+            if elems[1] == "file":
+                option.set_file_args(idx+1)
+                continue
+
+            if not option.has_option(idx+1):
+                option.set_unknown_args(idx+1)
+
+        return option
+
+    def get_command_options(self, command):
+        """ Returns the corresponding PossibleArgs for the command. """
+
+        # verify it's not a special option first
+        for soption in SPECIAL_OPTIONS:
+            if command == soption.command:
+                return self.parse_single_helper(soption)
+
+        return self.parse_single_helper(PossibleArgs(command))
+
+    def generate_completion_block(self, options):
+        commands = [o.command for o in options]
+        self.log.info(f"Generating part of the completion file for options {commands}")
+
+        if len(options) == 0:
+            return ""
+
+        generated = ""
+        max_pos_options = max(options, key=lambda o: o.get_num_args()).get_num_args()
+        for cword in range(max_pos_options, 0, -1):
+            this_options = [option for option in options if option.has_option(cword)]
+            if len(this_options) == 0:
+                continue
+
+            # group options by their arguments value
+            grouped_options = defaultdict(list)
+            for option in this_options:
+                arg = option.generate_autocomplete(cword)
+                grouped_options[arg].append(option)
+
+            # generate the cword block
+            indent = 12 if cword > 1 else 8
+            generated += generate_start_complete(cword)
+            for line, opt_gr in grouped_options.items():
+                opt_gr.sort(key=lambda o: o.command)  # show options alphabetically for clarity
+                args = '|'.join([o.command for o in opt_gr])
+                generated += f"\n{' '*indent}{args})\n"
+                generated += f"{' ' * (indent + 4)}{line}\n{' ' * (indent + 4)}return 0\n{' ' * (indent + 4)};;"
+            generated += generate_end_complete(cword)
+
+        return generated
+
+    def generate_completion_file(self, commands):
+        try:
+            with open(self.options.header, 'r', encoding='utf-8') as header_file:
+                header = header_file.read()
+
+            with open(self.options.footer, 'r', encoding='utf-8') as footer_file:
+                footer = footer_file.read()
+        except Exception as e:
+            raise AssertionError(
+                        f"Could not read header/footer ({self.options.header} and {self.options.footer}) files. "
+                        f"Tell the test where to find them using the --header/--footer parameters ({e})."
+            )
+        return COMPLETION_HEADER + header + commands + footer
+
+    def write_completion_file(self, new_file):
+        try:
+            with open(self.options.completion, 'w', encoding='utf-8') as completion_file:
+                completion_file.write(new_file)
+        except Exception as e:
+            raise AssertionError(
+                        f"Could not write the autocomplete file to {self.options.completion}. "
+                        f"Tell the test where to find it using the --completion parameters ({e})."
+            )
+
+    def read_completion_file(self):
+        try:
+            with open(self.options.completion, 'r', encoding='utf-8') as completion_file:
+                return completion_file.read()
+        except Exception as e:
+            raise AssertionError(
+                        f"Could not read the autocomplete file ({self.options.completion}) file. "
+                        f"Tell the test where to find it using the --completion parameters ({e})."
+            )
+
+
+    def run_test(self):
+        # self.config is not available in self.add_options, so complete filepaths here
+        src_dir = self.config["environment"]["SRCDIR"]
+        test_data_dir = path.join(src_dir, 'test', 'functional', 'data', 'completion')
+        if self.options.header is None or len(self.options.header) == 0:
+            self.options.header = path.join(test_data_dir, 'bitcoin-cli.header.bash-completion')
+
+        if self.options.footer is None or len(self.options.footer) == 0:
+            self.options.footer = path.join(test_data_dir, 'bitcoin-cli.footer.bash-completion')
+
+        if self.options.completion is None or len(self.options.completion) == 0:
+            self.options.completion = path.join(src_dir, 'contrib', 'completions', 'bash', 'bitcoin-cli.bash')
+
+        self.log.info('Parsing help commands to get all the command arguments...')
+        commands = self.nodes[0].help().split("\n")
+        commands = [c.split(' ')[0] for c in commands if not c.startswith("== ") and len(c) > 0]
+        commands = [self.get_command_options(c) for c in commands]
+
+        self.log.info('Generating new autocompletion file...')
+        commands = self.generate_completion_block(commands)
+        new_completion = self.generate_completion_file(commands)
+
+        if self.options.overwrite:
+            self.log.info("Overwriting the completion file...")
+            self.write_completion_file(new_completion)
+
+        self.log.info('Checking if the generated and the original completion files matches...')
+        completion = self.read_completion_file()
+        assert_equal(new_completion, completion)
+
+if __name__ == '__main__':
+    CliCompletionTest(__file__).main()