RPC: Introduce getblockstats to plot things #10757
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -13,6 +13,7 @@ | |
| #include <consensus/validation.h> | ||
| #include <validation.h> | ||
| #include <core_io.h> | ||
| #include <index/txindex.h> | ||
| #include <policy/feerate.h> | ||
| #include <policy/policy.h> | ||
| #include <primitives/transaction.h> | ||
|
|
@@ -31,6 +32,7 @@ | |
|
|
||
| #include <univalue.h> | ||
|
|
||
| #include <boost/algorithm/string.hpp> | ||
| #include <boost/thread/thread.hpp> // boost::thread::interrupt | ||
|
|
||
| #include <memory> | ||
|
|
@@ -737,6 +739,25 @@ static UniValue getblockheader(const JSONRPCRequest& request) | |
| return blockheaderToJSON(pblockindex); | ||
| } | ||
|
|
||
| static CBlock GetBlockChecked(const CBlockIndex* pblockindex) | ||
| { | ||
| CBlock block; | ||
| if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) { | ||
| throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); | ||
| } | ||
|
|
||
| if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) { | ||
| // Block not found on disk. This could be because we have the block | ||
| // header in our index but don't have the block (for example if a | ||
| // non-whitelisted node sends us an unrequested long chain of valid | ||
| // blocks, we add the headers to our index, but don't accept the | ||
| // block). | ||
| throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); | ||
| } | ||
|
|
||
| return block; | ||
| } | ||
|
|
||
| static UniValue getblock(const JSONRPCRequest& request) | ||
| { | ||
| if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) | ||
|
|
@@ -805,17 +826,7 @@ static UniValue getblock(const JSONRPCRequest& request) | |
| throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); | ||
| } | ||
|
|
||
| const CBlock block = GetBlockChecked(pblockindex); | ||
|
|
||
| if (verbosity <= 0) | ||
| { | ||
|
|
@@ -1614,6 +1625,284 @@ static UniValue getchaintxstats(const JSONRPCRequest& request) | |
| return ret; | ||
| } | ||
|
|
||
| template<typename T> | ||
| static T CalculateTruncatedMedian(std::vector<T>& scores) | ||
| { | ||
| size_t size = scores.size(); | ||
| if (size == 0) { | ||
| return 0; | ||
| } | ||
|
|
||
| std::sort(scores.begin(), scores.end()); | ||
| if (size % 2 == 0) { | ||
| return (scores[size / 2 - 1] + scores[size / 2]) / 2; | ||
| } else { | ||
| return scores[size / 2]; | ||
| } | ||
| } | ||
|
|
||
| template<typename T> | ||
| static inline bool SetHasKeys(const std::set<T>& set) {return false;} | ||
| template<typename T, typename Tk, typename... Args> | ||
| static inline bool SetHasKeys(const std::set<T>& set, const Tk& key, const Args&... args) | ||
| { | ||
| return (set.count(key) != 0) || SetHasKeys(set, args...); | ||
| } | ||
|
|
||
| // outpoint (needed for the utxo index) + nHeight + fCoinBase | ||
| static constexpr size_t PER_UTXO_OVERHEAD = sizeof(COutPoint) + sizeof(uint32_t) + sizeof(bool); | ||
|
|
||
| static UniValue getblockstats(const JSONRPCRequest& request) | ||
| { | ||
| if (request.fHelp || request.params.size() < 1 || request.params.size() > 4) { | ||
| throw std::runtime_error( | ||
| "getblockstats hash_or_height ( stats )\n" | ||
| "\nCompute per block statistics for a given window. All amounts are in satoshis.\n" | ||
| "It won't work for some heights with pruning.\n" | ||
| "It won't work without -txindex for utxo_size_inc, *fee or *feerate stats.\n" | ||
| "\nArguments:\n" | ||
| "1. \"hash_or_height\" (string or numeric, required) The block hash or height of the target block\n" | ||
| "2. \"stats\" (array, optional) Values to plot, by default all values (see result below)\n" | ||
| " [\n" | ||
| " \"height\", (string, optional) Selected statistic\n" | ||
| " \"time\", (string, optional) Selected statistic\n" | ||
| " ,...\n" | ||
| " ]\n" | ||
| "\nResult:\n" | ||
| "{ (json object)\n" | ||
| " \"avgfee\": xxxxx, (numeric) Average fee in the block\n" | ||
| " \"avgfeerate\": xxxxx, (numeric) Average feerate (in satoshis per virtual byte)\n" | ||
| " \"avgtxsize\": xxxxx, (numeric) Average transaction size\n" | ||
| " \"blockhash\": xxxxx, (string) The block hash (to check for potential reorgs)\n" | ||
| " \"height\": xxxxx, (numeric) The height of the block\n" | ||
| " \"ins\": xxxxx, (numeric) The number of inputs (excluding coinbase)\n" | ||
| " \"maxfee\": xxxxx, (numeric) Maximum fee in the block\n" | ||
| " \"maxfeerate\": xxxxx, (numeric) Maximum feerate (in satoshis per virtual byte)\n" | ||
| " \"maxtxsize\": xxxxx, (numeric) Maximum transaction size\n" | ||
| " \"medianfee\": xxxxx, (numeric) Truncated median fee in the block\n" | ||
| " \"medianfeerate\": xxxxx, (numeric) Truncated median feerate (in satoshis per virtual byte)\n" | ||
| " \"mediantime\": xxxxx, (numeric) The block median time past\n" | ||
| " \"mediantxsize\": xxxxx, (numeric) Truncated median transaction size\n" | ||
| " \"minfee\": xxxxx, (numeric) Minimum fee in the block\n" | ||
| " \"minfeerate\": xxxxx, (numeric) Minimum feerate (in satoshis per virtual byte)\n" | ||
| " \"mintxsize\": xxxxx, (numeric) Minimum transaction size\n" | ||
| " \"outs\": xxxxx, (numeric) The number of outputs\n" | ||
| " \"subsidy\": xxxxx, (numeric) The block subsidy\n" | ||
| " \"swtotal_size\": xxxxx, (numeric) Total size of all segwit transactions\n" | ||
| " \"swtotal_weight\": xxxxx, (numeric) Total weight of all segwit transactions divided by segwit scale factor (4)\n" | ||
| " \"swtxs\": xxxxx, (numeric) The number of segwit transactions\n" | ||
| " \"time\": xxxxx, (numeric) The block time\n" | ||
| " \"total_out\": xxxxx, (numeric) Total amount in all outputs (excluding coinbase and thus reward [ie subsidy + totalfee])\n" | ||
| " \"total_size\": xxxxx, (numeric) Total size of all non-coinbase transactions\n" | ||
| " \"total_weight\": xxxxx, (numeric) Total weight of all non-coinbase transactions divided by segwit scale factor (4)\n" | ||
| " \"totalfee\": xxxxx, (numeric) The fee total\n" | ||
| " \"txs\": xxxxx, (numeric) The number of transactions (excluding coinbase)\n" | ||
| " \"utxo_increase\": xxxxx, (numeric) The increase/decrease in the number of unspent outputs\n" | ||
| " \"utxo_size_inc\": xxxxx, (numeric) The increase/decrease in size for the utxo index (not discounting op_return and similar)\n" | ||
| "}\n" | ||
| "\nExamples:\n" | ||
| + HelpExampleCli("getblockstats", "1000 '[\"minfeerate\",\"avgfeerate\"]'") | ||
| + HelpExampleRpc("getblockstats", "1000 '[\"minfeerate\",\"avgfeerate\"]'") | ||
| ); | ||
| } | ||
|
|
||
| LOCK(cs_main); | ||
|
|
||
| CBlockIndex* pindex; | ||
| if (request.params[0].isNum()) { | ||
| const int height = request.params[0].get_int(); | ||
| const int current_tip = chainActive.Height(); | ||
| if (height < 0) { | ||
| throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d is negative", height)); | ||
| } | ||
| if (height > current_tip) { | ||
| throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d after current tip %d", height, current_tip)); | ||
| } | ||
|
|
||
| pindex = chainActive[height]; | ||
| } else { | ||
| const std::string strHash = request.params[0].get_str(); | ||
| const uint256 hash(uint256S(strHash)); | ||
| pindex = LookupBlockIndex(hash); | ||
| if (!pindex) { | ||
| throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); | ||
|
Contributor
There was a problem hiding this comment. Just a note here. In bitcoin/src/rpc/blockchain.cpp Lines 809 to 818 in 4cfe17c RPC_MISC_ERROR is used, while RPC_INVALID_ADDRESS_OR_KEY is used here. Not sure which is better, but I think they should both use the same error type. (This PR touches the other one so could easily tweak, I think.)
Contributor
There was a problem hiding this comment. I disagree. If the header is not found, that means the request was invalid, hence RPC_INVALID ADDRESS_OR KEY (key loosely taken to mean request argument, I take it). Whereas if the header is known but the block is not known, I do consider that to be a different error. In #13144, I introduce RPC_DATA_UNAVAILABLE, which may be more appropriate than RPC_MISC in the latter case. |
||
| } | ||
| if (!chainActive.Contains(pindex)) { | ||
| throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Block is not in chain %s", Params().NetworkIDString())); | ||
| } | ||
| } | ||
|
|
||
| assert(pindex != nullptr); | ||
|
|
||
| std::set<std::string> stats; | ||
| if (!request.params[1].isNull()) { | ||
| const UniValue stats_univalue = request.params[1].get_array(); | ||
| for (unsigned int i = 0; i < stats_univalue.size(); i++) { | ||
| const std::string stat = stats_univalue[i].get_str(); | ||
| stats.insert(stat); | ||
| } | ||
| } | ||
|
|
||
| const CBlock block = GetBlockChecked(pindex); | ||
|
|
||
| const bool do_all = stats.size() == 0; // Calculate everything if nothing selected (default) | ||
| const bool do_mediantxsize = do_all || stats.count("mediantxsize") != 0; | ||
| const bool do_medianfee = do_all || stats.count("medianfee") != 0; | ||
| const bool do_medianfeerate = do_all || stats.count("medianfeerate") != 0; | ||
| const bool loop_inputs = do_all || do_medianfee || do_medianfeerate || | ||
| SetHasKeys(stats, "utxo_size_inc", "totalfee", "avgfee", "avgfeerate", "minfee", "maxfee", "minfeerate", "maxfeerate"); | ||
| const bool loop_outputs = do_all || loop_inputs || stats.count("total_out"); | ||
| const bool do_calculate_size = do_mediantxsize || | ||
| SetHasKeys(stats, "total_size", "avgtxsize", "mintxsize", "maxtxsize", "swtotal_size"); | ||
| const bool do_calculate_weight = do_all || SetHasKeys(stats, "total_weight", "avgfeerate", "swtotal_weight", "avgfeerate", "medianfeerate", "minfeerate", "maxfeerate"); | ||
| const bool do_calculate_sw = do_all || SetHasKeys(stats, "swtxs", "swtotal_size", "swtotal_weight"); | ||
|
|
||
| CAmount maxfee = 0; | ||
| CAmount maxfeerate = 0; | ||
| CAmount minfee = MAX_MONEY; | ||
| CAmount minfeerate = MAX_MONEY; | ||
| CAmount total_out = 0; | ||
| CAmount totalfee = 0; | ||
| int64_t inputs = 0; | ||
| int64_t maxtxsize = 0; | ||
| int64_t mintxsize = MAX_BLOCK_SERIALIZED_SIZE; | ||
| int64_t outputs = 0; | ||
| int64_t swtotal_size = 0; | ||
| int64_t swtotal_weight = 0; | ||
| int64_t swtxs = 0; | ||
| int64_t total_size = 0; | ||
| int64_t total_weight = 0; | ||
| int64_t utxo_size_inc = 0; | ||
| std::vector<CAmount> fee_array; | ||
| std::vector<CAmount> feerate_array; | ||
| std::vector<int64_t> txsize_array; | ||
|
|
||
| for (const auto& tx : block.vtx) { | ||
| outputs += tx->vout.size(); | ||
|
|
||
| CAmount tx_total_out = 0; | ||
| if (loop_outputs) { | ||
| for (const CTxOut& out : tx->vout) { | ||
| tx_total_out += out.nValue; | ||
| utxo_size_inc += GetSerializeSize(out, SER_NETWORK, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD; | ||
| } | ||
| } | ||
|
|
||
| if (tx->IsCoinBase()) { | ||
| continue; | ||
| } | ||
|
|
||
| inputs += tx->vin.size(); // Don't count coinbase's fake input | ||
| total_out += tx_total_out; // Don't count coinbase reward | ||
|
|
||
| int64_t tx_size = 0; | ||
| if (do_calculate_size) { | ||
|
|
||
| tx_size = tx->GetTotalSize(); | ||
| if (do_mediantxsize) { | ||
| txsize_array.push_back(tx_size); | ||
| } | ||
| maxtxsize = std::max(maxtxsize, tx_size); | ||
| mintxsize = std::min(mintxsize, tx_size); | ||
| total_size += tx_size; | ||
| } | ||
|
|
||
| int64_t weight = 0; | ||
| if (do_calculate_weight) { | ||
| weight = GetTransactionWeight(*tx); | ||
| total_weight += weight; | ||
| } | ||
|
|
||
| if (do_calculate_sw && tx->HasWitness()) { | ||
| ++swtxs; | ||
| swtotal_size += tx_size; | ||
| swtotal_weight += weight; | ||
| } | ||
|
|
||
| if (loop_inputs) { | ||
|
|
||
| if (!g_txindex) { | ||
| throw JSONRPCError(RPC_INVALID_PARAMETER, "One or more of the selected stats requires -txindex enabled"); | ||
| } | ||
| CAmount tx_total_in = 0; | ||
| for (const CTxIn& in : tx->vin) { | ||
| CTransactionRef tx_in; | ||
| uint256 hashBlock; | ||
| if (!GetTransaction(in.prevout.hash, tx_in, Params().GetConsensus(), hashBlock, false)) { | ||
| throw JSONRPCError(RPC_INTERNAL_ERROR, std::string("Unexpected internal error (tx index seems corrupt)")); | ||
| } | ||
|
|
||
| CTxOut prevoutput = tx_in->vout[in.prevout.n]; | ||
|
|
||
| tx_total_in += prevoutput.nValue; | ||
| utxo_size_inc -= GetSerializeSize(prevoutput, SER_NETWORK, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD; | ||
| } | ||
|
|
||
| CAmount txfee = tx_total_in - tx_total_out; | ||
| assert(MoneyRange(txfee)); | ||
| if (do_medianfee) { | ||
| fee_array.push_back(txfee); | ||
| } | ||
| maxfee = std::max(maxfee, txfee); | ||
| minfee = std::min(minfee, txfee); | ||
| totalfee += txfee; | ||
|
|
||
| // New feerate uses satoshis per virtual byte instead of per serialized byte | ||
| CAmount feerate = weight ? (txfee * WITNESS_SCALE_FACTOR) / weight : 0; | ||
|
Contributor
There was a problem hiding this comment. May be ok, but I think
Contributor
There was a problem hiding this comment. That equation has truncated errors. Consider a tx with weight 3 and fee 100. The vsize is 1, so the fee rate should be 1, but your equation would give (100 * 4) / 6 = 66. You point out correctly though that vsize is computed incorrectly here by rounding down rather than up. This should use
Contributor
Author
There was a problem hiding this comment. Sorry, I don't follow. This is using GetTransactionWeight() which does GetVirtualTransactionSize does But I'm not sure why would we prefer that here.
Contributor
There was a problem hiding this comment. Sorry, I think I got things mixed up.
Contributor
There was a problem hiding this comment. The fee rate for a tx with weight 3 and fee 100 is the same as if there were 100 such tx's. That would have weight 300 or vsize 75, and fee 10,000, for a fee rate of 10,000/75 or 133.3, not 1, 100, or 66. :) Formula looks fine to me, though I guess estimatefee uses fee/vkbyte rather than fee/byte, which would be
Contributor
Author
There was a problem hiding this comment. Well, we can move from sat/vbyte to sat/vkbyte, sure. I t should be a simple change if people prefer that. |
||
| if (do_medianfeerate) { | ||
| feerate_array.push_back(feerate); | ||
| } | ||
| maxfeerate = std::max(maxfeerate, feerate); | ||
| minfeerate = std::min(minfeerate, feerate); | ||
| } | ||
| } | ||
|
|
||
| UniValue ret_all(UniValue::VOBJ); | ||
| ret_all.pushKV("avgfee", (block.vtx.size() > 1) ? totalfee / (block.vtx.size() - 1) : 0); | ||
| ret_all.pushKV("avgfeerate", total_weight ? (totalfee * WITNESS_SCALE_FACTOR) / total_weight : 0); // Unit: sat/vbyte | ||
| ret_all.pushKV("avgtxsize", (block.vtx.size() > 1) ? total_size / (block.vtx.size() - 1) : 0); | ||
| ret_all.pushKV("blockhash", pindex->GetBlockHash().GetHex()); | ||
| ret_all.pushKV("height", (int64_t)pindex->nHeight); | ||
| ret_all.pushKV("ins", inputs); | ||
| ret_all.pushKV("maxfee", maxfee); | ||
| ret_all.pushKV("maxfeerate", maxfeerate); | ||
| ret_all.pushKV("maxtxsize", maxtxsize); | ||
| ret_all.pushKV("medianfee", CalculateTruncatedMedian(fee_array)); | ||
| ret_all.pushKV("medianfeerate", CalculateTruncatedMedian(feerate_array)); | ||
| ret_all.pushKV("mediantime", pindex->GetMedianTimePast()); | ||
| ret_all.pushKV("mediantxsize", CalculateTruncatedMedian(txsize_array)); | ||
| ret_all.pushKV("minfee", (minfee == MAX_MONEY) ? 0 : minfee); | ||
| ret_all.pushKV("minfeerate", (minfeerate == MAX_MONEY) ? 0 : minfeerate); | ||
| ret_all.pushKV("mintxsize", mintxsize == MAX_BLOCK_SERIALIZED_SIZE ? 0 : mintxsize); | ||
| ret_all.pushKV("outs", outputs); | ||
| ret_all.pushKV("subsidy", GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())); | ||
| ret_all.pushKV("swtotal_size", swtotal_size); | ||
| ret_all.pushKV("swtotal_weight", swtotal_weight); | ||
| ret_all.pushKV("swtxs", swtxs); | ||
| ret_all.pushKV("time", pindex->GetBlockTime()); | ||
| ret_all.pushKV("total_out", total_out); | ||
| ret_all.pushKV("total_size", total_size); | ||
| ret_all.pushKV("total_weight", total_weight); | ||
| ret_all.pushKV("totalfee", totalfee); | ||
| ret_all.pushKV("txs", (int64_t)block.vtx.size()); | ||
| ret_all.pushKV("utxo_increase", outputs - inputs); | ||
| ret_all.pushKV("utxo_size_inc", utxo_size_inc); | ||
|
|
||
| if (do_all) { | ||
| return ret_all; | ||
| } | ||
|
|
||
| UniValue ret(UniValue::VOBJ); | ||
| for (const std::string& stat : stats) { | ||
| const UniValue& value = ret_all[stat]; | ||
| if (value.isNull()) { | ||
| throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid selected statistic %s", stat)); | ||
| } | ||
| ret.pushKV(stat, value); | ||
| } | ||
| return ret; | ||
| } | ||
|
|
||
| static UniValue savemempool(const JSONRPCRequest& request) | ||
| { | ||
| if (request.fHelp || request.params.size() != 0) { | ||
|
|
@@ -1642,6 +1931,7 @@ static const CRPCCommand commands[] = | |
| // --------------------- ------------------------ ----------------------- ---------- | ||
| { "blockchain", "getblockchaininfo", &getblockchaininfo, {} }, | ||
| { "blockchain", "getchaintxstats", &getchaintxstats, {"nblocks", "blockhash"} }, | ||
| { "blockchain", "getblockstats", &getblockstats, {"hash_or_height", "stats"} }, | ||
| { "blockchain", "getbestblockhash", &getbestblockhash, {} }, | ||
| { "blockchain", "getblockcount", &getblockcount, {} }, | ||
| { "blockchain", "getblock", &getblock, {"blockhash","verbosity|verbose"} }, | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I believe if the hash is invalid GetBlockChecked will crash trying to deref the blockindex nullptr.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is fixed.