bitcoin · jamesob · Mar 29, 2019 · Jan 7, 2022 · Jan 7, 2022 · Jan 7, 2022
diff --git a/contrib/devtools/test_utxo_snapshots.sh b/contrib/devtools/test_utxo_snapshots.sh
@@ -0,0 +1,201 @@
+#!/usr/bin/env bash
+# Demonstrate the creation and usage of UTXO snapshots.
+#
+# A server node starts up, IBDs up to a certain height, then generates a UTXO
+# snapshot at that point.
+#
+# The server then downloads more blocks (to create a diff from the snapshot).
+#
+# We bring a client up, load the UTXO snapshot, and we show the client sync to
+# the "network tip" and then start a background validation of the snapshot it
+# loaded. We see the background validation chainstate removed after validation
+# completes.
+#
+
+export LC_ALL=C
+set -e
+
+BASE_HEIGHT=${1:-30000}
+INCREMENTAL_HEIGHT=20000
+FINAL_HEIGHT=$(("$BASE_HEIGHT" + "$INCREMENTAL_HEIGHT"))
+
+SERVER_DATADIR="$(pwd)/utxodemo-data-server-$BASE_HEIGHT"
+CLIENT_DATADIR="$(pwd)/utxodemo-data-client-$BASE_HEIGHT"
+UTXO_DAT_FILE="$(pwd)/utxo.$BASE_HEIGHT.dat"
+
+# Chosen to try to not interfere with any running bitcoind processes.
+SERVER_PORT=8633
+SERVER_RPC_PORT=8632
+
+CLIENT_PORT=8733
+CLIENT_RPC_PORT=8732
+
+SERVER_PORTS="-port=${SERVER_PORT} -rpcport=${SERVER_RPC_PORT}"
+CLIENT_PORTS="-port=${CLIENT_PORT} -rpcport=${CLIENT_RPC_PORT}"
+
+# Ensure the client exercises all indexes.
+ALL_INDEXES="-txindex -coinstatsindex -blockfilterindex=1"
+
+if ! command -v jq >/dev/null ; then
+  echo "This script requires jq to parse JSON RPC output. Please install it."
+  echo "(e.g. sudo apt install jq)"
+  exit 1
+fi
+
+finish() {
+  echo
+  echo "Killing server and client PIDs ($SERVER_PID, $CLIENT_PID) and cleaning up datadirs"
+  echo
+  rm -f "$UTXO_DAT_FILE" "$DUMP_OUTPUT"
+  rm -rf "$SERVER_DATADIR" "$CLIENT_DATADIR"
+  kill -9 "$SERVER_PID" "$CLIENT_PID"
+}
+
+trap finish EXIT
+
+# Need to specify these to trick client into accepting server as a peer
+# it can IBD from.
+CHAIN_HACK_FLAGS="-maxtipage=9223372036854775207 -minimumchainwork=0x00"
+
+server_rpc() {
+  ./src/bitcoin-cli -rpcport=$SERVER_RPC_PORT -datadir="$SERVER_DATADIR" "$@"
+}
+client_rpc() {
+  ./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir="$CLIENT_DATADIR" "$@"
+}
+server_sleep_til_boot() {
+  while ! server_rpc ping >/dev/null 2>&1; do sleep 0.1; done
+}
+client_sleep_til_boot() {
+  while ! client_rpc ping >/dev/null 2>&1; do sleep 0.1; done
+}
+
+mkdir -p "$SERVER_DATADIR" "$CLIENT_DATADIR"
+
+echo "Hi, welcome to the assumeutxo demo/test"
+echo
+echo "We're going to"
+echo
+echo "  - start up a 'server' node, sync it via mainnet IBD to height ${BASE_HEIGHT}"
+echo "  - create a UTXO snapshot at that height"
+echo "  - IBD ${INCREMENTAL_HEIGHT} more blocks on top of that"
+echo
+echo "then we'll demonstrate assumeutxo by "
+echo
+echo "  - starting another node (the 'client') and loading the snapshot in"
+echo "    * first you'll have to modify the code slightly (chainparams) and recompile"
+echo "    * don't worry, we'll make it easy"
+echo "  - observing the client sync ${INCREMENTAL_HEIGHT} blocks on top of the snapshot from the server"
+echo "  - observing the client validate the snapshot chain via background IBD"
+echo
+read -p "Press [enter] to continue" _
+
+echo
+echo "-- Starting the demo. You might want to run the two following commands in"
+echo "   separate terminal windows:"
+echo
+echo "   watch -n0.1 tail -n 30 $SERVER_DATADIR/debug.log"
+echo "   watch -n0.1 tail -n 30 $CLIENT_DATADIR/debug.log"
+echo
+read -p "Press [enter] to continue" _
+
+echo
+echo "-- IBDing the blocks (height=$BASE_HEIGHT) required to the server node..."
+./src/bitcoind -logthreadnames=1 $SERVER_PORTS \
+    -datadir="$SERVER_DATADIR" $CHAIN_HACK_FLAGS -stopatheight="$BASE_HEIGHT" >/dev/null
+
+echo
+echo "-- Creating snapshot at ~ height $BASE_HEIGHT ($UTXO_DAT_FILE)..."
+sleep 2
+./src/bitcoind -logthreadnames=1 $SERVER_PORTS \
+    -datadir="$SERVER_DATADIR" $CHAIN_HACK_FLAGS -connect=0 -listen=0 >/dev/null &
+SERVER_PID="$!"
+
+DUMP_OUTPUT="dumptxoutset-output-$BASE_HEIGHT.json"
+
+server_sleep_til_boot
+server_rpc dumptxoutset "$UTXO_DAT_FILE" > "$DUMP_OUTPUT"
+cat "$DUMP_OUTPUT"
+kill -9 "$SERVER_PID"
+
+RPC_BASE_HEIGHT=$(jq -r .base_height < "$DUMP_OUTPUT")
+RPC_AU=$(jq -r .txoutset_hash < "$DUMP_OUTPUT")
+RPC_NCHAINTX=$(jq -r .nchaintx < "$DUMP_OUTPUT")
+
+# Wait for server to shutdown...
+while server_rpc ping >/dev/null 2>&1; do sleep 0.1; done
+
+echo
+echo "-- Now: add the following to CMainParams::m_assumeutxo_data"
+echo "   in src/kernel/chainparams.cpp, and recompile:"
+echo
+echo "     {"
+echo "         ${RPC_BASE_HEIGHT},"
+echo "         {AssumeutxoHash{uint256S(\"0x${RPC_AU}\")}, ${RPC_NCHAINTX}},"
+echo "     },"
+echo
+echo
+echo "-- IBDing more blocks to the server node (height=$FINAL_HEIGHT) so there is a diff between snapshot and tip..."
+./src/bitcoind $SERVER_PORTS -logthreadnames=1 -datadir="$SERVER_DATADIR" \
+    $CHAIN_HACK_FLAGS -stopatheight="$FINAL_HEIGHT" >/dev/null
+
+echo
+echo "-- Staring the server node to provide blocks to the client node..."
+./src/bitcoind $SERVER_PORTS -logthreadnames=1 -debug=net -datadir="$SERVER_DATADIR" \
+    $CHAIN_HACK_FLAGS -connect=0 -listen=1 >/dev/null &
+SERVER_PID="$!"
+server_sleep_til_boot
+
+echo
+echo "-- Okay, what you're about to see is the client starting up and activating the snapshot."
+echo "   I'm going to display the top 14 log lines from the client on top of an RPC called"
+echo "   getchainstates, which is like getblockchaininfo but for both the snapshot and "
+echo "   background validation chainstates."
+echo
+echo "   You're going to first see the snapshot chainstate sync to the server's tip, then"
+echo "   the background IBD chain kicks in to validate up to the base of the snapshot."
+echo
+echo "   Once validation of the snapshot is done, you should see log lines indicating"
+echo "   that we've deleted the background validation chainstate."
+echo
+echo "   Once everything completes, exit the watch command with CTRL+C."
+echo
+read -p "When you're ready for all this, hit [enter]" _
+
+echo
+echo "-- Starting the client node to get headers from the server, then load the snapshot..."
+./src/bitcoind $CLIENT_PORTS $ALL_INDEXES -logthreadnames=1 -datadir="$CLIENT_DATADIR" \
+    -connect=0 -addnode=127.0.0.1:$SERVER_PORT -debug=net $CHAIN_HACK_FLAGS >/dev/null &
+CLIENT_PID="$!"
+client_sleep_til_boot
+
+echo
+echo "-- Initial state of the client:"
+client_rpc getchainstates
+
+echo
+echo "-- Loading UTXO snapshot into client..."
+client_rpc loadtxoutset "$UTXO_DAT_FILE"
+
+watch -n 0.3 "( tail -n 14 $CLIENT_DATADIR/debug.log ; echo ; ./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir=$CLIENT_DATADIR getchainstates) | cat"
+
+echo
+echo "-- Okay, now I'm going to restart the client to make sure that the snapshot chain reloads "
+echo "   as the main chain properly..."
+echo
+echo "   Press CTRL+C after you're satisfied to exit the demo"
+echo
+read -p "Press [enter] to continue"
+
+while kill -0 "$CLIENT_PID"; do
+    sleep 1
+done
+./src/bitcoind $CLIENT_PORTS $ALL_INDEXES -logthreadnames=1 -datadir="$CLIENT_DATADIR" -connect=0 \
+    -addnode=127.0.0.1:$SERVER_PORT "$CHAIN_HACK_FLAGS" >/dev/null &
+CLIENT_PID="$!"
+client_sleep_til_boot
+
+watch -n 0.3 "( tail -n 14 $CLIENT_DATADIR/debug.log ; echo ; ./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir=$CLIENT_DATADIR getchainstates) | cat"
+
+echo
+echo "-- Done!"
diff --git a/doc/release-process.md b/doc/release-process.md
@@ -43,6 +43,11 @@ Release Process
     - On mainnet, the selected value must not be orphaned, so it may be useful to set the height two blocks back from the tip.
     - Testnet should be set with a height some tens of thousands back from the tip, due to reorgs there.
   - `nMinimumChainWork` with the "chainwork" value of RPC `getblockheader` using the same height as that selected for the previous step.
+  - `m_assumeutxo_data` with the updated assumeutxo hash and nChainTx count.
+    - You can obtain this information, and the corresponding snapshot, by running
+      `./contrib/devtools/utxo_snapshot.sh <blockheight> <snapshot-out-path>`.
+    - The height used should probably the be same as the assumevalid height chosen.
+    - Ensure the resulting snapshot is uploaded somewhere publicly accessible (torrent, HTTP server, etc.).
 - Clear the release notes and move them to the wiki (see "Write the release notes" below).
 - Translations on Transifex:
     - Pull translations from Transifex into the master branch.

diff --git a/src/bench/wallet_create_tx.cpp b/src/bench/wallet_create_tx.cpp
@@ -70,7 +70,7 @@ void generateFakeBlock(const CChainParams& params,
 
     // notify wallet
     const auto& pindex = WITH_LOCK(::cs_main, return context.chainman->ActiveChain().Tip());
-    wallet.blockConnected(kernel::MakeBlockInfo(pindex, &block));
+    wallet.blockConnected(ChainstateRole::NORMAL, kernel::MakeBlockInfo(pindex, &block));
 }
 
 struct PreSelectInputs {

diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp
@@ -188,7 +188,7 @@ int main(int argc, char* argv[])
             explicit submitblock_StateCatcher(const uint256& hashIn) : hash(hashIn), found(false), state() {}
 
         protected:
-            void BlockChecked(const CBlock& block, const BlockValidationState& stateIn) override
+            void BlockChecked(const ChainstateRole role, const CBlock& block, const BlockValidationState& stateIn) override
             {
                 if (block.GetHash() != hash)
                     return;

diff --git a/src/index/base.cpp b/src/index/base.cpp
@@ -90,7 +90,8 @@ bool BaseIndex::Init()
     }
 
     LOCK(cs_main);
-    CChain& active_chain = m_chainstate->m_chain;
+    CChain& index_chain = m_chainstate->m_chain;
+
     if (locator.IsNull()) {
         SetBestBlockIndex(nullptr);
     } else {
@@ -100,24 +101,25 @@ bool BaseIndex::Init()
     // Note: this will latch to true immediately if the user starts up with an empty
     // datadir and an index enabled. If this is the case, indexation will happen solely
     // via `BlockConnected` signals until, possibly, the next restart.
-    m_synced = m_best_block_index.load() == active_chain.Tip();
+    m_synced = m_best_block_index.load() == index_chain.Tip();
     if (!m_synced) {
         bool prune_violation = false;
         if (!m_best_block_index) {
             // index is not built yet
             // make sure we have all block data back to the genesis
-            prune_violation = m_chainstate->m_blockman.GetFirstStoredBlock(*active_chain.Tip()) != active_chain.Genesis();
+            prune_violation = m_chainstate->m_blockman.GetFirstStoredBlock(
+                *index_chain.Tip()) != index_chain.Genesis();
         }
         // in case the index has a best block set and is not fully synced
         // check if we have the required blocks to continue building the index
         else {
             const CBlockIndex* block_to_test = m_best_block_index.load();
-            if (!active_chain.Contains(block_to_test)) {
+            if (!index_chain.Contains(block_to_test)) {
                 // if the bestblock is not part of the mainchain, find the fork
                 // and make sure we have all data down to the fork
-                block_to_test = active_chain.FindFork(block_to_test);
+                block_to_test = index_chain.FindFork(block_to_test);
             }
-            const CBlockIndex* block = active_chain.Tip();
+            const CBlockIndex* block = index_chain.Tip();
             prune_violation = true;
             // check backwards from the tip if we have all block data until we reach the indexes bestblock
             while (block_to_test && block && (block->nStatus & BLOCK_HAVE_DATA)) {
@@ -165,6 +167,8 @@ void BaseIndex::ThreadSync()
         std::chrono::steady_clock::time_point last_locator_write_time{0s};
         while (true) {
             if (m_interrupt) {
+                LogPrintf("%s: m_interrupt set; exiting ThreadSync\n", GetName());
+
                 SetBestBlockIndex(pindex);
                 // No need to handle errors in Commit. If it fails, the error will be already be
                 // logged. The best way to recover is to continue, as index cannot be corrupted by
@@ -272,8 +276,19 @@ bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_ti
     return true;
 }
 
-void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex)
+void BaseIndex::BlockConnected(const ChainstateRole role, const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex)
 {
+    // Ignore events from the assumed-valid chain; we will process its blocks
+    // (sequentially) after it is fully verified by the background chainstate. This
+    // is to avoid any out-of-order indexation.
+    //
+    // TODO at some point we could parameterize whether a particular index can be
+    // built out of order, but for now just do the conservative simple thing.
+    if (role == ChainstateRole::ASSUMEDVALID) {
+        return;
+    }
+
+    // Ignore BlockConnected signals until we have fully indexed the chain.
     if (!m_synced) {
         return;
     }
@@ -318,8 +333,14 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const
     }
 }
 
-void BaseIndex::ChainStateFlushed(const CBlockLocator& locator)
+void BaseIndex::ChainStateFlushed(const ChainstateRole role, const CBlockLocator& locator)
 {
+    // Ignore events from the assumed-valid chain; we will process its blocks
+    // (sequentially) after it is fully verified by the background chainstate.
+    if (role == ChainstateRole::ASSUMEDVALID) {
+        return;
+    }
+
     if (!m_synced) {
         return;
     }
@@ -388,9 +409,15 @@ void BaseIndex::Interrupt()
 
 bool BaseIndex::Start()
 {
+    AssertLockNotHeld(cs_main);
+
+    // May need reset if index is being restarted.
+    m_interrupt.reset();
+
     // m_chainstate member gives indexing code access to node internals. It is
     // removed in followup https://github.com/bitcoin/bitcoin/pull/24230
-    m_chainstate = &m_chain->context()->chainman->ActiveChainstate();
+    m_chainstate = WITH_LOCK(::cs_main,
+        return &m_chain->context()->chainman->GetChainstateForIndexing());
     // Need to register this ValidationInterface before running Init(), so that
     // callbacks are not missed if Init sets m_synced to true.
     RegisterValidationInterface(this);

diff --git a/src/index/base.h b/src/index/base.h
@@ -15,6 +15,7 @@
 class CBlock;
 class CBlockIndex;
 class Chainstate;
+class ChainstateManager;
 namespace interfaces {
 class Chain;
 } // namespace interfaces
@@ -29,6 +30,11 @@ struct IndexSummary {
  * Base class for indices of blockchain data. This implements
  * CValidationInterface and ensures blocks are indexed sequentially according
  * to their position in the active chain.
+ *
+ * In the presence of multiple chainstates (i.e. if a UTXO snapshot is loaded),
+ * only the background "IBD" chainstate will be indexed to avoid building the
+ * index out of order. When the background chainstate completes validation, the
+ * index will be reinitialized and indexation will continue.
  */
 class BaseIndex : public CValidationInterface
 {
@@ -99,9 +105,9 @@ class BaseIndex : public CValidationInterface
     Chainstate* m_chainstate{nullptr};
     const std::string m_name;
 
-    void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) override;
+    void BlockConnected(const ChainstateRole role, const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) override;
 
-    void ChainStateFlushed(const CBlockLocator& locator) override;
+    void ChainStateFlushed(const ChainstateRole role, const CBlockLocator& locator) override;
 
     /// Initialize internal state from the database and block index.
     [[nodiscard]] virtual bool CustomInit(const std::optional<interfaces::BlockKey>& block) { return true; }
@@ -119,9 +125,6 @@ class BaseIndex : public CValidationInterface
 
     virtual DB& GetDB() const = 0;
 
-    /// Get the name of the index for display in logs.
-    const std::string& GetName() const LIFETIMEBOUND { return m_name; }
-
     /// Update the internal best block index as well as the prune lock.
     void SetBestBlockIndex(const CBlockIndex* block);
 
@@ -130,6 +133,9 @@ class BaseIndex : public CValidationInterface
     /// Destructor interrupts sync thread if running and blocks until it exits.
     virtual ~BaseIndex();
 
+    /// Get the name of the index for display in logs.
+    const std::string& GetName() const LIFETIMEBOUND { return m_name; }
+
     /// Blocks the current thread until the index is caught up to the current
     /// state of the block chain. This only blocks if the index has gotten in
     /// sync once and only needs to process blocks in the ValidationInterface