Bitcoin ABC  0.22.13
P2P Digital Currency
txindex.cpp
Go to the documentation of this file.
1 // Copyright (c) 2017-2018 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #include <index/txindex.h>
6 
7 #include <blockdb.h>
8 #include <chain.h>
9 #include <node/ui_interface.h>
10 #include <shutdown.h>
11 #include <util/system.h>
12 #include <util/translation.h>
13 #include <validation.h>
14 
15 #include <boost/thread.hpp>
16 
17 constexpr char DB_BEST_BLOCK = 'B';
18 constexpr char DB_TXINDEX = 't';
19 constexpr char DB_TXINDEX_BLOCK = 'T';
20 
21 std::unique_ptr<TxIndex> g_txindex;
22 
23 struct CDiskTxPos : public FlatFilePos {
24  unsigned int nTxOffset; // after header
25 
27 
28  template <typename Stream, typename Operation>
29  inline void SerializationOp(Stream &s, Operation ser_action) {
30  READWRITEAS(FlatFilePos, *this);
31  READWRITE(VARINT(nTxOffset));
32  }
33 
34  CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn)
35  : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {}
36 
38 
39  void SetNull() {
41  nTxOffset = 0;
42  }
43 };
44 
54 class TxIndex::DB : public BaseIndex::DB {
55 public:
56  explicit DB(size_t n_cache_size, bool f_memory = false,
57  bool f_wipe = false);
58 
61  bool ReadTxPos(const TxId &txid, CDiskTxPos &pos) const;
62 
64  bool WriteTxs(const std::vector<std::pair<TxId, CDiskTxPos>> &v_pos);
65 
68  bool MigrateData(CBlockTreeDB &block_tree_db,
69  const CBlockLocator &best_locator);
70 };
71 
72 TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe)
73  : BaseIndex::DB(GetDataDir() / "indexes" / "txindex", n_cache_size,
74  f_memory, f_wipe) {}
75 
76 bool TxIndex::DB::ReadTxPos(const TxId &txid, CDiskTxPos &pos) const {
77  return Read(std::make_pair(DB_TXINDEX, txid), pos);
78 }
79 
81  const std::vector<std::pair<TxId, CDiskTxPos>> &v_pos) {
82  CDBBatch batch(*this);
83  for (const auto &tuple : v_pos) {
84  batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second);
85  }
86  return WriteBatch(batch);
87 }
88 
89 /*
90  * Safely persist a transfer of data from the old txindex database to the new
91  * one, and compact the range of keys updated. This is used internally by
92  * MigrateData.
93  */
94 static void
96  CDBBatch &batch_newdb, CDBBatch &batch_olddb,
97  const std::pair<uint8_t, TxId> &begin_key,
98  const std::pair<uint8_t, TxId> &end_key) {
99  // Sync new DB changes to disk before deleting from old DB.
100  newdb.WriteBatch(batch_newdb, /*fSync=*/true);
101  olddb.WriteBatch(batch_olddb);
102  olddb.CompactRange(begin_key, end_key);
103 
104  batch_newdb.Clear();
105  batch_olddb.Clear();
106 }
107 
109  const CBlockLocator &best_locator) {
110  // The prior implementation of txindex was always in sync with block index
111  // and presence was indicated with a boolean DB flag. If the flag is set,
112  // this means the txindex from a previous version is valid and in sync with
113  // the chain tip. The first step of the migration is to unset the flag and
114  // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the
115  // index entries are copied over in batches to the new database. Finally,
116  // DB_TXINDEX_BLOCK is erased from the old database and the block hash is
117  // written to the new database.
118  //
119  // Unsetting the boolean flag ensures that if the node is downgraded to a
120  // previous version, it will not see a corrupted, partially migrated index
121  // -- it will see that the txindex is disabled. When the node is upgraded
122  // again, the migration will pick up where it left off and sync to the block
123  // with hash DB_TXINDEX_BLOCK.
124  bool f_legacy_flag = false;
125  block_tree_db.ReadFlag("txindex", f_legacy_flag);
126  if (f_legacy_flag) {
127  if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) {
128  return error("%s: cannot write block indicator", __func__);
129  }
130  if (!block_tree_db.WriteFlag("txindex", false)) {
131  return error("%s: cannot write block index db flag", __func__);
132  }
133  }
134 
135  CBlockLocator locator;
136  if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) {
137  return true;
138  }
139 
140  int64_t count = 0;
141  uiInterface.InitMessage(_("Upgrading txindex database").translated);
142  LogPrintf("Upgrading txindex database... [0%%]\n");
143  uiInterface.ShowProgress(_("Upgrading txindex database").translated, 0,
144  true);
145  int report_done = 0;
146  const size_t batch_size = 1 << 24; // 16 MiB
147 
148  CDBBatch batch_newdb(*this);
149  CDBBatch batch_olddb(block_tree_db);
150 
151  std::pair<uint8_t, TxId> key;
152  std::pair<uint8_t, TxId> begin_key{DB_TXINDEX, TxId()};
153  std::pair<uint8_t, TxId> prev_key = begin_key;
154 
155  bool interrupted = false;
156  std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator());
157  for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) {
158  boost::this_thread::interruption_point();
159  if (ShutdownRequested()) {
160  interrupted = true;
161  break;
162  }
163 
164  if (!cursor->GetKey(key)) {
165  return error("%s: cannot get key from valid cursor", __func__);
166  }
167  if (key.first != DB_TXINDEX) {
168  break;
169  }
170 
171  // Log progress every 10%.
172  if (++count % 256 == 0) {
173  // Since txids are uniformly random and traversed in increasing
174  // order, the high 16 bits of the ID can be used to estimate the
175  // current progress.
176  const TxId &txid = key.second;
177  uint32_t high_nibble =
178  (static_cast<uint32_t>(*(txid.begin() + 0)) << 8) +
179  (static_cast<uint32_t>(*(txid.begin() + 1)) << 0);
180  int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5);
181 
182  uiInterface.ShowProgress(_("Upgrading txindex database").translated,
183  percentage_done, true);
184  if (report_done < percentage_done / 10) {
185  LogPrintf("Upgrading txindex database... [%d%%]\n",
186  percentage_done);
187  report_done = percentage_done / 10;
188  }
189  }
190 
191  CDiskTxPos value;
192  if (!cursor->GetValue(value)) {
193  return error("%s: cannot parse txindex record", __func__);
194  }
195  batch_newdb.Write(key, value);
196  batch_olddb.Erase(key);
197 
198  if (batch_newdb.SizeEstimate() > batch_size ||
199  batch_olddb.SizeEstimate() > batch_size) {
200  // NOTE: it's OK to delete the key pointed at by the current DB
201  // cursor while iterating because LevelDB iterators are guaranteed
202  // to provide a consistent view of the underlying data, like a
203  // lightweight snapshot.
204  WriteTxIndexMigrationBatches(*this, block_tree_db, batch_newdb,
205  batch_olddb, prev_key, key);
206  prev_key = key;
207  }
208  }
209 
210  // If these final DB batches complete the migration, write the best block
211  // hash marker to the new database and delete from the old one. This signals
212  // that the former is fully caught up to that point in the blockchain and
213  // that all txindex entries have been removed from the latter.
214  if (!interrupted) {
215  batch_olddb.Erase(DB_TXINDEX_BLOCK);
216  batch_newdb.Write(DB_BEST_BLOCK, locator);
217  }
218 
219  WriteTxIndexMigrationBatches(*this, block_tree_db, batch_newdb, batch_olddb,
220  begin_key, key);
221 
222  if (interrupted) {
223  LogPrintf("[CANCELLED].\n");
224  return false;
225  }
226 
227  uiInterface.ShowProgress("", 100, false);
228 
229  LogPrintf("[DONE].\n");
230  return true;
231 }
232 
233 TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe)
234  : m_db(std::make_unique<TxIndex::DB>(n_cache_size, f_memory, f_wipe)) {}
235 
237 
239  LOCK(cs_main);
240 
241  // Attempt to migrate txindex from the old database to the new one. Even if
242  // chain_tip is null, the node could be reindexing and we still want to
243  // delete txindex records in the old database.
244  if (!m_db->MigrateData(*pblocktree, ::ChainActive().GetLocator())) {
245  return false;
246  }
247 
248  return BaseIndex::Init();
249 }
250 
251 bool TxIndex::WriteBlock(const CBlock &block, const CBlockIndex *pindex) {
252  // Exclude genesis block transaction because outputs are not spendable.
253  if (pindex->nHeight == 0) {
254  return true;
255  }
256 
257  CDiskTxPos pos(pindex->GetBlockPos(),
258  GetSizeOfCompactSize(block.vtx.size()));
259  std::vector<std::pair<TxId, CDiskTxPos>> vPos;
260  vPos.reserve(block.vtx.size());
261  for (const auto &tx : block.vtx) {
262  vPos.emplace_back(tx->GetId(), pos);
264  }
265  return m_db->WriteTxs(vPos);
266 }
267 
269  return *m_db;
270 }
271 
272 bool TxIndex::FindTx(const TxId &txid, BlockHash &block_hash,
273  CTransactionRef &tx) const {
274  CDiskTxPos postx;
275  if (!m_db->ReadTxPos(txid, postx)) {
276  return false;
277  }
278 
279  CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION);
280  if (file.IsNull()) {
281  return error("%s: OpenBlockFile failed", __func__);
282  }
283  CBlockHeader header;
284  try {
285  file >> header;
286  if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) {
287  return error("%s: fseek(...) failed", __func__);
288  }
289  file >> tx;
290  } catch (const std::exception &e) {
291  return error("%s: Deserialize or I/O error - %s", __func__, e.what());
292  }
293  if (tx->GetId() != txid) {
294  return error("%s: txid mismatch", __func__);
295  }
296  block_hash = header.GetHash();
297  return true;
298 }
std::shared_ptr< const CTransaction > CTransactionRef
Definition: transaction.h:336
void SerializationOp(Stream &s, Operation ser_action)
Definition: txindex.cpp:29
#define VARINT(obj)
Definition: serialize.h:608
bool WriteBlock(const CBlock &block, const CBlockIndex *pindex) override
Write update index entries for a newly connected block.
Definition: txindex.cpp:251
void Clear()
Definition: dbwrapper.h:68
bool ShutdownRequested()
Definition: shutdown.cpp:18
unsigned int nTxOffset
Definition: txindex.cpp:24
virtual bool Init()
Initialize internal state from the database and block index.
Definition: base.cpp:54
Describes a place in the block chain to another node such that if the other node doesn&#39;t have the sam...
Definition: block.h:110
constexpr char DB_TXINDEX_BLOCK
Definition: txindex.cpp:19
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:48
Definition: block.h:62
CChain & ChainActive()
Definition: validation.cpp:78
bool WriteTxs(const std::vector< std::pair< TxId, CDiskTxPos >> &v_pos)
Write a batch of transaction positions to the DB.
Definition: txindex.cpp:80
void Erase(const K &key)
Definition: dbwrapper.h:97
constexpr char DB_TXINDEX
Definition: txindex.cpp:18
static void LogPrintf(const char *fmt, const Args &... args)
Definition: logging.h:174
FlatFilePos GetBlockPos() const
Definition: blockindex.h:102
int nFile
Definition: flatfile.h:15
const std::unique_ptr< DB > m_db
Definition: txindex.h:20
#define READWRITEAS(type, obj)
Definition: serialize.h:192
bool Init() override
Override base class init to migrate from old database.
Definition: txindex.cpp:238
CDBIterator * NewIterator()
Definition: dbwrapper.h:297
bool IsNull() const
Return true if the wrapped FILE* is nullptr, false otherwise.
Definition: streams.h:626
Access to the txindex database (indexes/txindex/)
Definition: txindex.cpp:54
size_t GetSerializeSize(const T &t, int nVersion=0)
Definition: serialize.h:1193
bool FindTx(const TxId &txid, BlockHash &block_hash, CTransactionRef &tx) const
Look up a transaction by identifier.
Definition: txindex.cpp:272
Base class for indices of blockchain data.
Definition: base.h:21
Access to the block database (blocks/index/)
Definition: txdb.h:96
CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn)
Definition: txindex.cpp:34
bool MigrateData(CBlockTreeDB &block_tree_db, const CBlockLocator &best_locator)
Migrate txindex data from the block tree DB, where it may be for older nodes that have not been upgra...
Definition: txindex.cpp:108
#define LOCK(cs)
Definition: sync.h:230
std::unique_ptr< TxIndex > g_txindex
The global transaction index, used in GetTransaction. May be null.
Definition: txindex.cpp:21
bilingual_str _(const char *psz)
Translation function.
Definition: translation.h:55
const fs::path & GetDataDir(bool fNetSpecific)
Definition: system.cpp:760
ADD_SERIALIZE_METHODS
Definition: txindex.cpp:26
FILE * OpenBlockFile(const FlatFilePos &pos, bool fReadOnly)
Open a block file (blk?????.dat).
Definition: blockdb.cpp:20
RecursiveMutex cs_main
Global state.
Definition: validation.cpp:95
void SetNull()
Definition: flatfile.h:39
void Write(const K &key, const V &value)
Definition: dbwrapper.h:73
BaseIndex::DB & GetDB() const override
Definition: txindex.cpp:268
size_t SizeEstimate() const
Definition: dbwrapper.h:112
uint8_t * begin()
Definition: uint256.h:76
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:230
bool ReadFlag(const std::string &name, bool &fValue)
Definition: txdb.cpp:260
TxIndex(size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Constructs the index, which becomes available to be queried.
Definition: txindex.cpp:233
virtual ~TxIndex() override
Definition: txindex.cpp:236
bool Write(const K &key, const V &value, bool fSync=false)
Definition: dbwrapper.h:256
void CompactRange(const K &key_begin, const K &key_end) const
Compact a certain range of keys in the database.
Definition: dbwrapper.h:326
std::vector< CTransactionRef > vtx
Definition: block.h:65
FILE * Get() const
Get wrapped FILE* without transfer of ownership.
Definition: streams.h:623
static constexpr int CLIENT_VERSION
bitcoind-res.rc includes this file, but it cannot cope with real c++ code.
Definition: clientversion.h:44
A BlockHash is a unqiue identifier for a block.
Definition: blockhash.h:13
The block chain is a tree shaped structure starting with the genesis block at the root...
Definition: blockindex.h:23
BlockHash GetHash() const
Definition: block.cpp:11
A TxId is the identifier of a transaction.
Definition: txid.h:14
DB(size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Definition: txindex.cpp:72
constexpr char DB_BEST_BLOCK
Definition: txindex.cpp:17
std::unique_ptr< CBlockTreeDB > pblocktree
Global variable that points to the active block tree (protected by cs_main)
Definition: validation.cpp:171
static int count
Definition: tests.c:35
uint32_t GetSizeOfCompactSize(uint64_t nSize)
Compact Size size < 253 – 1 byte size <= USHRT_MAX – 3 bytes (253 + 2 bytes) size <= UINT_MAX – 5 ...
Definition: serialize.h:410
bool WriteFlag(const std::string &name, bool fValue)
Definition: txdb.cpp:256
static void WriteTxIndexMigrationBatches(CDBWrapper &newdb, CDBWrapper &olddb, CDBBatch &batch_newdb, CDBBatch &batch_olddb, const std::pair< uint8_t, TxId > &begin_key, const std::pair< uint8_t, TxId > &end_key)
Definition: txindex.cpp:95
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:186
CClientUIInterface uiInterface
int nHeight
height of the entry in the chain. The genesis block has height 0
Definition: blockindex.h:36
void SetNull()
Definition: txindex.cpp:39
#define READWRITE(...)
Definition: serialize.h:191
bool ReadTxPos(const TxId &txid, CDiskTxPos &pos) const
Read the disk location of the transaction data with the given ID.
Definition: txindex.cpp:76
CDiskTxPos()
Definition: txindex.cpp:37
unsigned int nPos
Definition: flatfile.h:16
bool error(const char *fmt, const Args &... args)
Definition: system.h:47
TxIndex is used to look up transactions included in the blockchain by ID.
Definition: txindex.h:18
Nodes collect new transactions into a block, hash them into a hash tree, and scan through nonce value...
Definition: block.h:22
Non-refcounted RAII wrapper for FILE*.
Definition: streams.h:580