Bitcoin ABC  0.22.13
P2P Digital Currency
blockfilterindex.cpp
Go to the documentation of this file.
1 // Copyright (c) 2018 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #include <dbwrapper.h>
7 #include <primitives/blockhash.h>
8 #include <util/system.h>
9 #include <validation.h>
10 
11 #include <map>
12 
32 constexpr char DB_BLOCK_HASH = 's';
33 constexpr char DB_BLOCK_HEIGHT = 't';
34 constexpr char DB_FILTER_POS = 'P';
35 
36 // 16 MiB
37 constexpr unsigned int MAX_FLTR_FILE_SIZE = 0x1000000;
39 // 1 MiB
40 constexpr unsigned int FLTR_FILE_CHUNK_SIZE = 0x100000;
41 
49 constexpr size_t CF_HEADERS_CACHE_MAX_SZ{2000};
50 
51 namespace {
52 
53 struct DBVal {
54  uint256 hash;
55  uint256 header;
56  FlatFilePos pos;
57 
59 
60  template <typename Stream, typename Operation>
61  inline void SerializationOp(Stream &s, Operation ser_action) {
62  READWRITE(hash);
63  READWRITE(header);
64  READWRITE(pos);
65  }
66 };
67 
68 struct DBHeightKey {
69  int height;
70 
71  DBHeightKey() : height(0) {}
72  explicit DBHeightKey(int height_in) : height(height_in) {}
73 
74  template <typename Stream> void Serialize(Stream &s) const {
76  ser_writedata32be(s, height);
77  }
78 
79  template <typename Stream> void Unserialize(Stream &s) {
80  char prefix = ser_readdata8(s);
81  if (prefix != DB_BLOCK_HEIGHT) {
82  throw std::ios_base::failure(
83  "Invalid format for block filter index DB height key");
84  }
85  height = ser_readdata32be(s);
86  }
87 };
88 
89 struct DBHashKey {
90  BlockHash hash;
91 
92  explicit DBHashKey(const BlockHash &hash_in) : hash(hash_in) {}
93 
95 
96  template <typename Stream, typename Operation>
97  inline void SerializationOp(Stream &s, Operation ser_action) {
98  char prefix = DB_BLOCK_HASH;
99  READWRITE(prefix);
100  if (prefix != DB_BLOCK_HASH) {
101  throw std::ios_base::failure(
102  "Invalid format for block filter index DB hash key");
103  }
104 
105  READWRITE(hash);
106  }
107 };
108 
109 }; // namespace
110 
111 static std::map<BlockFilterType, BlockFilterIndex> g_filter_indexes;
112 
114  size_t n_cache_size, bool f_memory,
115  bool f_wipe)
116  : m_filter_type(filter_type) {
117  const std::string &filter_name = BlockFilterTypeName(filter_type);
118  if (filter_name.empty()) {
119  throw std::invalid_argument("unknown filter_type");
120  }
121 
122  fs::path path = GetDataDir() / "indexes" / "blockfilter" / filter_name;
123  fs::create_directories(path);
124 
125  m_name = filter_name + " block filter index";
126  m_db = std::make_unique<BaseIndex::DB>(path / "db", n_cache_size, f_memory,
127  f_wipe);
128  m_filter_fileseq = std::make_unique<FlatFileSeq>(std::move(path), "fltr",
130 }
131 
133  if (!m_db->Read(DB_FILTER_POS, m_next_filter_pos)) {
134  // Check that the cause of the read failure is that the key does not
135  // exist. Any other errors indicate database corruption or a disk
136  // failure, and starting the index would cause further corruption.
137  if (m_db->Exists(DB_FILTER_POS)) {
138  return error(
139  "%s: Cannot read current %s state; index may be corrupted",
140  __func__, GetName());
141  }
142 
143  // If the DB_FILTER_POS is not set, then initialize to the first
144  // location.
147  }
148  return BaseIndex::Init();
149 }
150 
152  const FlatFilePos &pos = m_next_filter_pos;
153 
154  // Flush current filter file to disk.
156  if (file.IsNull()) {
157  return error("%s: Failed to open filter file %d", __func__, pos.nFile);
158  }
159  if (!FileCommit(file.Get())) {
160  return error("%s: Failed to commit filter file %d", __func__,
161  pos.nFile);
162  }
163 
164  batch.Write(DB_FILTER_POS, pos);
165  return BaseIndex::CommitInternal(batch);
166 }
167 
169  BlockFilter &filter) const {
170  CAutoFile filein(m_filter_fileseq->Open(pos, true), SER_DISK,
172  if (filein.IsNull()) {
173  return false;
174  }
175 
176  BlockHash block_hash;
177  std::vector<uint8_t> encoded_filter;
178  try {
179  filein >> block_hash >> encoded_filter;
180  filter =
181  BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter));
182  } catch (const std::exception &e) {
183  return error("%s: Failed to deserialize block filter from disk: %s",
184  __func__, e.what());
185  }
186 
187  return true;
188 }
189 
191  const BlockFilter &filter) {
192  assert(filter.GetFilterType() == GetFilterType());
193 
194  size_t data_size =
197 
198  // If writing the filter would overflow the file, flush and move to the next
199  // one.
200  if (pos.nPos + data_size > MAX_FLTR_FILE_SIZE) {
201  CAutoFile last_file(m_filter_fileseq->Open(pos), SER_DISK,
203  if (last_file.IsNull()) {
204  LogPrintf("%s: Failed to open filter file %d\n", __func__,
205  pos.nFile);
206  return 0;
207  }
208  if (!TruncateFile(last_file.Get(), pos.nPos)) {
209  LogPrintf("%s: Failed to truncate filter file %d\n", __func__,
210  pos.nFile);
211  return 0;
212  }
213  if (!FileCommit(last_file.Get())) {
214  LogPrintf("%s: Failed to commit filter file %d\n", __func__,
215  pos.nFile);
216  return 0;
217  }
218 
219  pos.nFile++;
220  pos.nPos = 0;
221  }
222 
223  // Pre-allocate sufficient space for filter data.
224  bool out_of_space;
225  m_filter_fileseq->Allocate(pos, data_size, out_of_space);
226  if (out_of_space) {
227  LogPrintf("%s: out of disk space\n", __func__);
228  return 0;
229  }
230 
231  CAutoFile fileout(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
232  if (fileout.IsNull()) {
233  LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile);
234  return 0;
235  }
236 
237  fileout << filter.GetBlockHash() << filter.GetEncodedFilter();
238  return data_size;
239 }
240 
242  const CBlockIndex *pindex) {
243  CBlockUndo block_undo;
244  uint256 prev_header;
245 
246  if (pindex->nHeight > 0) {
247  if (!UndoReadFromDisk(block_undo, pindex)) {
248  return false;
249  }
250 
251  std::pair<BlockHash, DBVal> read_out;
252  if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) {
253  return false;
254  }
255 
256  BlockHash expected_block_hash = pindex->pprev->GetBlockHash();
257  if (read_out.first != expected_block_hash) {
258  return error("%s: previous block header belongs to unexpected "
259  "block %s; expected %s",
260  __func__, read_out.first.ToString(),
261  expected_block_hash.ToString());
262  }
263 
264  prev_header = read_out.second.header;
265  }
266 
267  BlockFilter filter(m_filter_type, block, block_undo);
268 
269  size_t bytes_written = WriteFilterToDisk(m_next_filter_pos, filter);
270  if (bytes_written == 0) {
271  return false;
272  }
273 
274  std::pair<BlockHash, DBVal> value;
275  value.first = pindex->GetBlockHash();
276  value.second.hash = filter.GetHash();
277  value.second.header = filter.ComputeHeader(prev_header);
278  value.second.pos = m_next_filter_pos;
279 
280  if (!m_db->Write(DBHeightKey(pindex->nHeight), value)) {
281  return false;
282  }
283 
284  m_next_filter_pos.nPos += bytes_written;
285  return true;
286 }
287 
289  const std::string &index_name,
290  int start_height, int stop_height) {
291  DBHeightKey key(start_height);
292  db_it.Seek(key);
293 
294  for (int height = start_height; height <= stop_height; ++height) {
295  if (!db_it.GetKey(key) || key.height != height) {
296  return error("%s: unexpected key in %s: expected (%c, %d)",
297  __func__, index_name, DB_BLOCK_HEIGHT, height);
298  }
299 
300  std::pair<BlockHash, DBVal> value;
301  if (!db_it.GetValue(value)) {
302  return error("%s: unable to read value in %s at key (%c, %d)",
303  __func__, index_name, DB_BLOCK_HEIGHT, height);
304  }
305 
306  batch.Write(DBHashKey(value.first), std::move(value.second));
307 
308  db_it.Next();
309  }
310  return true;
311 }
312 
313 bool BlockFilterIndex::Rewind(const CBlockIndex *current_tip,
314  const CBlockIndex *new_tip) {
315  assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip);
316 
317  CDBBatch batch(*m_db);
318  std::unique_ptr<CDBIterator> db_it(m_db->NewIterator());
319 
320  // During a reorg, we need to copy all filters for blocks that are getting
321  // disconnected from the height index to the hash index so we can still find
322  // them when the height index entries are overwritten.
323  if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight,
324  current_tip->nHeight)) {
325  return false;
326  }
327 
328  // The latest filter position gets written in Commit by the call to the
329  // BaseIndex::Rewind. But since this creates new references to the filter,
330  // the position should get updated here atomically as well in case Commit
331  // fails.
332  batch.Write(DB_FILTER_POS, m_next_filter_pos);
333  if (!m_db->WriteBatch(batch)) {
334  return false;
335  }
336 
337  return BaseIndex::Rewind(current_tip, new_tip);
338 }
339 
340 static bool LookupOne(const CDBWrapper &db, const CBlockIndex *block_index,
341  DBVal &result) {
342  // First check if the result is stored under the height index and the value
343  // there matches the block hash. This should be the case if the block is on
344  // the active chain.
345  std::pair<BlockHash, DBVal> read_out;
346  if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) {
347  return false;
348  }
349  if (read_out.first == block_index->GetBlockHash()) {
350  result = std::move(read_out.second);
351  return true;
352  }
353 
354  // If value at the height index corresponds to an different block, the
355  // result will be stored in the hash index.
356  return db.Read(DBHashKey(block_index->GetBlockHash()), result);
357 }
358 
359 static bool LookupRange(CDBWrapper &db, const std::string &index_name,
360  int start_height, const CBlockIndex *stop_index,
361  std::vector<DBVal> &results) {
362  if (start_height < 0) {
363  return error("%s: start height (%d) is negative", __func__,
364  start_height);
365  }
366  if (start_height > stop_index->nHeight) {
367  return error("%s: start height (%d) is greater than stop height (%d)",
368  __func__, start_height, stop_index->nHeight);
369  }
370 
371  size_t results_size =
372  static_cast<size_t>(stop_index->nHeight - start_height + 1);
373  std::vector<std::pair<BlockHash, DBVal>> values(results_size);
374 
375  DBHeightKey key(start_height);
376  std::unique_ptr<CDBIterator> db_it(db.NewIterator());
377  db_it->Seek(DBHeightKey(start_height));
378  for (int height = start_height; height <= stop_index->nHeight; ++height) {
379  if (!db_it->Valid() || !db_it->GetKey(key) || key.height != height) {
380  return false;
381  }
382 
383  size_t i = static_cast<size_t>(height - start_height);
384  if (!db_it->GetValue(values[i])) {
385  return error("%s: unable to read value in %s at key (%c, %d)",
386  __func__, index_name, DB_BLOCK_HEIGHT, height);
387  }
388 
389  db_it->Next();
390  }
391 
392  results.resize(results_size);
393 
394  // Iterate backwards through block indexes collecting results in order to
395  // access the block hash of each entry in case we need to look it up in the
396  // hash index.
397  for (const CBlockIndex *block_index = stop_index;
398  block_index && block_index->nHeight >= start_height;
399  block_index = block_index->pprev) {
400  BlockHash block_hash = block_index->GetBlockHash();
401 
402  size_t i = static_cast<size_t>(block_index->nHeight - start_height);
403  if (block_hash == values[i].first) {
404  results[i] = std::move(values[i].second);
405  continue;
406  }
407 
408  if (!db.Read(DBHashKey(block_hash), results[i])) {
409  return error("%s: unable to read value in %s at key (%c, %s)",
410  __func__, index_name, DB_BLOCK_HASH,
411  block_hash.ToString());
412  }
413  }
414 
415  return true;
416 }
417 
419  BlockFilter &filter_out) const {
420  DBVal entry;
421  if (!LookupOne(*m_db, block_index, entry)) {
422  return false;
423  }
424 
425  return ReadFilterFromDisk(entry.pos, filter_out);
426 }
427 
429  uint256 &header_out) {
431 
432  bool is_checkpoint{block_index->nHeight % CFCHECKPT_INTERVAL == 0};
433 
434  if (is_checkpoint) {
435  // Try to find the block in the headers cache if this is a checkpoint
436  // height.
437  auto header = m_headers_cache.find(block_index->GetBlockHash());
438  if (header != m_headers_cache.end()) {
439  header_out = header->second;
440  return true;
441  }
442  }
443 
444  DBVal entry;
445  if (!LookupOne(*m_db, block_index, entry)) {
446  return false;
447  }
448 
449  if (is_checkpoint && m_headers_cache.size() < CF_HEADERS_CACHE_MAX_SZ) {
450  // Add to the headers cache if this is a checkpoint height.
451  m_headers_cache.emplace(block_index->GetBlockHash(), entry.header);
452  }
453 
454  header_out = entry.header;
455  return true;
456 }
457 
459  int start_height, const CBlockIndex *stop_index,
460  std::vector<BlockFilter> &filters_out) const {
461  std::vector<DBVal> entries;
462  if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) {
463  return false;
464  }
465 
466  filters_out.resize(entries.size());
467  auto filter_pos_it = filters_out.begin();
468  for (const auto &entry : entries) {
469  if (!ReadFilterFromDisk(entry.pos, *filter_pos_it)) {
470  return false;
471  }
472  ++filter_pos_it;
473  }
474 
475  return true;
476 }
477 
479  int start_height, const CBlockIndex *stop_index,
480  std::vector<uint256> &hashes_out) const
481 
482 {
483  std::vector<DBVal> entries;
484  if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) {
485  return false;
486  }
487 
488  hashes_out.clear();
489  hashes_out.reserve(entries.size());
490  for (const auto &entry : entries) {
491  hashes_out.push_back(entry.hash);
492  }
493  return true;
494 }
495 
497  auto it = g_filter_indexes.find(filter_type);
498  return it != g_filter_indexes.end() ? &it->second : nullptr;
499 }
500 
501 void ForEachBlockFilterIndex(std::function<void(BlockFilterIndex &)> fn) {
502  for (auto &entry : g_filter_indexes) {
503  fn(entry.second);
504  }
505 }
506 
507 bool InitBlockFilterIndex(BlockFilterType filter_type, size_t n_cache_size,
508  bool f_memory, bool f_wipe) {
509  auto result = g_filter_indexes.emplace(
510  std::piecewise_construct, std::forward_as_tuple(filter_type),
511  std::forward_as_tuple(filter_type, n_cache_size, f_memory, f_wipe));
512  return result.second;
513 }
514 
516  return g_filter_indexes.erase(filter_type);
517 }
518 
520  g_filter_indexes.clear();
521 }
bool GetKey(K &key)
Definition: dbwrapper.h:143
bool LookupFilter(const CBlockIndex *block_index, BlockFilter &filter_out) const
Get a single filter by block.
BlockFilterIndex is used to store and retrieve block filters, hashes, and headers for a range of bloc...
constexpr char DB_BLOCK_HEIGHT
bool FileCommit(FILE *file)
Definition: system.cpp:1120
constexpr char DB_BLOCK_HASH
The index database stores three items for each block: the disk location of the encoded filter...
virtual bool Init()
Initialize internal state from the database and block index.
Definition: base.cpp:54
uint8_t ser_readdata8(Stream &s)
Definition: serialize.h:109
CBlockIndex * pprev
pointer to the index of the predecessor of this block
Definition: blockindex.h:30
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:48
Definition: block.h:62
static bool LookupRange(CDBWrapper &db, const std::string &index_name, int start_height, const CBlockIndex *stop_index, std::vector< DBVal > &results)
BlockFilterIndex * GetBlockFilterIndex(BlockFilterType filter_type)
Get a block filter index by type.
const char * prefix
Definition: rest.cpp:755
void ForEachBlockFilterIndex(std::function< void(BlockFilterIndex &)> fn)
Iterate over all running block filter indexes, invoking fn on each.
constexpr size_t CF_HEADERS_CACHE_MAX_SZ
Maximum size of the cfheaders cache.
constexpr unsigned int MAX_FLTR_FILE_SIZE
bool Rewind(const CBlockIndex *current_tip, const CBlockIndex *new_tip) override
Rewind index to an earlier chain tip during a chain reorg.
const BlockHash & GetBlockHash() const
Definition: blockfilter.h:131
static void LogPrintf(const char *fmt, const Args &... args)
Definition: logging.h:174
bool WriteBlock(const CBlock &block, const CBlockIndex *pindex) override
Write update index entries for a newly connected block.
bool InitBlockFilterIndex(BlockFilterType filter_type, size_t n_cache_size, bool f_memory, bool f_wipe)
Initialize a block filter index for the given type if one does not already exist. ...
virtual bool Rewind(const CBlockIndex *current_tip, const CBlockIndex *new_tip)
Rewind index to an earlier chain tip during a chain reorg.
Definition: base.cpp:176
int nFile
Definition: flatfile.h:15
std::unique_ptr< BaseIndex::DB > m_db
CDBIterator * NewIterator()
Definition: dbwrapper.h:297
uint256 GetHash() const
Compute the filter hash.
size_t GetSerializeSize(const T &t, int nVersion=0)
Definition: serialize.h:1193
void Serialize(Stream &s, char a)
Definition: serialize.h:261
constexpr char DB_FILTER_POS
const std::vector< uint8_t > & GetEncodedFilter() const
Definition: blockfilter.h:134
bool GetValue(V &value)
Definition: dbwrapper.h:155
BlockHash GetBlockHash() const
Definition: blockindex.h:133
BlockFilterType
Definition: blockfilter.h:88
static bool CopyHeightIndexToHashIndex(CDBIterator &db_it, CDBBatch &batch, const std::string &index_name, int start_height, int stop_height)
bool ReadFilterFromDisk(const FlatFilePos &pos, BlockFilter &filter) const
size_t WriteFilterToDisk(FlatFilePos &pos, const BlockFilter &filter)
#define LOCK(cs)
Definition: sync.h:230
Complete block filter struct as defined in BIP 157.
Definition: blockfilter.h:111
ADD_SERIALIZE_METHODS
Definition: processor.cpp:266
bool UndoReadFromDisk(CBlockUndo &blockundo, const CBlockIndex *pindex)
std::string m_name
BlockFilterIndex(BlockFilterType filter_type, size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Constructs the index, which becomes available to be queried.
void DestroyAllBlockFilterIndexes()
Destroy all open block filter indexes.
const fs::path & GetDataDir(bool fNetSpecific)
Definition: system.cpp:760
constexpr unsigned int FLTR_FILE_CHUNK_SIZE
The pre-allocation chunk size for fltr?????.dat files.
void Write(const K &key, const V &value)
Definition: dbwrapper.h:73
bool LookupFilterHashRange(int start_height, const CBlockIndex *stop_index, std::vector< uint256 > &hashes_out) const
Get a range of filter hashes between two heights on a chain.
virtual bool CommitInternal(CDBBatch &batch)
Virtual method called internally by Commit that can be overridden to atomically commit more index sta...
Definition: base.cpp:169
std::string ToString() const
Definition: uint256.h:74
void ser_writedata32be(Stream &s, uint32_t obj)
Definition: serialize.h:100
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:230
BlockFilterType m_filter_type
static constexpr int CFCHECKPT_INTERVAL
Interval between compact filter checkpoints.
uint256 ComputeHeader(const uint256 &prev_header) const
Compute the filter header given the previous one.
void Next()
Definition: dbwrapper.cpp:248
256-bit opaque blob.
Definition: uint256.h:120
static constexpr int CLIENT_VERSION
bitcoind-res.rc includes this file, but it cannot cope with real c++ code.
Definition: clientversion.h:44
bool TruncateFile(FILE *file, unsigned int length)
Definition: system.cpp:1156
A BlockHash is a unqiue identifier for a block.
Definition: blockhash.h:13
The block chain is a tree shaped structure starting with the genesis block at the root...
Definition: blockindex.h:23
bool DestroyBlockFilterIndex(BlockFilterType filter_type)
Destroy the block filter index with the given type.
Undo information for a CBlock.
Definition: undo.h:73
FlatFilePos m_next_filter_pos
BlockFilterType GetFilterType() const
bool LookupFilterHeader(const CBlockIndex *block_index, uint256 &header_out)
Get a single filter header by block.
static std::map< BlockFilterType, BlockFilterIndex > g_filter_indexes
uint32_t ser_readdata32be(Stream &s)
Definition: serialize.h:129
void Seek(const K &key)
Definition: dbwrapper.h:133
void Unserialize(Stream &s, char &a)
Definition: serialize.h:313
bool CommitInternal(CDBBatch &batch) override
Virtual method called internally by Commit that can be overridden to atomically commit more index sta...
std::unique_ptr< FlatFileSeq > m_filter_fileseq
int nHeight
height of the entry in the chain. The genesis block has height 0
Definition: blockindex.h:36
CBlockIndex * GetAncestor(int height)
Efficiently find an ancestor of this block.
Definition: blockindex.cpp:71
#define READWRITE(...)
Definition: serialize.h:191
bool Init() override
Initialize internal state from the database and block index.
BlockFilterType GetFilterType() const
Definition: blockfilter.h:130
unsigned int nPos
Definition: flatfile.h:16
bool LookupFilterRange(int start_height, const CBlockIndex *stop_index, std::vector< BlockFilter > &filters_out) const
Get a range of filters between two heights on a chain.
bool error(const char *fmt, const Args &... args)
Definition: system.h:47
static bool LookupOne(const CDBWrapper &db, const CBlockIndex *block_index, DBVal &result)
Non-refcounted RAII wrapper for FILE*.
Definition: streams.h:580
void ser_writedata8(Stream &s, uint8_t obj)
Lowest-level serialization and conversion.
Definition: serialize.h:81
const char * GetName() const override
Get the name of the index for display in logs.
CAddrDb db
Definition: main.cpp:167
const std::string & BlockFilterTypeName(BlockFilterType filter_type)
Get the human-readable name for a filter type.