dbwrapper.cpp

Go to the documentation of this file.

// Copyright (c) 2012-2016 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
#include <dbwrapper.h>
 
#include <random.h>
#include <util/fs_helpers.h>
 
#include <leveldb/cache.h>
#include <leveldb/env.h>
#include <leveldb/filter_policy.h>
#include <memenv.h>
 
#include <algorithm>
#include <cstdint>
#include <memory>
 
class CBitcoinLevelDBLogger : public leveldb::Logger {
public:
    // This code is adapted from posix_logger.h, which is why it is using
    // vsprintf.
    // Please do not do this in normal code
    void Logv(const char *format, va_list ap) override {
        if (!LogAcceptCategory(BCLog::LEVELDB, BCLog::Level::Debug)) {
            return;
        }
        char buffer[500];
        for (int iter = 0; iter < 2; iter++) {
            char *base;
            int bufsize;
            if (iter == 0) {
                bufsize = sizeof(buffer);
                base = buffer;
            } else {
                bufsize = 30000;
                base = new char[bufsize];
            }
            char *p = base;
            char *limit = base + bufsize;
 
            // Print the message
            if (p < limit) {
                va_list backup_ap;
                va_copy(backup_ap, ap);
                // Do not use vsnprintf elsewhere in bitcoin source code, see
                // above.
                p += vsnprintf(p, limit - p, format, backup_ap);
                va_end(backup_ap);
            }
 
            // Truncate to available space if necessary
            if (p >= limit) {
                if (iter == 0) {
                    continue; // Try again with larger buffer
                } else {
                    p = limit - 1;
                }
            }
 
            // Add newline if necessary
            if (p == base || p[-1] != '\n') {
                *p++ = '\n';
            }
 
            assert(p <= limit);
            base[std::min(bufsize - 1, (int)(p - base))] = '\0';
            LogPrintLevelToBeContinued(BCLog::LEVELDB, BCLog::Level::Debug,
                                       "%s", base);
            if (base != buffer) {
                delete[] base;
            }
            break;
        }
    }
};
 
static void SetMaxOpenFiles(leveldb::Options *options) {
    // On most platforms the default setting of max_open_files (which is 1000)
    // is optimal. On Windows using a large file count is OK because the handles
    // do not interfere with select() loops. On 64-bit Unix hosts this value is
    // also OK, because up to that amount LevelDB will use an mmap
    // implementation that does not use extra file descriptors (the fds are
    // closed after being mmaped).
    //
    // Increasing the value beyond the default is dangerous because LevelDB will
    // fall back to a non-mmap implementation when the file count is too large.
    // On 32-bit Unix host we should decrease the value because the handles use
    // up real fds, and we want to avoid fd exhaustion issues.
    //
    // See PR #12495 for further discussion.
 
    int default_open_files = options->max_open_files;
#ifndef WIN32
    if (sizeof(void *) < 8) {
        options->max_open_files = 64;
    }
#endif
    LogPrint(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n",
             options->max_open_files, default_open_files);
}
 
static leveldb::Options GetOptions(size_t nCacheSize) {
    leveldb::Options options;
    options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
    // up to two write buffers may be held in memory simultaneously
    options.write_buffer_size = nCacheSize / 4;
    options.filter_policy = leveldb::NewBloomFilterPolicy(10);
    options.compression = leveldb::kNoCompression;
    options.info_log = new CBitcoinLevelDBLogger();
    if (leveldb::kMajorVersion > 1 ||
        (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
        // LevelDB versions before 1.16 consider short writes to be corruption.
        // Only trigger error on corruption in later versions.
        options.paranoid_checks = true;
    }
    options.max_file_size =
        std::max(options.max_file_size, DBWRAPPER_MAX_FILE_SIZE);
    SetMaxOpenFiles(&options);
    return options;
}
 
CDBWrapper::CDBWrapper(const DBParams &params)
    : m_name{fs::PathToString(params.path.stem())}, m_path{params.path},
      m_is_memory{params.memory_only} {
    penv = nullptr;
    readoptions.verify_checksums = true;
    iteroptions.verify_checksums = true;
    iteroptions.fill_cache = false;
    syncoptions.sync = true;
    options = GetOptions(params.cache_bytes);
    options.create_if_missing = true;
    if (params.memory_only) {
        penv = leveldb::NewMemEnv(leveldb::Env::Default());
        options.env = penv;
    } else {
        if (params.wipe_data) {
            LogPrintf("Wiping LevelDB in %s\n", fs::PathToString(params.path));
            leveldb::Status result =
                leveldb::DestroyDB(fs::PathToString(params.path), options);
            dbwrapper_private::HandleError(result);
        }
        TryCreateDirectories(params.path);
        LogPrintf("Opening LevelDB in %s\n", fs::PathToString(params.path));
    }
    // PathToString() return value is safe to pass to leveldb open function,
    // because on POSIX leveldb passes the byte string directly to ::open(), and
    // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW
    // (see env_posix.cc and env_windows.cc).
    leveldb::Status status =
        leveldb::DB::Open(options, fs::PathToString(params.path), &pdb);
    dbwrapper_private::HandleError(status);
    LogPrintf("Opened LevelDB successfully\n");
 
    if (params.options.force_compact) {
        LogPrintf("Starting database compaction of %s\n",
                  fs::PathToString(params.path));
        pdb->CompactRange(nullptr, nullptr);
        LogPrintf("Finished database compaction of %s\n",
                  fs::PathToString(params.path));
    }
 
    // The base-case obfuscation key, which is a noop.
    obfuscate_key = std::vector<uint8_t>(OBFUSCATE_KEY_NUM_BYTES, '\000');
 
    bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);
 
    if (!key_exists && params.obfuscate && IsEmpty()) {
        // Initialize non-degenerate obfuscation if it won't upset existing,
        // non-obfuscated data.
        std::vector<uint8_t> new_key = CreateObfuscateKey();
 
        // Write `new_key` so we don't obfuscate the key with itself
        Write(OBFUSCATE_KEY_KEY, new_key);
        obfuscate_key = new_key;
 
        LogPrintf("Wrote new obfuscate key for %s: %s\n",
                  fs::PathToString(params.path), HexStr(obfuscate_key));
    }
 
    LogPrintf("Using obfuscation key for %s: %s\n",
              fs::PathToString(params.path), HexStr(obfuscate_key));
}
 
CDBWrapper::~CDBWrapper() {
    delete pdb;
    pdb = nullptr;
    delete options.filter_policy;
    options.filter_policy = nullptr;
    delete options.info_log;
    options.info_log = nullptr;
    delete options.block_cache;
    options.block_cache = nullptr;
    delete penv;
    options.env = nullptr;
}
 
void CDBWrapper::WriteBatch(CDBBatch &batch, bool fSync) {
    const bool log_memory =
        LogAcceptCategory(BCLog::LEVELDB, BCLog::Level::Debug);
    double mem_before = 0;
    if (log_memory) {
        mem_before = DynamicMemoryUsage() / 1024.0 / 1024;
    }
    leveldb::Status status =
        pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch);
    dbwrapper_private::HandleError(status);
    if (log_memory) {
        double mem_after = DynamicMemoryUsage() / 1024.0 / 1024;
        LogPrint(
            BCLog::LEVELDB,
            "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
            m_name, mem_before, mem_after);
    }
}
 
size_t CDBWrapper::DynamicMemoryUsage() const {
    std::string memory;
    if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory)) {
        LogPrint(BCLog::LEVELDB,
                 "Failed to get approximate-memory-usage property\n");
        return 0;
    }
    return stoul(memory);
}
 
// Prefixed with null character to avoid collisions with other keys
//
// We must use a string constructor which specifies length so that we copy past
// the null-terminator.
const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);
 
const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
 
std::vector<uint8_t> CDBWrapper::CreateObfuscateKey() const {
    std::vector<uint8_t> ret(OBFUSCATE_KEY_NUM_BYTES);
    GetRandBytes(ret);
    return ret;
}
 
bool CDBWrapper::IsEmpty() {
    std::unique_ptr<CDBIterator> it(NewIterator());
    it->SeekToFirst();
    return !(it->Valid());
}
 
CDBIterator::~CDBIterator() {
    delete piter;
}
bool CDBIterator::Valid() const {
    return piter->Valid();
}
void CDBIterator::SeekToFirst() {
    piter->SeekToFirst();
}
void CDBIterator::Next() {
    piter->Next();
}
 
namespace dbwrapper_private {
 
void HandleError(const leveldb::Status &status) {
    if (status.ok()) {
        return;
    }
    const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
    LogPrintf("%s\n", errmsg);
    LogPrintf("You can use -debug=leveldb to get more complete diagnostic "
              "messages\n");
    throw dbwrapper_error(errmsg);
}
 
const std::vector<uint8_t> &GetObfuscateKey(const CDBWrapper &w) {
    return w.obfuscate_key;
}
}; // namespace dbwrapper_private