Bitcoin ABC  0.22.13
P2P Digital Currency
lockedpool.cpp
Go to the documentation of this file.
1 // Copyright (c) 2016 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #include <support/cleanse.h>
6 #include <support/lockedpool.h>
7 
8 #if defined(HAVE_CONFIG_H)
9 #include <config/bitcoin-config.h>
10 #endif
11 
12 #ifdef WIN32
13 #ifndef NOMINMAX
14 #define NOMINMAX
15 #endif
16 #include <windows.h>
17 #else
18 #include <climits> // for PAGESIZE
19 #include <sys/mman.h> // for mmap
20 #include <sys/resource.h> // for getrlimit
21 #include <unistd.h> // for sysconf
22 #endif
23 
24 #include <algorithm>
25 #ifdef ARENA_DEBUG
26 #include <iomanip>
27 #include <iostream>
28 #endif
29 
31 std::once_flag LockedPoolManager::init_flag;
32 
33 /*******************************************************************************/
34 // Utilities
35 //
37 static inline size_t align_up(size_t x, size_t align) {
38  return (x + align - 1) & ~(align - 1);
39 }
40 
41 /*******************************************************************************/
42 // Implementation: Arena
43 
44 Arena::Arena(void *base_in, size_t size_in, size_t alignment_in)
45  : base(static_cast<char *>(base_in)),
46  end(static_cast<char *>(base_in) + size_in), alignment(alignment_in) {
47  // Start with one free chunk that covers the entire arena
48  auto it = size_to_free_chunk.emplace(size_in, base);
49  chunks_free.emplace(base, it);
50  chunks_free_end.emplace(base + size_in, it);
51 }
52 
54 
55 void *Arena::alloc(size_t size) {
56  // Round to next multiple of alignment
57  size = align_up(size, alignment);
58 
59  // Don't handle zero-sized chunks
60  if (size == 0) {
61  return nullptr;
62  }
63 
64  // Pick a large enough free-chunk. Returns an iterator pointing to the first
65  // element that is not less than key. This allocation strategy is best-fit.
66  // According to "Dynamic Storage Allocation: A Survey and Critical Review",
67  // Wilson et. al. 1995,
68  // http://www.scs.stanford.edu/14wi-cs140/sched/readings/wilson.pdf,
69  // best-fit and first-fit policies seem to work well in practice.
70  auto size_ptr_it = size_to_free_chunk.lower_bound(size);
71  if (size_ptr_it == size_to_free_chunk.end()) {
72  return nullptr;
73  }
74 
75  // Create the used-chunk, taking its space from the end of the free-chunk
76  const size_t size_remaining = size_ptr_it->first - size;
77  auto alloced =
78  chunks_used.emplace(size_ptr_it->second + size_remaining, size).first;
79  chunks_free_end.erase(size_ptr_it->second + size_ptr_it->first);
80  if (size_ptr_it->first == size) {
81  // whole chunk is used up
82  chunks_free.erase(size_ptr_it->second);
83  } else {
84  // still some memory left in the chunk
85  auto it_remaining =
86  size_to_free_chunk.emplace(size_remaining, size_ptr_it->second);
87  chunks_free[size_ptr_it->second] = it_remaining;
88  chunks_free_end.emplace(size_ptr_it->second + size_remaining,
89  it_remaining);
90  }
91  size_to_free_chunk.erase(size_ptr_it);
92 
93  return reinterpret_cast<void *>(alloced->first);
94 }
95 
96 void Arena::free(void *ptr) {
97  // Freeing the nullptr pointer is OK.
98  if (ptr == nullptr) {
99  return;
100  }
101 
102  // Remove chunk from used map
103  auto i = chunks_used.find(static_cast<char *>(ptr));
104  if (i == chunks_used.end()) {
105  throw std::runtime_error("Arena: invalid or double free");
106  }
107  std::pair<char *, size_t> freed = *i;
108  chunks_used.erase(i);
109 
110  // coalesce freed with previous chunk
111  auto prev = chunks_free_end.find(freed.first);
112  if (prev != chunks_free_end.end()) {
113  freed.first -= prev->second->first;
114  freed.second += prev->second->first;
115  size_to_free_chunk.erase(prev->second);
116  chunks_free_end.erase(prev);
117  }
118 
119  // coalesce freed with chunk after freed
120  auto next = chunks_free.find(freed.first + freed.second);
121  if (next != chunks_free.end()) {
122  freed.second += next->second->first;
123  size_to_free_chunk.erase(next->second);
124  chunks_free.erase(next);
125  }
126 
127  // Add/set space with coalesced free chunk
128  auto it = size_to_free_chunk.emplace(freed.second, freed.first);
129  chunks_free[freed.first] = it;
130  chunks_free_end[freed.first + freed.second] = it;
131 }
132 
134  Arena::Stats r{0, 0, 0, chunks_used.size(), chunks_free.size()};
135  for (const auto &chunk : chunks_used) {
136  r.used += chunk.second;
137  }
138  for (const auto &chunk : chunks_free) {
139  r.free += chunk.second->first;
140  }
141  r.total = r.used + r.free;
142  return r;
143 }
144 
145 #ifdef ARENA_DEBUG
146 static void printchunk(void *base, size_t sz, bool used) {
147  std::cout << "0x" << std::hex << std::setw(16) << std::setfill('0') << base
148  << " 0x" << std::hex << std::setw(16) << std::setfill('0') << sz
149  << " 0x" << used << std::endl;
150 }
151 void Arena::walk() const {
152  for (const auto &chunk : chunks_used) {
153  printchunk(chunk.first, chunk.second, true);
154  }
155  std::cout << std::endl;
156  for (const auto &chunk : chunks_free) {
157  printchunk(chunk.first, chunk.second->first, false);
158  }
159  std::cout << std::endl;
160 }
161 #endif
162 
163 /*******************************************************************************/
164 // Implementation: Win32LockedPageAllocator
165 
166 #ifdef WIN32
167 
170 class Win32LockedPageAllocator : public LockedPageAllocator {
171 public:
172  Win32LockedPageAllocator();
173  void *AllocateLocked(size_t len, bool *lockingSuccess) override;
174  void FreeLocked(void *addr, size_t len) override;
175  size_t GetLimit() override;
176 
177 private:
178  size_t page_size;
179 };
180 
181 Win32LockedPageAllocator::Win32LockedPageAllocator() {
182  // Determine system page size in bytes
183  SYSTEM_INFO sSysInfo;
184  GetSystemInfo(&sSysInfo);
185  page_size = sSysInfo.dwPageSize;
186 }
187 void *Win32LockedPageAllocator::AllocateLocked(size_t len,
188  bool *lockingSuccess) {
189  len = align_up(len, page_size);
190  void *addr =
191  VirtualAlloc(nullptr, len, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
192  if (addr) {
193  // VirtualLock is used to attempt to keep keying material out of swap.
194  // Note that it does not provide this as a guarantee, but, in practice,
195  // memory that has been VirtualLock'd almost never gets written to the
196  // pagefile except in rare circumstances where memory is extremely low.
197  *lockingSuccess = VirtualLock(const_cast<void *>(addr), len) != 0;
198  }
199  return addr;
200 }
201 void Win32LockedPageAllocator::FreeLocked(void *addr, size_t len) {
202  len = align_up(len, page_size);
203  memory_cleanse(addr, len);
204  VirtualUnlock(const_cast<void *>(addr), len);
205 }
206 
207 size_t Win32LockedPageAllocator::GetLimit() {
208  // TODO is there a limit on Windows, how to get it?
209  return std::numeric_limits<size_t>::max();
210 }
211 #endif
212 
213 /*******************************************************************************/
214 // Implementation: PosixLockedPageAllocator
215 
216 #ifndef WIN32
217 
222 public:
224  void *AllocateLocked(size_t len, bool *lockingSuccess) override;
225  void FreeLocked(void *addr, size_t len) override;
226  size_t GetLimit() override;
227 
228 private:
229  size_t page_size;
230 };
231 
233 // Determine system page size in bytes
234 #if defined(PAGESIZE) // defined in climits
235  page_size = PAGESIZE;
236 #else // assume some POSIX OS
237  page_size = sysconf(_SC_PAGESIZE);
238 #endif
239 }
240 
241 // Some systems (at least OS X) do not define MAP_ANONYMOUS yet and define
242 // MAP_ANON which is deprecated
243 #ifndef MAP_ANONYMOUS
244 #define MAP_ANONYMOUS MAP_ANON
245 #endif
246 
248  bool *lockingSuccess) {
249  void *addr;
250  len = align_up(len, page_size);
251  addr = mmap(nullptr, len, PROT_READ | PROT_WRITE,
252  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
253  if (addr == MAP_FAILED) {
254  return nullptr;
255  }
256  if (addr) {
257  *lockingSuccess = mlock(addr, len) == 0;
258 #if defined(MADV_DONTDUMP) // Linux
259  madvise(addr, len, MADV_DONTDUMP);
260 #elif defined(MADV_NOCORE) // FreeBSD
261  madvise(addr, len, MADV_NOCORE);
262 #endif
263  }
264  return addr;
265 }
266 void PosixLockedPageAllocator::FreeLocked(void *addr, size_t len) {
267  len = align_up(len, page_size);
268  memory_cleanse(addr, len);
269  munlock(addr, len);
270  munmap(addr, len);
271 }
273 #ifdef RLIMIT_MEMLOCK
274  struct rlimit rlim;
275  if (getrlimit(RLIMIT_MEMLOCK, &rlim) == 0) {
276  if (rlim.rlim_cur != RLIM_INFINITY) {
277  return rlim.rlim_cur;
278  }
279  }
280 #endif
281  return std::numeric_limits<size_t>::max();
282 }
283 #endif
284 
285 /*******************************************************************************/
286 // Implementation: LockedPool
287 
288 LockedPool::LockedPool(std::unique_ptr<LockedPageAllocator> allocator_in,
289  LockingFailed_Callback lf_cb_in)
290  : allocator(std::move(allocator_in)), lf_cb(lf_cb_in),
291  cumulative_bytes_locked(0) {}
292 
294 void *LockedPool::alloc(size_t size) {
295  std::lock_guard<std::mutex> lock(mutex);
296 
297  // Don't handle impossible sizes
298  if (size == 0 || size > ARENA_SIZE) {
299  return nullptr;
300  }
301 
302  // Try allocating from each current arena
303  for (auto &arena : arenas) {
304  void *addr = arena.alloc(size);
305  if (addr) {
306  return addr;
307  }
308  }
309  // If that fails, create a new one
311  return arenas.back().alloc(size);
312  }
313  return nullptr;
314 }
315 
316 void LockedPool::free(void *ptr) {
317  std::lock_guard<std::mutex> lock(mutex);
318  // TODO we can do better than this linear search by keeping a map of arena
319  // extents to arena, and looking up the address.
320  for (auto &arena : arenas) {
321  if (arena.addressInArena(ptr)) {
322  arena.free(ptr);
323  return;
324  }
325  }
326  throw std::runtime_error(
327  "LockedPool: invalid address not pointing to any arena");
328 }
329 
331  std::lock_guard<std::mutex> lock(mutex);
332  LockedPool::Stats r{0, 0, 0, cumulative_bytes_locked, 0, 0};
333  for (const auto &arena : arenas) {
334  Arena::Stats i = arena.stats();
335  r.used += i.used;
336  r.free += i.free;
337  r.total += i.total;
338  r.chunks_used += i.chunks_used;
339  r.chunks_free += i.chunks_free;
340  }
341  return r;
342 }
343 
344 bool LockedPool::new_arena(size_t size, size_t align) {
345  bool locked;
346  // If this is the first arena, handle this specially: Cap the upper size by
347  // the process limit. This makes sure that the first arena will at least be
348  // locked. An exception to this is if the process limit is 0: in this case
349  // no memory can be locked at all so we'll skip past this logic.
350  if (arenas.empty()) {
351  size_t limit = allocator->GetLimit();
352  if (limit > 0) {
353  size = std::min(size, limit);
354  }
355  }
356  void *addr = allocator->AllocateLocked(size, &locked);
357  if (!addr) {
358  return false;
359  }
360  if (locked) {
361  cumulative_bytes_locked += size;
362  } else if (lf_cb) {
363  // Call the locking-failed callback if locking failed
364  if (!lf_cb()) {
365  // If the callback returns false, free the memory and fail,
366  // otherwise consider the user warned and proceed.
367  allocator->FreeLocked(addr, size);
368  return false;
369  }
370  }
371  arenas.emplace_back(allocator.get(), addr, size, align);
372  return true;
373 }
374 
376  void *base_in, size_t size_in,
377  size_t align_in)
378  : Arena(base_in, size_in, align_in), base(base_in), size(size_in),
379  allocator(allocator_in) {}
382 }
383 
384 /*******************************************************************************/
385 // Implementation: LockedPoolManager
386 //
388  std::unique_ptr<LockedPageAllocator> allocator_in)
389  : LockedPool(std::move(allocator_in), &LockedPoolManager::LockingFailed) {}
390 
392  // TODO: log something but how? without including util.h
393  return true;
394 }
395 
397 // Using a local static instance guarantees that the object is initialized when
398 // it's first needed and also deinitialized after all objects that use it are
399 // done with it. I can think of one unlikely scenario where we may have a static
400 // deinitialization order/problem, but the check in LockedPoolManagerBase's
401 // destructor helps us detect if that ever happens.
402 #ifdef WIN32
403  std::unique_ptr<LockedPageAllocator> allocator(
404  new Win32LockedPageAllocator());
405 #else
406  std::unique_ptr<LockedPageAllocator> allocator(
408 #endif
409  static LockedPoolManager instance(std::move(allocator));
410  LockedPoolManager::_instance = &instance;
411 }
size_t chunks_free
Definition: lockedpool.h:64
size_t chunks_used
Definition: lockedpool.h:63
static std::once_flag init_flag
Definition: lockedpool.h:251
size_t used
Definition: lockedpool.h:60
size_t alignment
Minimum chunk alignment.
Definition: lockedpool.h:115
std::mutex mutex
Mutex protects access to this pool&#39;s data structures, including arenas.
Definition: lockedpool.h:219
void * AllocateLocked(size_t len, bool *lockingSuccess) override
Allocate and lock memory pages.
Definition: lockedpool.cpp:247
std::list< LockedPageArena > arenas
Definition: lockedpool.h:213
static const size_t ARENA_ALIGN
Chunk alignment.
Definition: lockedpool.h:145
std::unordered_map< char *, size_t > chunks_used
Map from begin of used chunk to its size.
Definition: lockedpool.h:108
LockedPool(std::unique_ptr< LockedPageAllocator > allocator, LockingFailed_Callback lf_cb_in=nullptr)
Create a new LockedPool.
Definition: lockedpool.cpp:288
ChunkToSizeMap chunks_free
Map from begin of free chunk to its node in size_to_free_chunk.
Definition: lockedpool.h:103
LockingFailed_Callback lf_cb
Definition: lockedpool.h:214
size_t total
Definition: lockedpool.h:62
SizeToChunkSortedMap size_to_free_chunk
Map to enable O(log(n)) best-fit allocation, as it&#39;s sorted by size.
Definition: lockedpool.h:98
LockedPageArena(LockedPageAllocator *alloc_in, void *base_in, size_t size, size_t align)
Definition: lockedpool.cpp:375
OS-dependent allocation and deallocation of locked/pinned memory pages.
Definition: lockedpool.h:19
LockedPageAllocator * allocator
Definition: lockedpool.h:208
Singleton class to keep track of locked (ie, non-swappable) memory, for use in std::allocator templat...
Definition: lockedpool.h:233
void * alloc(size_t size)
Allocate size bytes from this arena.
Definition: lockedpool.cpp:55
void memory_cleanse(void *ptr, size_t len)
Secure overwrite a buffer (possibly containing secret data) with zero-bytes.
Definition: cleanse.cpp:14
void FreeLocked(void *addr, size_t len) override
Unlock and free memory pages.
Definition: lockedpool.cpp:266
Stats stats() const
Get arena usage statistics.
Definition: lockedpool.cpp:133
static LockedPoolManager * _instance
Definition: lockedpool.h:250
void * alloc(size_t size)
Allocate size bytes from this arena.
Definition: lockedpool.cpp:294
virtual ~Arena()
Definition: lockedpool.cpp:53
virtual void FreeLocked(void *addr, size_t len)=0
Unlock and free memory pages.
static size_t align_up(size_t x, size_t align)
Align up to power of 2.
Definition: lockedpool.cpp:37
static const size_t ARENA_SIZE
Size of one arena of locked memory.
Definition: lockedpool.h:140
static bool LockingFailed()
Called when locking fails, warn the user here.
Definition: lockedpool.cpp:391
size_t free
Definition: lockedpool.h:61
Pool for locked memory chunks.
Definition: lockedpool.h:132
size_t GetLimit() override
Get the total limit on the amount of memory that may be locked by this process, in bytes...
Definition: lockedpool.cpp:272
void free(void *ptr)
Free a previously allocated chunk of memory.
Definition: lockedpool.cpp:96
void free(void *ptr)
Free a previously allocated chunk of memory.
Definition: lockedpool.cpp:316
char * base
Base address of arena.
Definition: lockedpool.h:111
LockedPageAllocator specialized for OSes that don&#39;t try to be special snowflakes. ...
Definition: lockedpool.cpp:221
bool new_arena(size_t size, size_t align)
Definition: lockedpool.cpp:344
Memory statistics.
Definition: lockedpool.h:59
LockedPoolManager(std::unique_ptr< LockedPageAllocator > allocator)
Definition: lockedpool.cpp:387
Stats stats() const
Get pool usage statistics.
Definition: lockedpool.cpp:330
static void CreateInstance()
Create a new LockedPoolManager specialized to the OS.
Definition: lockedpool.cpp:396
An arena manages a contiguous region of memory by dividing it into chunks.
Definition: lockedpool.h:50
size_t cumulative_bytes_locked
Definition: lockedpool.h:215
Arena(void *base, size_t size, size_t alignment)
Definition: lockedpool.cpp:44
ChunkToSizeMap chunks_free_end
Map from end of free chunk to its node in size_to_free_chunk.
Definition: lockedpool.h:105
#define MAP_ANONYMOUS
Definition: lockedpool.cpp:244
std::unique_ptr< LockedPageAllocator > allocator
Definition: lockedpool.h:196
Memory statistics.
Definition: lockedpool.h:153