Bitcoin ABC  0.29.4
P2P Digital Currency
serialize.h
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2016 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #ifndef BITCOIN_SERIALIZE_H
7 #define BITCOIN_SERIALIZE_H
8 
9 #include <compat/endian.h>
10 #include <prevector.h>
11 #include <rcu.h>
12 #include <span.h>
13 
14 #include <algorithm>
15 #include <array>
16 #include <cstdint>
17 #include <cstring>
18 #include <ios>
19 #include <limits>
20 #include <map>
21 #include <memory>
22 #include <set>
23 #include <string>
24 #include <utility>
25 #include <vector>
26 
31 static constexpr uint64_t MAX_SIZE = 0x02000000;
32 
37 static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;
38 
49 struct deserialize_type {};
51 
55 template <typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj) {
56  s.write(AsBytes(Span{&obj, 1}));
57 }
58 template <typename Stream>
59 inline void ser_writedata16(Stream &s, uint16_t obj) {
60  obj = htole16(obj);
61  s.write(AsBytes(Span{&obj, 1}));
62 }
63 template <typename Stream>
64 inline void ser_writedata16be(Stream &s, uint16_t obj) {
65  obj = htobe16(obj);
66  s.write(AsBytes(Span{&obj, 1}));
67 }
68 template <typename Stream>
69 inline void ser_writedata32(Stream &s, uint32_t obj) {
70  obj = htole32(obj);
71  s.write(AsBytes(Span{&obj, 1}));
72 }
73 template <typename Stream>
74 inline void ser_writedata32be(Stream &s, uint32_t obj) {
75  obj = htobe32(obj);
76  s.write(AsBytes(Span{&obj, 1}));
77 }
78 template <typename Stream>
79 inline void ser_writedata64(Stream &s, uint64_t obj) {
80  obj = htole64(obj);
81  s.write(AsBytes(Span{&obj, 1}));
82 }
83 template <typename Stream> inline uint8_t ser_readdata8(Stream &s) {
84  uint8_t obj;
85  s.read(AsWritableBytes(Span{&obj, 1}));
86  return obj;
87 }
88 template <typename Stream> inline uint16_t ser_readdata16(Stream &s) {
89  uint16_t obj;
90  s.read(AsWritableBytes(Span{&obj, 1}));
91  return le16toh(obj);
92 }
93 template <typename Stream> inline uint16_t ser_readdata16be(Stream &s) {
94  uint16_t obj;
95  s.read(AsWritableBytes(Span{&obj, 1}));
96  return be16toh(obj);
97 }
98 template <typename Stream> inline uint32_t ser_readdata32(Stream &s) {
99  uint32_t obj;
100  s.read(AsWritableBytes(Span{&obj, 1}));
101  return le32toh(obj);
102 }
103 template <typename Stream> inline uint32_t ser_readdata32be(Stream &s) {
104  uint32_t obj;
105  s.read(AsWritableBytes(Span{&obj, 1}));
106  return be32toh(obj);
107 }
108 template <typename Stream> inline uint64_t ser_readdata64(Stream &s) {
109  uint64_t obj;
110  s.read(AsWritableBytes(Span{&obj, 1}));
111  return le64toh(obj);
112 }
113 inline uint64_t ser_double_to_uint64(double x) {
114  uint64_t tmp;
115  std::memcpy(&tmp, &x, sizeof(x));
116  static_assert(sizeof(tmp) == sizeof(x),
117  "double and uint64_t assumed to have the same size");
118  return tmp;
119 }
120 inline uint32_t ser_float_to_uint32(float x) {
121  uint32_t tmp;
122  std::memcpy(&tmp, &x, sizeof(x));
123  static_assert(sizeof(tmp) == sizeof(x),
124  "float and uint32_t assumed to have the same size");
125  return tmp;
126 }
127 inline double ser_uint64_to_double(uint64_t y) {
128  double tmp;
129  std::memcpy(&tmp, &y, sizeof(y));
130  static_assert(sizeof(tmp) == sizeof(y),
131  "double and uint64_t assumed to have the same size");
132  return tmp;
133 }
134 inline float ser_uint32_to_float(uint32_t y) {
135  float tmp;
136  std::memcpy(&tmp, &y, sizeof(y));
137  static_assert(sizeof(tmp) == sizeof(y),
138  "float and uint32_t assumed to have the same size");
139  return tmp;
140 }
141 
143 //
144 // Templates for serializing to anything that looks like a stream,
145 // i.e. anything that supports .read(Span<std::byte>) and .write(Span<const
146 // std::byte>)
147 //
148 class CSizeComputer;
149 
150 enum {
151  // primary actions
152  SER_NETWORK = (1 << 0),
153  SER_DISK = (1 << 1),
154  SER_GETHASH = (1 << 2),
155 };
156 
159 template <typename X> X &ReadWriteAsHelper(X &x) {
160  return x;
161 }
162 template <typename X> const X &ReadWriteAsHelper(const X &x) {
163  return x;
164 }
165 
166 #define READWRITE(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__))
167 #define READWRITEAS(type, obj) \
168  (::SerReadWriteMany(s, ser_action, ReadWriteAsHelper<type>(obj)))
169 #define SER_READ(obj, code) \
170  ::SerRead( \
171  s, ser_action, obj, \
172  [&](Stream &s, typename std::remove_const<Type>::type &obj) { code; })
173 #define SER_WRITE(obj, code) \
174  ::SerWrite(s, ser_action, obj, [&](Stream &s, const Type &obj) { code; })
175 
194 #define FORMATTER_METHODS(cls, obj) \
195  template <typename Stream> static void Ser(Stream &s, const cls &obj) { \
196  SerializationOps(obj, s, CSerActionSerialize()); \
197  } \
198  template <typename Stream> static void Unser(Stream &s, cls &obj) { \
199  SerializationOps(obj, s, CSerActionUnserialize()); \
200  } \
201  template <typename Stream, typename Type, typename Operation> \
202  static inline void SerializationOps(Type &obj, Stream &s, \
203  Operation ser_action)
204 
213 #define SERIALIZE_METHODS(cls, obj) \
214  template <typename Stream> void Serialize(Stream &s) const { \
215  static_assert(std::is_same<const cls &, decltype(*this)>::value, \
216  "Serialize type mismatch"); \
217  Ser(s, *this); \
218  } \
219  template <typename Stream> void Unserialize(Stream &s) { \
220  static_assert(std::is_same<cls &, decltype(*this)>::value, \
221  "Unserialize type mismatch"); \
222  Unser(s, *this); \
223  } \
224  FORMATTER_METHODS(cls, obj)
225 
226 #ifndef CHAR_EQUALS_INT8
227 // char serialization forbidden. Use uint8_t or int8_t
228 template <typename Stream> void Serialize(Stream &, char) = delete;
229 #endif
230 template <typename Stream> inline void Serialize(Stream &s, int8_t a) {
231  ser_writedata8(s, a);
232 }
233 template <typename Stream> inline void Serialize(Stream &s, uint8_t a) {
234  ser_writedata8(s, a);
235 }
236 template <typename Stream> inline void Serialize(Stream &s, int16_t a) {
237  ser_writedata16(s, a);
238 }
239 template <typename Stream> inline void Serialize(Stream &s, uint16_t a) {
240  ser_writedata16(s, a);
241 }
242 template <typename Stream> inline void Serialize(Stream &s, int32_t a) {
243  ser_writedata32(s, a);
244 }
245 template <typename Stream> inline void Serialize(Stream &s, uint32_t a) {
246  ser_writedata32(s, a);
247 }
248 template <typename Stream> inline void Serialize(Stream &s, int64_t a) {
249  ser_writedata64(s, a);
250 }
251 template <typename Stream> inline void Serialize(Stream &s, uint64_t a) {
252  ser_writedata64(s, a);
253 }
254 template <typename Stream> inline void Serialize(Stream &s, float a) {
256 }
257 template <typename Stream> inline void Serialize(Stream &s, double a) {
259 }
260 template <typename Stream, size_t N>
261 inline void Serialize(Stream &s, const int8_t (&a)[N]) {
262  s.write(a, N);
263 }
264 template <typename Stream, size_t N>
265 inline void Serialize(Stream &s, const uint8_t (&a)[N]) {
266  s.write(MakeByteSpan(a));
267 }
268 template <typename Stream, size_t N>
269 inline void Serialize(Stream &s, const std::array<int8_t, N> &a) {
270  s.write(a.data(), N);
271 }
272 template <typename Stream, size_t N>
273 inline void Serialize(Stream &s, const std::array<uint8_t, N> &a) {
274  s.write(MakeByteSpan(a));
275 }
276 #ifndef CHAR_EQUALS_INT8
277 // char serialization forbidden. Use uint8_t or int8_t
278 template <typename Stream> void Unserialize(Stream &, char) = delete;
279 template <typename Stream, size_t N>
280 inline void Serialize(Stream &s, const char (&a)[N]) {
281  s.write(MakeByteSpan(a));
282 }
283 template <typename Stream, size_t N>
284 inline void Serialize(Stream &s, const std::array<char, N> &a) {
285  s.write(MakeByteSpan(a));
286 }
287 #endif
288 template <typename Stream>
289 inline void Serialize(Stream &s, const Span<const uint8_t> &span) {
290  s.write(AsBytes(span));
291 }
292 template <typename Stream>
293 inline void Serialize(Stream &s, const Span<uint8_t> &span) {
294  s.write(AsBytes(span));
295 }
296 template <typename Stream> inline void Unserialize(Stream &s, int8_t &a) {
297  a = ser_readdata8(s);
298 }
299 template <typename Stream> inline void Unserialize(Stream &s, uint8_t &a) {
300  a = ser_readdata8(s);
301 }
302 template <typename Stream> inline void Unserialize(Stream &s, int16_t &a) {
303  a = ser_readdata16(s);
304 }
305 template <typename Stream> inline void Unserialize(Stream &s, uint16_t &a) {
306  a = ser_readdata16(s);
307 }
308 template <typename Stream> inline void Unserialize(Stream &s, int32_t &a) {
309  a = ser_readdata32(s);
310 }
311 template <typename Stream> inline void Unserialize(Stream &s, uint32_t &a) {
312  a = ser_readdata32(s);
313 }
314 template <typename Stream> inline void Unserialize(Stream &s, int64_t &a) {
315  a = ser_readdata64(s);
316 }
317 template <typename Stream> inline void Unserialize(Stream &s, uint64_t &a) {
318  a = ser_readdata64(s);
319 }
320 template <typename Stream> inline void Unserialize(Stream &s, float &a) {
322 }
323 template <typename Stream> inline void Unserialize(Stream &s, double &a) {
325 }
326 template <typename Stream, size_t N>
327 inline void Unserialize(Stream &s, int8_t (&a)[N]) {
328  s.read(MakeWritableByteSpan(a));
329 }
330 template <typename Stream, size_t N>
331 inline void Unserialize(Stream &s, uint8_t (&a)[N]) {
332  s.read(MakeWritableByteSpan(a));
333 }
334 template <typename Stream, size_t N>
335 inline void Unserialize(Stream &s, std::array<int8_t, N> &a) {
336  s.read(a.data(), N);
337 }
338 template <typename Stream, size_t N>
339 inline void Unserialize(Stream &s, std::array<uint8_t, N> &a) {
340  s.read(MakeWritableByteSpan(a));
341 }
342 #ifndef CHAR_EQUALS_INT8
343 template <typename Stream, size_t N>
344 inline void Unserialize(Stream &s, char (&a)[N]) {
345  s.read(MakeWritableByteSpan(a));
346 }
347 template <typename Stream, size_t N>
348 inline void Unserialize(Stream &s, std::array<char, N> &a) {
349  s.read(MakeWritableByteSpan(a));
350 }
351 #endif
352 
353 template <typename Stream> inline void Serialize(Stream &s, bool a) {
354  char f = a;
355  ser_writedata8(s, f);
356 }
357 template <typename Stream> inline void Unserialize(Stream &s, bool &a) {
358  char f = ser_readdata8(s);
359  a = f;
360 }
361 template <typename Stream>
362 inline void Unserialize(Stream &s, Span<uint8_t> &span) {
363  s.read(AsWritableBytes(span));
364 }
365 
373 inline uint32_t GetSizeOfCompactSize(uint64_t nSize) {
374  if (nSize < 253) {
375  return sizeof(uint8_t);
376  }
377  if (nSize <= std::numeric_limits<uint16_t>::max()) {
378  return sizeof(uint8_t) + sizeof(uint16_t);
379  }
380  if (nSize <= std::numeric_limits<uint32_t>::max()) {
381  return sizeof(uint8_t) + sizeof(uint32_t);
382  }
383 
384  return sizeof(uint8_t) + sizeof(uint64_t);
385 }
386 
387 inline void WriteCompactSize(CSizeComputer &os, uint64_t nSize);
388 
389 template <typename Stream> void WriteCompactSize(Stream &os, uint64_t nSize) {
390  if (nSize < 253) {
391  ser_writedata8(os, nSize);
392  } else if (nSize <= std::numeric_limits<uint16_t>::max()) {
393  ser_writedata8(os, 253);
394  ser_writedata16(os, nSize);
395  } else if (nSize <= std::numeric_limits<uint32_t>::max()) {
396  ser_writedata8(os, 254);
397  ser_writedata32(os, nSize);
398  } else {
399  ser_writedata8(os, 255);
400  ser_writedata64(os, nSize);
401  }
402  return;
403 }
404 
412 template <typename Stream>
413 uint64_t ReadCompactSize(Stream &is, bool range_check = true) {
414  uint8_t chSize = ser_readdata8(is);
415  uint64_t nSizeRet = 0;
416  if (chSize < 253) {
417  nSizeRet = chSize;
418  } else if (chSize == 253) {
419  nSizeRet = ser_readdata16(is);
420  if (nSizeRet < 253) {
421  throw std::ios_base::failure("non-canonical ReadCompactSize()");
422  }
423  } else if (chSize == 254) {
424  nSizeRet = ser_readdata32(is);
425  if (nSizeRet < 0x10000u) {
426  throw std::ios_base::failure("non-canonical ReadCompactSize()");
427  }
428  } else {
429  nSizeRet = ser_readdata64(is);
430  if (nSizeRet < 0x100000000ULL) {
431  throw std::ios_base::failure("non-canonical ReadCompactSize()");
432  }
433  }
434  if (range_check && nSizeRet > MAX_SIZE) {
435  throw std::ios_base::failure("ReadCompactSize(): size too large");
436  }
437  return nSizeRet;
438 }
439 
475 
476 template <VarIntMode Mode, typename I> struct CheckVarIntMode {
477  constexpr CheckVarIntMode() {
478  static_assert(Mode != VarIntMode::DEFAULT || std::is_unsigned<I>::value,
479  "Unsigned type required with mode DEFAULT.");
480  static_assert(Mode != VarIntMode::NONNEGATIVE_SIGNED ||
481  std::is_signed<I>::value,
482  "Signed type required with mode NONNEGATIVE_SIGNED.");
483  }
484 };
485 
486 template <VarIntMode Mode, typename I>
487 inline unsigned int GetSizeOfVarInt(I n) {
489  int nRet = 0;
490  while (true) {
491  nRet++;
492  if (n <= 0x7F) {
493  return nRet;
494  }
495  n = (n >> 7) - 1;
496  }
497 }
498 
499 template <typename I> inline void WriteVarInt(CSizeComputer &os, I n);
500 
501 template <typename Stream, VarIntMode Mode, typename I>
502 void WriteVarInt(Stream &os, I n) {
504  uint8_t tmp[(sizeof(n) * 8 + 6) / 7];
505  int len = 0;
506  while (true) {
507  tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00);
508  if (n <= 0x7F) {
509  break;
510  }
511  n = (n >> 7) - 1;
512  len++;
513  }
514  do {
515  ser_writedata8(os, tmp[len]);
516  } while (len--);
517 }
518 
519 template <typename Stream, VarIntMode Mode, typename I>
520 I ReadVarInt(Stream &is) {
522  I n = 0;
523  while (true) {
524  uint8_t chData = ser_readdata8(is);
525  if (n > (std::numeric_limits<I>::max() >> 7)) {
526  throw std::ios_base::failure("ReadVarInt(): size too large");
527  }
528  n = (n << 7) | (chData & 0x7F);
529  if ((chData & 0x80) == 0) {
530  return n;
531  }
532  if (n == std::numeric_limits<I>::max()) {
533  throw std::ios_base::failure("ReadVarInt(): size too large");
534  }
535  n++;
536  }
537 }
538 
543 template <typename Formatter, typename T> class Wrapper {
544  static_assert(std::is_lvalue_reference<T>::value,
545  "Wrapper needs an lvalue reference type T");
546 
547 protected:
549 
550 public:
551  explicit Wrapper(T obj) : m_object(obj) {}
552  template <typename Stream> void Serialize(Stream &s) const {
553  Formatter().Ser(s, m_object);
554  }
555  template <typename Stream> void Unserialize(Stream &s) {
556  Formatter().Unser(s, m_object);
557  }
558 };
559 
573 template <typename Formatter, typename T>
574 static inline Wrapper<Formatter, T &> Using(T &&t) {
575  return Wrapper<Formatter, T &>(t);
576 }
577 
578 #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj)
579 #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj)
580 #define COMPACTSIZE(obj) Using<CompactSizeFormatter<true>>(obj)
581 #define LIMITED_STRING(obj, n) Using<LimitedStringFormatter<n>>(obj)
582 
586 template <VarIntMode Mode> struct VarIntFormatter {
587  template <typename Stream, typename I> void Ser(Stream &s, I v) {
588  WriteVarInt<Stream, Mode, typename std::remove_cv<I>::type>(s, v);
589  }
590 
591  template <typename Stream, typename I> void Unser(Stream &s, I &v) {
592  v = ReadVarInt<Stream, Mode, typename std::remove_cv<I>::type>(s);
593  }
594 };
595 
606 template <int Bytes, bool BigEndian = false> struct CustomUintFormatter {
607  static_assert(Bytes > 0 && Bytes <= 8,
608  "CustomUintFormatter Bytes out of range");
609  static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes));
610 
611  template <typename Stream, typename I> void Ser(Stream &s, I v) {
612  if (v < 0 || v > MAX) {
613  throw std::ios_base::failure(
614  "CustomUintFormatter value out of range");
615  }
616  if (BigEndian) {
617  uint64_t raw = htobe64(v);
618  s.write({BytePtr(&raw) + 8 - Bytes, Bytes});
619  } else {
620  uint64_t raw = htole64(v);
621  s.write({BytePtr(&raw), Bytes});
622  }
623  }
624 
625  template <typename Stream, typename I> void Unser(Stream &s, I &v) {
626  using U = typename std::conditional<std::is_enum<I>::value,
627  std::underlying_type<I>,
628  std::common_type<I>>::type::type;
629  static_assert(std::numeric_limits<U>::max() >= MAX &&
630  std::numeric_limits<U>::min() <= 0,
631  "Assigned type too small");
632  uint64_t raw = 0;
633  if (BigEndian) {
634  s.read({BytePtr(&raw) + 8 - Bytes, Bytes});
635  v = static_cast<I>(be64toh(raw));
636  } else {
637  s.read({BytePtr(&raw), Bytes});
638  v = static_cast<I>(le64toh(raw));
639  }
640  }
641 };
642 
643 template <int Bytes>
645 
647 template <bool RangeCheck> struct CompactSizeFormatter {
648  template <typename Stream, typename I> void Unser(Stream &s, I &v) {
649  uint64_t n = ReadCompactSize<Stream>(s, RangeCheck);
650  if (n < std::numeric_limits<I>::min() ||
651  n > std::numeric_limits<I>::max()) {
652  throw std::ios_base::failure("CompactSize exceeds limit of type");
653  }
654  v = n;
655  }
656 
657  template <typename Stream, typename I> void Ser(Stream &s, I v) {
658  static_assert(std::is_unsigned<I>::value,
659  "CompactSize only supported for unsigned integers");
660  static_assert(std::numeric_limits<I>::max() <=
661  std::numeric_limits<uint64_t>::max(),
662  "CompactSize only supports 64-bit integers and below");
663 
664  WriteCompactSize<Stream>(s, v);
665  }
666 };
667 
668 template <typename U, bool LOSSY = false> struct ChronoFormatter {
669  template <typename Stream, typename Tp> void Unser(Stream &s, Tp &tp) {
670  U u;
671  s >> u;
672  // Lossy deserialization does not make sense, so force Wnarrowing
673  tp = Tp{typename Tp::duration{typename Tp::duration::rep{u}}};
674  }
675  template <typename Stream, typename Tp> void Ser(Stream &s, Tp tp) {
676  if constexpr (LOSSY) {
677  s << U(tp.time_since_epoch().count());
678  } else {
679  s << U{tp.time_since_epoch().count()};
680  }
681  }
682 };
683 template <typename U> using LossyChronoFormatter = ChronoFormatter<U, true>;
684 
685 template <size_t Limit> struct LimitedStringFormatter {
686  template <typename Stream> void Unser(Stream &s, std::string &v) {
687  size_t size = ReadCompactSize(s);
688  if (size > Limit) {
689  throw std::ios_base::failure("String length limit exceeded");
690  }
691  v.resize(size);
692  if (size != 0) {
693  s.read(MakeWritableByteSpan(v));
694  }
695  }
696 
697  template <typename Stream> void Ser(Stream &s, const std::string &v) {
698  s << v;
699  }
700 };
701 
718 template <class Formatter> struct VectorFormatter {
719  template <typename Stream, typename V> void Ser(Stream &s, const V &v) {
720  Formatter formatter;
721  WriteCompactSize(s, v.size());
722  for (const typename V::value_type &elem : v) {
723  formatter.Ser(s, elem);
724  }
725  }
726 
727  template <typename Stream, typename V> void Unser(Stream &s, V &v) {
728  Formatter formatter;
729  v.clear();
730  size_t size = ReadCompactSize(s);
731  size_t allocated = 0;
732  while (allocated < size) {
733  // For DoS prevention, do not blindly allocate as much as the stream
734  // claims to contain. Instead, allocate in 5MiB batches, so that an
735  // attacker actually needs to provide X MiB of data to make us
736  // allocate X+5 Mib.
737  static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE,
738  "Vector element size too large");
739  allocated =
740  std::min(size, allocated + MAX_VECTOR_ALLOCATE /
741  sizeof(typename V::value_type));
742  v.reserve(allocated);
743  while (v.size() < allocated) {
744  v.emplace_back();
745  formatter.Unser(s, v.back());
746  }
747  }
748  };
749 };
750 
762  uint64_t m_shift = 0;
763 
764 public:
765  template <typename Stream, typename I> void Ser(Stream &s, I v) {
766  if (v < m_shift || v >= std::numeric_limits<uint64_t>::max()) {
767  throw std::ios_base::failure("differential value overflow");
768  }
769  WriteCompactSize(s, v - m_shift);
770  m_shift = uint64_t(v) + 1;
771  }
772  template <typename Stream, typename I> void Unser(Stream &s, I &v) {
773  uint64_t n = ReadCompactSize(s);
774  m_shift += n;
775  if (m_shift < n || m_shift >= std::numeric_limits<uint64_t>::max() ||
776  m_shift < std::numeric_limits<I>::min() ||
777  m_shift > std::numeric_limits<I>::max()) {
778  throw std::ios_base::failure("differential value overflow");
779  }
780  v = I(m_shift++);
781  }
782 };
783 
795  template <typename Stream, typename T> void Ser(Stream &s, T v) {
796  DifferenceFormatter::Ser(s, v.index);
797  v.SerData(s);
798  }
799 
800  template <typename Stream, typename T> void Unser(Stream &s, T &v) {
801  DifferenceFormatter::Unser(s, v.index);
802  v.UnserData(s);
803  }
804 };
805 
813 template <typename Stream, typename C>
814 void Serialize(Stream &os, const std::basic_string<C> &str);
815 template <typename Stream, typename C>
816 void Unserialize(Stream &is, std::basic_string<C> &str);
817 
823 template <typename Stream, unsigned int N, typename T>
824 void Serialize_impl(Stream &os, const prevector<N, T> &v, const uint8_t &);
825 template <typename Stream, unsigned int N, typename T, typename V>
826 void Serialize_impl(Stream &os, const prevector<N, T> &v, const V &);
827 template <typename Stream, unsigned int N, typename T>
828 inline void Serialize(Stream &os, const prevector<N, T> &v);
829 template <typename Stream, unsigned int N, typename T>
830 void Unserialize_impl(Stream &is, prevector<N, T> &v, const uint8_t &);
831 template <typename Stream, unsigned int N, typename T, typename V>
832 void Unserialize_impl(Stream &is, prevector<N, T> &v, const V &);
833 template <typename Stream, unsigned int N, typename T>
834 inline void Unserialize(Stream &is, prevector<N, T> &v);
835 
841 template <typename Stream, typename T, typename A>
842 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const uint8_t &);
843 template <typename Stream, typename T, typename A>
844 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const bool &);
845 template <typename Stream, typename T, typename A, typename V>
846 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const V &);
847 template <typename Stream, typename T, typename A>
848 inline void Serialize(Stream &os, const std::vector<T, A> &v);
849 template <typename Stream, typename T, typename A>
850 void Unserialize_impl(Stream &is, std::vector<T, A> &v, const uint8_t &);
851 template <typename Stream, typename T, typename A, typename V>
852 void Unserialize_impl(Stream &is, std::vector<T, A> &v, const V &);
853 template <typename Stream, typename T, typename A>
854 inline void Unserialize(Stream &is, std::vector<T, A> &v);
855 
859 template <typename Stream, typename K, typename T>
860 void Serialize(Stream &os, const std::pair<K, T> &item);
861 template <typename Stream, typename K, typename T>
862 void Unserialize(Stream &is, std::pair<K, T> &item);
863 
867 template <typename Stream, typename K, typename T, typename Pred, typename A>
868 void Serialize(Stream &os, const std::map<K, T, Pred, A> &m);
869 template <typename Stream, typename K, typename T, typename Pred, typename A>
870 void Unserialize(Stream &is, std::map<K, T, Pred, A> &m);
871 
875 template <typename Stream, typename K, typename Pred, typename A>
876 void Serialize(Stream &os, const std::set<K, Pred, A> &m);
877 template <typename Stream, typename K, typename Pred, typename A>
878 void Unserialize(Stream &is, std::set<K, Pred, A> &m);
879 
883 template <typename Stream, typename T>
884 void Serialize(Stream &os, const std::shared_ptr<const T> &p);
885 template <typename Stream, typename T>
886 void Unserialize(Stream &os, std::shared_ptr<const T> &p);
887 
891 template <typename Stream, typename T>
892 void Serialize(Stream &os, const std::unique_ptr<const T> &p);
893 template <typename Stream, typename T>
894 void Unserialize(Stream &os, std::unique_ptr<const T> &p);
895 
899 template <typename Stream, typename T>
900 void Serialize(Stream &os, const RCUPtr<const T> &p);
901 template <typename Stream, typename T>
902 void Unserialize(Stream &os, RCUPtr<const T> &p);
903 
908 template <typename Stream, typename T>
909 inline void Serialize(Stream &os, const T &a) {
910  a.Serialize(os);
911 }
912 
913 template <typename Stream, typename T>
914 inline void Unserialize(Stream &is, T &&a) {
915  a.Unserialize(is);
916 }
917 
925  template <typename Stream, typename T>
926  static void Ser(Stream &s, const T &t) {
927  Serialize(s, t);
928  }
929 
930  template <typename Stream, typename T> static void Unser(Stream &s, T &t) {
931  Unserialize(s, t);
932  }
933 };
934 
938 template <typename Stream, typename C>
939 void Serialize(Stream &os, const std::basic_string<C> &str) {
940  WriteCompactSize(os, str.size());
941  if (!str.empty()) {
942  os.write(MakeByteSpan(str));
943  }
944 }
945 
946 template <typename Stream, typename C>
947 void Unserialize(Stream &is, std::basic_string<C> &str) {
948  size_t nSize = ReadCompactSize(is);
949  str.resize(nSize);
950  if (nSize != 0) {
951  is.read(MakeWritableByteSpan(str));
952  }
953 }
954 
958 template <typename Stream, unsigned int N, typename T>
959 void Serialize_impl(Stream &os, const prevector<N, T> &v, const uint8_t &) {
960  WriteCompactSize(os, v.size());
961  if (!v.empty()) {
962  os.write(MakeByteSpan(v));
963  }
964 }
965 
966 template <typename Stream, unsigned int N, typename T, typename V>
967 void Serialize_impl(Stream &os, const prevector<N, T> &v, const V &) {
969 }
970 
971 template <typename Stream, unsigned int N, typename T>
972 inline void Serialize(Stream &os, const prevector<N, T> &v) {
973  Serialize_impl(os, v, T());
974 }
975 
976 template <typename Stream, unsigned int N, typename T>
977 void Unserialize_impl(Stream &is, prevector<N, T> &v, const uint8_t &) {
978  // Limit size per read so bogus size value won't cause out of memory
979  v.clear();
980  size_t nSize = ReadCompactSize(is);
981  size_t i = 0;
982  while (i < nSize) {
983  size_t blk = std::min(nSize - i, size_t(1 + 4999999 / sizeof(T)));
984  v.resize_uninitialized(i + blk);
985  is.read(AsWritableBytes(Span{&v[i], blk}));
986  i += blk;
987  }
988 }
989 
990 template <typename Stream, unsigned int N, typename T, typename V>
991 void Unserialize_impl(Stream &is, prevector<N, T> &v, const V &) {
993 }
994 
995 template <typename Stream, unsigned int N, typename T>
996 inline void Unserialize(Stream &is, prevector<N, T> &v) {
997  Unserialize_impl(is, v, T());
998 }
999 
1003 template <typename Stream, typename T, typename A>
1004 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const uint8_t &) {
1005  WriteCompactSize(os, v.size());
1006  if (!v.empty()) {
1007  os.write(MakeByteSpan(v));
1008  }
1009 }
1010 
1011 template <typename Stream, typename T, typename A>
1012 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const bool &) {
1013  // A special case for std::vector<bool>, as dereferencing
1014  // std::vector<bool>::const_iterator does not result in a const bool&
1015  // due to std::vector's special casing for bool arguments.
1016  WriteCompactSize(os, v.size());
1017  for (bool elem : v) {
1018  ::Serialize(os, elem);
1019  }
1020 }
1021 
1022 template <typename Stream, typename T, typename A, typename V>
1023 void Serialize_impl(Stream &os, const std::vector<T, A> &v, const V &) {
1025 }
1026 
1027 template <typename Stream, typename T, typename A>
1028 inline void Serialize(Stream &os, const std::vector<T, A> &v) {
1029  Serialize_impl(os, v, T());
1030 }
1031 
1032 template <typename Stream, typename T, typename A>
1033 void Unserialize_impl(Stream &is, std::vector<T, A> &v, const uint8_t &) {
1034  // Limit size per read so bogus size value won't cause out of memory
1035  v.clear();
1036  size_t nSize = ReadCompactSize(is);
1037  size_t i = 0;
1038  while (i < nSize) {
1039  size_t blk = std::min(nSize - i, size_t(1 + 4999999 / sizeof(T)));
1040  v.resize(i + blk);
1041  is.read(AsWritableBytes(Span{&v[i], blk}));
1042  i += blk;
1043  }
1044 }
1045 
1046 template <typename Stream, typename T, typename A, typename V>
1047 void Unserialize_impl(Stream &is, std::vector<T, A> &v, const V &) {
1049 }
1050 
1051 template <typename Stream, typename T, typename A>
1052 inline void Unserialize(Stream &is, std::vector<T, A> &v) {
1053  Unserialize_impl(is, v, T());
1054 }
1055 
1059 template <typename Stream, typename K, typename T>
1060 void Serialize(Stream &os, const std::pair<K, T> &item) {
1061  Serialize(os, item.first);
1062  Serialize(os, item.second);
1063 }
1064 
1065 template <typename Stream, typename K, typename T>
1066 void Unserialize(Stream &is, std::pair<K, T> &item) {
1067  Unserialize(is, item.first);
1068  Unserialize(is, item.second);
1069 }
1070 
1074 template <typename Stream, typename K, typename T, typename Pred, typename A>
1075 void Serialize(Stream &os, const std::map<K, T, Pred, A> &m) {
1076  WriteCompactSize(os, m.size());
1077  for (const auto &entry : m) {
1078  Serialize(os, entry);
1079  }
1080 }
1081 
1082 template <typename Stream, typename K, typename T, typename Pred, typename A>
1083 void Unserialize(Stream &is, std::map<K, T, Pred, A> &m) {
1084  m.clear();
1085  size_t nSize = ReadCompactSize(is);
1086  typename std::map<K, T, Pred, A>::iterator mi = m.begin();
1087  for (size_t i = 0; i < nSize; i++) {
1088  std::pair<K, T> item;
1089  Unserialize(is, item);
1090  mi = m.insert(mi, item);
1091  }
1092 }
1093 
1097 template <typename Stream, typename K, typename Pred, typename A>
1098 void Serialize(Stream &os, const std::set<K, Pred, A> &m) {
1099  WriteCompactSize(os, m.size());
1100  for (const K &i : m) {
1101  Serialize(os, i);
1102  }
1103 }
1104 
1105 template <typename Stream, typename K, typename Pred, typename A>
1106 void Unserialize(Stream &is, std::set<K, Pred, A> &m) {
1107  m.clear();
1108  size_t nSize = ReadCompactSize(is);
1109  typename std::set<K, Pred, A>::iterator it = m.begin();
1110  for (size_t i = 0; i < nSize; i++) {
1111  K key;
1112  Unserialize(is, key);
1113  it = m.insert(it, key);
1114  }
1115 }
1116 
1120 template <typename Stream, typename T>
1121 void Serialize(Stream &os, const std::unique_ptr<const T> &p) {
1122  Serialize(os, *p);
1123 }
1124 
1125 template <typename Stream, typename T>
1126 void Unserialize(Stream &is, std::unique_ptr<const T> &p) {
1127  p.reset(new T(deserialize, is));
1128 }
1129 
1133 template <typename Stream, typename T>
1134 void Serialize(Stream &os, const std::shared_ptr<const T> &p) {
1135  Serialize(os, *p);
1136 }
1137 
1138 template <typename Stream, typename T>
1139 void Unserialize(Stream &is, std::shared_ptr<const T> &p) {
1140  p = std::make_shared<const T>(deserialize, is);
1141 }
1142 
1146 template <typename Stream, typename T>
1147 void Serialize(Stream &os, const RCUPtr<const T> &p) {
1148  Serialize(os, *p);
1149 }
1150 
1151 template <typename Stream, typename T>
1152 void Unserialize(Stream &is, RCUPtr<const T> &p) {
1154 }
1155 
1160  constexpr bool ForRead() const { return false; }
1161 };
1163  constexpr bool ForRead() const { return true; }
1164 };
1165 
1179 protected:
1180  size_t nSize;
1181 
1182  const int nVersion;
1183 
1184 public:
1185  explicit CSizeComputer(int nVersionIn) : nSize(0), nVersion(nVersionIn) {}
1186 
1187  void write(Span<const std::byte> src) { this->nSize += src.size(); }
1188 
1190  void seek(size_t _nSize) { this->nSize += _nSize; }
1191 
1192  template <typename T> CSizeComputer &operator<<(const T &obj) {
1193  ::Serialize(*this, obj);
1194  return (*this);
1195  }
1196 
1197  size_t size() const { return nSize; }
1198 
1199  int GetVersion() const { return nVersion; }
1200 };
1201 
1202 template <typename Stream> void SerializeMany(Stream &s) {}
1203 
1204 template <typename Stream, typename Arg, typename... Args>
1205 void SerializeMany(Stream &s, const Arg &arg, const Args &...args) {
1206  ::Serialize(s, arg);
1207  ::SerializeMany(s, args...);
1208 }
1209 
1210 template <typename Stream> inline void UnserializeMany(Stream &s) {}
1211 
1212 template <typename Stream, typename Arg, typename... Args>
1213 inline void UnserializeMany(Stream &s, Arg &&arg, Args &&...args) {
1214  ::Unserialize(s, arg);
1215  ::UnserializeMany(s, args...);
1216 }
1217 
1218 template <typename Stream, typename... Args>
1219 inline void SerReadWriteMany(Stream &s, CSerActionSerialize ser_action,
1220  const Args &...args) {
1221  ::SerializeMany(s, args...);
1222 }
1223 
1224 template <typename Stream, typename... Args>
1225 inline void SerReadWriteMany(Stream &s, CSerActionUnserialize ser_action,
1226  Args &&...args) {
1227  ::UnserializeMany(s, args...);
1228 }
1229 
1230 template <typename Stream, typename Type, typename Fn>
1231 inline void SerRead(Stream &s, CSerActionSerialize ser_action, Type &&, Fn &&) {
1232 }
1233 
1234 template <typename Stream, typename Type, typename Fn>
1235 inline void SerRead(Stream &s, CSerActionUnserialize ser_action, Type &&obj,
1236  Fn &&fn) {
1237  fn(s, std::forward<Type>(obj));
1238 }
1239 
1240 template <typename Stream, typename Type, typename Fn>
1241 inline void SerWrite(Stream &s, CSerActionSerialize ser_action, Type &&obj,
1242  Fn &&fn) {
1243  fn(s, std::forward<Type>(obj));
1244 }
1245 
1246 template <typename Stream, typename Type, typename Fn>
1247 inline void SerWrite(Stream &s, CSerActionUnserialize ser_action, Type &&,
1248  Fn &&) {}
1249 
1250 template <typename I> inline void WriteVarInt(CSizeComputer &s, I n) {
1251  s.seek(GetSizeOfVarInt<I>(n));
1252 }
1253 
1254 inline void WriteCompactSize(CSizeComputer &s, uint64_t nSize) {
1255  s.seek(GetSizeOfCompactSize(nSize));
1256 }
1257 
1258 template <typename T> size_t GetSerializeSize(const T &t, int nVersion = 0) {
1259  return (CSizeComputer(nVersion) << t).size();
1260 }
1261 
1262 template <typename... T>
1263 size_t GetSerializeSizeMany(int nVersion, const T &...t) {
1264  CSizeComputer sc(nVersion);
1265  SerializeMany(sc, t...);
1266  return sc.size();
1267 }
1268 
1269 #endif // BITCOIN_SERIALIZE_H
GetSerializeSize implementations
Definition: serialize.h:1178
CSizeComputer & operator<<(const T &obj)
Definition: serialize.h:1192
void write(Span< const std::byte > src)
Definition: serialize.h:1187
CSizeComputer(int nVersionIn)
Definition: serialize.h:1185
size_t nSize
Definition: serialize.h:1180
size_t size() const
Definition: serialize.h:1197
const int nVersion
Definition: serialize.h:1182
void seek(size_t _nSize)
Pretend _nSize bytes are written, without specifying them.
Definition: serialize.h:1190
int GetVersion() const
Definition: serialize.h:1199
Helper for differentially encoded Compact Size integers in lists.
Definition: serialize.h:761
void Unser(Stream &s, I &v)
Definition: serialize.h:772
void Ser(Stream &s, I v)
Definition: serialize.h:765
Definition: rcu.h:85
static RCUPtr make(Args &&...args)
Construct a new object that is owned by the pointer.
Definition: rcu.h:112
A Span is an object that can refer to a contiguous sequence of objects.
Definition: span.h:93
constexpr std::size_t size() const noexcept
Definition: span.h:209
Simple wrapper class to serialize objects using a formatter; used by Using().
Definition: serialize.h:543
Wrapper(T obj)
Definition: serialize.h:551
T m_object
Definition: serialize.h:545
void Serialize(Stream &s) const
Definition: serialize.h:552
void Unserialize(Stream &s)
Definition: serialize.h:555
Implements a drop-in replacement for std::vector<T> which stores up to N elements directly (without h...
Definition: prevector.h:38
bool empty() const
Definition: prevector.h:388
void clear()
Definition: prevector.h:441
size_type size() const
Definition: prevector.h:386
void resize_uninitialized(size_type new_size)
Definition: prevector.h:482
uint16_t be16toh(uint16_t big_endian_16bits)
Definition: endian.h:156
uint16_t htobe16(uint16_t host_16bits)
Definition: endian.h:144
uint32_t le32toh(uint32_t little_endian_32bits)
Definition: endian.h:186
uint32_t htobe32(uint32_t host_32bits)
Definition: endian.h:168
uint16_t le16toh(uint16_t little_endian_16bits)
Definition: endian.h:162
uint64_t htobe64(uint64_t host_64bits)
Definition: endian.h:192
uint64_t be64toh(uint64_t big_endian_64bits)
Definition: endian.h:204
uint32_t be32toh(uint32_t big_endian_32bits)
Definition: endian.h:180
uint64_t htole64(uint64_t host_64bits)
Definition: endian.h:198
uint32_t htole32(uint32_t host_32bits)
Definition: endian.h:174
uint16_t htole16(uint16_t host_16bits)
Definition: endian.h:150
uint64_t le64toh(uint64_t little_endian_64bits)
Definition: endian.h:210
void SerializeMany(Stream &s)
Definition: serialize.h:1202
static const unsigned int MAX_VECTOR_ALLOCATE
Maximum amount of memory (in bytes) to allocate at once when deserializing vectors.
Definition: serialize.h:37
uint8_t ser_readdata8(Stream &s)
Definition: serialize.h:83
float ser_uint32_to_float(uint32_t y)
Definition: serialize.h:134
I ReadVarInt(Stream &is)
Definition: serialize.h:520
void ser_writedata32be(Stream &s, uint32_t obj)
Definition: serialize.h:74
void WriteVarInt(CSizeComputer &os, I n)
Definition: serialize.h:1250
VarIntMode
Variable-length integers: bytes are a MSB base-128 encoding of the number.
Definition: serialize.h:474
@ NONNEGATIVE_SIGNED
void ser_writedata32(Stream &s, uint32_t obj)
Definition: serialize.h:69
size_t GetSerializeSizeMany(int nVersion, const T &...t)
Definition: serialize.h:1263
uint32_t GetSizeOfCompactSize(uint64_t nSize)
Compact Size size < 253 – 1 byte size <= USHRT_MAX – 3 bytes (253 + 2 bytes) size <= UINT_MAX – 5 byt...
Definition: serialize.h:373
static constexpr uint64_t MAX_SIZE
The maximum size of a serialized object in bytes or number of elements (for eg vectors) when the size...
Definition: serialize.h:31
constexpr deserialize_type deserialize
Definition: serialize.h:50
void ser_writedata16(Stream &s, uint16_t obj)
Definition: serialize.h:59
uint32_t ser_float_to_uint32(float x)
Definition: serialize.h:120
static Wrapper< Formatter, T & > Using(T &&t)
Cause serialization/deserialization of an object to be done using a specified formatter class.
Definition: serialize.h:574
X & ReadWriteAsHelper(X &x)
Convert the reference base type to X, without changing constness or reference type.
Definition: serialize.h:159
@ SER_DISK
Definition: serialize.h:153
@ SER_NETWORK
Definition: serialize.h:152
@ SER_GETHASH
Definition: serialize.h:154
void Unserialize(Stream &, char)=delete
void Unserialize_impl(Stream &is, prevector< N, T > &v, const uint8_t &)
Definition: serialize.h:977
void SerReadWriteMany(Stream &s, CSerActionSerialize ser_action, const Args &...args)
Definition: serialize.h:1219
void SerWrite(Stream &s, CSerActionSerialize ser_action, Type &&obj, Fn &&fn)
Definition: serialize.h:1241
void ser_writedata16be(Stream &s, uint16_t obj)
Definition: serialize.h:64
uint16_t ser_readdata16(Stream &s)
Definition: serialize.h:88
uint64_t ser_readdata64(Stream &s)
Definition: serialize.h:108
double ser_uint64_to_double(uint64_t y)
Definition: serialize.h:127
void ser_writedata8(Stream &s, uint8_t obj)
Lowest-level serialization and conversion.
Definition: serialize.h:55
uint64_t ReadCompactSize(Stream &is, bool range_check=true)
Decode a CompactSize-encoded variable-length integer.
Definition: serialize.h:413
unsigned int GetSizeOfVarInt(I n)
Definition: serialize.h:487
uint32_t ser_readdata32(Stream &s)
Definition: serialize.h:98
uint16_t ser_readdata16be(Stream &s)
Definition: serialize.h:93
void SerRead(Stream &s, CSerActionSerialize ser_action, Type &&, Fn &&)
Definition: serialize.h:1231
void Serialize_impl(Stream &os, const prevector< N, T > &v, const uint8_t &)
prevector prevectors of uint8_t are a special case and are intended to be serialized as a single opaq...
Definition: serialize.h:959
size_t GetSerializeSize(const T &t, int nVersion=0)
Definition: serialize.h:1258
void UnserializeMany(Stream &s)
Definition: serialize.h:1210
uint64_t ser_double_to_uint64(double x)
Definition: serialize.h:113
void ser_writedata64(Stream &s, uint64_t obj)
Definition: serialize.h:79
uint32_t ser_readdata32be(Stream &s)
Definition: serialize.h:103
void Serialize(Stream &, char)=delete
void WriteCompactSize(CSizeComputer &os, uint64_t nSize)
Definition: serialize.h:1254
const std::byte * BytePtr(const void *data)
Convert a data pointer to a std::byte data pointer.
Definition: span.h:286
Span< std::byte > AsWritableBytes(Span< T > s) noexcept
Definition: span.h:297
Span< const std::byte > MakeByteSpan(V &&v) noexcept
Definition: span.h:301
Span< const std::byte > AsBytes(Span< T > s) noexcept
Definition: span.h:294
Span< std::byte > MakeWritableByteSpan(V &&v) noexcept
Definition: span.h:304
Support for SERIALIZE_METHODS and READWRITE macro.
Definition: serialize.h:1159
constexpr bool ForRead() const
Definition: serialize.h:1160
constexpr bool ForRead() const
Definition: serialize.h:1163
constexpr CheckVarIntMode()
Definition: serialize.h:477
void Unser(Stream &s, Tp &tp)
Definition: serialize.h:669
void Ser(Stream &s, Tp tp)
Definition: serialize.h:675
Formatter for integers in CompactSize format.
Definition: serialize.h:647
void Unser(Stream &s, I &v)
Definition: serialize.h:648
void Ser(Stream &s, I v)
Definition: serialize.h:657
Serialization wrapper class for custom integers and enums.
Definition: serialize.h:606
static constexpr uint64_t MAX
Definition: serialize.h:609
void Ser(Stream &s, I v)
Definition: serialize.h:611
void Unser(Stream &s, I &v)
Definition: serialize.h:625
Default formatter.
Definition: serialize.h:924
static void Ser(Stream &s, const T &t)
Definition: serialize.h:926
static void Unser(Stream &s, T &t)
Definition: serialize.h:930
Helper for a list of items containing a differentially encoded index as their first member.
Definition: serialize.h:794
void Unser(Stream &s, T &v)
Definition: serialize.h:800
void Ser(Stream &s, T v)
Definition: serialize.h:795
void Unser(Stream &s, std::string &v)
Definition: serialize.h:686
void Ser(Stream &s, const std::string &v)
Definition: serialize.h:697
Serialization wrapper class for integers in VarInt format.
Definition: serialize.h:586
void Ser(Stream &s, I v)
Definition: serialize.h:587
void Unser(Stream &s, I &v)
Definition: serialize.h:591
Formatter to serialize/deserialize vector elements using another formatter.
Definition: serialize.h:718
void Unser(Stream &s, V &v)
Definition: serialize.h:727
void Ser(Stream &s, const V &v)
Definition: serialize.h:719
Dummy data type to identify deserializing constructors.
Definition: serialize.h:49