tdutils: update from other project

GitOrigin-RevId: 4a0a7ed6fff6af9b498122c66de9576939dce523

tdutils/CMakeLists.txt

@@ -16,6 +16,10 @@ if (ZLIB_FOUND)
   endif()
 endif()
 
+if (CRC32C_FOUND)
+  set(TD_HAVE_CRC32C 1)
+endif()
+
 configure_file(td/utils/config.h.in td/utils/config.h @ONLY)
 
 add_subdirectory(generate)
@@ -83,6 +87,7 @@ set(TDUTILS_SOURCE
   td/utils/StringBuilder.cpp
   td/utils/Time.cpp
   td/utils/Timer.cpp
+  td/utils/tests.cpp
   td/utils/tl_parsers.cpp
   td/utils/translit.cpp
   td/utils/unicode.cpp
@@ -131,6 +136,7 @@ set(TDUTILS_SOURCE
   td/utils/base64.h
   td/utils/benchmark.h
   td/utils/BigNum.h
+  td/utils/bits.h
   td/utils/buffer.h
   td/utils/BufferedFd.h
   td/utils/BufferedReader.h
@@ -257,6 +263,10 @@ if (ZLIB_FOUND)
   target_include_directories(tdutils SYSTEM PRIVATE ${ZLIB_INCLUDE_DIR})
 endif()
 
+if (CRC32C_FOUND)
+  target_link_libraries(tdutils PRIVATE crc32c)
+endif()
+
 install(TARGETS tdutils EXPORT TdTargets
   LIBRARY DESTINATION lib
   ARCHIVE DESTINATION lib

tdutils/td/utils/BigNum.cpp

@@ -103,10 +103,12 @@ Result<BigNum> BigNum::from_decimal(CSlice str) {
   return result;
 }
 
-BigNum BigNum::from_hex(CSlice str) {
+Result<BigNum> BigNum::from_hex(CSlice str) {
   BigNum result;
-  int err = BN_hex2bn(&result.impl_->big_num, str.c_str());
+  int res = BN_hex2bn(&result.impl_->big_num, str.c_str());
-  LOG_IF(FATAL, err == 0);
+  if (res == 0 || static_cast<size_t>(res) != str.size()) {
+    return Status::Error(PSLICE() << "Failed to parse \"" << str << "\" as hexadecimal BigNum");
+  }
   return result;
 }
 

tdutils/td/utils/BigNum.h

@@ -48,7 +48,7 @@ class BigNum {
 
   static Result<BigNum> from_decimal(CSlice str);
 
-  static BigNum from_hex(CSlice str);
+  static Result<BigNum> from_hex(CSlice str);
 
   static BigNum from_raw(void *openssl_big_num);
 

tdutils/td/utils/BufferedStdin.cpp

@@ -0,0 +1,20 @@
+//
+// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+#include "td/utils/BufferedStdin.h"
+#include "td/utils/buffer.h"
+#include "td/utils/port/detail/PollableFd.h"
+
+namespace td {
+class BufferedStdin {
+ public:
+ private:
+  PollableFdInfo info_;
+  ChainBufferWriter writer_;
+  ChainBufferReader reader_ = writer_.extract_reader();
+};
+}  // namespace td
+

tdutils/td/utils/BufferedStdin.h

@@ -0,0 +1,9 @@
+//
+// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+#pragma once
+
+namespace td {}

tdutils/td/utils/Closure.h

@@ -68,16 +68,17 @@ class ImmediateClosure {
   friend Delayed;
   using ActorType = ActorT;
 
-  auto run(ActorT *actor) {
-    return mem_call_tuple(actor, std::move(args));
-  }
-
   // no &&. just save references as references.
   explicit ImmediateClosure(FunctionT func, ArgsT... args) : args(func, std::forward<ArgsT>(args)...) {
   }
 
  private:
   std::tuple<FunctionT, ArgsT...> args;
+
+ public:
+  auto run(ActorT *actor) -> decltype(mem_call_tuple(actor, std::move(args))) {
+    return mem_call_tuple(actor, std::move(args));
+  }
 };
 
 template <class ActorT, class ResultT, class... DestArgsT, class... SrcArgsT>
@@ -93,10 +94,6 @@ class DelayedClosure {
   using ActorType = ActorT;
   using Delayed = DelayedClosure<ActorT, FunctionT, ArgsT...>;
 
-  auto run(ActorT *actor) {
-    return mem_call_tuple(actor, std::move(args));
-  }
-
   DelayedClosure clone() const {
     return do_clone(*this);
   }
@@ -145,6 +142,11 @@ class DelayedClosure {
   do_clone(const DelayedClosure<FromActorT, FromFunctionT, FromArgsT...> &value) const {
     return DelayedClosure<FromActorT, FromFunctionT, FromArgsT...>(value);
   }
+
+ public:
+  auto run(ActorT *actor) -> decltype(mem_call_tuple(actor, std::move(args))) {
+    return mem_call_tuple(actor, std::move(args));
+  }
 };
 
 template <class... ArgsT>

tdutils/td/utils/DecTree.h

@@ -111,6 +111,31 @@ class DecTree {
     }
   }
 
+  static const ValueType *get_node(const unique_ptr<Node> &Tree, const KeyType &key) {
+    if (Tree == nullptr) {
+      return nullptr;
+    }
+    if (Compare()(key, Tree->key_)) {
+      return get_node(Tree->left_, key);
+    } else if (Compare()(Tree->key_, key)) {
+      return get_node(Tree->right_, key);
+    } else {
+      return &Tree->value_;
+    }
+  }
+
+  static const ValueType *get_node_by_idx(const unique_ptr<Node> &Tree, size_t idx) {
+    CHECK(Tree != nullptr);
+    auto s = (Tree->left_ != nullptr) ? Tree->left_->size_ : 0;
+    if (idx < s) {
+      return get_node_by_idx(Tree->left_, idx);
+    } else if (idx == s) {
+      return &Tree->value_;
+    } else {
+      return get_node_by_idx(Tree->right_, idx - s - 1);
+    }
+  }
+
   static std::pair<unique_ptr<Node>, unique_ptr<Node>> split_node(unique_ptr<Node> Tree, const KeyType &key) {
     if (Tree == nullptr) {
       return {nullptr, nullptr};
@@ -128,7 +153,7 @@ class DecTree {
       P.first = std::move(Tree);
       return P;
     }
-  }
+  }  // namespace td
 
   static unique_ptr<Node> merge_node(unique_ptr<Node> left, unique_ptr<Node> right) {
     if (left == nullptr) {
@@ -172,6 +197,16 @@ class DecTree {
       return get_node_by_idx(root_, td::Random::fast_uint32() % size());
     }
   }
+  const ValueType *get(const KeyType &key) const {
+    return get_node(root_, key);
+  }
+  const ValueType *get_random() const {
+    if (size() == 0) {
+      return nullptr;
+    } else {
+      return get_node_by_idx(root_, td::Random::fast_uint32() % size());
+    }
+  }
   bool exists(const KeyType &key) const {
     return get_node(root_, key) != nullptr;
   }

tdutils/td/utils/Parser.h

@@ -146,6 +146,10 @@ class Parser {
   void skip_whitespaces() {
     skip_till_not(" \t\r\n");
   }
+  MutableSlice read_word() {
+    skip_whitespaces();
+    return read_till_nofail(" \t\r\n");
+  }
 
   MutableSlice data() const {
     return MutableSlice(ptr_, end_);

tdutils/td/utils/Random.cpp

@@ -148,5 +148,8 @@ uint64 Random::Xorshift128plus::operator()() {
   seed_[1] = x ^ y ^ (x >> 17) ^ (y >> 26);
   return seed_[1] + y;
 }
+int Random::Xorshift128plus::fast(int min, int max) {
+  return static_cast<int>((*this)() % (max - min + 1) + min);
+}
 
 }  // namespace td

tdutils/td/utils/Random.h

@@ -34,6 +34,7 @@ class Random {
     explicit Xorshift128plus(uint64 seed);
     Xorshift128plus(uint64 seed_a, uint64 seed_b);
     uint64 operator()();
+    int fast(int min, int max);
 
    private:
     uint64 seed_[2];

tdutils/td/utils/Slice.h

@@ -301,13 +301,28 @@ inline std::size_t SliceHash::operator()(Slice slice) const {
   return result;
 }
 
+inline Slice as_slice(Slice slice) {
+  return slice;
+}
+inline MutableSlice as_slice(MutableSlice slice) {
+  return slice;
+}
+
 template <size_t N>
-td::Slice as_slice(const td::UInt<N> &value) {
+Slice UInt<N>::as_slice() const {
-  return td::Slice(value.raw, N / 8);
+  return Slice(raw, N / 8);
 }
 template <size_t N>
-td::MutableSlice as_slice(td::UInt<N> &value) {
+MutableSlice UInt<N>::as_slice() {
-  return td::MutableSlice(value.raw, N / 8);
+  return MutableSlice(raw, N / 8);
+}
+template <size_t N>
+Slice as_slice(const UInt<N> &value) {
+  return value.as_slice();
+}
+template <size_t N>
+MutableSlice as_slice(UInt<N> &value) {
+  return value.as_slice();
 }
 
 }  // namespace td

tdutils/td/utils/Span.h

@@ -36,6 +36,12 @@ class SpanImpl {
   template <size_t N>
   SpanImpl(std::array<T, N> &arr) : SpanImpl(arr.data(), arr.size()) {
   }
+  template <size_t N>
+  SpanImpl(const T (&arr)[N]) : SpanImpl(arr, N) {
+  }
+  template <size_t N>
+  SpanImpl(T (&arr)[N]) : SpanImpl(arr, N) {
+  }
   SpanImpl(const vector<T> &v) : SpanImpl(v.data(), v.size()) {
   }
   SpanImpl(vector<T> &v) : SpanImpl(v.data(), v.size()) {
@@ -52,6 +58,11 @@ class SpanImpl {
     return data_[i];
   }
 
+  const InnerT &operator[](size_t i) const {
+    DCHECK(i < size());
+    return data_[i];
+  }
+
   InnerT *data() const {
     return data_;
   }
@@ -75,6 +86,11 @@ class SpanImpl {
     CHECK(offset <= size_);
     return SpanImpl(begin() + offset, size_ - offset);
   }
+  SpanImpl substr(size_t offset, size_t size) const {
+    CHECK(offset <= size_);
+    CHECK(size_ - offset >= size);
+    return SpanImpl(begin() + offset, size);
+  }
 };
 }  // namespace detail
 

tdutils/td/utils/Time.h

@@ -52,6 +52,9 @@ class Timestamp {
   static Timestamp at(double timeout) {
     return Timestamp{timeout};
   }
+  static Timestamp at_unix(double timeout) {
+    return Timestamp{timeout - td::Clocks::system() + Time::now()};
+  }
 
   static Timestamp in(double timeout) {
     return Timestamp{Time::now_cached() + timeout};

tdutils/td/utils/Timer.cpp

@@ -30,12 +30,16 @@ PerfWarningTimer::PerfWarningTimer(PerfWarningTimer &&other)
 }
 
 PerfWarningTimer::~PerfWarningTimer() {
+  reset();
+}
+void PerfWarningTimer::reset() {
   if (start_at_ == 0) {
     return;
   }
   double duration = Time::now() - start_at_;
   LOG_IF(WARNING, duration > max_duration_)
       << "SLOW: " << tag("name", name_) << tag("duration", format::as_time(duration));
+  start_at_ = 0;
 }
 
 }  // namespace td

tdutils/td/utils/Timer.h

@@ -28,6 +28,7 @@ class PerfWarningTimer {
   PerfWarningTimer(PerfWarningTimer &&other);
   PerfWarningTimer &operator=(PerfWarningTimer &&) = delete;
   ~PerfWarningTimer();
+  void reset();
 
  private:
   string name_;

tdutils/td/utils/bits.h

@@ -0,0 +1,140 @@
+//
+// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+#pragma once
+#include "td/utils/logging.h"
+#if TD_MSVC
+#include <intrin.h>
+#endif
+namespace td {
+int32 count_leading_zeroes32(uint32 x);
+int32 count_leading_zeroes64(uint64 x);
+int32 count_trailing_zeroes32(uint32 x);
+int32 count_trailing_zeroes64(uint64 x);
+uint32 bswap32(uint32 x);
+uint64 bswap64(uint64 x);
+int32 count_bits32(uint32 x);
+int32 count_bits64(uint64 x);
+
+//TODO: optimize
+inline int32 count_leading_zeroes_non_zero32(uint32 x) {
+  DCHECK(x != 0);
+  return count_leading_zeroes32(x);
+}
+inline int32 count_leading_zeroes_non_zero64(uint64 x) {
+  DCHECK(x != 0);
+  return count_leading_zeroes64(x);
+}
+inline int32 count_trailing_zeroes_non_zero32(uint32 x) {
+  DCHECK(x != 0);
+  return count_trailing_zeroes32(x);
+}
+inline int32 count_trailing_zeroes_non_zero64(uint64 x) {
+  DCHECK(x != 0);
+  return count_trailing_zeroes64(x);
+}
+
+#if TD_MSVC
+inline int32 count_leading_zeroes32(uint32 x) {
+  unsigned long res = 0;
+  if (_BitScanReverse(&res, x)) {
+    return 31 - res;
+  }
+  return 32;
+}
+inline int32 count_leading_zeroes64(uint64 x) {
+#if defined(_M_X64)
+  unsigned long res = 0;
+  if (_BitScanReverse64(&res, x)) {
+    return 63 - res;
+  }
+  return 64;
+#else
+  if ((x >> 32) == 0) {
+    return count_leading_zeroes32(static_cast<uint32>(x)) + 32;
+  } else {
+    return count_leading_zeroes32(static_cast<uint32>(x >> 32));
+  }
+#endif
+}
+inline int32 count_trailing_zeroes32(uint32 x) {
+  unsigned long res = 0;
+  if (_BitScanForward(&res, x)) {
+    return res;
+  }
+  return 32;
+}
+inline int32 count_trailing_zeroes64(uint64 x) {
+#if defined(_M_X64)
+  unsigned long res = 0;
+  if (_BitScanForward64(&res, x)) {
+    return res;
+  }
+  return 64;
+#else
+  if (static_cast<uint32>(x) == 0) {
+    return count_trailing_zeroes32(static_cast<uint32>(x >> 32)) + 32;
+  } else {
+    return count_trailing_zeroes32(static_cast<uint32>(x));
+  }
+#endif
+}
+inline uint32 bswap32(uint32 x) {
+  return _byteswap_ulong(x);
+}
+inline uint64 bswap64(uint64 x) {
+  return _byteswap_uint64(x);
+}
+inline int32 count_bits32(uint32 x) {
+  return __popcnt(x);
+}
+inline int32 count_bits64(uint64 x) {
+#if defined(_M_X64)
+  return __popcnt64(x);
+#else
+  return count_bits32(static_cast<uint32>(x >> 32)) + count_bits32(static_cast<uint32>(x));
+#endif
+}
+#else
+inline int32 count_leading_zeroes32(uint32 x) {
+  if (x == 0) {
+    return 32;
+  }
+  return __builtin_clz(x);
+}
+inline int32 count_leading_zeroes64(uint64 x) {
+  if (x == 0) {
+    return 64;
+  }
+  return __builtin_clzll(x);
+}
+inline int32 count_trailing_zeroes32(uint32 x) {
+  if (x == 0) {
+    return 32;
+  }
+  return __builtin_ctz(x);
+}
+inline int32 count_trailing_zeroes64(uint64 x) {
+  if (x == 0) {
+    return 64;
+  }
+  return __builtin_ctzll(x);
+}
+inline uint32 bswap32(uint32 x) {
+  return __builtin_bswap32(x);
+}
+inline uint64 bswap64(uint64 x) {
+  return __builtin_bswap64(x);
+}
+inline int32 count_bits32(uint32 x) {
+  return __builtin_popcount(x);
+}
+inline int32 count_bits64(uint64 x) {
+  return __builtin_popcountll(x);
+}
+#endif
+
+}  // namespace td

tdutils/td/utils/buffer.h

@@ -762,4 +762,11 @@ class BufferBuilder {
   void prepend_slow(BufferSlice slice);
 };
 
+inline Slice as_slice(const BufferSlice &value) {
+  return value.as_slice();
+}
+inline MutableSlice as_slice(BufferSlice &value) {
+  return value.as_slice();
+}
+
 }  // namespace td

tdutils/td/utils/common.h

@@ -107,14 +107,55 @@ struct Auto {
   }
 };
 
+template <class T>
+class As {
+ public:
+  As(void *ptr) : ptr_(ptr) {
+  }
+  As(As &&) = default;
+  const As<T> &operator=(const As &new_value) const {
+    memcpy(ptr_, new_value.ptr_, sizeof(T));
+    return *this;
+  }
+  const As<T> &operator=(const T new_value) const {
+    memcpy(ptr_, &new_value, sizeof(T));
+    return *this;
+  }
+
+  operator T() const {
+    T res;
+    memcpy(&res, ptr_, sizeof(T));
+    return res;
+  }
+
+ private:
+  void *ptr_;
+};
+
+template <class T>
+class ConstAs {
+ public:
+  ConstAs(const void *ptr) : ptr_(ptr) {
+  }
+
+  operator T() const {
+    T res;
+    memcpy(&res, ptr_, sizeof(T));
+    return res;
+  }
+
+ private:
+  const void *ptr_;
+};
+
 template <class ToT, class FromT>
-ToT &as(FromT *from) {
+As<ToT> as(FromT *from) {
-  return *reinterpret_cast<ToT *>(from);
+  return As<ToT>(from);
 }
 
 template <class ToT, class FromT>
-const ToT &as(const FromT *from) {
+const ConstAs<ToT> as(const FromT *from) {
-  return *reinterpret_cast<const ToT *>(from);
+  return ConstAs<ToT>(from);
 }
 
 }  // namespace td

tdutils/td/utils/config.h.in

@@ -2,3 +2,4 @@
 
 #cmakedefine01 TD_HAVE_OPENSSL
 #cmakedefine01 TD_HAVE_ZLIB
+#cmakedefine01 TD_HAVE_CRC32C

tdutils/td/utils/crypto.cpp

@@ -30,6 +30,10 @@
 #include <zlib.h>
 #endif
 
+#if TD_HAVE_CRC32C
+#include "crc32c/crc32c.h"
+#endif
+
 #include <algorithm>
 #include <cstring>
 #include <mutex>
@@ -631,6 +635,12 @@ uint32 crc32(Slice data) {
 }
 #endif
 
+#if TD_HAVE_CRC32C
+uint32 crc32c(Slice data) {
+  return crc32c::Crc32c(data.data(), data.size());
+}
+#endif
+
 static const uint64 crc64_table[256] = {
     0x0000000000000000, 0xb32e4cbe03a75f6f, 0xf4843657a840a05b, 0x47aa7ae9abe7ff34, 0x7bd0c384ff8f5e33,
     0xc8fe8f3afc28015c, 0x8f54f5d357cffe68, 0x3c7ab96d5468a107, 0xf7a18709ff1ebc66, 0x448fcbb7fcb9e309,

tdutils/td/utils/crypto.h

@@ -100,6 +100,10 @@ void init_openssl_threads();
 uint32 crc32(Slice data);
 #endif
 
+#if TD_HAVE_CRC32C
+uint32 crc32c(Slice data);
+#endif
+
 uint64 crc64(Slice data);
 
 }  // namespace td

tdutils/td/utils/filesystem.cpp

@@ -17,19 +17,38 @@
 
 namespace td {
 
-Result<BufferSlice> read_file(CSlice path, int64 size) {
+Result<BufferSlice> read_file(CSlice path, int64 size, int64 offset) {
   TRY_RESULT(from_file, FileFd::open(path, FileFd::Read));
   if (size == -1) {
-    size = from_file.get_size();
+    size = from_file.get_size() - offset;
+  }
+  if (size < 0) {
+    return Status::Error("Failed to read file: invalid size or offset");
   }
   BufferWriter content{static_cast<size_t>(size), 0, 0};
-  TRY_RESULT(got_size, from_file.read(content.as_slice()));
+  TRY_RESULT(got_size, from_file.pread(content.as_slice(), offset));
   if (got_size != static_cast<size_t>(size)) {
     return Status::Error("Failed to read file");
   }
   from_file.close();
   return content.as_buffer_slice();
 }
+Result<std::string> read_file_str(CSlice path, int64 size, int64 offset) {
+  TRY_RESULT(from_file, FileFd::open(path, FileFd::Read));
+  if (size == -1) {
+    size = from_file.get_size() - offset;
+  }
+  if (size < 0) {
+    return Status::Error("Failed to read file: invalid size or offset");
+  }
+  std::string content(static_cast<size_t>(size), '\0');
+  TRY_RESULT(got_size, from_file.pread(content, offset));
+  if (got_size != static_cast<size_t>(size)) {
+    return Status::Error("Failed to read file");
+  }
+  from_file.close();
+  return std::move(content);
+}
 
 // Very straightforward function. Don't expect much of it.
 Status copy_file(CSlice from, CSlice to, int64 size) {

tdutils/td/utils/filesystem.h

@@ -12,7 +12,8 @@
 
 namespace td {
 
-Result<BufferSlice> read_file(CSlice path, int64 size = -1);
+Result<BufferSlice> read_file(CSlice path, int64 size = -1, int64 offset = 0);
+Result<std::string> read_file_str(CSlice path, int64 size = -1, int64 offset = 0);
 
 Status copy_file(CSlice from, CSlice to, int64 size = -1) TD_WARN_UNUSED_RESULT;
 

tdutils/td/utils/int_types.h

@@ -43,10 +43,33 @@ static_assert(static_cast<char>(-256) == 0, "Unexpected cast to char implementat
 #pragma warning(pop)
 #endif
 
+class Slice;
+class MutableSlice;
 template <size_t size>
 struct UInt {
   static_assert(size % 8 == 0, "size should be divisible by 8");
   uint8 raw[size / 8];
+  Slice as_slice() const;
+  MutableSlice as_slice();
+
+  bool is_zero() const {
+    for (size_t i = 0; i * 8 < size; i++) {
+      if (raw[i] != 0) {
+        return false;
+      }
+    }
+    return true;
+  }
+  void set_zero() {
+    for (size_t i = 0; i * 8 < size; i++) {
+      raw[i] = 0;
+    }
+  }
+  static UInt zero() {
+    UInt v;
+    v.set_zero();
+    return v;
+  }
 };
 
 template <size_t size>

tdutils/td/utils/port/IPAddress.cpp

@@ -375,7 +375,7 @@ Status IPAddress::init_host_port(CSlice host, CSlice port, bool prefer_ipv6) {
   hints.ai_family = AF_UNSPEC;
   hints.ai_socktype = SOCK_STREAM;
   hints.ai_protocol = IPPROTO_TCP;
-  LOG(INFO) << "Try to init IP address of " << host << " with port " << port;
+  LOG(DEBUG + 10) << "Try to init IP address of " << host << " with port " << port;
   auto err = getaddrinfo(host.c_str(), port.c_str(), &hints, &info);
   if (err != 0) {
 #if TD_WINDOWS
@@ -440,7 +440,7 @@ Status IPAddress::init_sockaddr(sockaddr *addr, socklen_t len) {
   }
 
   is_valid_ = true;
-  LOG(INFO) << "Have address " << get_ip_str() << " with port " << get_port();
+  LOG(DEBUG + 10) << "Have address " << get_ip_str() << " with port " << get_port();
   return Status::OK();
 }
 

tdutils/td/utils/port/Stat.cpp

@@ -123,8 +123,10 @@ Status update_atime(int native_fd) {
   timespec times[2];
   // access time
   times[0].tv_nsec = UTIME_NOW;
+  times[0].tv_sec = 0;
   // modify time
   times[1].tv_nsec = UTIME_OMIT;
+  times[1].tv_sec = 0;
   if (futimens(native_fd, times) < 0) {
     auto status = OS_ERROR(PSLICE() << "futimens " << tag("fd", native_fd));
     LOG(WARNING) << status;

tdutils/td/utils/port/UdpSocketFd.cpp

@@ -69,9 +69,6 @@ class UdpSocketReceiveHelper {
     CHECK(message_size <= message.data.size());
     message.data.truncate(message_size);
     CHECK(message_size == message.data.size());
-    if (message_size >= 1500) {
-      LOG(ERROR) << "Received datagram of size " << message_size;
-    }
   }
 
  private:
@@ -433,9 +430,6 @@ class UdpSocketReceiveHelper {
     CHECK(message_size <= message.data.size());
     message.data.truncate(message_size);
     CHECK(message_size == message.data.size());
-    if (message_size >= 1500) {
-      LOG(ERROR) << "received datagram of size " << message_size;
-    }
   }
 
  private:
@@ -765,7 +759,6 @@ Result<UdpSocketFd> UdpSocketFd::open(const IPAddress &address) {
   BOOL flags = TRUE;
 #endif
   setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<const char *>(&flags), sizeof(flags));
-  setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, reinterpret_cast<const char *>(&flags), sizeof(flags));
   // TODO: SO_REUSEADDR, SO_KEEPALIVE, TCP_NODELAY, SO_SNDBUF, SO_RCVBUF, TCP_QUICKACK, SO_LINGER
 
   auto bind_addr = address.get_any_addr();

tdutils/td/utils/tests.h

@@ -7,15 +7,19 @@
 #pragma once
 
 #include "td/utils/common.h"
+#include "td/utils/Context.h"
 #include "td/utils/format.h"
 #include "td/utils/List.h"
 #include "td/utils/logging.h"
 #include "td/utils/port/thread.h"
 #include "td/utils/Random.h"
+#include "td/utils/optional.h"
 #include "td/utils/Slice.h"
+#include "td/utils/Status.h"
 #include "td/utils/Time.h"
 
 #include <atomic>
+#include <map>
 
 #define REGISTER_TESTS(x)                \
   void TD_CONCAT(register_tests_, x)() { \
@@ -25,113 +29,75 @@
 
 namespace td {
 
-class Test : private ListNode {
+class RegressionTester {
  public:
-  explicit Test(CSlice name) : name_(name) {
+  virtual ~RegressionTester() = default;
-    get_tests_list()->put_back(this);
+  static void destroy(CSlice db_path);
-  }
+  static std::unique_ptr<RegressionTester> create(string db_path, string db_cache_dir = "");
 
-  Test(const Test &) = delete;
+  virtual Status verify_test(Slice name, Slice result) = 0;
-  Test &operator=(const Test &) = delete;
+  virtual void save_db() = 0;
-  Test(Test &&) = delete;
+};
-  Test &operator=(Test &&) = delete;
+
+class Test {
+ public:
   virtual ~Test() = default;
-
-  static void add_substr_filter(std::string str) {
-    if (str[0] != '+' && str[0] != '-') {
-      str = "+" + str;
-    }
-    get_substr_filters()->push_back(std::move(str));
-  }
-  static void set_stress_flag(bool flag) {
-    get_stress_flag() = flag;
-  }
-  static void run_all() {
-    while (run_all_step()) {
-    }
-  }
-
-  static bool run_all_step() {
-    auto *state = get_state();
-    if (state->it == nullptr) {
-      state->end = get_tests_list();
-      state->it = state->end->next;
-    }
-
-    while (state->it != state->end) {
-      auto test = static_cast<td::Test *>(state->it);
-      if (!state->is_running) {
-        bool ok = true;
-        for (const auto &filter : *get_substr_filters()) {
-          bool is_match = test->name_.str().find(filter.substr(1)) != std::string::npos;
-          if (is_match != (filter[0] == '+')) {
-            ok = false;
-            break;
-          }
-        }
-        if (!ok) {
-          state->it = state->it->next;
-          continue;
-        }
-        LOG(ERROR) << "Run test " << tag("name", test->name_);
-        state->start = Time::now();
-        state->is_running = true;
-      }
-
-      if (test->step()) {
-        break;
-      }
-
-      LOG(ERROR) << format::as_time(Time::now() - state->start);
-      state->is_running = false;
-      state->it = state->it->next;
-    }
-
-    auto ret = state->it != state->end;
-    if (!ret) {
-      *state = State();
-    }
-    return ret || get_stress_flag();
-  }
-
- private:
-  CSlice name_;
-  struct State {
-    ListNode *it = nullptr;
-    bool is_running = false;
-    double start = 0;
-    ListNode *end = nullptr;
-  };
-  static State *get_state() {
-    static State state;
-    return &state;
-  }
-  static std::vector<std::string> *get_substr_filters() {
-    static std::vector<std::string> substr_filters_;
-    return &substr_filters_;
-  }
-
-  static ListNode *get_tests_list() {
-    static ListNode root;
-    return &root;
-  }
-  static bool &get_ok_flag() {
-    static bool is_ok = true;
-    return is_ok;
-  }
-  static bool &get_stress_flag() {
-    static bool stress_flag = false;
-    return stress_flag;
-  }
   virtual void run() {
     while (step()) {
     }
   }
-
   virtual bool step() {
     run();
     return false;
   }
+  Test() = default;
+  Test(const Test &) = delete;
+  Test &operator=(const Test &) = delete;
+  Test(Test &&) = delete;
+  Test &operator=(Test &&) = delete;
+};
+
+class TestContext : public Context<TestContext> {
+ public:
+  virtual ~TestContext() = default;
+  virtual Slice name() = 0;
+  virtual Status verify(Slice data) = 0;
+};
+
+class TestsRunner : public TestContext {
+ public:
+  static TestsRunner &get_default();
+
+  void add_test(string name, std::unique_ptr<Test> test);
+  void add_substr_filter(std::string str);
+  void set_stress_flag(bool flag);
+  void run_all();
+  bool run_all_step();
+  void set_regression_tester(std::unique_ptr<RegressionTester> regression_tester);
+
+ private:
+  struct State {
+    size_t it{0};
+    bool is_running = false;
+    double start{0};
+    size_t end{0};
+  };
+  bool stress_flag_{false};
+  std::vector<std::string> substr_filters_;
+  std::vector<std::pair<string, std::unique_ptr<Test>>> tests_;
+  State state_;
+  std::unique_ptr<RegressionTester> regression_tester_;
+
+  Slice name() override;
+  Status verify(Slice data) override;
+};
+
+template <class T>
+class RegisterTest {
+ public:
+  RegisterTest(string name, TestsRunner &runner = TestsRunner::get_default()) {
+    runner.add_test(name, std::make_unique<T>());
+  }
 };
 
 class Stage {
@@ -190,16 +156,18 @@ void assert_true_impl(const T &got, const char *file, int line) {
 #define ASSERT_STREQ(expected, got) \
   ::td::assert_eq_impl(::td::Slice((expected)), ::td::Slice((got)), __FILE__, __LINE__)
 
+#define REGRESSION_VERIFY(data) ::td::TestContext::get()->verify(data).ensure()
+
 #define TEST_NAME(test_case_name, test_name) \
   TD_CONCAT(Test, TD_CONCAT(_, TD_CONCAT(test_case_name, TD_CONCAT(_, test_name))))
 
 #define TEST(test_case_name, test_name) TEST_IMPL(TEST_NAME(test_case_name, test_name))
 
-#define TEST_IMPL(test_name)                                                                     \
+#define TEST_IMPL(test_name)                                                                                         \
-  class test_name : public ::td::Test {                                                          \
+  class test_name : public ::td::Test {                                                                              \
-   public:                                                                                       \
+   public:                                                                                                           \
-    using Test::Test;                                                                            \
+    using Test::Test;                                                                                                \
-    void run() final;                                                                            \
+    void run() final;                                                                                                \
-  };                                                                                             \
+  };                                                                                                                 \
-  test_name TD_CONCAT(test_instance_, TD_CONCAT(test_name, __LINE__))(TD_DEFINE_STR(test_name)); \
+  ::td::RegisterTest<test_name> TD_CONCAT(test_instance_, TD_CONCAT(test_name, __LINE__))(TD_DEFINE_STR(test_name)); \
   void test_name::run()

tdutils/test/crypto.cpp

@@ -157,6 +157,16 @@ TEST(Crypto, crc32) {
 }
 #endif
 
+#if TD_HAVE_CRC32C
+TEST(Crypto, crc32c) {
+  td::vector<td::uint32> answers{0u, 2432014819u, 1077264849u, 1131405888u};
+
+  for (std::size_t i = 0; i < strings.size(); i++) {
+    ASSERT_EQ(answers[i], td::crc32c(strings[i]));
+  }
+}
+#endif
+
 TEST(Crypto, crc64) {
   td::vector<td::uint64> answers{0ull, 3039664240384658157ull, 17549519902062861804ull, 8794730974279819706ull};
 

tdutils/test/misc.cpp

@@ -5,6 +5,7 @@
 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 //
 #include "td/utils/base64.h"
+#include "td/utils/bits.h"
 #include "td/utils/BigNum.h"
 #include "td/utils/HttpUrl.h"
 #include "td/utils/invoke.h"
@@ -585,3 +586,75 @@ TEST(Misc, StringBuilder) {
     }
   }
 }
+
+TEST(Misc, As) {
+  char buf[100];
+  as<int>(buf) = 123;
+  ASSERT_EQ(123, as<int>((const char *)buf));
+  ASSERT_EQ(123, as<int>((char *)buf));
+  char buf2[100];
+  as<int>(buf2) = as<int>(buf);
+  ASSERT_EQ(123, as<int>((const char *)buf2));
+  ASSERT_EQ(123, as<int>((char *)buf2));
+}
+
+TEST(Misc, Regression) {
+  string name = "regression_db";
+  RegressionTester::destroy(name);
+
+  {
+    auto tester = RegressionTester::create(name);
+    tester->save_db();
+    tester->verify_test("one_plus_one", "two").ensure();
+    tester->verify_test("one_plus_one", "two").ensure();
+    tester->verify_test("two_plus_one", "three").ensure();
+    tester->verify_test("one_plus_one", "two").ensure();
+    tester->verify_test("two_plus_one", "three").ensure();
+    tester->save_db();
+  }
+  {
+    auto tester = RegressionTester::create(name);
+    tester->save_db();
+    tester->verify_test("one_plus_one", "two").ensure();
+    tester->verify_test("one_plus_one", "two").ensure();
+    tester->verify_test("two_plus_one", "three").ensure();
+    tester->verify_test("one_plus_one", "two").ensure();
+    tester->verify_test("two_plus_one", "three").ensure();
+    tester->save_db();
+    tester->verify_test("one_plus_one", "three").ensure_error();
+    tester->verify_test("two_plus_one", "two").ensure_error();
+  }
+  {
+    auto tester = RegressionTester::create(name);
+    tester->verify_test("one_plus_one", "three").ensure_error();
+    tester->verify_test("two_plus_one", "two").ensure_error();
+  }
+}
+
+TEST(Misc, Bits) {
+  ASSERT_EQ(32, count_leading_zeroes32(0));
+  ASSERT_EQ(64, count_leading_zeroes64(0));
+  ASSERT_EQ(32, count_trailing_zeroes32(0));
+  ASSERT_EQ(64, count_trailing_zeroes64(0));
+
+  for (int i = 0; i < 32; i++) {
+    ASSERT_EQ(31 - i, count_leading_zeroes32(1u << i));
+    ASSERT_EQ(i, count_trailing_zeroes32(1u << i));
+    ASSERT_EQ(31 - i, count_leading_zeroes_non_zero32(1u << i));
+    ASSERT_EQ(i, count_trailing_zeroes_non_zero32(1u << i));
+  }
+  for (int i = 0; i < 64; i++) {
+    ASSERT_EQ(63 - i, count_leading_zeroes64(1ull << i));
+    ASSERT_EQ(i, count_trailing_zeroes64(1ull << i));
+    ASSERT_EQ(63 - i, count_leading_zeroes_non_zero64(1ull << i));
+    ASSERT_EQ(i, count_trailing_zeroes_non_zero64(1ull << i));
+  }
+
+  ASSERT_EQ(0x12345678u, bswap32(0x78563412u));
+  ASSERT_EQ(0x12345678abcdef67ull, bswap64(0x67efcdab78563412ull));
+
+  ASSERT_EQ(0, count_bits32(0));
+  ASSERT_EQ(0, count_bits64(0));
+  ASSERT_EQ(4, count_bits32((1u << 31) | 7));
+  ASSERT_EQ(4, count_bits64((1ull << 63) | 7));
+}