tdutils: import changes from other project

GitOrigin-RevId: 0cfbe2418933cffdce4f0d1b8461c247a814d868

tdutils/CMakeLists.txt

@@ -99,12 +99,15 @@ set(TDUTILS_SOURCE
   td/utils/MpmcQueue.cpp
   td/utils/OptionsParser.cpp
   td/utils/Random.cpp
+  td/utils/Slice.cpp
+  td/utils/SharedSlice.cpp
   td/utils/StackAllocator.cpp
   td/utils/Status.cpp
   td/utils/StringBuilder.cpp
-  td/utils/tests.cpp
   td/utils/Time.cpp
   td/utils/Timer.cpp
+  td/utils/TsFileLog.cpp
+  td/utils/tests.cpp
   td/utils/tl_parsers.cpp
   td/utils/translit.cpp
   td/utils/unicode.cpp
@@ -231,6 +234,7 @@ set(TDUTILS_SOURCE
   td/utils/Time.h
   td/utils/TimedStat.h
   td/utils/Timer.h
+  td/utils/TsFileLog.h
   td/utils/tl_helpers.h
   td/utils/tl_parsers.h
   td/utils/tl_storers.h
@@ -250,12 +254,12 @@ set(TDUTILS_TEST_SOURCE
   ${CMAKE_CURRENT_SOURCE_DIR}/test/ConcurrentHashMap.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/crypto.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/Enumerator.cpp
-  ${CMAKE_CURRENT_SOURCE_DIR}/test/EpochBasedMemoryReclamation.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/filesystem.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/gzip.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/HazardPointers.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/heap.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/json.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/test/log.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/misc.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcQueue.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcWaiter.cpp
@@ -264,6 +268,7 @@ set(TDUTILS_TEST_SOURCE
   ${CMAKE_CURRENT_SOURCE_DIR}/test/port.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/pq.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/SharedObjectPool.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/test/SharedSlice.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/test/variant.cpp
   PARENT_SCOPE
 )
@@ -300,7 +305,7 @@ if (CRC32C_FOUND)
   target_link_libraries(tdutils PRIVATE crc32c)
 endif()
 if (ABSL_FOUND)
-  target_link_libraries(tdutils PUBLIC absl::base absl::container absl::hash)
+  target_link_libraries_system(tdutils PUBLIC absl::flat_hash_map absl::flat_hash_set absl::hash)
 endif()
 
 if (WIN32 AND WINGETOPT_FOUND)

tdutils/td/utils/DecTree.h

@@ -185,6 +185,9 @@ class DecTree {
   void remove(const KeyType &key) {
     root_ = remove_node(std::move(root_), key);
   }
+  void reset() {
+    root_ = nullptr;
+  }
   ValueType *get(const KeyType &key) {
     return get_node(root_, key);
   }

tdutils/td/utils/FileLog.cpp

@@ -17,7 +17,7 @@
 
 namespace td {
 
-Status FileLog::init(string path, int64 rotate_threshold) {
+Status FileLog::init(string path, int64 rotate_threshold, bool redirect_stderr) {
   if (path == path_) {
     set_rotate_threshold(rotate_threshold);
     return Status::OK();
@@ -30,7 +30,7 @@ Status FileLog::init(string path, int64 rotate_threshold) {
 
   fd_.close();
   fd_ = std::move(fd);
-  if (!Stderr().empty()) {
+  if (!Stderr().empty() && redirect_stderr) {
     fd_.get_native_fd().duplicate(Stderr().get_native_fd()).ignore();
   }
 
@@ -43,6 +43,7 @@ Status FileLog::init(string path, int64 rotate_threshold) {
   TRY_RESULT(size, fd_.get_size());
   size_ = size;
   rotate_threshold_ = rotate_threshold;
+  redirect_stderr_ = redirect_stderr;
   return Status::OK();
 }
 
@@ -108,7 +109,7 @@ void FileLog::do_rotate() {
     process_fatal_error(PSLICE() << r_fd.error() << " in " << __FILE__ << " at " << __LINE__);
   }
   fd_ = r_fd.move_as_ok();
-  if (!Stderr().empty()) {
+  if (!Stderr().empty() && redirect_stderr_) {
     fd_.get_native_fd().duplicate(Stderr().get_native_fd()).ignore();
   }
   size_ = 0;

tdutils/td/utils/FileLog.h

@@ -18,7 +18,7 @@ class FileLog : public LogInterface {
   static constexpr int64 DEFAULT_ROTATE_THRESHOLD = 10 * (1 << 20);
 
  public:
-  Status init(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD);
+  Status init(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD, bool redirect_stderr = true);
 
   Slice get_path() const;
 
@@ -37,6 +37,7 @@ class FileLog : public LogInterface {
   string path_;
   int64 size_ = 0;
   int64 rotate_threshold_ = 0;
+  bool redirect_stderr_;
 
   void do_rotate();
 };

tdutils/td/utils/JsonBuilder.h

@@ -181,7 +181,7 @@ class JsonObjectScope;
 
 class JsonBuilder {
  public:
-  explicit JsonBuilder(StringBuilder &&sb) : sb_(std::move(sb)) {
+  explicit JsonBuilder(StringBuilder &&sb, int32 offset = -1) : sb_(std::move(sb)), offset_(offset) {
   }
   StringBuilder &string_builder() {
     return sb_;
@@ -191,9 +191,33 @@ class JsonBuilder {
   JsonArrayScope enter_array() TD_WARN_UNUSED_RESULT;
   JsonObjectScope enter_object() TD_WARN_UNUSED_RESULT;
 
+  int32 offset() const {
+    return offset_;
+  }
+  bool is_pretty() const {
+    return offset_ >= 0;
+  }
+  void print_offset() {
+    for (int x = 0; x < offset_; x++) {
+      sb_ << "   ";
+    }
+  }
+  void dec_offset() {
+    if (offset_ >= 0) {
+      CHECK(offset_ > 0);
+      offset_--;
+    }
+  }
+  void inc_offset() {
+    if (offset_ >= 0) {
+      offset_++;
+    }
+  }
+
  private:
   StringBuilder sb_;
   JsonScope *scope_ = nullptr;
+  int32 offset_;
 };
 
 class Jsonable {};
@@ -328,6 +352,7 @@ class JsonValueScope : public JsonScope {
 class JsonArrayScope : public JsonScope {
  public:
   explicit JsonArrayScope(JsonBuilder *jb) : JsonScope(jb) {
+    jb->inc_offset();
     *sb_ << "[";
   }
   JsonArrayScope(JsonArrayScope &&other) = default;
@@ -337,6 +362,11 @@ class JsonArrayScope : public JsonScope {
     }
   }
   void leave() {
+    jb_->dec_offset();
+    if (jb_->is_pretty()) {
+      *sb_ << "\n";
+      jb_->print_offset();
+    }
     *sb_ << "]";
   }
   template <class T>
@@ -355,6 +385,10 @@ class JsonArrayScope : public JsonScope {
     } else {
       is_first_ = true;
     }
+    if (jb_->is_pretty()) {
+      *sb_ << "\n";
+      jb_->print_offset();
+    }
     return jb_->enter_value();
   }
 
@@ -365,6 +399,7 @@ class JsonArrayScope : public JsonScope {
 class JsonObjectScope : public JsonScope {
  public:
   explicit JsonObjectScope(JsonBuilder *jb) : JsonScope(jb) {
+    jb->inc_offset();
     *sb_ << "{";
   }
   JsonObjectScope(JsonObjectScope &&other) = default;
@@ -374,6 +409,11 @@ class JsonObjectScope : public JsonScope {
     }
   }
   void leave() {
+    jb_->dec_offset();
+    if (jb_->is_pretty()) {
+      *sb_ << "\n";
+      jb_->print_offset();
+    }
     *sb_ << "}";
   }
   template <class S, class T>
@@ -392,8 +432,16 @@ class JsonObjectScope : public JsonScope {
     } else {
       is_first_ = true;
     }
+    if (jb_->is_pretty()) {
+      *sb_ << "\n";
+      jb_->print_offset();
+    }
     jb_->enter_value() << key;
+    if (jb_->is_pretty()) {
+      *sb_ << " : ";
+    } else {
       *sb_ << ":";
+    }
     jb_->enter_value() << value;
     return *this;
   }
@@ -737,11 +785,14 @@ inline Result<JsonValue> json_decode(MutableSlice json) {
 }
 
 template <class StrT, class ValT>
-StrT json_encode(const ValT &val) {
+StrT json_encode(const ValT &val, bool pretty = false) {
   auto buf_len = 1 << 18;
   auto buf = StackAllocator::alloc(buf_len);
-  JsonBuilder jb(StringBuilder(buf.as_slice(), true));
+  JsonBuilder jb(StringBuilder(buf.as_slice(), true), pretty ? 0 : -1);
   jb.enter_value() << val;
+  if (pretty) {
+    jb.string_builder() << "\n";
+  }
   LOG_IF(ERROR, jb.string_builder().is_error()) << "JSON buffer overflow";
   auto slice = jb.string_builder().as_cslice();
   return StrT(slice.begin(), slice.size());

tdutils/td/utils/MpmcWaiter.h

@@ -90,7 +90,8 @@ class MpmcWaiter {
       return (state >> 1) == (worker + 1);
     }
   };
-  enum { RoundsTillSleepy = 32, RoundsTillAsleep = 64 };
+  //enum { RoundsTillSleepy = 32, RoundsTillAsleep = 64 };
+  enum { RoundsTillSleepy = 1, RoundsTillAsleep = 2 };
   std::atomic<uint32> state_{State::awake()};
   std::mutex mutex_;
   std::condition_variable condition_variable_;

tdutils/td/utils/SharedSlice.cpp

@@ -0,0 +1,17 @@
+//
+// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+#include "td/utils/SharedSlice.h"
+
+#include "td/utils/buffer.h"
+
+namespace td {
+
+BufferSlice SharedSlice::clone_as_buffer_slice() const {
+  return BufferSlice{as_slice().str()};
+}
+
+}  // namespace td

tdutils/td/utils/SharedSlice.h

@@ -0,0 +1,377 @@
+//
+// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+#pragma once
+
+#include "td/utils/Slice.h"
+
+#include <atomic>
+#include <memory>
+
+namespace td {
+
+namespace detail {
+struct SharedSliceHeader {
+  explicit SharedSliceHeader(size_t size) : refcnt_{1}, size_{size} {
+  }
+
+  void inc() {
+    refcnt_.fetch_add(1, std::memory_order_relaxed);
+  }
+
+  bool dec() {
+    return refcnt_.fetch_sub(1, std::memory_order_acq_rel) == 1;
+  }
+
+  bool is_unique() const {
+    // NB: race if std::memory_order_relaxed is used
+    // reader may see a change by a new writer
+    return refcnt_.load(std::memory_order_acquire) == 1;
+  }
+
+  size_t size() const {
+    return size_;
+  }
+
+ private:
+  std::atomic<uint64> refcnt_;
+  size_t size_;
+};
+
+struct UniqueSliceHeader {
+  explicit UniqueSliceHeader(size_t size) : size_{size} {
+  }
+
+  void inc() {
+  }
+
+  bool dec() {
+    return true;
+  }
+
+  bool is_unique() const {
+    return true;
+  }
+
+  size_t size() const {
+    return size_;
+  }
+
+ private:
+  size_t size_;
+};
+
+template <class HeaderT, bool zero_on_destruct = false>
+class UnsafeSharedSlice {
+ public:
+  UnsafeSharedSlice() = default;
+  UnsafeSharedSlice clone() const {
+    header()->inc();
+    return UnsafeSharedSlice(ptr_.get());
+  }
+
+  bool is_null() const {
+    return !ptr_;
+  }
+
+  bool is_unique() const {
+    if (is_null()) {
+      return true;
+    }
+    return header()->is_unique();
+  }
+
+  MutableSlice as_mutable_slice() {
+    if (is_null()) {
+      return MutableSlice();
+    }
+    return MutableSlice(ptr_.get() + sizeof(HeaderT), header()->size());
+  }
+
+  Slice as_slice() const {
+    if (is_null()) {
+      return Slice();
+    }
+    return Slice(ptr_.get() + sizeof(HeaderT), header()->size());
+  }
+
+  size_t size() const {
+    if (is_null()) {
+      return 0;
+    }
+    return header()->size();
+  }
+
+  static UnsafeSharedSlice create(size_t size) {
+    static_assert(std::is_standard_layout<HeaderT>::value, "HeaderT must have statdard layout");
+    auto ptr = std::make_unique<char[]>(size + sizeof(HeaderT));
+    auto header_ptr = new (ptr.get()) HeaderT(size);
+    CHECK(reinterpret_cast<char *>(header_ptr) == ptr.get());
+
+    return UnsafeSharedSlice(std::move(ptr));
+  }
+
+  static UnsafeSharedSlice create(Slice slice) {
+    auto res = create(slice.size());
+    res.as_mutable_slice().copy_from(slice);
+    return res;
+  }
+
+  void clear() {
+    ptr_.reset();
+  }
+
+ private:
+  explicit UnsafeSharedSlice(char *ptr) : ptr_(ptr) {
+  }
+  explicit UnsafeSharedSlice(std::unique_ptr<char[]> from) : ptr_(from.release()) {
+  }
+
+  HeaderT *header() const {
+    return reinterpret_cast<HeaderT *>(ptr_.get());
+  }
+
+  struct Destructor {
+    void operator()(char *ptr) {
+      auto header = reinterpret_cast<HeaderT *>(ptr);
+      if (header->dec()) {
+        if (zero_on_destruct) {
+          MutableSlice(ptr, sizeof(HeaderT) + header->size()).fill_zero_secure();
+        }
+        std::default_delete<char[]>()(ptr);
+      }
+    }
+  };
+
+  std::unique_ptr<char[], Destructor> ptr_;
+};
+}  // namespace detail
+
+class BufferSlice;
+
+class UniqueSharedSlice;
+
+class SharedSlice {
+  using Impl = detail::UnsafeSharedSlice<detail::SharedSliceHeader>;
+
+ public:
+  SharedSlice() = default;
+
+  explicit SharedSlice(Slice slice) : impl_(Impl::create(slice.size())) {
+    impl_.as_mutable_slice().copy_from(slice);
+  }
+
+  explicit SharedSlice(UniqueSharedSlice from);
+
+  SharedSlice(const char *ptr, size_t size) : SharedSlice(Slice(ptr, size)) {
+  }
+
+  SharedSlice clone() const {
+    return SharedSlice(impl_.clone());
+  }
+
+  Slice as_slice() const {
+    return impl_.as_slice();
+  }
+
+  BufferSlice clone_as_buffer_slice() const;
+
+  operator Slice() const {
+    return as_slice();
+  }
+
+  // like in std::string
+  const char *data() const {
+    return as_slice().data();
+  }
+
+  char operator[](size_t at) const {
+    return as_slice()[at];
+  }
+
+  bool empty() const {
+    return size() == 0;
+  }
+
+  size_t size() const {
+    return impl_.size();
+  }
+
+  // like in std::string
+  size_t length() const {
+    return size();
+  }
+
+  void clear() {
+    impl_.clear();
+  }
+
+ private:
+  friend class UniqueSharedSlice;
+  explicit SharedSlice(Impl impl) : impl_(std::move(impl)) {
+  }
+  Impl impl_;
+};
+
+class UniqueSharedSlice {
+  using Impl = detail::UnsafeSharedSlice<detail::SharedSliceHeader>;
+
+ public:
+  UniqueSharedSlice() = default;
+
+  explicit UniqueSharedSlice(size_t size) : impl_(Impl::create(size)) {
+  }
+  explicit UniqueSharedSlice(Slice slice) : impl_(Impl::create(slice)) {
+  }
+
+  UniqueSharedSlice(const char *ptr, size_t size) : UniqueSharedSlice(Slice(ptr, size)) {
+  }
+  explicit UniqueSharedSlice(SharedSlice from) : impl_() {
+    if (from.impl_.is_unique()) {
+      impl_ = std::move(from.impl_);
+    } else {
+      impl_ = Impl::create(from.as_slice());
+    }
+  }
+
+  UniqueSharedSlice copy() const {
+    return UniqueSharedSlice(as_slice());
+  }
+
+  Slice as_slice() const {
+    return impl_.as_slice();
+  }
+
+  MutableSlice as_mutable_slice() {
+    return impl_.as_mutable_slice();
+  }
+
+  operator Slice() const {
+    return as_slice();
+  }
+
+  // like in std::string
+  char *data() {
+    return as_mutable_slice().data();
+  }
+  const char *data() const {
+    return as_slice().data();
+  }
+  char operator[](size_t at) const {
+    return as_slice()[at];
+  }
+
+  bool empty() const {
+    return size() == 0;
+  }
+
+  size_t size() const {
+    return impl_.size();
+  }
+
+  // like in std::string
+  size_t length() const {
+    return size();
+  }
+
+  void clear() {
+    impl_.clear();
+  }
+
+ private:
+  friend class SharedSlice;
+  explicit UniqueSharedSlice(Impl impl) : impl_(std::move(impl)) {
+  }
+  Impl impl_;
+};
+
+inline SharedSlice::SharedSlice(UniqueSharedSlice from) : impl_(std::move(from.impl_)) {
+}
+
+template <bool zero_on_destruct>
+class UniqueSliceImpl {
+  using Impl = detail::UnsafeSharedSlice<detail::UniqueSliceHeader, zero_on_destruct>;
+
+ public:
+  UniqueSliceImpl() = default;
+
+  explicit UniqueSliceImpl(size_t size) : impl_(Impl::create(size)) {
+  }
+  UniqueSliceImpl(size_t size, char c) : impl_(Impl::create(size)) {
+    std::memset(as_mutable_slice().data(), c, size);
+  }
+  explicit UniqueSliceImpl(Slice slice) : impl_(Impl::create(slice)) {
+  }
+
+  UniqueSliceImpl(const char *ptr, size_t size) : UniqueSliceImpl(Slice(ptr, size)) {
+  }
+
+  UniqueSliceImpl copy() const {
+    return UniqueSliceImpl(as_slice());
+  }
+
+  Slice as_slice() const {
+    return impl_.as_slice();
+  }
+
+  MutableSlice as_mutable_slice() {
+    return impl_.as_mutable_slice();
+  }
+
+  operator Slice() const {
+    return as_slice();
+  }
+
+  // like in std::string
+  char *data() {
+    return as_mutable_slice().data();
+  }
+  const char *data() const {
+    return as_slice().data();
+  }
+  char operator[](size_t at) const {
+    return as_slice()[at];
+  }
+
+  bool empty() const {
+    return size() == 0;
+  }
+
+  size_t size() const {
+    return impl_.size();
+  }
+
+  // like in std::string
+  size_t length() const {
+    return size();
+  }
+
+  void clear() {
+    impl_.clear();
+  }
+
+ private:
+  explicit UniqueSliceImpl(Impl impl) : impl_(std::move(impl)) {
+  }
+  Impl impl_;
+};
+
+using UniqueSlice = UniqueSliceImpl<false>;
+using SecureString = UniqueSliceImpl<true>;
+
+inline MutableSlice as_mutable_slice(UniqueSharedSlice &unique_shared_slice) {
+  return unique_shared_slice.as_mutable_slice();
+}
+
+inline MutableSlice as_mutable_slice(UniqueSlice &unique_slice) {
+  return unique_slice.as_mutable_slice();
+}
+
+inline MutableSlice as_mutable_slice(SecureString &secure_string) {
+  return secure_string.as_mutable_slice();
+}
+
+}  // namespace td

tdutils/td/utils/Slice-decl.h

@@ -52,6 +52,9 @@ class MutableSlice {
   MutableSlice substr(size_t from, size_t size) const;
   size_t find(char c) const;
   size_t rfind(char c) const;
+  void fill(char c);
+  void fill_zero();
+  void fill_zero_secure();
 
   void copy_from(Slice from);
 

tdutils/td/utils/Slice.cpp

@@ -0,0 +1,31 @@
+//
+// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+#include "td/utils/Slice.h"
+#if TD_HAVE_OPENSSL
+#include <openssl/crypto.h>
+#endif
+
+namespace td {
+
+void MutableSlice::fill(char c) {
+  std::memset(data(), c, size());
+}
+void MutableSlice::fill_zero() {
+  fill(0);
+}
+void MutableSlice::fill_zero_secure() {
+#if TD_HAVE_OPENSSL
+  OPENSSL_cleanse(begin(), size());
+#else
+  volatile char *ptr = begin();
+  for (size_t i = 0; i < size(); i++) {
+    ptr[i] = 0;
+  }
+#endif
+}
+
+}  // namespace td

tdutils/td/utils/Slice.h

@@ -316,4 +316,12 @@ inline MutableSlice as_slice(string &str) {
   return str;
 }
 
+inline MutableSlice as_mutable_slice(MutableSlice slice) {
+  return slice;
+}
+
+inline MutableSlice as_mutable_slice(string &str) {
+  return str;
+}
+
 }  // namespace td

tdutils/td/utils/ThreadLocalStorage.h

@@ -0,0 +1,54 @@
+//
+// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+#pragma once
+
+#include "td/utils/port/thread.h"
+#include "td/utils/port/thread_local.h"
+#include "td/utils/int_types.h"
+
+#include <atomic>
+#include <array>
+
+namespace td {
+template <class T>
+class ThreadLocalStorage {
+ public:
+  T& get() {
+    return thread_local_node().value;
+  }
+
+  template <class F>
+  void for_each(F&& f) {
+    int n = max_thread_id_.load();
+    for (int i = 0; i < n; i++) {
+      f(nodes_[i].value);
+    }
+  }
+  template <class F>
+  void for_each(F&& f) const {
+    int n = max_thread_id_.load();
+    for (int i = 0; i < n; i++) {
+      f(nodes_[i].value);
+    }
+  }
+
+ private:
+  struct Node {
+    T value{};
+    char padding[128];
+  };
+  static constexpr int MAX_THREAD_ID = 128;
+  std::atomic<int> max_thread_id_{MAX_THREAD_ID};
+  std::array<Node, MAX_THREAD_ID> nodes_;
+
+  Node& thread_local_node() {
+    auto thread_id = get_thread_id();
+    CHECK(0 <= thread_id && static_cast<size_t>(thread_id) < nodes_.size());
+    return nodes_[thread_id];
+  }
+};
+}  // namespace td

tdutils/td/utils/ThreadSafeCounter.h

@@ -5,45 +5,107 @@
 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 //
 #pragma once
-
+#include "td/utils/ThreadLocalStorage.h"
-#include "td/utils/common.h"
+#include "td/utils/StringBuilder.h"
-#include "td/utils/port/thread_local.h"
 
 #include <array>
-#include <atomic>
+#include <mutex>
 
 namespace td {
-
+template <size_t N>
-class ThreadSafeCounter {
+class ThreadSafeMultiCounter {
  public:
-  void add(int64 diff) {
+  void add(size_t index, int64 diff) {
-    auto &node = thread_local_node();
+    CHECK(index < N);
-    node.count_.store(node.count_.load(std::memory_order_relaxed) + diff, std::memory_order_relaxed);
+    tls_.get()[index].fetch_add(diff, std::memory_order_relaxed);
   }
 
-  int64 sum() const {
+  int64 sum(size_t index) const {
-    int n = max_thread_id_.load();
+    CHECK(index < N);
     int64 res = 0;
-    for (int i = 0; i < n; i++) {
+    tls_.for_each([&](auto &value) { res += value[index].load(); });
-      res += nodes_[i].count_.load();
-    }
     return res;
   }
 
  private:
-  struct Node {
+  ThreadLocalStorage<std::array<std::atomic<int64>, N>> tls_;
-    std::atomic<int64> count_{0};
+};
-    char padding[128];
-  };
-  static constexpr int MAX_THREAD_ID = 128;
-  std::atomic<int> max_thread_id_{MAX_THREAD_ID};
-  std::array<Node, MAX_THREAD_ID> nodes_;
 
-  Node &thread_local_node() {
+class ThreadSafeCounter {
-    auto thread_id = get_thread_id();
+ public:
-    CHECK(static_cast<size_t>(thread_id) < nodes_.size());
+  void add(int64 diff) {
-    return nodes_[thread_id];
+    counter_.add(0, diff);
   }
+
+  int64 sum() const {
+    return counter_.sum(0);
+  }
+
+ private:
+  ThreadSafeMultiCounter<1> counter_;
+};
+
+class NamedThreadSafeCounter {
+  static constexpr int N = 128;
+  using Counter = ThreadSafeMultiCounter<N>;
+
+ public:
+  class CounterRef {
+   public:
+    CounterRef() = default;
+    CounterRef(size_t index, Counter *counter) : index_(index), counter_(counter) {
+    }
+    void add(int64 diff) {
+      counter_->add(index_, diff);
+    }
+    int64 sum() const {
+      return counter_->sum(index_);
+    }
+
+   private:
+    size_t index_{0};
+    Counter *counter_{nullptr};
+  };
+
+  CounterRef get_counter(Slice name) {
+    std::unique_lock<std::mutex> guard(mutex_);
+    for (size_t i = 0; i < names_.size(); i++) {
+      if (names_[i] == name) {
+        return get_counter_ref(i);
+      }
+    }
+    CHECK(names_.size() < N);
+    names_.push_back(name.str());
+    return get_counter_ref(names_.size() - 1);
+  }
+
+  CounterRef get_counter_ref(size_t index) {
+    return CounterRef(index, &counter_);
+  }
+
+  static NamedThreadSafeCounter &get_default() {
+    static NamedThreadSafeCounter res;
+    return res;
+  }
+
+  template <class F>
+  void for_each(F &&f) const {
+    std::unique_lock<std::mutex> guard(mutex_);
+    for (size_t i = 0; i < names_.size(); i++) {
+      f(names_[i], counter_.sum(i));
+    }
+  }
+
+  friend StringBuilder &operator<<(StringBuilder &sb, const NamedThreadSafeCounter &counter) {
+    counter.for_each([&sb](Slice name, int64 cnt) { sb << name << ": " << cnt << "\n"; });
+    return sb;
+  }
+
+ private:
+  mutable std::mutex mutex_;
+  std::vector<std::string> names_;
+
+  Counter counter_;
 };
 
 }  // namespace td

tdutils/td/utils/TsFileLog.cpp

@@ -0,0 +1,78 @@
+//
+// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+#include "TsFileLog.h"
+
+namespace td {
+namespace detail {
+class TsFileLog : public LogInterface {
+ public:
+  Status init(string path) {
+    path_ = std::move(path);
+    for (int i = 0; i < (int)logs_.size(); i++) {
+      logs_[i].id = i;
+    }
+    return init_info(&logs_[0]);
+  }
+
+  vector<string> get_file_paths() override {
+    vector<string> res;
+    for (auto &log : logs_) {
+      res.push_back(get_path(&log));
+    }
+    return res;
+  }
+
+  void append(CSlice cslice) override {
+    return append(cslice, -1);
+  }
+  void append(CSlice cslice, int log_level) override {
+    get_current_logger()->append(cslice, log_level);
+  }
+
+ private:
+  struct Info {
+    FileLog log;
+    bool is_inited;
+    int id;
+  };
+  static constexpr int MAX_THREAD_ID = 128;
+  std::string path_;
+  std::array<Info, MAX_THREAD_ID> logs_;
+
+  LogInterface *get_current_logger() {
+    auto *info = get_current_info();
+    if (!info->is_inited) {
+      CHECK(init_info(info).is_ok());
+    }
+    return &info->log;
+  }
+
+  Info *get_current_info() {
+    return &logs_[get_thread_id()];
+  }
+
+  Status init_info(Info *info) {
+    TRY_STATUS(info->log.init(get_path(info), std::numeric_limits<int64>::max(), info->id == 0));
+    info->is_inited = true;
+    return Status::OK();
+  }
+
+  string get_path(Info *info) {
+    if (info->id == 0) {
+      return path_;
+    }
+    return PSTRING() << path_ << "." << info->id;
+  }
+};
+}  // namespace detail
+
+Result<td::unique_ptr<LogInterface>> TsFileLog::create(string path) {
+  auto res = td::make_unique<detail::TsFileLog>();
+  TRY_STATUS(res->init(path));
+  return std::move(res);
+}
+}  // namespace td

tdutils/td/utils/TsFileLog.h

@@ -0,0 +1,16 @@
+//
+// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+#pragma once
+
+#include "td/utils/FileLog.h"
+
+namespace td {
+class TsFileLog {
+ public:
+  static Result<td::unique_ptr<LogInterface>> create(string path);
+};
+}  // namespace td

tdutils/td/utils/base64.cpp

@@ -10,6 +10,8 @@
 #include "td/utils/Slice.h"
 #include "td/utils/Status.h"
 
+#include "td/utils/format.h"
+
 #include <algorithm>
 #include <iterator>
 
@@ -48,7 +50,7 @@ string base64_encode(Slice input) {
 
 static unsigned char char_to_value[256];
 static void init_base64_table() {
-  static bool is_inited = []() {
+  static bool is_inited = [] {
     std::fill(std::begin(char_to_value), std::end(char_to_value), static_cast<unsigned char>(64));
     for (unsigned char i = 0; i < 64; i++) {
       char_to_value[static_cast<size_t>(symbols64[i])] = i;
@@ -58,9 +60,7 @@ static void init_base64_table() {
   CHECK(is_inited);
 }
 
-Result<string> base64_decode(Slice base64) {
+Result<Slice> base64_drop_padding(Slice base64) {
-  init_base64_table();
-
   if ((base64.size() & 3) != 0) {
     return Status::Error("Wrong string length");
   }
@@ -73,9 +73,11 @@ Result<string> base64_decode(Slice base64) {
   if (padding_length >= 3) {
     return Status::Error("Wrong string padding");
   }
+  return base64;
+}
 
-  string output;
+template <class F>
-  output.reserve(((base64.size() + 3) >> 2) * 3);
+Status base64_do_decode(Slice base64, F &&append) {
   for (size_t i = 0; i < base64.size();) {
     size_t left = min(base64.size() - i, static_cast<size_t>(4));
     int c = 0;
@@ -86,25 +88,53 @@ Result<string> base64_decode(Slice base64) {
       }
       c |= value << ((3 - t) * 6);
     }
-    output += static_cast<char>(static_cast<unsigned char>(c >> 16));  // implementation-defined
+    append(static_cast<char>(static_cast<unsigned char>(c >> 16)));  // implementation-defined
     if (left == 2) {
       if ((c & ((1 << 16) - 1)) != 0) {
         return Status::Error("Wrong padding in the string");
       }
     } else {
-      output += static_cast<char>(static_cast<unsigned char>(c >> 8));  // implementation-defined
+      append(static_cast<char>(static_cast<unsigned char>(c >> 8)));  // implementation-defined
       if (left == 3) {
         if ((c & ((1 << 8) - 1)) != 0) {
           return Status::Error("Wrong padding in the string");
         }
       } else {
-        output += static_cast<char>(static_cast<unsigned char>(c));  // implementation-defined
+        append(static_cast<char>(static_cast<unsigned char>(c)));  // implementation-defined
       }
     }
   }
+  return Status::OK();
+}
+
+Result<string> base64_decode(Slice base64) {
+  init_base64_table();
+
+  TRY_RESULT(tmp, base64_drop_padding(base64));
+  base64 = tmp;
+
+  string output;
+  output.reserve(((base64.size() + 3) >> 2) * 3);
+  TRY_STATUS(base64_do_decode(base64, [&output](char c) { output += c; }));
   return output;
 }
 
+Result<SecureString> base64_decode_secure(Slice base64) {
+  init_base64_table();
+
+  TRY_RESULT(tmp, base64_drop_padding(base64));
+  base64 = tmp;
+
+  SecureString output(((base64.size() + 3) >> 2) * 3);
+  char *ptr = output.as_mutable_slice().begin();
+  TRY_STATUS(base64_do_decode(base64, [&ptr](char c) { *ptr++ = c; }));
+  size_t size = ptr - output.as_mutable_slice().begin();
+  if (size == output.size()) {
+    return std::move(output);
+  }
+  return SecureString(output.as_slice().substr(0, size));
+}
+
 static const char *const url_symbols64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
 
 string base64url_encode(Slice input) {
@@ -133,7 +163,7 @@ string base64url_encode(Slice input) {
 
 static unsigned char url_char_to_value[256];
 static void init_base64url_table() {
-  static bool is_inited = []() {
+  static bool is_inited = [] {
     std::fill(std::begin(url_char_to_value), std::end(url_char_to_value), static_cast<unsigned char>(64));
     for (unsigned char i = 0; i < 64; i++) {
       url_char_to_value[static_cast<size_t>(url_symbols64[i])] = i;

tdutils/td/utils/base64.h

@@ -8,12 +8,14 @@
 
 #include "td/utils/common.h"
 #include "td/utils/Slice.h"
+#include "td/utils/SharedSlice.h"
 #include "td/utils/Status.h"
 
 namespace td {
 
 string base64_encode(Slice input);
 Result<string> base64_decode(Slice base64);
+Result<SecureString> base64_decode_secure(Slice base64);
 
 string base64url_encode(Slice input);
 Result<string> base64url_decode(Slice base64);

tdutils/td/utils/buffer.h

@@ -796,5 +796,8 @@ inline Slice as_slice(const BufferSlice &value) {
 inline MutableSlice as_slice(BufferSlice &value) {
   return value.as_slice();
 }
+inline MutableSlice as_mutable_slice(BufferSlice &value) {
+  return value.as_slice();
+}
 
 }  // namespace td

tdutils/td/utils/crypto.cpp

@@ -241,66 +241,74 @@ int pq_factorize(Slice pq_str, string *p_str, string *q_str) {
   return 0;
 }
 
-static void aes_ige_xcrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to, bool encrypt_flag) {
+static void aes_ige_xcrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to, bool encrypt_flag) {
+  CHECK(aes_key.size() == 32);
+  CHECK(aes_iv.size() == 16);
   AES_KEY key;
   int err;
   if (encrypt_flag) {
-    err = AES_set_encrypt_key(aes_key.raw, 256, &key);
+    err = AES_set_encrypt_key(aes_key.ubegin(), 256, &key);
   } else {
-    err = AES_set_decrypt_key(aes_key.raw, 256, &key);
+    err = AES_set_decrypt_key(aes_key.ubegin(), 256, &key);
   }
   LOG_IF(FATAL, err != 0);
   CHECK(from.size() <= to.size());
-  AES_ige_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv->raw, encrypt_flag);
+  AES_ige_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv.ubegin(), encrypt_flag);
 }
 
-void aes_ige_encrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to) {
+void aes_ige_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) {
   aes_ige_xcrypt(aes_key, aes_iv, from, to, true);
 }
 
-void aes_ige_decrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to) {
+void aes_ige_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) {
   aes_ige_xcrypt(aes_key, aes_iv, from, to, false);
 }
 
-static void aes_cbc_xcrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to, bool encrypt_flag) {
+static void aes_cbc_xcrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to, bool encrypt_flag) {
+  CHECK(aes_key.size() == 32);
+  CHECK(aes_iv.size() == 16);
   AES_KEY key;
   int err;
   if (encrypt_flag) {
-    err = AES_set_encrypt_key(aes_key.raw, 256, &key);
+    err = AES_set_encrypt_key(aes_key.ubegin(), 256, &key);
   } else {
-    err = AES_set_decrypt_key(aes_key.raw, 256, &key);
+    err = AES_set_decrypt_key(aes_key.ubegin(), 256, &key);
   }
   LOG_IF(FATAL, err != 0);
   CHECK(from.size() <= to.size());
-  AES_cbc_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv->raw, encrypt_flag);
+  AES_cbc_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv.ubegin(), encrypt_flag);
 }
 
-void aes_cbc_encrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to) {
+void aes_cbc_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) {
   aes_cbc_xcrypt(aes_key, aes_iv, from, to, true);
 }
 
-void aes_cbc_decrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to) {
+void aes_cbc_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) {
   aes_cbc_xcrypt(aes_key, aes_iv, from, to, false);
 }
 
-AesCbcState::AesCbcState(const UInt256 &key, const UInt128 &iv) : key_(key), iv_(iv) {
+AesCbcState::AesCbcState(Slice key256, Slice iv128) : key_(key256), iv_(iv128) {
+  CHECK(key_.size() == 32);
+  CHECK(iv_.size() == 16);
 }
 
 void AesCbcState::encrypt(Slice from, MutableSlice to) {
-  ::td::aes_cbc_encrypt(key_, &iv_, from, to);
+  ::td::aes_cbc_encrypt(key_.as_slice(), iv_.as_mutable_slice(), from, to);
 }
 void AesCbcState::decrypt(Slice from, MutableSlice to) {
-  ::td::aes_cbc_decrypt(key_, &iv_, from, to);
+  ::td::aes_cbc_decrypt(key_.as_slice(), iv_.as_mutable_slice(), from, to);
 }
 
 class AesCtrState::Impl {
  public:
-  Impl(const UInt256 &key, const UInt128 &iv) {
+  Impl(Slice key, Slice iv) {
+    CHECK(key.size() == 32);
+    CHECK(iv.size() == 16);
     static_assert(AES_BLOCK_SIZE == 16, "");
-    if (AES_set_encrypt_key(key.raw, 256, &aes_key) < 0) {
+    if (AES_set_encrypt_key(key.ubegin(), 256, &aes_key) < 0) {
       LOG(FATAL) << "Failed to set encrypt key";
     }
-    MutableSlice(counter, AES_BLOCK_SIZE).copy_from(as_slice(iv));
+    counter.as_mutable_slice().copy_from(as_slice(iv));
     current_pos = 0;
   }
 
@@ -308,9 +316,10 @@ class AesCtrState::Impl {
     CHECK(to.size() >= from.size());
     for (size_t i = 0; i < from.size(); i++) {
       if (current_pos == 0) {
-        AES_encrypt(counter, encrypted_counter, &aes_key);
+        AES_encrypt(counter.as_slice().ubegin(), encrypted_counter.as_mutable_slice().ubegin(), &aes_key);
+        uint8 *ptr = counter.as_mutable_slice().ubegin();
         for (int j = 15; j >= 0; j--) {
-          if (++counter[j] != 0) {
+          if (++ptr[j] != 0) {
             break;
           }
         }
@@ -322,8 +331,8 @@ class AesCtrState::Impl {
 
  private:
   AES_KEY aes_key;
-  uint8 counter[AES_BLOCK_SIZE];
+  SecureString counter{AES_BLOCK_SIZE};
-  uint8 encrypted_counter[AES_BLOCK_SIZE];
+  SecureString encrypted_counter{AES_BLOCK_SIZE};
   uint8 current_pos;
 };
 
@@ -332,7 +341,7 @@ AesCtrState::AesCtrState(AesCtrState &&from) = default;
 AesCtrState &AesCtrState::operator=(AesCtrState &&from) = default;
 AesCtrState::~AesCtrState() = default;
 
-void AesCtrState::init(const UInt256 &key, const UInt128 &iv) {
+void AesCtrState::init(Slice key, Slice iv) {
   ctx_ = make_unique<AesCtrState::Impl>(key, iv);
 }
 

tdutils/td/utils/crypto.h

@@ -9,8 +9,8 @@
 #include "td/utils/buffer.h"
 #include "td/utils/common.h"
 #include "td/utils/Slice.h"
+#include "td/utils/SharedSlice.h"
 #include "td/utils/Status.h"
-#include "td/utils/UInt.h"
 
 namespace td {
 
@@ -21,11 +21,11 @@ void init_crypto();
 
 int pq_factorize(Slice pq_str, string *p_str, string *q_str);
 
-void aes_ige_encrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to);
+void aes_ige_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to);
-void aes_ige_decrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to);
+void aes_ige_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to);
 
-void aes_cbc_encrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to);
+void aes_cbc_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to);
-void aes_cbc_decrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to);
+void aes_cbc_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to);
 
 class AesCtrState {
  public:
@@ -36,7 +36,7 @@ class AesCtrState {
   AesCtrState &operator=(AesCtrState &&from);
   ~AesCtrState();
 
-  void init(const UInt256 &key, const UInt128 &iv);
+  void init(Slice key, Slice iv);
 
   void encrypt(Slice from, MutableSlice to);
 
@@ -49,14 +49,14 @@ class AesCtrState {
 
 class AesCbcState {
  public:
-  AesCbcState(const UInt256 &key, const UInt128 &iv);
+  AesCbcState(Slice key256, Slice iv128);
 
   void encrypt(Slice from, MutableSlice to);
   void decrypt(Slice from, MutableSlice to);
 
  private:
-  UInt256 key_;
+  SecureString key_;
-  UInt128 iv_;
+  SecureString iv_;
 };
 
 void sha1(Slice data, unsigned char output[20]);

tdutils/td/utils/filesystem.cpp

@@ -32,6 +32,10 @@ template <>
 BufferSlice create_empty<BufferSlice>(size_t size) {
   return BufferSlice{size};
 }
+template <>
+SecureString create_empty<SecureString>(size_t size) {
+  return SecureString{size};
+}
 
 template <class T>
 Result<T> read_file_impl(CSlice path, int64 size, int64 offset) {
@@ -48,7 +52,7 @@ Result<T> read_file_impl(CSlice path, int64 size, int64 offset) {
   }
   size -= offset;
   auto content = create_empty<T>(narrow_cast<size_t>(size));
-  TRY_RESULT(got_size, from_file.pread(as_slice(content), offset));
+  TRY_RESULT(got_size, from_file.pread(as_mutable_slice(content), offset));
   if (got_size != static_cast<size_t>(size)) {
     return Status::Error("Failed to read file");
   }
@@ -66,6 +70,10 @@ Result<string> read_file_str(CSlice path, int64 size, int64 offset) {
   return read_file_impl<string>(path, size, offset);
 }
 
+Result<SecureString> read_file_secure(CSlice path, int64 size, int64 offset) {
+  return read_file_impl<SecureString>(path, size, offset);
+}
+
 // Very straightforward function. Don't expect much of it.
 Status copy_file(CSlice from, CSlice to, int64 size) {
   TRY_RESULT(content, read_file(from, size));

tdutils/td/utils/filesystem.h

@@ -8,12 +8,14 @@
 
 #include "td/utils/buffer.h"
 #include "td/utils/Slice.h"
+#include "td/utils/SharedSlice.h"
 #include "td/utils/Status.h"
 
 namespace td {
 
 Result<BufferSlice> read_file(CSlice path, int64 size = -1, int64 offset = 0);
 Result<string> read_file_str(CSlice path, int64 size = -1, int64 offset = 0);
+Result<SecureString> read_file_secure(CSlice path, int64 size = -1, int64 offset = 0);
 
 Status copy_file(CSlice from, CSlice to, int64 size = -1) TD_WARN_UNUSED_RESULT;
 

tdutils/td/utils/port/detail/ThreadStl.h

@@ -30,7 +30,9 @@ class ThreadStl {
   ThreadStl &operator=(const ThreadStl &other) = delete;
   ThreadStl(ThreadStl &&) = default;
   ThreadStl &operator=(ThreadStl &&) = default;
-  ~ThreadStl() = default;
+  ~ThreadStl() {
+    join();
+  }
   template <class Function, class... Args>
   explicit ThreadStl(Function &&f, Args &&... args) {
     thread_ = std::thread([args = std::make_tuple(decay_copy(std::forward<Function>(f)),
@@ -42,11 +44,15 @@ class ThreadStl {
   }
 
   void join() {
+    if (thread_.joinable()) {
       thread_.join();
     }
+  }
   void detach() {
+    if (thread_.joinable()) {
       thread_.detach();
     }
+  }
   void set_name(CSlice name) {
   }
 

tdutils/td/utils/tl_helpers.h

@@ -10,6 +10,7 @@
 #include "td/utils/logging.h"
 #include "td/utils/misc.h"
 #include "td/utils/Slice.h"
+#include "td/utils/SharedSlice.h"
 #include "td/utils/StackAllocator.h"
 #include "td/utils/Status.h"
 #include "td/utils/tl_parsers.h"
@@ -115,11 +116,20 @@ template <class StorerT>
 void store(const string &x, StorerT &storer) {
   storer.store_string(x);
 }
+template <class StorerT>
+void store(const SecureString &x, StorerT &storer) {
+  storer.store_string(x.as_slice());
+}
 template <class ParserT>
 void parse(string &x, ParserT &parser) {
   x = parser.template fetch_string<string>();
 }
 
+template <class ParserT>
+void parse(SecureString &x, ParserT &parser) {
+  x = parser.template fetch_string<SecureString>();
+}
+
 template <class T, class StorerT>
 void store(const vector<T> &vec, StorerT &storer) {
   storer.store_binary(narrow_cast<int32>(vec.size()));
@@ -228,6 +238,21 @@ string serialize(const T &object) {
   return key;
 }
 
+template <class T>
+SecureString serialize_secure(const T &object) {
+  TlStorerCalcLength calc_length;
+  store(object, calc_length);
+  size_t length = calc_length.get_length();
+
+  SecureString key(length, '\0');
+  CHECK(is_aligned_pointer<4>(key.data()));
+  MutableSlice data = key.as_mutable_slice();
+  TlStorerUnsafe storer(data.ubegin());
+  store(object, storer);
+  CHECK(storer.get_buf() == data.uend());
+  return key;
+}
+
 template <class T>
 TD_WARN_UNUSED_RESULT Status unserialize(T &object, Slice data) {
   TlParser parser(data);

tdutils/td/utils/tl_parsers.h

@@ -148,16 +148,29 @@ class TlParser {
     if (result_len < 254) {
       result_begin = reinterpret_cast<const char *>(data + 1);
       result_aligned_len = (result_len >> 2) << 2;
+      data += sizeof(int32);
     } else if (result_len == 254) {
       result_len = data[1] + (data[2] << 8) + (data[3] << 16);
       result_begin = reinterpret_cast<const char *>(data + 4);
       result_aligned_len = ((result_len + 3) >> 2) << 2;
+      data += sizeof(int32);
     } else {
-      set_error("Can't fetch string, 255 found");
+      check_len(sizeof(int32));
+      result_len = data[1] + (data[2] << 8) + (data[3] << 16) + (data[4] << 24) + (static_cast<uint64>(data[5]) << 32) +
+                   (static_cast<uint64>(data[6]) << 40) + (static_cast<uint64>(data[7]) << 48);
+      if (result_len > std::numeric_limits<size_t>::max() - 3) {
+        set_error("Too big string found");
         return T();
       }
+      result_begin = reinterpret_cast<const char *>(data + 8);
+      result_aligned_len = ((result_len + 3) >> 2) << 2;
+      data += sizeof(int64);
+    }
     check_len(result_aligned_len);
-    data += result_aligned_len + sizeof(int32);
+    if (!error.empty()) {
+      return T();
+    }
+    data += result_aligned_len;
     return T(result_begin, result_len);
   }
 
@@ -165,6 +178,9 @@ class TlParser {
   T fetch_string_raw(const size_t size) {
     //CHECK(size % sizeof(int32) == 0);
     check_len(size);
+    if (!error.empty()) {
+      return T();
+    }
     const char *result = reinterpret_cast<const char *>(data);
     data += size;
     return T(result, size);

tdutils/td/utils/tl_storers.h

@@ -62,6 +62,15 @@ class TlStorerUnsafe {
       *buf_++ = static_cast<unsigned char>(len & 255);
       *buf_++ = static_cast<unsigned char>((len >> 8) & 255);
       *buf_++ = static_cast<unsigned char>(len >> 16);
+    } else if (len < (1ull << 32)) {
+      *buf_++ = static_cast<unsigned char>(255);
+      *buf_++ = static_cast<unsigned char>(len & 255);
+      *buf_++ = static_cast<unsigned char>((len >> 8) & 255);
+      *buf_++ = static_cast<unsigned char>((len >> 16) & 255);
+      *buf_++ = static_cast<unsigned char>((len >> 24) & 255);
+      *buf_++ = static_cast<unsigned char>(0);
+      *buf_++ = static_cast<unsigned char>(0);
+      *buf_++ = static_cast<unsigned char>(0);
     } else {
       LOG(FATAL) << "String size " << len << " is too big to be stored";
     }
@@ -119,8 +128,10 @@ class TlStorerCalcLength {
     size_t add = str.size();
     if (add < 254) {
       add += 1;
-    } else {
+    } else if (add < (1 << 24)) {
       add += 4;
+    } else {
+      add += 8;
     }
     add = (add + 3) & -4;
     length += add;

tdutils/test/SharedSlice.cpp

@@ -0,0 +1,85 @@
+//
+// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+#include "td/utils/tests.h"
+#include "td/utils/SharedSlice.h"
+
+using namespace td;
+
+TEST(SharedSlice, Hands) {
+  {
+    SharedSlice h("hello");
+    ASSERT_EQ("hello", h.as_slice());
+    // auto g = h; // CE
+    auto g = h.clone();
+    ASSERT_EQ("hello", g.as_slice());
+  }
+
+  {
+    SharedSlice h("hello");
+    UniqueSharedSlice g(std::move(h));
+    ASSERT_EQ("", h.as_slice());
+    ASSERT_EQ("hello", g.as_slice());
+  }
+  {
+    SharedSlice h("hello");
+    SharedSlice t = h.clone();
+    UniqueSharedSlice g(std::move(h));
+    ASSERT_EQ("", h.as_slice());
+    ASSERT_EQ("hello", g.as_slice());
+    ASSERT_EQ("hello", t.as_slice());
+  }
+
+  {
+    UniqueSharedSlice g(5);
+    g.as_mutable_slice().copy_from("hello");
+    SharedSlice h(std::move(g));
+    ASSERT_EQ("hello", h);
+    ASSERT_EQ("", g);
+  }
+
+  {
+    UniqueSlice h("hello");
+    UniqueSlice g(std::move(h));
+    ASSERT_EQ("", h.as_slice());
+    ASSERT_EQ("hello", g.as_slice());
+  }
+
+  {
+    SecureString h("hello");
+    SecureString g(std::move(h));
+    ASSERT_EQ("", h.as_slice());
+    ASSERT_EQ("hello", g.as_slice());
+  }
+
+  {
+    Stage stage;
+    SharedSlice a, b;
+    std::vector<td::thread> threads(2);
+    for (int i = 0; i < 2; i++) {
+      threads[i] = td::thread([i, &stage, &a, &b] {
+        for (int j = 0; j < 10000; j++) {
+          if (i == 0) {
+            a = SharedSlice("hello");
+            b = a.clone();
+          }
+          stage.wait((2 * j + 1) * 2);
+          if (i == 0) {
+            ASSERT_EQ('h', a[0]);
+            a.clear();
+          } else {
+            UniqueSharedSlice c(std::move(b));
+            c.as_mutable_slice()[0] = '!';
+          }
+          stage.wait((2 * j + 2) * 2);
+        }
+      });
+    }
+    for (auto &thread : threads) {
+      thread.join();
+    }
+  }
+}

tdutils/test/crypto.cpp

@@ -45,18 +45,18 @@ TEST(Crypto, AesCtrState) {
     }
 
     td::AesCtrState state;
-    state.init(key, iv);
+    state.init(as_slice(key), as_slice(iv));
     td::string t(length, '\0');
     state.encrypt(s, t);
     ASSERT_EQ(answers1[i], td::crc32(t));
-    state.init(key, iv);
+    state.init(as_slice(key), as_slice(iv));
     state.decrypt(t, t);
     ASSERT_STREQ(s, t);
 
     for (auto &c : iv.raw) {
       c = 0xFF;
     }
-    state.init(key, iv);
+    state.init(as_slice(key), as_slice(iv));
     state.encrypt(s, t);
     ASSERT_EQ(answers2[i], td::crc32(t));
 

tdutils/test/log.cpp

@@ -0,0 +1,122 @@
+//
+// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+#include "td/utils/tests.h"
+#include "td/utils/FileLog.h"
+#include "td/utils/TsFileLog.h"
+#include "td/utils/logging.h"
+#include "td/utils/port/path.h"
+#include "td/utils/benchmark.h"
+
+// Thread safe logging with tests
+//
+// LOG uses thread local LogInterface
+// void append(CSlice slice, int log_level);
+//
+
+template <class Log>
+class LogBenchmark : public td::Benchmark {
+ public:
+  explicit LogBenchmark(std::string name, int threads_n, std::function<td::unique_ptr<Log>()> creator)
+      : name_(std::move(name)), threads_n_(threads_n), creator_(std::move(creator)) {
+  }
+  std::string get_description() const override {
+    return PSTRING() << name_ << " " << td::tag("threads_n", threads_n_);
+  }
+  void start_up() override {
+    log_ = creator_();
+    threads_.resize(threads_n_);
+  }
+  void tear_down() override {
+    for (auto path : log_->get_file_paths()) {
+      td::unlink(path).ignore();
+    }
+    log_.reset();
+  }
+  void run(int n) override {
+    for (auto &thread : threads_) {
+      thread = td::thread([this, n] { this->run_thread(n); });
+    }
+    for (auto &thread : threads_) {
+      thread.join();
+    }
+  }
+
+  void run_thread(int n) {
+    auto str = PSTRING() << "#" << n << " : fsjklfdjsklfjdsklfjdksl\n";
+    for (int i = 0; i < n; i++) {
+      log_->append(str);
+    }
+  }
+
+ private:
+  std::string name_;
+  td::unique_ptr<Log> log_;
+  int threads_n_{0};
+  std::function<td::unique_ptr<Log>()> creator_;
+  std::vector<td::thread> threads_;
+};
+
+template <class F>
+void bench_log(std::string name, int threads_n, F &&f) {
+  bench(LogBenchmark<typename decltype(f())::element_type>(std::move(name), threads_n, std::move(f)));
+};
+
+TEST(Log, TsLogger) {
+  bench_log("NewTsFileLog", 4, [] { return td::TsFileLog::create("tmplog").move_as_ok(); });
+  bench_log("TsFileLog", 8, [] {
+    class FileLog : public td::LogInterface {
+     public:
+      FileLog() {
+        file_log_.init("tmplog", std::numeric_limits<td::int64>::max(), false);
+        ts_log_.init(&file_log_);
+      }
+      ~FileLog() {
+      }
+      void append(td::CSlice slice) override {
+        ts_log_.append(slice, -1);
+      }
+      std::vector<std::string> get_file_paths() override {
+        return file_log_.get_file_paths();
+      }
+
+     private:
+      td::FileLog file_log_;
+      td::TsLog ts_log_{nullptr};
+    };
+    return td::make_unique<FileLog>();
+  });
+
+  bench_log("noop", 4, [] {
+    class NoopLog : public td::LogInterface {
+     public:
+      void append(td::CSlice slice) override {
+      }
+    };
+    return td::make_unique<NoopLog>();
+  });
+
+  bench_log("FileLog", 4, [] {
+    class FileLog : public td::LogInterface {
+     public:
+      FileLog() {
+        file_log_.init("tmplog", std::numeric_limits<td::int64>::max(), false);
+      }
+      ~FileLog() {
+      }
+      void append(td::CSlice slice) override {
+        file_log_.append(slice, -1);
+      }
+      std::vector<std::string> get_file_paths() override {
+        return file_log_.get_file_paths();
+      }
+
+     private:
+      td::FileLog file_log_;
+    };
+    return td::make_unique<FileLog>();
+  });
+}

tdutils/test/misc.cpp

@@ -895,6 +895,9 @@ static void test_hash() {
   test_hash<HashT, BadValue>({BadValue{1}, BadValue{2}}).ensure_error();
   test_hash<HashT, ValueA>({ValueA{1}, ValueA{2}}).ensure();
   test_hash<HashT, ValueB>({ValueB{1}, ValueB{2}}).ensure();
+  test_hash<HashT, std::pair<int, int>>({{1, 1}, {1, 2}}).ensure();
+  // FIXME: use some better hash
+  //test_hash<HashT, std::pair<int, int>>({{1, 1}, {1, 2}, {2, 1}, {2, 2}}).ensure();
 }
 
 TEST(Misc, Hasher) {