From f63ce74feff1f411f0cc15d8ee277e034e1aaeb4 Mon Sep 17 00:00:00 2001 From: Arseny Smirnov Date: Mon, 12 Aug 2019 14:45:57 +0300 Subject: [PATCH] tdutils: import changes from other project GitOrigin-RevId: 0cfbe2418933cffdce4f0d1b8461c247a814d868 --- tdutils/CMakeLists.txt | 11 +- tdutils/td/utils/DecTree.h | 3 + tdutils/td/utils/FileLog.cpp | 7 +- tdutils/td/utils/FileLog.h | 3 +- tdutils/td/utils/JsonBuilder.h | 59 +++- tdutils/td/utils/MpmcWaiter.h | 3 +- tdutils/td/utils/SharedSlice.cpp | 17 + tdutils/td/utils/SharedSlice.h | 377 +++++++++++++++++++++++ tdutils/td/utils/Slice-decl.h | 3 + tdutils/td/utils/Slice.cpp | 31 ++ tdutils/td/utils/Slice.h | 8 + tdutils/td/utils/ThreadLocalStorage.h | 54 ++++ tdutils/td/utils/ThreadSafeCounter.h | 112 +++++-- tdutils/td/utils/TsFileLog.cpp | 78 +++++ tdutils/td/utils/TsFileLog.h | 16 + tdutils/td/utils/base64.cpp | 50 ++- tdutils/td/utils/base64.h | 2 + tdutils/td/utils/buffer.h | 3 + tdutils/td/utils/crypto.cpp | 55 ++-- tdutils/td/utils/crypto.h | 18 +- tdutils/td/utils/filesystem.cpp | 10 +- tdutils/td/utils/filesystem.h | 2 + tdutils/td/utils/port/detail/ThreadStl.h | 12 +- tdutils/td/utils/tl_helpers.h | 25 ++ tdutils/td/utils/tl_parsers.h | 22 +- tdutils/td/utils/tl_storers.h | 15 +- tdutils/test/SharedSlice.cpp | 85 +++++ tdutils/test/crypto.cpp | 6 +- tdutils/test/log.cpp | 122 ++++++++ tdutils/test/misc.cpp | 3 + 30 files changed, 1121 insertions(+), 91 deletions(-) create mode 100644 tdutils/td/utils/SharedSlice.cpp create mode 100644 tdutils/td/utils/SharedSlice.h create mode 100644 tdutils/td/utils/Slice.cpp create mode 100644 tdutils/td/utils/ThreadLocalStorage.h create mode 100644 tdutils/td/utils/TsFileLog.cpp create mode 100644 tdutils/td/utils/TsFileLog.h create mode 100644 tdutils/test/SharedSlice.cpp create mode 100644 tdutils/test/log.cpp diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index 7c9469d6c..4375a479e 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -99,12 +99,15 @@ set(TDUTILS_SOURCE td/utils/MpmcQueue.cpp td/utils/OptionsParser.cpp td/utils/Random.cpp + td/utils/Slice.cpp + td/utils/SharedSlice.cpp td/utils/StackAllocator.cpp td/utils/Status.cpp td/utils/StringBuilder.cpp - td/utils/tests.cpp td/utils/Time.cpp td/utils/Timer.cpp + td/utils/TsFileLog.cpp + td/utils/tests.cpp td/utils/tl_parsers.cpp td/utils/translit.cpp td/utils/unicode.cpp @@ -231,6 +234,7 @@ set(TDUTILS_SOURCE td/utils/Time.h td/utils/TimedStat.h td/utils/Timer.h + td/utils/TsFileLog.h td/utils/tl_helpers.h td/utils/tl_parsers.h td/utils/tl_storers.h @@ -250,12 +254,12 @@ set(TDUTILS_TEST_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/test/ConcurrentHashMap.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/crypto.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/Enumerator.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/test/EpochBasedMemoryReclamation.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/filesystem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/gzip.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/HazardPointers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/heap.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/json.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/log.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/misc.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcQueue.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcWaiter.cpp @@ -264,6 +268,7 @@ set(TDUTILS_TEST_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/test/port.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/pq.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/SharedObjectPool.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/SharedSlice.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/variant.cpp PARENT_SCOPE ) @@ -300,7 +305,7 @@ if (CRC32C_FOUND) target_link_libraries(tdutils PRIVATE crc32c) endif() if (ABSL_FOUND) - target_link_libraries(tdutils PUBLIC absl::base absl::container absl::hash) + target_link_libraries_system(tdutils PUBLIC absl::flat_hash_map absl::flat_hash_set absl::hash) endif() if (WIN32 AND WINGETOPT_FOUND) diff --git a/tdutils/td/utils/DecTree.h b/tdutils/td/utils/DecTree.h index a03c3ae0a..467158d0d 100644 --- a/tdutils/td/utils/DecTree.h +++ b/tdutils/td/utils/DecTree.h @@ -185,6 +185,9 @@ class DecTree { void remove(const KeyType &key) { root_ = remove_node(std::move(root_), key); } + void reset() { + root_ = nullptr; + } ValueType *get(const KeyType &key) { return get_node(root_, key); } diff --git a/tdutils/td/utils/FileLog.cpp b/tdutils/td/utils/FileLog.cpp index 3e892d250..89ddf4ac0 100644 --- a/tdutils/td/utils/FileLog.cpp +++ b/tdutils/td/utils/FileLog.cpp @@ -17,7 +17,7 @@ namespace td { -Status FileLog::init(string path, int64 rotate_threshold) { +Status FileLog::init(string path, int64 rotate_threshold, bool redirect_stderr) { if (path == path_) { set_rotate_threshold(rotate_threshold); return Status::OK(); @@ -30,7 +30,7 @@ Status FileLog::init(string path, int64 rotate_threshold) { fd_.close(); fd_ = std::move(fd); - if (!Stderr().empty()) { + if (!Stderr().empty() && redirect_stderr) { fd_.get_native_fd().duplicate(Stderr().get_native_fd()).ignore(); } @@ -43,6 +43,7 @@ Status FileLog::init(string path, int64 rotate_threshold) { TRY_RESULT(size, fd_.get_size()); size_ = size; rotate_threshold_ = rotate_threshold; + redirect_stderr_ = redirect_stderr; return Status::OK(); } @@ -108,7 +109,7 @@ void FileLog::do_rotate() { process_fatal_error(PSLICE() << r_fd.error() << " in " << __FILE__ << " at " << __LINE__); } fd_ = r_fd.move_as_ok(); - if (!Stderr().empty()) { + if (!Stderr().empty() && redirect_stderr_) { fd_.get_native_fd().duplicate(Stderr().get_native_fd()).ignore(); } size_ = 0; diff --git a/tdutils/td/utils/FileLog.h b/tdutils/td/utils/FileLog.h index e015d96c3..145d6ac82 100644 --- a/tdutils/td/utils/FileLog.h +++ b/tdutils/td/utils/FileLog.h @@ -18,7 +18,7 @@ class FileLog : public LogInterface { static constexpr int64 DEFAULT_ROTATE_THRESHOLD = 10 * (1 << 20); public: - Status init(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD); + Status init(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD, bool redirect_stderr = true); Slice get_path() const; @@ -37,6 +37,7 @@ class FileLog : public LogInterface { string path_; int64 size_ = 0; int64 rotate_threshold_ = 0; + bool redirect_stderr_; void do_rotate(); }; diff --git a/tdutils/td/utils/JsonBuilder.h b/tdutils/td/utils/JsonBuilder.h index 3aabe1a8f..3526787fa 100644 --- a/tdutils/td/utils/JsonBuilder.h +++ b/tdutils/td/utils/JsonBuilder.h @@ -181,7 +181,7 @@ class JsonObjectScope; class JsonBuilder { public: - explicit JsonBuilder(StringBuilder &&sb) : sb_(std::move(sb)) { + explicit JsonBuilder(StringBuilder &&sb, int32 offset = -1) : sb_(std::move(sb)), offset_(offset) { } StringBuilder &string_builder() { return sb_; @@ -191,9 +191,33 @@ class JsonBuilder { JsonArrayScope enter_array() TD_WARN_UNUSED_RESULT; JsonObjectScope enter_object() TD_WARN_UNUSED_RESULT; + int32 offset() const { + return offset_; + } + bool is_pretty() const { + return offset_ >= 0; + } + void print_offset() { + for (int x = 0; x < offset_; x++) { + sb_ << " "; + } + } + void dec_offset() { + if (offset_ >= 0) { + CHECK(offset_ > 0); + offset_--; + } + } + void inc_offset() { + if (offset_ >= 0) { + offset_++; + } + } + private: StringBuilder sb_; JsonScope *scope_ = nullptr; + int32 offset_; }; class Jsonable {}; @@ -328,6 +352,7 @@ class JsonValueScope : public JsonScope { class JsonArrayScope : public JsonScope { public: explicit JsonArrayScope(JsonBuilder *jb) : JsonScope(jb) { + jb->inc_offset(); *sb_ << "["; } JsonArrayScope(JsonArrayScope &&other) = default; @@ -337,6 +362,11 @@ class JsonArrayScope : public JsonScope { } } void leave() { + jb_->dec_offset(); + if (jb_->is_pretty()) { + *sb_ << "\n"; + jb_->print_offset(); + } *sb_ << "]"; } template @@ -355,6 +385,10 @@ class JsonArrayScope : public JsonScope { } else { is_first_ = true; } + if (jb_->is_pretty()) { + *sb_ << "\n"; + jb_->print_offset(); + } return jb_->enter_value(); } @@ -365,6 +399,7 @@ class JsonArrayScope : public JsonScope { class JsonObjectScope : public JsonScope { public: explicit JsonObjectScope(JsonBuilder *jb) : JsonScope(jb) { + jb->inc_offset(); *sb_ << "{"; } JsonObjectScope(JsonObjectScope &&other) = default; @@ -374,6 +409,11 @@ class JsonObjectScope : public JsonScope { } } void leave() { + jb_->dec_offset(); + if (jb_->is_pretty()) { + *sb_ << "\n"; + jb_->print_offset(); + } *sb_ << "}"; } template @@ -392,8 +432,16 @@ class JsonObjectScope : public JsonScope { } else { is_first_ = true; } + if (jb_->is_pretty()) { + *sb_ << "\n"; + jb_->print_offset(); + } jb_->enter_value() << key; - *sb_ << ":"; + if (jb_->is_pretty()) { + *sb_ << " : "; + } else { + *sb_ << ":"; + } jb_->enter_value() << value; return *this; } @@ -737,11 +785,14 @@ inline Result json_decode(MutableSlice json) { } template -StrT json_encode(const ValT &val) { +StrT json_encode(const ValT &val, bool pretty = false) { auto buf_len = 1 << 18; auto buf = StackAllocator::alloc(buf_len); - JsonBuilder jb(StringBuilder(buf.as_slice(), true)); + JsonBuilder jb(StringBuilder(buf.as_slice(), true), pretty ? 0 : -1); jb.enter_value() << val; + if (pretty) { + jb.string_builder() << "\n"; + } LOG_IF(ERROR, jb.string_builder().is_error()) << "JSON buffer overflow"; auto slice = jb.string_builder().as_cslice(); return StrT(slice.begin(), slice.size()); diff --git a/tdutils/td/utils/MpmcWaiter.h b/tdutils/td/utils/MpmcWaiter.h index f71613fc1..1afbd1471 100644 --- a/tdutils/td/utils/MpmcWaiter.h +++ b/tdutils/td/utils/MpmcWaiter.h @@ -90,7 +90,8 @@ class MpmcWaiter { return (state >> 1) == (worker + 1); } }; - enum { RoundsTillSleepy = 32, RoundsTillAsleep = 64 }; + //enum { RoundsTillSleepy = 32, RoundsTillAsleep = 64 }; + enum { RoundsTillSleepy = 1, RoundsTillAsleep = 2 }; std::atomic state_{State::awake()}; std::mutex mutex_; std::condition_variable condition_variable_; diff --git a/tdutils/td/utils/SharedSlice.cpp b/tdutils/td/utils/SharedSlice.cpp new file mode 100644 index 000000000..17b937cf3 --- /dev/null +++ b/tdutils/td/utils/SharedSlice.cpp @@ -0,0 +1,17 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/SharedSlice.h" + +#include "td/utils/buffer.h" + +namespace td { + +BufferSlice SharedSlice::clone_as_buffer_slice() const { + return BufferSlice{as_slice().str()}; +} + +} // namespace td diff --git a/tdutils/td/utils/SharedSlice.h b/tdutils/td/utils/SharedSlice.h new file mode 100644 index 000000000..39735aa6f --- /dev/null +++ b/tdutils/td/utils/SharedSlice.h @@ -0,0 +1,377 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/Slice.h" + +#include +#include + +namespace td { + +namespace detail { +struct SharedSliceHeader { + explicit SharedSliceHeader(size_t size) : refcnt_{1}, size_{size} { + } + + void inc() { + refcnt_.fetch_add(1, std::memory_order_relaxed); + } + + bool dec() { + return refcnt_.fetch_sub(1, std::memory_order_acq_rel) == 1; + } + + bool is_unique() const { + // NB: race if std::memory_order_relaxed is used + // reader may see a change by a new writer + return refcnt_.load(std::memory_order_acquire) == 1; + } + + size_t size() const { + return size_; + } + + private: + std::atomic refcnt_; + size_t size_; +}; + +struct UniqueSliceHeader { + explicit UniqueSliceHeader(size_t size) : size_{size} { + } + + void inc() { + } + + bool dec() { + return true; + } + + bool is_unique() const { + return true; + } + + size_t size() const { + return size_; + } + + private: + size_t size_; +}; + +template +class UnsafeSharedSlice { + public: + UnsafeSharedSlice() = default; + UnsafeSharedSlice clone() const { + header()->inc(); + return UnsafeSharedSlice(ptr_.get()); + } + + bool is_null() const { + return !ptr_; + } + + bool is_unique() const { + if (is_null()) { + return true; + } + return header()->is_unique(); + } + + MutableSlice as_mutable_slice() { + if (is_null()) { + return MutableSlice(); + } + return MutableSlice(ptr_.get() + sizeof(HeaderT), header()->size()); + } + + Slice as_slice() const { + if (is_null()) { + return Slice(); + } + return Slice(ptr_.get() + sizeof(HeaderT), header()->size()); + } + + size_t size() const { + if (is_null()) { + return 0; + } + return header()->size(); + } + + static UnsafeSharedSlice create(size_t size) { + static_assert(std::is_standard_layout::value, "HeaderT must have statdard layout"); + auto ptr = std::make_unique(size + sizeof(HeaderT)); + auto header_ptr = new (ptr.get()) HeaderT(size); + CHECK(reinterpret_cast(header_ptr) == ptr.get()); + + return UnsafeSharedSlice(std::move(ptr)); + } + + static UnsafeSharedSlice create(Slice slice) { + auto res = create(slice.size()); + res.as_mutable_slice().copy_from(slice); + return res; + } + + void clear() { + ptr_.reset(); + } + + private: + explicit UnsafeSharedSlice(char *ptr) : ptr_(ptr) { + } + explicit UnsafeSharedSlice(std::unique_ptr from) : ptr_(from.release()) { + } + + HeaderT *header() const { + return reinterpret_cast(ptr_.get()); + } + + struct Destructor { + void operator()(char *ptr) { + auto header = reinterpret_cast(ptr); + if (header->dec()) { + if (zero_on_destruct) { + MutableSlice(ptr, sizeof(HeaderT) + header->size()).fill_zero_secure(); + } + std::default_delete()(ptr); + } + } + }; + + std::unique_ptr ptr_; +}; +} // namespace detail + +class BufferSlice; + +class UniqueSharedSlice; + +class SharedSlice { + using Impl = detail::UnsafeSharedSlice; + + public: + SharedSlice() = default; + + explicit SharedSlice(Slice slice) : impl_(Impl::create(slice.size())) { + impl_.as_mutable_slice().copy_from(slice); + } + + explicit SharedSlice(UniqueSharedSlice from); + + SharedSlice(const char *ptr, size_t size) : SharedSlice(Slice(ptr, size)) { + } + + SharedSlice clone() const { + return SharedSlice(impl_.clone()); + } + + Slice as_slice() const { + return impl_.as_slice(); + } + + BufferSlice clone_as_buffer_slice() const; + + operator Slice() const { + return as_slice(); + } + + // like in std::string + const char *data() const { + return as_slice().data(); + } + + char operator[](size_t at) const { + return as_slice()[at]; + } + + bool empty() const { + return size() == 0; + } + + size_t size() const { + return impl_.size(); + } + + // like in std::string + size_t length() const { + return size(); + } + + void clear() { + impl_.clear(); + } + + private: + friend class UniqueSharedSlice; + explicit SharedSlice(Impl impl) : impl_(std::move(impl)) { + } + Impl impl_; +}; + +class UniqueSharedSlice { + using Impl = detail::UnsafeSharedSlice; + + public: + UniqueSharedSlice() = default; + + explicit UniqueSharedSlice(size_t size) : impl_(Impl::create(size)) { + } + explicit UniqueSharedSlice(Slice slice) : impl_(Impl::create(slice)) { + } + + UniqueSharedSlice(const char *ptr, size_t size) : UniqueSharedSlice(Slice(ptr, size)) { + } + explicit UniqueSharedSlice(SharedSlice from) : impl_() { + if (from.impl_.is_unique()) { + impl_ = std::move(from.impl_); + } else { + impl_ = Impl::create(from.as_slice()); + } + } + + UniqueSharedSlice copy() const { + return UniqueSharedSlice(as_slice()); + } + + Slice as_slice() const { + return impl_.as_slice(); + } + + MutableSlice as_mutable_slice() { + return impl_.as_mutable_slice(); + } + + operator Slice() const { + return as_slice(); + } + + // like in std::string + char *data() { + return as_mutable_slice().data(); + } + const char *data() const { + return as_slice().data(); + } + char operator[](size_t at) const { + return as_slice()[at]; + } + + bool empty() const { + return size() == 0; + } + + size_t size() const { + return impl_.size(); + } + + // like in std::string + size_t length() const { + return size(); + } + + void clear() { + impl_.clear(); + } + + private: + friend class SharedSlice; + explicit UniqueSharedSlice(Impl impl) : impl_(std::move(impl)) { + } + Impl impl_; +}; + +inline SharedSlice::SharedSlice(UniqueSharedSlice from) : impl_(std::move(from.impl_)) { +} + +template +class UniqueSliceImpl { + using Impl = detail::UnsafeSharedSlice; + + public: + UniqueSliceImpl() = default; + + explicit UniqueSliceImpl(size_t size) : impl_(Impl::create(size)) { + } + UniqueSliceImpl(size_t size, char c) : impl_(Impl::create(size)) { + std::memset(as_mutable_slice().data(), c, size); + } + explicit UniqueSliceImpl(Slice slice) : impl_(Impl::create(slice)) { + } + + UniqueSliceImpl(const char *ptr, size_t size) : UniqueSliceImpl(Slice(ptr, size)) { + } + + UniqueSliceImpl copy() const { + return UniqueSliceImpl(as_slice()); + } + + Slice as_slice() const { + return impl_.as_slice(); + } + + MutableSlice as_mutable_slice() { + return impl_.as_mutable_slice(); + } + + operator Slice() const { + return as_slice(); + } + + // like in std::string + char *data() { + return as_mutable_slice().data(); + } + const char *data() const { + return as_slice().data(); + } + char operator[](size_t at) const { + return as_slice()[at]; + } + + bool empty() const { + return size() == 0; + } + + size_t size() const { + return impl_.size(); + } + + // like in std::string + size_t length() const { + return size(); + } + + void clear() { + impl_.clear(); + } + + private: + explicit UniqueSliceImpl(Impl impl) : impl_(std::move(impl)) { + } + Impl impl_; +}; + +using UniqueSlice = UniqueSliceImpl; +using SecureString = UniqueSliceImpl; + +inline MutableSlice as_mutable_slice(UniqueSharedSlice &unique_shared_slice) { + return unique_shared_slice.as_mutable_slice(); +} + +inline MutableSlice as_mutable_slice(UniqueSlice &unique_slice) { + return unique_slice.as_mutable_slice(); +} + +inline MutableSlice as_mutable_slice(SecureString &secure_string) { + return secure_string.as_mutable_slice(); +} + +} // namespace td diff --git a/tdutils/td/utils/Slice-decl.h b/tdutils/td/utils/Slice-decl.h index 4a7a7c368..1224b7b30 100644 --- a/tdutils/td/utils/Slice-decl.h +++ b/tdutils/td/utils/Slice-decl.h @@ -52,6 +52,9 @@ class MutableSlice { MutableSlice substr(size_t from, size_t size) const; size_t find(char c) const; size_t rfind(char c) const; + void fill(char c); + void fill_zero(); + void fill_zero_secure(); void copy_from(Slice from); diff --git a/tdutils/td/utils/Slice.cpp b/tdutils/td/utils/Slice.cpp new file mode 100644 index 000000000..e965ab858 --- /dev/null +++ b/tdutils/td/utils/Slice.cpp @@ -0,0 +1,31 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/Slice.h" +#if TD_HAVE_OPENSSL +#include +#endif + +namespace td { + +void MutableSlice::fill(char c) { + std::memset(data(), c, size()); +} +void MutableSlice::fill_zero() { + fill(0); +} +void MutableSlice::fill_zero_secure() { +#if TD_HAVE_OPENSSL + OPENSSL_cleanse(begin(), size()); +#else + volatile char *ptr = begin(); + for (size_t i = 0; i < size(); i++) { + ptr[i] = 0; + } +#endif +} + +} // namespace td diff --git a/tdutils/td/utils/Slice.h b/tdutils/td/utils/Slice.h index a755ed0cd..93a50600c 100644 --- a/tdutils/td/utils/Slice.h +++ b/tdutils/td/utils/Slice.h @@ -316,4 +316,12 @@ inline MutableSlice as_slice(string &str) { return str; } +inline MutableSlice as_mutable_slice(MutableSlice slice) { + return slice; +} + +inline MutableSlice as_mutable_slice(string &str) { + return str; +} + } // namespace td diff --git a/tdutils/td/utils/ThreadLocalStorage.h b/tdutils/td/utils/ThreadLocalStorage.h new file mode 100644 index 000000000..06ab5e0f9 --- /dev/null +++ b/tdutils/td/utils/ThreadLocalStorage.h @@ -0,0 +1,54 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/thread.h" +#include "td/utils/port/thread_local.h" +#include "td/utils/int_types.h" + +#include +#include + +namespace td { +template +class ThreadLocalStorage { + public: + T& get() { + return thread_local_node().value; + } + + template + void for_each(F&& f) { + int n = max_thread_id_.load(); + for (int i = 0; i < n; i++) { + f(nodes_[i].value); + } + } + template + void for_each(F&& f) const { + int n = max_thread_id_.load(); + for (int i = 0; i < n; i++) { + f(nodes_[i].value); + } + } + + private: + struct Node { + T value{}; + char padding[128]; + }; + static constexpr int MAX_THREAD_ID = 128; + std::atomic max_thread_id_{MAX_THREAD_ID}; + std::array nodes_; + + Node& thread_local_node() { + auto thread_id = get_thread_id(); + CHECK(0 <= thread_id && static_cast(thread_id) < nodes_.size()); + return nodes_[thread_id]; + } +}; +} // namespace td diff --git a/tdutils/td/utils/ThreadSafeCounter.h b/tdutils/td/utils/ThreadSafeCounter.h index 6d72f21a4..c1fd3c75a 100644 --- a/tdutils/td/utils/ThreadSafeCounter.h +++ b/tdutils/td/utils/ThreadSafeCounter.h @@ -5,45 +5,107 @@ // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // #pragma once - -#include "td/utils/common.h" -#include "td/utils/port/thread_local.h" +#include "td/utils/ThreadLocalStorage.h" +#include "td/utils/StringBuilder.h" #include -#include +#include namespace td { - -class ThreadSafeCounter { +template +class ThreadSafeMultiCounter { public: - void add(int64 diff) { - auto &node = thread_local_node(); - node.count_.store(node.count_.load(std::memory_order_relaxed) + diff, std::memory_order_relaxed); + void add(size_t index, int64 diff) { + CHECK(index < N); + tls_.get()[index].fetch_add(diff, std::memory_order_relaxed); } - int64 sum() const { - int n = max_thread_id_.load(); + int64 sum(size_t index) const { + CHECK(index < N); int64 res = 0; - for (int i = 0; i < n; i++) { - res += nodes_[i].count_.load(); - } + tls_.for_each([&](auto &value) { res += value[index].load(); }); return res; } private: - struct Node { - std::atomic count_{0}; - char padding[128]; - }; - static constexpr int MAX_THREAD_ID = 128; - std::atomic max_thread_id_{MAX_THREAD_ID}; - std::array nodes_; + ThreadLocalStorage, N>> tls_; +}; - Node &thread_local_node() { - auto thread_id = get_thread_id(); - CHECK(static_cast(thread_id) < nodes_.size()); - return nodes_[thread_id]; +class ThreadSafeCounter { + public: + void add(int64 diff) { + counter_.add(0, diff); } + + int64 sum() const { + return counter_.sum(0); + } + + private: + ThreadSafeMultiCounter<1> counter_; +}; + +class NamedThreadSafeCounter { + static constexpr int N = 128; + using Counter = ThreadSafeMultiCounter; + + public: + class CounterRef { + public: + CounterRef() = default; + CounterRef(size_t index, Counter *counter) : index_(index), counter_(counter) { + } + void add(int64 diff) { + counter_->add(index_, diff); + } + int64 sum() const { + return counter_->sum(index_); + } + + private: + size_t index_{0}; + Counter *counter_{nullptr}; + }; + + CounterRef get_counter(Slice name) { + std::unique_lock guard(mutex_); + for (size_t i = 0; i < names_.size(); i++) { + if (names_[i] == name) { + return get_counter_ref(i); + } + } + CHECK(names_.size() < N); + names_.push_back(name.str()); + return get_counter_ref(names_.size() - 1); + } + + CounterRef get_counter_ref(size_t index) { + return CounterRef(index, &counter_); + } + + static NamedThreadSafeCounter &get_default() { + static NamedThreadSafeCounter res; + return res; + } + + template + void for_each(F &&f) const { + std::unique_lock guard(mutex_); + for (size_t i = 0; i < names_.size(); i++) { + f(names_[i], counter_.sum(i)); + } + } + + friend StringBuilder &operator<<(StringBuilder &sb, const NamedThreadSafeCounter &counter) { + counter.for_each([&sb](Slice name, int64 cnt) { sb << name << ": " << cnt << "\n"; }); + return sb; + } + + private: + mutable std::mutex mutex_; + std::vector names_; + + Counter counter_; }; } // namespace td diff --git a/tdutils/td/utils/TsFileLog.cpp b/tdutils/td/utils/TsFileLog.cpp new file mode 100644 index 000000000..57aac34b9 --- /dev/null +++ b/tdutils/td/utils/TsFileLog.cpp @@ -0,0 +1,78 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "TsFileLog.h" + +namespace td { +namespace detail { +class TsFileLog : public LogInterface { + public: + Status init(string path) { + path_ = std::move(path); + for (int i = 0; i < (int)logs_.size(); i++) { + logs_[i].id = i; + } + return init_info(&logs_[0]); + } + + vector get_file_paths() override { + vector res; + for (auto &log : logs_) { + res.push_back(get_path(&log)); + } + return res; + } + + void append(CSlice cslice) override { + return append(cslice, -1); + } + void append(CSlice cslice, int log_level) override { + get_current_logger()->append(cslice, log_level); + } + + private: + struct Info { + FileLog log; + bool is_inited; + int id; + }; + static constexpr int MAX_THREAD_ID = 128; + std::string path_; + std::array logs_; + + LogInterface *get_current_logger() { + auto *info = get_current_info(); + if (!info->is_inited) { + CHECK(init_info(info).is_ok()); + } + return &info->log; + } + + Info *get_current_info() { + return &logs_[get_thread_id()]; + } + + Status init_info(Info *info) { + TRY_STATUS(info->log.init(get_path(info), std::numeric_limits::max(), info->id == 0)); + info->is_inited = true; + return Status::OK(); + } + + string get_path(Info *info) { + if (info->id == 0) { + return path_; + } + return PSTRING() << path_ << "." << info->id; + } +}; +} // namespace detail + +Result> TsFileLog::create(string path) { + auto res = td::make_unique(); + TRY_STATUS(res->init(path)); + return std::move(res); +} +} // namespace td diff --git a/tdutils/td/utils/TsFileLog.h b/tdutils/td/utils/TsFileLog.h new file mode 100644 index 000000000..ac6aa1b69 --- /dev/null +++ b/tdutils/td/utils/TsFileLog.h @@ -0,0 +1,16 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/FileLog.h" + +namespace td { +class TsFileLog { + public: + static Result> create(string path); +}; +} // namespace td diff --git a/tdutils/td/utils/base64.cpp b/tdutils/td/utils/base64.cpp index 92ab1c134..ba0562e90 100644 --- a/tdutils/td/utils/base64.cpp +++ b/tdutils/td/utils/base64.cpp @@ -10,6 +10,8 @@ #include "td/utils/Slice.h" #include "td/utils/Status.h" +#include "td/utils/format.h" + #include #include @@ -48,7 +50,7 @@ string base64_encode(Slice input) { static unsigned char char_to_value[256]; static void init_base64_table() { - static bool is_inited = []() { + static bool is_inited = [] { std::fill(std::begin(char_to_value), std::end(char_to_value), static_cast(64)); for (unsigned char i = 0; i < 64; i++) { char_to_value[static_cast(symbols64[i])] = i; @@ -58,9 +60,7 @@ static void init_base64_table() { CHECK(is_inited); } -Result base64_decode(Slice base64) { - init_base64_table(); - +Result base64_drop_padding(Slice base64) { if ((base64.size() & 3) != 0) { return Status::Error("Wrong string length"); } @@ -73,9 +73,11 @@ Result base64_decode(Slice base64) { if (padding_length >= 3) { return Status::Error("Wrong string padding"); } + return base64; +} - string output; - output.reserve(((base64.size() + 3) >> 2) * 3); +template +Status base64_do_decode(Slice base64, F &&append) { for (size_t i = 0; i < base64.size();) { size_t left = min(base64.size() - i, static_cast(4)); int c = 0; @@ -86,25 +88,53 @@ Result base64_decode(Slice base64) { } c |= value << ((3 - t) * 6); } - output += static_cast(static_cast(c >> 16)); // implementation-defined + append(static_cast(static_cast(c >> 16))); // implementation-defined if (left == 2) { if ((c & ((1 << 16) - 1)) != 0) { return Status::Error("Wrong padding in the string"); } } else { - output += static_cast(static_cast(c >> 8)); // implementation-defined + append(static_cast(static_cast(c >> 8))); // implementation-defined if (left == 3) { if ((c & ((1 << 8) - 1)) != 0) { return Status::Error("Wrong padding in the string"); } } else { - output += static_cast(static_cast(c)); // implementation-defined + append(static_cast(static_cast(c))); // implementation-defined } } } + return Status::OK(); +} + +Result base64_decode(Slice base64) { + init_base64_table(); + + TRY_RESULT(tmp, base64_drop_padding(base64)); + base64 = tmp; + + string output; + output.reserve(((base64.size() + 3) >> 2) * 3); + TRY_STATUS(base64_do_decode(base64, [&output](char c) { output += c; })); return output; } +Result base64_decode_secure(Slice base64) { + init_base64_table(); + + TRY_RESULT(tmp, base64_drop_padding(base64)); + base64 = tmp; + + SecureString output(((base64.size() + 3) >> 2) * 3); + char *ptr = output.as_mutable_slice().begin(); + TRY_STATUS(base64_do_decode(base64, [&ptr](char c) { *ptr++ = c; })); + size_t size = ptr - output.as_mutable_slice().begin(); + if (size == output.size()) { + return std::move(output); + } + return SecureString(output.as_slice().substr(0, size)); +} + static const char *const url_symbols64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; string base64url_encode(Slice input) { @@ -133,7 +163,7 @@ string base64url_encode(Slice input) { static unsigned char url_char_to_value[256]; static void init_base64url_table() { - static bool is_inited = []() { + static bool is_inited = [] { std::fill(std::begin(url_char_to_value), std::end(url_char_to_value), static_cast(64)); for (unsigned char i = 0; i < 64; i++) { url_char_to_value[static_cast(url_symbols64[i])] = i; diff --git a/tdutils/td/utils/base64.h b/tdutils/td/utils/base64.h index e7d3dfccf..d4ed52c11 100644 --- a/tdutils/td/utils/base64.h +++ b/tdutils/td/utils/base64.h @@ -8,12 +8,14 @@ #include "td/utils/common.h" #include "td/utils/Slice.h" +#include "td/utils/SharedSlice.h" #include "td/utils/Status.h" namespace td { string base64_encode(Slice input); Result base64_decode(Slice base64); +Result base64_decode_secure(Slice base64); string base64url_encode(Slice input); Result base64url_decode(Slice base64); diff --git a/tdutils/td/utils/buffer.h b/tdutils/td/utils/buffer.h index 85d78efea..d540f6099 100644 --- a/tdutils/td/utils/buffer.h +++ b/tdutils/td/utils/buffer.h @@ -796,5 +796,8 @@ inline Slice as_slice(const BufferSlice &value) { inline MutableSlice as_slice(BufferSlice &value) { return value.as_slice(); } +inline MutableSlice as_mutable_slice(BufferSlice &value) { + return value.as_slice(); +} } // namespace td diff --git a/tdutils/td/utils/crypto.cpp b/tdutils/td/utils/crypto.cpp index f98d5d7a9..f827762e0 100644 --- a/tdutils/td/utils/crypto.cpp +++ b/tdutils/td/utils/crypto.cpp @@ -241,66 +241,74 @@ int pq_factorize(Slice pq_str, string *p_str, string *q_str) { return 0; } -static void aes_ige_xcrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to, bool encrypt_flag) { +static void aes_ige_xcrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to, bool encrypt_flag) { + CHECK(aes_key.size() == 32); + CHECK(aes_iv.size() == 16); AES_KEY key; int err; if (encrypt_flag) { - err = AES_set_encrypt_key(aes_key.raw, 256, &key); + err = AES_set_encrypt_key(aes_key.ubegin(), 256, &key); } else { - err = AES_set_decrypt_key(aes_key.raw, 256, &key); + err = AES_set_decrypt_key(aes_key.ubegin(), 256, &key); } LOG_IF(FATAL, err != 0); CHECK(from.size() <= to.size()); - AES_ige_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv->raw, encrypt_flag); + AES_ige_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv.ubegin(), encrypt_flag); } -void aes_ige_encrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to) { +void aes_ige_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) { aes_ige_xcrypt(aes_key, aes_iv, from, to, true); } -void aes_ige_decrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to) { +void aes_ige_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) { aes_ige_xcrypt(aes_key, aes_iv, from, to, false); } -static void aes_cbc_xcrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to, bool encrypt_flag) { +static void aes_cbc_xcrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to, bool encrypt_flag) { + CHECK(aes_key.size() == 32); + CHECK(aes_iv.size() == 16); AES_KEY key; int err; if (encrypt_flag) { - err = AES_set_encrypt_key(aes_key.raw, 256, &key); + err = AES_set_encrypt_key(aes_key.ubegin(), 256, &key); } else { - err = AES_set_decrypt_key(aes_key.raw, 256, &key); + err = AES_set_decrypt_key(aes_key.ubegin(), 256, &key); } LOG_IF(FATAL, err != 0); CHECK(from.size() <= to.size()); - AES_cbc_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv->raw, encrypt_flag); + AES_cbc_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv.ubegin(), encrypt_flag); } -void aes_cbc_encrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to) { +void aes_cbc_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) { aes_cbc_xcrypt(aes_key, aes_iv, from, to, true); } -void aes_cbc_decrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to) { +void aes_cbc_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) { aes_cbc_xcrypt(aes_key, aes_iv, from, to, false); } -AesCbcState::AesCbcState(const UInt256 &key, const UInt128 &iv) : key_(key), iv_(iv) { +AesCbcState::AesCbcState(Slice key256, Slice iv128) : key_(key256), iv_(iv128) { + CHECK(key_.size() == 32); + CHECK(iv_.size() == 16); } void AesCbcState::encrypt(Slice from, MutableSlice to) { - ::td::aes_cbc_encrypt(key_, &iv_, from, to); + ::td::aes_cbc_encrypt(key_.as_slice(), iv_.as_mutable_slice(), from, to); } void AesCbcState::decrypt(Slice from, MutableSlice to) { - ::td::aes_cbc_decrypt(key_, &iv_, from, to); + ::td::aes_cbc_decrypt(key_.as_slice(), iv_.as_mutable_slice(), from, to); } class AesCtrState::Impl { public: - Impl(const UInt256 &key, const UInt128 &iv) { + Impl(Slice key, Slice iv) { + CHECK(key.size() == 32); + CHECK(iv.size() == 16); static_assert(AES_BLOCK_SIZE == 16, ""); - if (AES_set_encrypt_key(key.raw, 256, &aes_key) < 0) { + if (AES_set_encrypt_key(key.ubegin(), 256, &aes_key) < 0) { LOG(FATAL) << "Failed to set encrypt key"; } - MutableSlice(counter, AES_BLOCK_SIZE).copy_from(as_slice(iv)); + counter.as_mutable_slice().copy_from(as_slice(iv)); current_pos = 0; } @@ -308,9 +316,10 @@ class AesCtrState::Impl { CHECK(to.size() >= from.size()); for (size_t i = 0; i < from.size(); i++) { if (current_pos == 0) { - AES_encrypt(counter, encrypted_counter, &aes_key); + AES_encrypt(counter.as_slice().ubegin(), encrypted_counter.as_mutable_slice().ubegin(), &aes_key); + uint8 *ptr = counter.as_mutable_slice().ubegin(); for (int j = 15; j >= 0; j--) { - if (++counter[j] != 0) { + if (++ptr[j] != 0) { break; } } @@ -322,8 +331,8 @@ class AesCtrState::Impl { private: AES_KEY aes_key; - uint8 counter[AES_BLOCK_SIZE]; - uint8 encrypted_counter[AES_BLOCK_SIZE]; + SecureString counter{AES_BLOCK_SIZE}; + SecureString encrypted_counter{AES_BLOCK_SIZE}; uint8 current_pos; }; @@ -332,7 +341,7 @@ AesCtrState::AesCtrState(AesCtrState &&from) = default; AesCtrState &AesCtrState::operator=(AesCtrState &&from) = default; AesCtrState::~AesCtrState() = default; -void AesCtrState::init(const UInt256 &key, const UInt128 &iv) { +void AesCtrState::init(Slice key, Slice iv) { ctx_ = make_unique(key, iv); } diff --git a/tdutils/td/utils/crypto.h b/tdutils/td/utils/crypto.h index 2a081b4a0..8edcb9dd7 100644 --- a/tdutils/td/utils/crypto.h +++ b/tdutils/td/utils/crypto.h @@ -9,8 +9,8 @@ #include "td/utils/buffer.h" #include "td/utils/common.h" #include "td/utils/Slice.h" +#include "td/utils/SharedSlice.h" #include "td/utils/Status.h" -#include "td/utils/UInt.h" namespace td { @@ -21,11 +21,11 @@ void init_crypto(); int pq_factorize(Slice pq_str, string *p_str, string *q_str); -void aes_ige_encrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to); -void aes_ige_decrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to); +void aes_ige_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to); +void aes_ige_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to); -void aes_cbc_encrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to); -void aes_cbc_decrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to); +void aes_cbc_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to); +void aes_cbc_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to); class AesCtrState { public: @@ -36,7 +36,7 @@ class AesCtrState { AesCtrState &operator=(AesCtrState &&from); ~AesCtrState(); - void init(const UInt256 &key, const UInt128 &iv); + void init(Slice key, Slice iv); void encrypt(Slice from, MutableSlice to); @@ -49,14 +49,14 @@ class AesCtrState { class AesCbcState { public: - AesCbcState(const UInt256 &key, const UInt128 &iv); + AesCbcState(Slice key256, Slice iv128); void encrypt(Slice from, MutableSlice to); void decrypt(Slice from, MutableSlice to); private: - UInt256 key_; - UInt128 iv_; + SecureString key_; + SecureString iv_; }; void sha1(Slice data, unsigned char output[20]); diff --git a/tdutils/td/utils/filesystem.cpp b/tdutils/td/utils/filesystem.cpp index b9bd84010..071c9a389 100644 --- a/tdutils/td/utils/filesystem.cpp +++ b/tdutils/td/utils/filesystem.cpp @@ -32,6 +32,10 @@ template <> BufferSlice create_empty(size_t size) { return BufferSlice{size}; } +template <> +SecureString create_empty(size_t size) { + return SecureString{size}; +} template Result read_file_impl(CSlice path, int64 size, int64 offset) { @@ -48,7 +52,7 @@ Result read_file_impl(CSlice path, int64 size, int64 offset) { } size -= offset; auto content = create_empty(narrow_cast(size)); - TRY_RESULT(got_size, from_file.pread(as_slice(content), offset)); + TRY_RESULT(got_size, from_file.pread(as_mutable_slice(content), offset)); if (got_size != static_cast(size)) { return Status::Error("Failed to read file"); } @@ -66,6 +70,10 @@ Result read_file_str(CSlice path, int64 size, int64 offset) { return read_file_impl(path, size, offset); } +Result read_file_secure(CSlice path, int64 size, int64 offset) { + return read_file_impl(path, size, offset); +} + // Very straightforward function. Don't expect much of it. Status copy_file(CSlice from, CSlice to, int64 size) { TRY_RESULT(content, read_file(from, size)); diff --git a/tdutils/td/utils/filesystem.h b/tdutils/td/utils/filesystem.h index d7d613181..182a4e62e 100644 --- a/tdutils/td/utils/filesystem.h +++ b/tdutils/td/utils/filesystem.h @@ -8,12 +8,14 @@ #include "td/utils/buffer.h" #include "td/utils/Slice.h" +#include "td/utils/SharedSlice.h" #include "td/utils/Status.h" namespace td { Result read_file(CSlice path, int64 size = -1, int64 offset = 0); Result read_file_str(CSlice path, int64 size = -1, int64 offset = 0); +Result read_file_secure(CSlice path, int64 size = -1, int64 offset = 0); Status copy_file(CSlice from, CSlice to, int64 size = -1) TD_WARN_UNUSED_RESULT; diff --git a/tdutils/td/utils/port/detail/ThreadStl.h b/tdutils/td/utils/port/detail/ThreadStl.h index 1e0f758a9..246e14877 100644 --- a/tdutils/td/utils/port/detail/ThreadStl.h +++ b/tdutils/td/utils/port/detail/ThreadStl.h @@ -30,7 +30,9 @@ class ThreadStl { ThreadStl &operator=(const ThreadStl &other) = delete; ThreadStl(ThreadStl &&) = default; ThreadStl &operator=(ThreadStl &&) = default; - ~ThreadStl() = default; + ~ThreadStl() { + join(); + } template explicit ThreadStl(Function &&f, Args &&... args) { thread_ = std::thread([args = std::make_tuple(decay_copy(std::forward(f)), @@ -42,10 +44,14 @@ class ThreadStl { } void join() { - thread_.join(); + if (thread_.joinable()) { + thread_.join(); + } } void detach() { - thread_.detach(); + if (thread_.joinable()) { + thread_.detach(); + } } void set_name(CSlice name) { } diff --git a/tdutils/td/utils/tl_helpers.h b/tdutils/td/utils/tl_helpers.h index 3fed15127..2c7b2a3f7 100644 --- a/tdutils/td/utils/tl_helpers.h +++ b/tdutils/td/utils/tl_helpers.h @@ -10,6 +10,7 @@ #include "td/utils/logging.h" #include "td/utils/misc.h" #include "td/utils/Slice.h" +#include "td/utils/SharedSlice.h" #include "td/utils/StackAllocator.h" #include "td/utils/Status.h" #include "td/utils/tl_parsers.h" @@ -115,11 +116,20 @@ template void store(const string &x, StorerT &storer) { storer.store_string(x); } +template +void store(const SecureString &x, StorerT &storer) { + storer.store_string(x.as_slice()); +} template void parse(string &x, ParserT &parser) { x = parser.template fetch_string(); } +template +void parse(SecureString &x, ParserT &parser) { + x = parser.template fetch_string(); +} + template void store(const vector &vec, StorerT &storer) { storer.store_binary(narrow_cast(vec.size())); @@ -228,6 +238,21 @@ string serialize(const T &object) { return key; } +template +SecureString serialize_secure(const T &object) { + TlStorerCalcLength calc_length; + store(object, calc_length); + size_t length = calc_length.get_length(); + + SecureString key(length, '\0'); + CHECK(is_aligned_pointer<4>(key.data())); + MutableSlice data = key.as_mutable_slice(); + TlStorerUnsafe storer(data.ubegin()); + store(object, storer); + CHECK(storer.get_buf() == data.uend()); + return key; +} + template TD_WARN_UNUSED_RESULT Status unserialize(T &object, Slice data) { TlParser parser(data); diff --git a/tdutils/td/utils/tl_parsers.h b/tdutils/td/utils/tl_parsers.h index 10b92f8e2..15f6d1ac9 100644 --- a/tdutils/td/utils/tl_parsers.h +++ b/tdutils/td/utils/tl_parsers.h @@ -148,16 +148,29 @@ class TlParser { if (result_len < 254) { result_begin = reinterpret_cast(data + 1); result_aligned_len = (result_len >> 2) << 2; + data += sizeof(int32); } else if (result_len == 254) { result_len = data[1] + (data[2] << 8) + (data[3] << 16); result_begin = reinterpret_cast(data + 4); result_aligned_len = ((result_len + 3) >> 2) << 2; + data += sizeof(int32); } else { - set_error("Can't fetch string, 255 found"); - return T(); + check_len(sizeof(int32)); + result_len = data[1] + (data[2] << 8) + (data[3] << 16) + (data[4] << 24) + (static_cast(data[5]) << 32) + + (static_cast(data[6]) << 40) + (static_cast(data[7]) << 48); + if (result_len > std::numeric_limits::max() - 3) { + set_error("Too big string found"); + return T(); + } + result_begin = reinterpret_cast(data + 8); + result_aligned_len = ((result_len + 3) >> 2) << 2; + data += sizeof(int64); } check_len(result_aligned_len); - data += result_aligned_len + sizeof(int32); + if (!error.empty()) { + return T(); + } + data += result_aligned_len; return T(result_begin, result_len); } @@ -165,6 +178,9 @@ class TlParser { T fetch_string_raw(const size_t size) { //CHECK(size % sizeof(int32) == 0); check_len(size); + if (!error.empty()) { + return T(); + } const char *result = reinterpret_cast(data); data += size; return T(result, size); diff --git a/tdutils/td/utils/tl_storers.h b/tdutils/td/utils/tl_storers.h index cd1258bac..7c22d741f 100644 --- a/tdutils/td/utils/tl_storers.h +++ b/tdutils/td/utils/tl_storers.h @@ -62,6 +62,15 @@ class TlStorerUnsafe { *buf_++ = static_cast(len & 255); *buf_++ = static_cast((len >> 8) & 255); *buf_++ = static_cast(len >> 16); + } else if (len < (1ull << 32)) { + *buf_++ = static_cast(255); + *buf_++ = static_cast(len & 255); + *buf_++ = static_cast((len >> 8) & 255); + *buf_++ = static_cast((len >> 16) & 255); + *buf_++ = static_cast((len >> 24) & 255); + *buf_++ = static_cast(0); + *buf_++ = static_cast(0); + *buf_++ = static_cast(0); } else { LOG(FATAL) << "String size " << len << " is too big to be stored"; } @@ -71,7 +80,7 @@ class TlStorerUnsafe { switch (len & 3) { case 1: *buf_++ = 0; - // fallthrough + // fallthrough case 2: *buf_++ = 0; // fallthrough @@ -119,8 +128,10 @@ class TlStorerCalcLength { size_t add = str.size(); if (add < 254) { add += 1; - } else { + } else if (add < (1 << 24)) { add += 4; + } else { + add += 8; } add = (add + 3) & -4; length += add; diff --git a/tdutils/test/SharedSlice.cpp b/tdutils/test/SharedSlice.cpp new file mode 100644 index 000000000..67ff1d881 --- /dev/null +++ b/tdutils/test/SharedSlice.cpp @@ -0,0 +1,85 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/tests.h" +#include "td/utils/SharedSlice.h" + +using namespace td; + +TEST(SharedSlice, Hands) { + { + SharedSlice h("hello"); + ASSERT_EQ("hello", h.as_slice()); + // auto g = h; // CE + auto g = h.clone(); + ASSERT_EQ("hello", g.as_slice()); + } + + { + SharedSlice h("hello"); + UniqueSharedSlice g(std::move(h)); + ASSERT_EQ("", h.as_slice()); + ASSERT_EQ("hello", g.as_slice()); + } + { + SharedSlice h("hello"); + SharedSlice t = h.clone(); + UniqueSharedSlice g(std::move(h)); + ASSERT_EQ("", h.as_slice()); + ASSERT_EQ("hello", g.as_slice()); + ASSERT_EQ("hello", t.as_slice()); + } + + { + UniqueSharedSlice g(5); + g.as_mutable_slice().copy_from("hello"); + SharedSlice h(std::move(g)); + ASSERT_EQ("hello", h); + ASSERT_EQ("", g); + } + + { + UniqueSlice h("hello"); + UniqueSlice g(std::move(h)); + ASSERT_EQ("", h.as_slice()); + ASSERT_EQ("hello", g.as_slice()); + } + + { + SecureString h("hello"); + SecureString g(std::move(h)); + ASSERT_EQ("", h.as_slice()); + ASSERT_EQ("hello", g.as_slice()); + } + + { + Stage stage; + SharedSlice a, b; + std::vector threads(2); + for (int i = 0; i < 2; i++) { + threads[i] = td::thread([i, &stage, &a, &b] { + for (int j = 0; j < 10000; j++) { + if (i == 0) { + a = SharedSlice("hello"); + b = a.clone(); + } + stage.wait((2 * j + 1) * 2); + if (i == 0) { + ASSERT_EQ('h', a[0]); + a.clear(); + } else { + UniqueSharedSlice c(std::move(b)); + c.as_mutable_slice()[0] = '!'; + } + stage.wait((2 * j + 2) * 2); + } + }); + } + for (auto &thread : threads) { + thread.join(); + } + } +} diff --git a/tdutils/test/crypto.cpp b/tdutils/test/crypto.cpp index 2f4755676..c5afefad4 100644 --- a/tdutils/test/crypto.cpp +++ b/tdutils/test/crypto.cpp @@ -45,18 +45,18 @@ TEST(Crypto, AesCtrState) { } td::AesCtrState state; - state.init(key, iv); + state.init(as_slice(key), as_slice(iv)); td::string t(length, '\0'); state.encrypt(s, t); ASSERT_EQ(answers1[i], td::crc32(t)); - state.init(key, iv); + state.init(as_slice(key), as_slice(iv)); state.decrypt(t, t); ASSERT_STREQ(s, t); for (auto &c : iv.raw) { c = 0xFF; } - state.init(key, iv); + state.init(as_slice(key), as_slice(iv)); state.encrypt(s, t); ASSERT_EQ(answers2[i], td::crc32(t)); diff --git a/tdutils/test/log.cpp b/tdutils/test/log.cpp new file mode 100644 index 000000000..990a4514b --- /dev/null +++ b/tdutils/test/log.cpp @@ -0,0 +1,122 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/tests.h" +#include "td/utils/FileLog.h" +#include "td/utils/TsFileLog.h" +#include "td/utils/logging.h" +#include "td/utils/port/path.h" +#include "td/utils/benchmark.h" + +// Thread safe logging with tests +// +// LOG uses thread local LogInterface +// void append(CSlice slice, int log_level); +// + +template +class LogBenchmark : public td::Benchmark { + public: + explicit LogBenchmark(std::string name, int threads_n, std::function()> creator) + : name_(std::move(name)), threads_n_(threads_n), creator_(std::move(creator)) { + } + std::string get_description() const override { + return PSTRING() << name_ << " " << td::tag("threads_n", threads_n_); + } + void start_up() override { + log_ = creator_(); + threads_.resize(threads_n_); + } + void tear_down() override { + for (auto path : log_->get_file_paths()) { + td::unlink(path).ignore(); + } + log_.reset(); + } + void run(int n) override { + for (auto &thread : threads_) { + thread = td::thread([this, n] { this->run_thread(n); }); + } + for (auto &thread : threads_) { + thread.join(); + } + } + + void run_thread(int n) { + auto str = PSTRING() << "#" << n << " : fsjklfdjsklfjdsklfjdksl\n"; + for (int i = 0; i < n; i++) { + log_->append(str); + } + } + + private: + std::string name_; + td::unique_ptr log_; + int threads_n_{0}; + std::function()> creator_; + std::vector threads_; +}; + +template +void bench_log(std::string name, int threads_n, F &&f) { + bench(LogBenchmark(std::move(name), threads_n, std::move(f))); +}; + +TEST(Log, TsLogger) { + bench_log("NewTsFileLog", 4, [] { return td::TsFileLog::create("tmplog").move_as_ok(); }); + bench_log("TsFileLog", 8, [] { + class FileLog : public td::LogInterface { + public: + FileLog() { + file_log_.init("tmplog", std::numeric_limits::max(), false); + ts_log_.init(&file_log_); + } + ~FileLog() { + } + void append(td::CSlice slice) override { + ts_log_.append(slice, -1); + } + std::vector get_file_paths() override { + return file_log_.get_file_paths(); + } + + private: + td::FileLog file_log_; + td::TsLog ts_log_{nullptr}; + }; + return td::make_unique(); + }); + + bench_log("noop", 4, [] { + class NoopLog : public td::LogInterface { + public: + void append(td::CSlice slice) override { + } + }; + return td::make_unique(); + }); + + bench_log("FileLog", 4, [] { + class FileLog : public td::LogInterface { + public: + FileLog() { + file_log_.init("tmplog", std::numeric_limits::max(), false); + } + ~FileLog() { + } + void append(td::CSlice slice) override { + file_log_.append(slice, -1); + } + std::vector get_file_paths() override { + return file_log_.get_file_paths(); + } + + private: + td::FileLog file_log_; + }; + return td::make_unique(); + }); +} diff --git a/tdutils/test/misc.cpp b/tdutils/test/misc.cpp index 84fa67f19..724dc5b05 100644 --- a/tdutils/test/misc.cpp +++ b/tdutils/test/misc.cpp @@ -895,6 +895,9 @@ static void test_hash() { test_hash({BadValue{1}, BadValue{2}}).ensure_error(); test_hash({ValueA{1}, ValueA{2}}).ensure(); test_hash({ValueB{1}, ValueB{2}}).ensure(); + test_hash>({{1, 1}, {1, 2}}).ensure(); + // FIXME: use some better hash + //test_hash>({{1, 1}, {1, 2}, {2, 1}, {2, 2}}).ensure(); } TEST(Misc, Hasher) {