andreacavalli/tdlib-fork
Archived
1
0
Fork 0
Code Pull Requests Releases Activity

tdutils: import changes from other project

Browse Source 
GitOrigin-RevId: 0cfbe2418933cffdce4f0d1b8461c247a814d868
This commit is contained in:
Arseny Smirnov 2019-08-12 14:45:57 +03:00
parent 7a863daa50
commit f63ce74fef
30 changed files with 1121 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
tdutils/CMakeLists.txt
View File

@ -99,12 +99,15 @@ set(TDUTILS_SOURCE
td/utils/MpmcQueue.cpp
td/utils/OptionsParser.cpp
td/utils/Random.cpp
td/utils/Slice.cpp
td/utils/SharedSlice.cpp
td/utils/StackAllocator.cpp
td/utils/Status.cpp
td/utils/StringBuilder.cpp
td/utils/tests.cpp
td/utils/Time.cpp
td/utils/Timer.cpp
td/utils/TsFileLog.cpp
td/utils/tests.cpp
td/utils/tl_parsers.cpp
td/utils/translit.cpp
td/utils/unicode.cpp
@ -231,6 +234,7 @@ set(TDUTILS_SOURCE
td/utils/Time.h
td/utils/TimedStat.h
td/utils/Timer.h
td/utils/TsFileLog.h
td/utils/tl_helpers.h
td/utils/tl_parsers.h
td/utils/tl_storers.h
@ -250,12 +254,12 @@ set(TDUTILS_TEST_SOURCE
${CMAKE_CURRENT_SOURCE_DIR}/test/ConcurrentHashMap.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/crypto.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/Enumerator.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/EpochBasedMemoryReclamation.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/filesystem.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/gzip.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/HazardPointers.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/heap.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/json.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/log.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/misc.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcQueue.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcWaiter.cpp
@ -264,6 +268,7 @@ set(TDUTILS_TEST_SOURCE
${CMAKE_CURRENT_SOURCE_DIR}/test/port.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/pq.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/SharedObjectPool.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/SharedSlice.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/variant.cpp
PARENT_SCOPE
)
@ -300,7 +305,7 @@ if (CRC32C_FOUND)
target_link_libraries(tdutils PRIVATE crc32c)
endif()
if (ABSL_FOUND)
target_link_libraries(tdutils PUBLIC absl::base absl::container absl::hash)
target_link_libraries_system(tdutils PUBLIC absl::flat_hash_map absl::flat_hash_set absl::hash)
endif()
if (WIN32 AND WINGETOPT_FOUND)

3
tdutils/td/utils/DecTree.h
View File

@ -185,6 +185,9 @@ class DecTree {
void remove(const KeyType &key) {
root_ = remove_node(std::move(root_), key);
}
void reset() {
root_ = nullptr;
}
ValueType *get(const KeyType &key) {
return get_node(root_, key);
}

7
tdutils/td/utils/FileLog.cpp
View File

@ -17,7 +17,7 @@
namespace td {
Status FileLog::init(string path, int64 rotate_threshold) {
Status FileLog::init(string path, int64 rotate_threshold, bool redirect_stderr) {
if (path == path_) {
set_rotate_threshold(rotate_threshold);
return Status::OK();
@ -30,7 +30,7 @@ Status FileLog::init(string path, int64 rotate_threshold) {
fd_.close();
fd_ = std::move(fd);
if (!Stderr().empty()) {
if (!Stderr().empty() && redirect_stderr) {
fd_.get_native_fd().duplicate(Stderr().get_native_fd()).ignore();
}
@ -43,6 +43,7 @@ Status FileLog::init(string path, int64 rotate_threshold) {
TRY_RESULT(size, fd_.get_size());
size_ = size;
rotate_threshold_ = rotate_threshold;
redirect_stderr_ = redirect_stderr;
return Status::OK();
}
@ -108,7 +109,7 @@ void FileLog::do_rotate() {
process_fatal_error(PSLICE() << r_fd.error() << " in " << __FILE__ << " at " << __LINE__);
}
fd_ = r_fd.move_as_ok();
if (!Stderr().empty()) {
if (!Stderr().empty() && redirect_stderr_) {
fd_.get_native_fd().duplicate(Stderr().get_native_fd()).ignore();
}
size_ = 0;

3
tdutils/td/utils/FileLog.h
View File

@ -18,7 +18,7 @@ class FileLog : public LogInterface {
static constexpr int64 DEFAULT_ROTATE_THRESHOLD = 10 * (1 << 20);
public:
Status init(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD);
Status init(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD, bool redirect_stderr = true);
Slice get_path() const;
@ -37,6 +37,7 @@ class FileLog : public LogInterface {
string path_;
int64 size_ = 0;
int64 rotate_threshold_ = 0;
bool redirect_stderr_;
void do_rotate();
};

59
tdutils/td/utils/JsonBuilder.h
View File

@ -181,7 +181,7 @@ class JsonObjectScope;
class JsonBuilder {
public:
explicit JsonBuilder(StringBuilder &&sb) : sb_(std::move(sb)) {
explicit JsonBuilder(StringBuilder &&sb, int32 offset = -1) : sb_(std::move(sb)), offset_(offset) {
}
StringBuilder &string_builder() {
return sb_;
@ -191,9 +191,33 @@ class JsonBuilder {
JsonArrayScope enter_array() TD_WARN_UNUSED_RESULT;
JsonObjectScope enter_object() TD_WARN_UNUSED_RESULT;
int32 offset() const {
return offset_;
}
bool is_pretty() const {
return offset_ >= 0;
}
void print_offset() {
for (int x = 0; x < offset_; x++) {
sb_ << " ";
}
}
void dec_offset() {
if (offset_ >= 0) {
CHECK(offset_ > 0);
offset_--;
}
}
void inc_offset() {
if (offset_ >= 0) {
offset_++;
}
}
private:
StringBuilder sb_;
JsonScope *scope_ = nullptr;
int32 offset_;
};
class Jsonable {};
@ -328,6 +352,7 @@ class JsonValueScope : public JsonScope {
class JsonArrayScope : public JsonScope {
public:
explicit JsonArrayScope(JsonBuilder *jb) : JsonScope(jb) {
jb->inc_offset();
*sb_ << "[";
}
JsonArrayScope(JsonArrayScope &&other) = default;
@ -337,6 +362,11 @@ class JsonArrayScope : public JsonScope {
}
}
void leave() {
jb_->dec_offset();
if (jb_->is_pretty()) {
*sb_ << "\n";
jb_->print_offset();
}
*sb_ << "]";
}
template <class T>
@ -355,6 +385,10 @@ class JsonArrayScope : public JsonScope {
} else {
is_first_ = true;
}
if (jb_->is_pretty()) {
*sb_ << "\n";
jb_->print_offset();
}
return jb_->enter_value();
}
@ -365,6 +399,7 @@ class JsonArrayScope : public JsonScope {
class JsonObjectScope : public JsonScope {
public:
explicit JsonObjectScope(JsonBuilder *jb) : JsonScope(jb) {
jb->inc_offset();
*sb_ << "{";
}
JsonObjectScope(JsonObjectScope &&other) = default;
@ -374,6 +409,11 @@ class JsonObjectScope : public JsonScope {
}
}
void leave() {
jb_->dec_offset();
if (jb_->is_pretty()) {
*sb_ << "\n";
jb_->print_offset();
}
*sb_ << "}";
}
template <class S, class T>
@ -392,8 +432,16 @@ class JsonObjectScope : public JsonScope {
} else {
is_first_ = true;
}
if (jb_->is_pretty()) {
*sb_ << "\n";
jb_->print_offset();
}
jb_->enter_value() << key;
*sb_ << ":";
if (jb_->is_pretty()) {
*sb_ << " : ";
} else {
*sb_ << ":";
}
jb_->enter_value() << value;
return *this;
}
@ -737,11 +785,14 @@ inline Result<JsonValue> json_decode(MutableSlice json) {
}
template <class StrT, class ValT>
StrT json_encode(const ValT &val) {
StrT json_encode(const ValT &val, bool pretty = false) {
auto buf_len = 1 << 18;
auto buf = StackAllocator::alloc(buf_len);
JsonBuilder jb(StringBuilder(buf.as_slice(), true));
JsonBuilder jb(StringBuilder(buf.as_slice(), true), pretty ? 0 : -1);
jb.enter_value() << val;
if (pretty) {
jb.string_builder() << "\n";
}
LOG_IF(ERROR, jb.string_builder().is_error()) << "JSON buffer overflow";
auto slice = jb.string_builder().as_cslice();
return StrT(slice.begin(), slice.size());

3
tdutils/td/utils/MpmcWaiter.h
View File

@ -90,7 +90,8 @@ class MpmcWaiter {
return (state >> 1) == (worker + 1);
}
};
enum { RoundsTillSleepy = 32, RoundsTillAsleep = 64 };
//enum { RoundsTillSleepy = 32, RoundsTillAsleep = 64 };
enum { RoundsTillSleepy = 1, RoundsTillAsleep = 2 };
std::atomic<uint32> state_{State::awake()};
std::mutex mutex_;
std::condition_variable condition_variable_;

17
tdutils/td/utils/SharedSlice.cpp Normal file
View File

@ -0,0 +1,17 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/SharedSlice.h"
#include "td/utils/buffer.h"
namespace td {
BufferSlice SharedSlice::clone_as_buffer_slice() const {
return BufferSlice{as_slice().str()};
}
} // namespace td

377
tdutils/td/utils/SharedSlice.h Normal file
View File

@ -0,0 +1,377 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/Slice.h"
#include <atomic>
#include <memory>
namespace td {
namespace detail {
struct SharedSliceHeader {
explicit SharedSliceHeader(size_t size) : refcnt_{1}, size_{size} {
}
void inc() {
refcnt_.fetch_add(1, std::memory_order_relaxed);
}
bool dec() {
return refcnt_.fetch_sub(1, std::memory_order_acq_rel) == 1;
}
bool is_unique() const {
// NB: race if std::memory_order_relaxed is used
// reader may see a change by a new writer
return refcnt_.load(std::memory_order_acquire) == 1;
}
size_t size() const {
return size_;
}
private:
std::atomic<uint64> refcnt_;
size_t size_;
};
struct UniqueSliceHeader {
explicit UniqueSliceHeader(size_t size) : size_{size} {
}
void inc() {
}
bool dec() {
return true;
}
bool is_unique() const {
return true;
}
size_t size() const {
return size_;
}
private:
size_t size_;
};
template <class HeaderT, bool zero_on_destruct = false>
class UnsafeSharedSlice {
public:
UnsafeSharedSlice() = default;
UnsafeSharedSlice clone() const {
header()->inc();
return UnsafeSharedSlice(ptr_.get());
}
bool is_null() const {
return !ptr_;
}
bool is_unique() const {
if (is_null()) {
return true;
}
return header()->is_unique();
}
MutableSlice as_mutable_slice() {
if (is_null()) {
return MutableSlice();
}
return MutableSlice(ptr_.get() + sizeof(HeaderT), header()->size());
}
Slice as_slice() const {
if (is_null()) {
return Slice();
}
return Slice(ptr_.get() + sizeof(HeaderT), header()->size());
}
size_t size() const {
if (is_null()) {
return 0;
}
return header()->size();
}
static UnsafeSharedSlice create(size_t size) {
static_assert(std::is_standard_layout<HeaderT>::value, "HeaderT must have statdard layout");
auto ptr = std::make_unique<char[]>(size + sizeof(HeaderT));
auto header_ptr = new (ptr.get()) HeaderT(size);
CHECK(reinterpret_cast<char *>(header_ptr) == ptr.get());
return UnsafeSharedSlice(std::move(ptr));
}
static UnsafeSharedSlice create(Slice slice) {
auto res = create(slice.size());
res.as_mutable_slice().copy_from(slice);
return res;
}
void clear() {
ptr_.reset();
}
private:
explicit UnsafeSharedSlice(char *ptr) : ptr_(ptr) {
}
explicit UnsafeSharedSlice(std::unique_ptr<char[]> from) : ptr_(from.release()) {
}
HeaderT *header() const {
return reinterpret_cast<HeaderT *>(ptr_.get());
}
struct Destructor {
void operator()(char *ptr) {
auto header = reinterpret_cast<HeaderT *>(ptr);
if (header->dec()) {
if (zero_on_destruct) {
MutableSlice(ptr, sizeof(HeaderT) + header->size()).fill_zero_secure();
}
std::default_delete<char[]>()(ptr);
}
}
};
std::unique_ptr<char[], Destructor> ptr_;
};
} // namespace detail
class BufferSlice;
class UniqueSharedSlice;
class SharedSlice {
using Impl = detail::UnsafeSharedSlice<detail::SharedSliceHeader>;
public:
SharedSlice() = default;
explicit SharedSlice(Slice slice) : impl_(Impl::create(slice.size())) {
impl_.as_mutable_slice().copy_from(slice);
}
explicit SharedSlice(UniqueSharedSlice from);
SharedSlice(const char *ptr, size_t size) : SharedSlice(Slice(ptr, size)) {
}
SharedSlice clone() const {
return SharedSlice(impl_.clone());
}
Slice as_slice() const {
return impl_.as_slice();
}
BufferSlice clone_as_buffer_slice() const;
operator Slice() const {
return as_slice();
}
// like in std::string
const char *data() const {
return as_slice().data();
}
char operator[](size_t at) const {
return as_slice()[at];
}
bool empty() const {
return size() == 0;
}
size_t size() const {
return impl_.size();
}
// like in std::string
size_t length() const {
return size();
}
void clear() {
impl_.clear();
}
private:
friend class UniqueSharedSlice;
explicit SharedSlice(Impl impl) : impl_(std::move(impl)) {
}
Impl impl_;
};
class UniqueSharedSlice {
using Impl = detail::UnsafeSharedSlice<detail::SharedSliceHeader>;
public:
UniqueSharedSlice() = default;
explicit UniqueSharedSlice(size_t size) : impl_(Impl::create(size)) {
}
explicit UniqueSharedSlice(Slice slice) : impl_(Impl::create(slice)) {
}
UniqueSharedSlice(const char *ptr, size_t size) : UniqueSharedSlice(Slice(ptr, size)) {
}
explicit UniqueSharedSlice(SharedSlice from) : impl_() {
if (from.impl_.is_unique()) {
impl_ = std::move(from.impl_);
} else {
impl_ = Impl::create(from.as_slice());
}
}
UniqueSharedSlice copy() const {
return UniqueSharedSlice(as_slice());
}
Slice as_slice() const {
return impl_.as_slice();
}
MutableSlice as_mutable_slice() {
return impl_.as_mutable_slice();
}
operator Slice() const {
return as_slice();
}
// like in std::string
char *data() {
return as_mutable_slice().data();
}
const char *data() const {
return as_slice().data();
}
char operator[](size_t at) const {
return as_slice()[at];
}
bool empty() const {
return size() == 0;
}
size_t size() const {
return impl_.size();
}
// like in std::string
size_t length() const {
return size();
}
void clear() {
impl_.clear();
}
private:
friend class SharedSlice;
explicit UniqueSharedSlice(Impl impl) : impl_(std::move(impl)) {
}
Impl impl_;
};
inline SharedSlice::SharedSlice(UniqueSharedSlice from) : impl_(std::move(from.impl_)) {
}
template <bool zero_on_destruct>
class UniqueSliceImpl {
using Impl = detail::UnsafeSharedSlice<detail::UniqueSliceHeader, zero_on_destruct>;
public:
UniqueSliceImpl() = default;
explicit UniqueSliceImpl(size_t size) : impl_(Impl::create(size)) {
}
UniqueSliceImpl(size_t size, char c) : impl_(Impl::create(size)) {
std::memset(as_mutable_slice().data(), c, size);
}
explicit UniqueSliceImpl(Slice slice) : impl_(Impl::create(slice)) {
}
UniqueSliceImpl(const char *ptr, size_t size) : UniqueSliceImpl(Slice(ptr, size)) {
}
UniqueSliceImpl copy() const {
return UniqueSliceImpl(as_slice());
}
Slice as_slice() const {
return impl_.as_slice();
}
MutableSlice as_mutable_slice() {
return impl_.as_mutable_slice();
}
operator Slice() const {
return as_slice();
}
// like in std::string
char *data() {
return as_mutable_slice().data();
}
const char *data() const {
return as_slice().data();
}
char operator[](size_t at) const {
return as_slice()[at];
}
bool empty() const {
return size() == 0;
}
size_t size() const {
return impl_.size();
}
// like in std::string
size_t length() const {
return size();
}
void clear() {
impl_.clear();
}
private:
explicit UniqueSliceImpl(Impl impl) : impl_(std::move(impl)) {
}
Impl impl_;
};
using UniqueSlice = UniqueSliceImpl<false>;
using SecureString = UniqueSliceImpl<true>;
inline MutableSlice as_mutable_slice(UniqueSharedSlice &unique_shared_slice) {
return unique_shared_slice.as_mutable_slice();
}
inline MutableSlice as_mutable_slice(UniqueSlice &unique_slice) {
return unique_slice.as_mutable_slice();
}
inline MutableSlice as_mutable_slice(SecureString &secure_string) {
return secure_string.as_mutable_slice();
}
} // namespace td

3
tdutils/td/utils/Slice-decl.h
View File

@ -52,6 +52,9 @@ class MutableSlice {
MutableSlice substr(size_t from, size_t size) const;
size_t find(char c) const;
size_t rfind(char c) const;
void fill(char c);
void fill_zero();
void fill_zero_secure();
void copy_from(Slice from);

31
tdutils/td/utils/Slice.cpp Normal file
View File

@ -0,0 +1,31 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/Slice.h"
#if TD_HAVE_OPENSSL
#include <openssl/crypto.h>
#endif
namespace td {
void MutableSlice::fill(char c) {
std::memset(data(), c, size());
}
void MutableSlice::fill_zero() {
fill(0);
}
void MutableSlice::fill_zero_secure() {
#if TD_HAVE_OPENSSL
OPENSSL_cleanse(begin(), size());
#else
volatile char *ptr = begin();
for (size_t i = 0; i < size(); i++) {
ptr[i] = 0;
}
#endif
}
} // namespace td

8
tdutils/td/utils/Slice.h
View File

@ -316,4 +316,12 @@ inline MutableSlice as_slice(string &str) {
return str;
}
inline MutableSlice as_mutable_slice(MutableSlice slice) {
return slice;
}
inline MutableSlice as_mutable_slice(string &str) {
return str;
}
} // namespace td

54
tdutils/td/utils/ThreadLocalStorage.h Normal file
View File

@ -0,0 +1,54 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/port/thread.h"
#include "td/utils/port/thread_local.h"
#include "td/utils/int_types.h"
#include <atomic>
#include <array>
namespace td {
template <class T>
class ThreadLocalStorage {
public:
T& get() {
return thread_local_node().value;
}
template <class F>
void for_each(F&& f) {
int n = max_thread_id_.load();
for (int i = 0; i < n; i++) {
f(nodes_[i].value);
}
}
template <class F>
void for_each(F&& f) const {
int n = max_thread_id_.load();
for (int i = 0; i < n; i++) {
f(nodes_[i].value);
}
}
private:
struct Node {
T value{};
char padding[128];
};
static constexpr int MAX_THREAD_ID = 128;
std::atomic<int> max_thread_id_{MAX_THREAD_ID};
std::array<Node, MAX_THREAD_ID> nodes_;
Node& thread_local_node() {
auto thread_id = get_thread_id();
CHECK(0 <= thread_id && static_cast<size_t>(thread_id) < nodes_.size());
return nodes_[thread_id];
}
};
} // namespace td

112
tdutils/td/utils/ThreadSafeCounter.h
View File

@ -5,45 +5,107 @@
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/port/thread_local.h"
#include "td/utils/ThreadLocalStorage.h"
#include "td/utils/StringBuilder.h"
#include <array>
#include <atomic>
#include <mutex>
namespace td {
class ThreadSafeCounter {
template <size_t N>
class ThreadSafeMultiCounter {
public:
void add(int64 diff) {
auto &node = thread_local_node();
node.count_.store(node.count_.load(std::memory_order_relaxed) + diff, std::memory_order_relaxed);
void add(size_t index, int64 diff) {
CHECK(index < N);
tls_.get()[index].fetch_add(diff, std::memory_order_relaxed);
}
int64 sum() const {
int n = max_thread_id_.load();
int64 sum(size_t index) const {
CHECK(index < N);
int64 res = 0;
for (int i = 0; i < n; i++) {
res += nodes_[i].count_.load();
}
tls_.for_each([&](auto &value) { res += value[index].load(); });
return res;
}
private:
struct Node {
std::atomic<int64> count_{0};
char padding[128];
};
static constexpr int MAX_THREAD_ID = 128;
std::atomic<int> max_thread_id_{MAX_THREAD_ID};
std::array<Node, MAX_THREAD_ID> nodes_;
ThreadLocalStorage<std::array<std::atomic<int64>, N>> tls_;
};
Node &thread_local_node() {
auto thread_id = get_thread_id();
CHECK(static_cast<size_t>(thread_id) < nodes_.size());
return nodes_[thread_id];
class ThreadSafeCounter {
public:
void add(int64 diff) {
counter_.add(0, diff);
}
int64 sum() const {
return counter_.sum(0);
}
private:
ThreadSafeMultiCounter<1> counter_;
};
class NamedThreadSafeCounter {
static constexpr int N = 128;
using Counter = ThreadSafeMultiCounter<N>;
public:
class CounterRef {
public:
CounterRef() = default;
CounterRef(size_t index, Counter *counter) : index_(index), counter_(counter) {
}
void add(int64 diff) {
counter_->add(index_, diff);
}
int64 sum() const {
return counter_->sum(index_);
}
private:
size_t index_{0};
Counter *counter_{nullptr};
};
CounterRef get_counter(Slice name) {
std::unique_lock<std::mutex> guard(mutex_);
for (size_t i = 0; i < names_.size(); i++) {
if (names_[i] == name) {
return get_counter_ref(i);
}
}
CHECK(names_.size() < N);
names_.push_back(name.str());
return get_counter_ref(names_.size() - 1);
}
CounterRef get_counter_ref(size_t index) {
return CounterRef(index, &counter_);
}
static NamedThreadSafeCounter &get_default() {
static NamedThreadSafeCounter res;
return res;
}
template <class F>
void for_each(F &&f) const {
std::unique_lock<std::mutex> guard(mutex_);
for (size_t i = 0; i < names_.size(); i++) {
f(names_[i], counter_.sum(i));
}
}
friend StringBuilder &operator<<(StringBuilder &sb, const NamedThreadSafeCounter &counter) {
counter.for_each([&sb](Slice name, int64 cnt) { sb << name << ": " << cnt << "\n"; });
return sb;
}
private:
mutable std::mutex mutex_;
std::vector<std::string> names_;
Counter counter_;
};
} // namespace td

78
tdutils/td/utils/TsFileLog.cpp Normal file
View File

@ -0,0 +1,78 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "TsFileLog.h"
namespace td {
namespace detail {
class TsFileLog : public LogInterface {
public:
Status init(string path) {
path_ = std::move(path);
for (int i = 0; i < (int)logs_.size(); i++) {
logs_[i].id = i;
}
return init_info(&logs_[0]);
}
vector<string> get_file_paths() override {
vector<string> res;
for (auto &log : logs_) {
res.push_back(get_path(&log));
}
return res;
}
void append(CSlice cslice) override {
return append(cslice, -1);
}
void append(CSlice cslice, int log_level) override {
get_current_logger()->append(cslice, log_level);
}
private:
struct Info {
FileLog log;
bool is_inited;
int id;
};
static constexpr int MAX_THREAD_ID = 128;
std::string path_;
std::array<Info, MAX_THREAD_ID> logs_;
LogInterface *get_current_logger() {
auto *info = get_current_info();
if (!info->is_inited) {
CHECK(init_info(info).is_ok());
}
return &info->log;
}
Info *get_current_info() {
return &logs_[get_thread_id()];
}
Status init_info(Info *info) {
TRY_STATUS(info->log.init(get_path(info), std::numeric_limits<int64>::max(), info->id == 0));
info->is_inited = true;
return Status::OK();
}
string get_path(Info *info) {
if (info->id == 0) {
return path_;
}
return PSTRING() << path_ << "." << info->id;
}
};
} // namespace detail
Result<td::unique_ptr<LogInterface>> TsFileLog::create(string path) {
auto res = td::make_unique<detail::TsFileLog>();
TRY_STATUS(res->init(path));
return std::move(res);
}
} // namespace td

16
tdutils/td/utils/TsFileLog.h Normal file
View File

@ -0,0 +1,16 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/FileLog.h"
namespace td {
class TsFileLog {
public:
static Result<td::unique_ptr<LogInterface>> create(string path);
};
} // namespace td

50
tdutils/td/utils/base64.cpp
View File

@ -10,6 +10,8 @@
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include "td/utils/format.h"
#include <algorithm>
#include <iterator>
@ -48,7 +50,7 @@ string base64_encode(Slice input) {
static unsigned char char_to_value[256];
static void init_base64_table() {
static bool is_inited = []() {
static bool is_inited = [] {
std::fill(std::begin(char_to_value), std::end(char_to_value), static_cast<unsigned char>(64));
for (unsigned char i = 0; i < 64; i++) {
char_to_value[static_cast<size_t>(symbols64[i])] = i;
@ -58,9 +60,7 @@ static void init_base64_table() {
CHECK(is_inited);
}
Result<string> base64_decode(Slice base64) {
init_base64_table();
Result<Slice> base64_drop_padding(Slice base64) {
if ((base64.size() & 3) != 0) {
return Status::Error("Wrong string length");
}
@ -73,9 +73,11 @@ Result<string> base64_decode(Slice base64) {
if (padding_length >= 3) {
return Status::Error("Wrong string padding");
}
return base64;
}
string output;
output.reserve(((base64.size() + 3) >> 2) * 3);
template <class F>
Status base64_do_decode(Slice base64, F &&append) {
for (size_t i = 0; i < base64.size();) {
size_t left = min(base64.size() - i, static_cast<size_t>(4));
int c = 0;
@ -86,25 +88,53 @@ Result<string> base64_decode(Slice base64) {
}
c |= value << ((3 - t) * 6);
}
output += static_cast<char>(static_cast<unsigned char>(c >> 16)); // implementation-defined
append(static_cast<char>(static_cast<unsigned char>(c >> 16))); // implementation-defined
if (left == 2) {
if ((c & ((1 << 16) - 1)) != 0) {
return Status::Error("Wrong padding in the string");
}
} else {
output += static_cast<char>(static_cast<unsigned char>(c >> 8)); // implementation-defined
append(static_cast<char>(static_cast<unsigned char>(c >> 8))); // implementation-defined
if (left == 3) {
if ((c & ((1 << 8) - 1)) != 0) {
return Status::Error("Wrong padding in the string");
}
} else {
output += static_cast<char>(static_cast<unsigned char>(c)); // implementation-defined
append(static_cast<char>(static_cast<unsigned char>(c))); // implementation-defined
}
}
}
return Status::OK();
}
Result<string> base64_decode(Slice base64) {
init_base64_table();
TRY_RESULT(tmp, base64_drop_padding(base64));
base64 = tmp;
string output;
output.reserve(((base64.size() + 3) >> 2) * 3);
TRY_STATUS(base64_do_decode(base64, [&output](char c) { output += c; }));
return output;
}
Result<SecureString> base64_decode_secure(Slice base64) {
init_base64_table();
TRY_RESULT(tmp, base64_drop_padding(base64));
base64 = tmp;
SecureString output(((base64.size() + 3) >> 2) * 3);
char *ptr = output.as_mutable_slice().begin();
TRY_STATUS(base64_do_decode(base64, [&ptr](char c) { *ptr++ = c; }));
size_t size = ptr - output.as_mutable_slice().begin();
if (size == output.size()) {
return std::move(output);
}
return SecureString(output.as_slice().substr(0, size));
}
static const char *const url_symbols64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
string base64url_encode(Slice input) {
@ -133,7 +163,7 @@ string base64url_encode(Slice input) {
static unsigned char url_char_to_value[256];
static void init_base64url_table() {
static bool is_inited = []() {
static bool is_inited = [] {
std::fill(std::begin(url_char_to_value), std::end(url_char_to_value), static_cast<unsigned char>(64));
for (unsigned char i = 0; i < 64; i++) {
url_char_to_value[static_cast<size_t>(url_symbols64[i])] = i;

2
tdutils/td/utils/base64.h
View File

@ -8,12 +8,14 @@
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include "td/utils/SharedSlice.h"
#include "td/utils/Status.h"
namespace td {
string base64_encode(Slice input);
Result<string> base64_decode(Slice base64);
Result<SecureString> base64_decode_secure(Slice base64);
string base64url_encode(Slice input);
Result<string> base64url_decode(Slice base64);

3
tdutils/td/utils/buffer.h
View File

@ -796,5 +796,8 @@ inline Slice as_slice(const BufferSlice &value) {
inline MutableSlice as_slice(BufferSlice &value) {
return value.as_slice();
}
inline MutableSlice as_mutable_slice(BufferSlice &value) {
return value.as_slice();
}
} // namespace td

55
tdutils/td/utils/crypto.cpp
View File

@ -241,66 +241,74 @@ int pq_factorize(Slice pq_str, string *p_str, string *q_str) {
return 0;
}
static void aes_ige_xcrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to, bool encrypt_flag) {
static void aes_ige_xcrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to, bool encrypt_flag) {
CHECK(aes_key.size() == 32);
CHECK(aes_iv.size() == 16);
AES_KEY key;
int err;
if (encrypt_flag) {
err = AES_set_encrypt_key(aes_key.raw, 256, &key);
err = AES_set_encrypt_key(aes_key.ubegin(), 256, &key);
} else {
err = AES_set_decrypt_key(aes_key.raw, 256, &key);
err = AES_set_decrypt_key(aes_key.ubegin(), 256, &key);
}
LOG_IF(FATAL, err != 0);
CHECK(from.size() <= to.size());
AES_ige_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv->raw, encrypt_flag);
AES_ige_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv.ubegin(), encrypt_flag);
}
void aes_ige_encrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to) {
void aes_ige_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) {
aes_ige_xcrypt(aes_key, aes_iv, from, to, true);
}
void aes_ige_decrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to) {
void aes_ige_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) {
aes_ige_xcrypt(aes_key, aes_iv, from, to, false);
}
static void aes_cbc_xcrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to, bool encrypt_flag) {
static void aes_cbc_xcrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to, bool encrypt_flag) {
CHECK(aes_key.size() == 32);
CHECK(aes_iv.size() == 16);
AES_KEY key;
int err;
if (encrypt_flag) {
err = AES_set_encrypt_key(aes_key.raw, 256, &key);
err = AES_set_encrypt_key(aes_key.ubegin(), 256, &key);
} else {
err = AES_set_decrypt_key(aes_key.raw, 256, &key);
err = AES_set_decrypt_key(aes_key.ubegin(), 256, &key);
}
LOG_IF(FATAL, err != 0);
CHECK(from.size() <= to.size());
AES_cbc_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv->raw, encrypt_flag);
AES_cbc_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv.ubegin(), encrypt_flag);
}
void aes_cbc_encrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to) {
void aes_cbc_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) {
aes_cbc_xcrypt(aes_key, aes_iv, from, to, true);
}
void aes_cbc_decrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to) {
void aes_cbc_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) {
aes_cbc_xcrypt(aes_key, aes_iv, from, to, false);
}
AesCbcState::AesCbcState(const UInt256 &key, const UInt128 &iv) : key_(key), iv_(iv) {
AesCbcState::AesCbcState(Slice key256, Slice iv128) : key_(key256), iv_(iv128) {
CHECK(key_.size() == 32);
CHECK(iv_.size() == 16);
}
void AesCbcState::encrypt(Slice from, MutableSlice to) {
::td::aes_cbc_encrypt(key_, &iv_, from, to);
::td::aes_cbc_encrypt(key_.as_slice(), iv_.as_mutable_slice(), from, to);
}
void AesCbcState::decrypt(Slice from, MutableSlice to) {
::td::aes_cbc_decrypt(key_, &iv_, from, to);
::td::aes_cbc_decrypt(key_.as_slice(), iv_.as_mutable_slice(), from, to);
}
class AesCtrState::Impl {
public:
Impl(const UInt256 &key, const UInt128 &iv) {
Impl(Slice key, Slice iv) {
CHECK(key.size() == 32);
CHECK(iv.size() == 16);
static_assert(AES_BLOCK_SIZE == 16, "");
if (AES_set_encrypt_key(key.raw, 256, &aes_key) < 0) {
if (AES_set_encrypt_key(key.ubegin(), 256, &aes_key) < 0) {
LOG(FATAL) << "Failed to set encrypt key";
}
MutableSlice(counter, AES_BLOCK_SIZE).copy_from(as_slice(iv));
counter.as_mutable_slice().copy_from(as_slice(iv));
current_pos = 0;
}
@ -308,9 +316,10 @@ class AesCtrState::Impl {
CHECK(to.size() >= from.size());
for (size_t i = 0; i < from.size(); i++) {
if (current_pos == 0) {
AES_encrypt(counter, encrypted_counter, &aes_key);
AES_encrypt(counter.as_slice().ubegin(), encrypted_counter.as_mutable_slice().ubegin(), &aes_key);
uint8 *ptr = counter.as_mutable_slice().ubegin();
for (int j = 15; j >= 0; j--) {
if (++counter[j] != 0) {
if (++ptr[j] != 0) {
break;
}
}
@ -322,8 +331,8 @@ class AesCtrState::Impl {
private:
AES_KEY aes_key;
uint8 counter[AES_BLOCK_SIZE];
uint8 encrypted_counter[AES_BLOCK_SIZE];
SecureString counter{AES_BLOCK_SIZE};
SecureString encrypted_counter{AES_BLOCK_SIZE};
uint8 current_pos;
};
@ -332,7 +341,7 @@ AesCtrState::AesCtrState(AesCtrState &&from) = default;
AesCtrState &AesCtrState::operator=(AesCtrState &&from) = default;
AesCtrState::~AesCtrState() = default;
void AesCtrState::init(const UInt256 &key, const UInt128 &iv) {
void AesCtrState::init(Slice key, Slice iv) {
ctx_ = make_unique<AesCtrState::Impl>(key, iv);
}

18
tdutils/td/utils/crypto.h
View File

@ -9,8 +9,8 @@
#include "td/utils/buffer.h"
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include "td/utils/SharedSlice.h"
#include "td/utils/Status.h"
#include "td/utils/UInt.h"
namespace td {
@ -21,11 +21,11 @@ void init_crypto();
int pq_factorize(Slice pq_str, string *p_str, string *q_str);
void aes_ige_encrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to);
void aes_ige_decrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to);
void aes_ige_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to);
void aes_ige_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to);
void aes_cbc_encrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to);
void aes_cbc_decrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to);
void aes_cbc_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to);
void aes_cbc_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to);
class AesCtrState {
public:
@ -36,7 +36,7 @@ class AesCtrState {
AesCtrState &operator=(AesCtrState &&from);
~AesCtrState();
void init(const UInt256 &key, const UInt128 &iv);
void init(Slice key, Slice iv);
void encrypt(Slice from, MutableSlice to);
@ -49,14 +49,14 @@ class AesCtrState {
class AesCbcState {
public:
AesCbcState(const UInt256 &key, const UInt128 &iv);
AesCbcState(Slice key256, Slice iv128);
void encrypt(Slice from, MutableSlice to);
void decrypt(Slice from, MutableSlice to);
private:
UInt256 key_;
UInt128 iv_;
SecureString key_;
SecureString iv_;
};
void sha1(Slice data, unsigned char output[20]);

10
tdutils/td/utils/filesystem.cpp
View File

@ -32,6 +32,10 @@ template <>
BufferSlice create_empty<BufferSlice>(size_t size) {
return BufferSlice{size};
}
template <>
SecureString create_empty<SecureString>(size_t size) {
return SecureString{size};
}
template <class T>
Result<T> read_file_impl(CSlice path, int64 size, int64 offset) {
@ -48,7 +52,7 @@ Result<T> read_file_impl(CSlice path, int64 size, int64 offset) {
}
size -= offset;
auto content = create_empty<T>(narrow_cast<size_t>(size));
TRY_RESULT(got_size, from_file.pread(as_slice(content), offset));
TRY_RESULT(got_size, from_file.pread(as_mutable_slice(content), offset));
if (got_size != static_cast<size_t>(size)) {
return Status::Error("Failed to read file");
}
@ -66,6 +70,10 @@ Result<string> read_file_str(CSlice path, int64 size, int64 offset) {
return read_file_impl<string>(path, size, offset);
}
Result<SecureString> read_file_secure(CSlice path, int64 size, int64 offset) {
return read_file_impl<SecureString>(path, size, offset);
}
// Very straightforward function. Don't expect much of it.
Status copy_file(CSlice from, CSlice to, int64 size) {
TRY_RESULT(content, read_file(from, size));

2
tdutils/td/utils/filesystem.h
View File

@ -8,12 +8,14 @@
#include "td/utils/buffer.h"
#include "td/utils/Slice.h"
#include "td/utils/SharedSlice.h"
#include "td/utils/Status.h"
namespace td {
Result<BufferSlice> read_file(CSlice path, int64 size = -1, int64 offset = 0);
Result<string> read_file_str(CSlice path, int64 size = -1, int64 offset = 0);
Result<SecureString> read_file_secure(CSlice path, int64 size = -1, int64 offset = 0);
Status copy_file(CSlice from, CSlice to, int64 size = -1) TD_WARN_UNUSED_RESULT;

12
tdutils/td/utils/port/detail/ThreadStl.h
View File

@ -30,7 +30,9 @@ class ThreadStl {
ThreadStl &operator=(const ThreadStl &other) = delete;
ThreadStl(ThreadStl &&) = default;
ThreadStl &operator=(ThreadStl &&) = default;
~ThreadStl() = default;
~ThreadStl() {
join();
}
template <class Function, class... Args>
explicit ThreadStl(Function &&f, Args &&... args) {
thread_ = std::thread([args = std::make_tuple(decay_copy(std::forward<Function>(f)),
@ -42,10 +44,14 @@ class ThreadStl {
}
void join() {
thread_.join();
if (thread_.joinable()) {
thread_.join();
}
}
void detach() {
thread_.detach();
if (thread_.joinable()) {
thread_.detach();
}
}
void set_name(CSlice name) {
}

25
tdutils/td/utils/tl_helpers.h
View File

@ -10,6 +10,7 @@
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include "td/utils/Slice.h"
#include "td/utils/SharedSlice.h"
#include "td/utils/StackAllocator.h"
#include "td/utils/Status.h"
#include "td/utils/tl_parsers.h"
@ -115,11 +116,20 @@ template <class StorerT>
void store(const string &x, StorerT &storer) {
storer.store_string(x);
}
template <class StorerT>
void store(const SecureString &x, StorerT &storer) {
storer.store_string(x.as_slice());
}
template <class ParserT>
void parse(string &x, ParserT &parser) {
x = parser.template fetch_string<string>();
}
template <class ParserT>
void parse(SecureString &x, ParserT &parser) {
x = parser.template fetch_string<SecureString>();
}
template <class T, class StorerT>
void store(const vector<T> &vec, StorerT &storer) {
storer.store_binary(narrow_cast<int32>(vec.size()));
@ -228,6 +238,21 @@ string serialize(const T &object) {
return key;
}
template <class T>
SecureString serialize_secure(const T &object) {
TlStorerCalcLength calc_length;
store(object, calc_length);
size_t length = calc_length.get_length();
SecureString key(length, '\0');
CHECK(is_aligned_pointer<4>(key.data()));
MutableSlice data = key.as_mutable_slice();
TlStorerUnsafe storer(data.ubegin());
store(object, storer);
CHECK(storer.get_buf() == data.uend());
return key;
}
template <class T>
TD_WARN_UNUSED_RESULT Status unserialize(T &object, Slice data) {
TlParser parser(data);

22
tdutils/td/utils/tl_parsers.h
View File

@ -148,16 +148,29 @@ class TlParser {
if (result_len < 254) {
result_begin = reinterpret_cast<const char *>(data + 1);
result_aligned_len = (result_len >> 2) << 2;
data += sizeof(int32);
} else if (result_len == 254) {
result_len = data[1] + (data[2] << 8) + (data[3] << 16);
result_begin = reinterpret_cast<const char *>(data + 4);
result_aligned_len = ((result_len + 3) >> 2) << 2;
data += sizeof(int32);
} else {
set_error("Can't fetch string, 255 found");
return T();
check_len(sizeof(int32));
result_len = data[1] + (data[2] << 8) + (data[3] << 16) + (data[4] << 24) + (static_cast<uint64>(data[5]) << 32) +
(static_cast<uint64>(data[6]) << 40) + (static_cast<uint64>(data[7]) << 48);
if (result_len > std::numeric_limits<size_t>::max() - 3) {
set_error("Too big string found");
return T();
}
result_begin = reinterpret_cast<const char *>(data + 8);
result_aligned_len = ((result_len + 3) >> 2) << 2;
data += sizeof(int64);
}
check_len(result_aligned_len);
data += result_aligned_len + sizeof(int32);
if (!error.empty()) {
return T();
}
data += result_aligned_len;
return T(result_begin, result_len);
}
@ -165,6 +178,9 @@ class TlParser {
T fetch_string_raw(const size_t size) {
//CHECK(size % sizeof(int32) == 0);
check_len(size);
if (!error.empty()) {
return T();
}
const char *result = reinterpret_cast<const char *>(data);
data += size;
return T(result, size);

15
tdutils/td/utils/tl_storers.h
View File

@ -62,6 +62,15 @@ class TlStorerUnsafe {
*buf_++ = static_cast<unsigned char>(len & 255);
*buf_++ = static_cast<unsigned char>((len >> 8) & 255);
*buf_++ = static_cast<unsigned char>(len >> 16);
} else if (len < (1ull << 32)) {
*buf_++ = static_cast<unsigned char>(255);
*buf_++ = static_cast<unsigned char>(len & 255);
*buf_++ = static_cast<unsigned char>((len >> 8) & 255);
*buf_++ = static_cast<unsigned char>((len >> 16) & 255);
*buf_++ = static_cast<unsigned char>((len >> 24) & 255);
*buf_++ = static_cast<unsigned char>(0);
*buf_++ = static_cast<unsigned char>(0);
*buf_++ = static_cast<unsigned char>(0);
} else {
LOG(FATAL) << "String size " << len << " is too big to be stored";
}
@ -71,7 +80,7 @@ class TlStorerUnsafe {
switch (len & 3) {
case 1:
*buf_++ = 0;
// fallthrough
// fallthrough
case 2:
*buf_++ = 0;
// fallthrough
@ -119,8 +128,10 @@ class TlStorerCalcLength {
size_t add = str.size();
if (add < 254) {
add += 1;
} else {
} else if (add < (1 << 24)) {
add += 4;
} else {
add += 8;
}
add = (add + 3) & -4;
length += add;

85
tdutils/test/SharedSlice.cpp Normal file
View File

@ -0,0 +1,85 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/tests.h"
#include "td/utils/SharedSlice.h"
using namespace td;
TEST(SharedSlice, Hands) {
{
SharedSlice h("hello");
ASSERT_EQ("hello", h.as_slice());
// auto g = h; // CE
auto g = h.clone();
ASSERT_EQ("hello", g.as_slice());
}
{
SharedSlice h("hello");
UniqueSharedSlice g(std::move(h));
ASSERT_EQ("", h.as_slice());
ASSERT_EQ("hello", g.as_slice());
}
{
SharedSlice h("hello");
SharedSlice t = h.clone();
UniqueSharedSlice g(std::move(h));
ASSERT_EQ("", h.as_slice());
ASSERT_EQ("hello", g.as_slice());
ASSERT_EQ("hello", t.as_slice());
}
{
UniqueSharedSlice g(5);
g.as_mutable_slice().copy_from("hello");
SharedSlice h(std::move(g));
ASSERT_EQ("hello", h);
ASSERT_EQ("", g);
}
{
UniqueSlice h("hello");
UniqueSlice g(std::move(h));
ASSERT_EQ("", h.as_slice());
ASSERT_EQ("hello", g.as_slice());
}
{
SecureString h("hello");
SecureString g(std::move(h));
ASSERT_EQ("", h.as_slice());
ASSERT_EQ("hello", g.as_slice());
}
{
Stage stage;
SharedSlice a, b;
std::vector<td::thread> threads(2);
for (int i = 0; i < 2; i++) {
threads[i] = td::thread([i, &stage, &a, &b] {
for (int j = 0; j < 10000; j++) {
if (i == 0) {
a = SharedSlice("hello");
b = a.clone();
}
stage.wait((2 * j + 1) * 2);
if (i == 0) {
ASSERT_EQ('h', a[0]);
a.clear();
} else {
UniqueSharedSlice c(std::move(b));
c.as_mutable_slice()[0] = '!';
}
stage.wait((2 * j + 2) * 2);
}
});
}
for (auto &thread : threads) {
thread.join();
}
}
}

6
tdutils/test/crypto.cpp
View File

@ -45,18 +45,18 @@ TEST(Crypto, AesCtrState) {
}
td::AesCtrState state;
state.init(key, iv);
state.init(as_slice(key), as_slice(iv));
td::string t(length, '\0');
state.encrypt(s, t);
ASSERT_EQ(answers1[i], td::crc32(t));
state.init(key, iv);
state.init(as_slice(key), as_slice(iv));
state.decrypt(t, t);
ASSERT_STREQ(s, t);
for (auto &c : iv.raw) {
c = 0xFF;
}
state.init(key, iv);
state.init(as_slice(key), as_slice(iv));
state.encrypt(s, t);
ASSERT_EQ(answers2[i], td::crc32(t));

122
tdutils/test/log.cpp Normal file
View File

@ -0,0 +1,122 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/tests.h"
#include "td/utils/FileLog.h"
#include "td/utils/TsFileLog.h"
#include "td/utils/logging.h"
#include "td/utils/port/path.h"
#include "td/utils/benchmark.h"
// Thread safe logging with tests
//
// LOG uses thread local LogInterface
// void append(CSlice slice, int log_level);
//
template <class Log>
class LogBenchmark : public td::Benchmark {
public:
explicit LogBenchmark(std::string name, int threads_n, std::function<td::unique_ptr<Log>()> creator)
: name_(std::move(name)), threads_n_(threads_n), creator_(std::move(creator)) {
}
std::string get_description() const override {
return PSTRING() << name_ << " " << td::tag("threads_n", threads_n_);
}
void start_up() override {
log_ = creator_();
threads_.resize(threads_n_);
}
void tear_down() override {
for (auto path : log_->get_file_paths()) {
td::unlink(path).ignore();
}
log_.reset();
}
void run(int n) override {
for (auto &thread : threads_) {
thread = td::thread([this, n] { this->run_thread(n); });
}
for (auto &thread : threads_) {
thread.join();
}
}
void run_thread(int n) {
auto str = PSTRING() << "#" << n << " : fsjklfdjsklfjdsklfjdksl\n";
for (int i = 0; i < n; i++) {
log_->append(str);
}
}
private:
std::string name_;
td::unique_ptr<Log> log_;
int threads_n_{0};
std::function<td::unique_ptr<Log>()> creator_;
std::vector<td::thread> threads_;
};
template <class F>
void bench_log(std::string name, int threads_n, F &&f) {
bench(LogBenchmark<typename decltype(f())::element_type>(std::move(name), threads_n, std::move(f)));
};
TEST(Log, TsLogger) {
bench_log("NewTsFileLog", 4, [] { return td::TsFileLog::create("tmplog").move_as_ok(); });
bench_log("TsFileLog", 8, [] {
class FileLog : public td::LogInterface {
public:
FileLog() {
file_log_.init("tmplog", std::numeric_limits<td::int64>::max(), false);
ts_log_.init(&file_log_);
}
~FileLog() {
}
void append(td::CSlice slice) override {
ts_log_.append(slice, -1);
}
std::vector<std::string> get_file_paths() override {
return file_log_.get_file_paths();
}
private:
td::FileLog file_log_;
td::TsLog ts_log_{nullptr};
};
return td::make_unique<FileLog>();
});
bench_log("noop", 4, [] {
class NoopLog : public td::LogInterface {
public:
void append(td::CSlice slice) override {
}
};
return td::make_unique<NoopLog>();
});
bench_log("FileLog", 4, [] {
class FileLog : public td::LogInterface {
public:
FileLog() {
file_log_.init("tmplog", std::numeric_limits<td::int64>::max(), false);
}
~FileLog() {
}
void append(td::CSlice slice) override {
file_log_.append(slice, -1);
}
std::vector<std::string> get_file_paths() override {
return file_log_.get_file_paths();
}
private:
td::FileLog file_log_;
};
return td::make_unique<FileLog>();
});
}

3
tdutils/test/misc.cpp
View File

@ -895,6 +895,9 @@ static void test_hash() {
test_hash<HashT, BadValue>({BadValue{1}, BadValue{2}}).ensure_error();
test_hash<HashT, ValueA>({ValueA{1}, ValueA{2}}).ensure();
test_hash<HashT, ValueB>({ValueB{1}, ValueB{2}}).ensure();
test_hash<HashT, std::pair<int, int>>({{1, 1}, {1, 2}}).ensure();
// FIXME: use some better hash
//test_hash<HashT, std::pair<int, int>>({{1, 1}, {1, 2}, {2, 1}, {2, 2}}).ensure();
}
TEST(Misc, Hasher) {