From 8872fbf6aca12242ee8ab945bb3822f790cbb478 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 26 Jun 2020 02:24:13 +0300 Subject: [PATCH] Fix tdutils after merge. GitOrigin-RevId: 5b9a863e405a9e3782157570156fda71bf2bb589 --- CMakeLists.txt | 9 +- td/telegram/StorageManager.cpp | 2 +- td/telegram/StorageManager.h | 6 +- td/telegram/cli.cpp | 3 +- td/telegram/files/FileStats.cpp | 1 + td/telegram/files/FileStatsWorker.cpp | 1 - td/telegram/net/NetStatsManager.cpp | 2 +- tddb/td/db/TQueue.cpp | 3 +- tddb/td/db/binlog/BinlogHelper.h | 5 +- tdnet/td/net/HttpQuery.cpp | 4 +- tdnet/td/net/HttpQuery.h | 10 +- tdutils/CMakeLists.txt | 22 +-- tdutils/td/utils/AtomicRead.h | 79 ++++++++ tdutils/td/utils/BigNum.cpp | 8 +- tdutils/td/utils/BufferedFd.h | 5 +- tdutils/td/utils/CancellationToken.h | 2 +- tdutils/td/utils/ConcurrentHashTable.h | 2 +- tdutils/td/utils/FileLog.cpp | 6 +- tdutils/td/utils/FileLog.h | 8 +- tdutils/td/utils/Heap.h | 2 +- tdutils/td/utils/JsonBuilder.h | 20 +- tdutils/td/utils/MpmcWaiter.h | 32 ++-- tdutils/td/utils/OptionParser.cpp | 7 +- tdutils/td/utils/PathView.cpp | 14 +- tdutils/td/utils/PathView.h | 4 +- tdutils/td/utils/Random.cpp | 7 +- tdutils/td/utils/Random.h | 11 +- tdutils/td/utils/SharedObjectPool.h | 4 +- tdutils/td/utils/SharedSlice.cpp | 1 - tdutils/td/utils/SharedSlice.h | 2 +- tdutils/td/utils/Slice.cpp | 1 - tdutils/td/utils/Span.h | 9 +- tdutils/td/utils/Status.h | 11 +- tdutils/td/utils/StealingQueue.h | 125 +++++++++++++ tdutils/td/utils/ThreadLocalStorage.h | 3 +- tdutils/td/utils/ThreadSafeCounter.h | 6 +- tdutils/td/utils/Time.cpp | 11 +- tdutils/td/utils/Time.h | 4 +- tdutils/td/utils/TimedStat.h | 12 +- tdutils/td/utils/Timer.cpp | 10 +- tdutils/td/utils/Timer.h | 8 +- tdutils/td/utils/TsFileLog.cpp | 30 +-- tdutils/td/utils/TsFileLog.h | 7 +- tdutils/td/utils/TsList.h | 1 - tdutils/td/utils/VectorQueue.h | 21 ++- tdutils/td/utils/base64.cpp | 50 ++--- tdutils/td/utils/base64.h | 3 +- tdutils/td/utils/bits.h | 13 +- tdutils/td/utils/crypto.cpp | 2 +- tdutils/td/utils/crypto.h | 3 +- tdutils/td/utils/filesystem.cpp | 11 +- tdutils/td/utils/filesystem.h | 7 +- tdutils/td/utils/format.h | 2 +- tdutils/td/utils/logging.cpp | 3 +- tdutils/td/utils/logging.h | 7 + tdutils/td/utils/misc.h | 23 ++- tdutils/td/utils/optional.h | 2 +- tdutils/td/utils/port/StdStreams.cpp | 9 +- tdutils/td/utils/port/detail/Epoll.cpp | 2 - .../td/utils/port/detail/ThreadPthread.cpp | 1 - tdutils/td/utils/port/detail/skip_eintr.h | 1 - tdutils/td/utils/port/rlimit.cpp | 132 ++++++------- tdutils/td/utils/port/rlimit.h | 9 +- tdutils/td/utils/port/signals.cpp | 2 +- tdutils/td/utils/port/uname.h | 2 +- tdutils/td/utils/port/user.cpp | 3 +- tdutils/td/utils/port/user.h | 5 +- tdutils/td/utils/tests.cpp | 2 +- tdutils/td/utils/tests.h | 17 +- tdutils/td/utils/tl_helpers.h | 1 - tdutils/td/utils/tl_storers.h | 3 +- tdutils/test/List.cpp | 1 - tdutils/test/MpmcWaiter.cpp | 8 +- tdutils/test/OptionParser.cpp | 2 - tdutils/test/SharedSlice.cpp | 1 - tdutils/test/StealingQueue.cpp | 173 ++++++++++++++++++ tdutils/test/heap.cpp | 3 +- tdutils/test/log.cpp | 1 - tdutils/test/misc.cpp | 43 +++-- test/main.cpp | 3 - 80 files changed, 750 insertions(+), 341 deletions(-) create mode 100644 tdutils/td/utils/AtomicRead.h create mode 100644 tdutils/td/utils/StealingQueue.h create mode 100644 tdutils/test/StealingQueue.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 622fefc41..719292ea7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -276,7 +276,7 @@ if (NOT CMAKE_CROSSCOMPILING) endif() if (NOT OPENSSL_FOUND) - message(WARNING "Not found OpenSSL: skip TDLib, tdactor, tdnet, tddb") + message(WARNING "Can't find OpenSSL: stop TDLib building") return() endif() @@ -284,7 +284,12 @@ if (NOT ZLIB_FOUND) find_package(ZLIB) endif() if (NOT ZLIB_FOUND) - message(WARNING "Not found zlib: skip TDLib, tdactor, tdnet, tddb") + message(WARNING "Can't find zlib: stop TDLib building") + return() +endif() + +if (NOT TDUTILS_MIME_TYPE) + message(WARNING "Option TDUTILS_MIME_TYPE must not be disabled: stop TDLib building") return() endif() diff --git a/td/telegram/StorageManager.cpp b/td/telegram/StorageManager.cpp index 3de3ed427..270d778bc 100644 --- a/td/telegram/StorageManager.cpp +++ b/td/telegram/StorageManager.cpp @@ -340,7 +340,7 @@ void StorageManager::schedule_next_gc() { if (next_gc_at > sys_time + GC_EACH) { next_gc_at = sys_time + GC_EACH; } - next_gc_at += Random::fast(static_cast(GC_DELAY), static_cast(GC_DELAY + GC_RAND_DELAY)); + next_gc_at += Random::fast(GC_DELAY, GC_DELAY + GC_RAND_DELAY); CHECK(next_gc_at >= sys_time); auto next_gc_in = next_gc_at - sys_time; diff --git a/td/telegram/StorageManager.h b/td/telegram/StorageManager.h index a6a07071e..f2b74091d 100644 --- a/td/telegram/StorageManager.h +++ b/td/telegram/StorageManager.h @@ -41,9 +41,9 @@ class StorageManager : public Actor { void on_new_file(int64 size, int64 real_size, int32 cnt); private: - static constexpr uint32 GC_EACH = 60 * 60 * 24; // 1 day - static constexpr uint32 GC_DELAY = 60; - static constexpr uint32 GC_RAND_DELAY = 60 * 15; + static constexpr int GC_EACH = 60 * 60 * 24; // 1 day + static constexpr int GC_DELAY = 60; + static constexpr int GC_RAND_DELAY = 60 * 15; ActorShared<> parent_; diff --git a/td/telegram/cli.cpp b/td/telegram/cli.cpp index a20a70328..07bc81792 100644 --- a/td/telegram/cli.cpp +++ b/td/telegram/cli.cpp @@ -46,7 +46,6 @@ #include #include #include -#include #include #include #include @@ -4330,7 +4329,7 @@ void main(int argc, char **argv) { return std::string(); }(std::getenv("TD_API_HASH")); - td::OptionParser options; + OptionParser options; options.set_description("TDLib test client"); options.add_option('\0', "test", "Use test DC", [&] { use_test_dc = true; }); options.add_option('v', "verbosity", "Set verbosity level", [&](Slice level) { diff --git a/td/telegram/files/FileStats.cpp b/td/telegram/files/FileStats.cpp index 5c9393010..69adf5f0a 100644 --- a/td/telegram/files/FileStats.cpp +++ b/td/telegram/files/FileStats.cpp @@ -12,6 +12,7 @@ #include "td/utils/common.h" #include "td/utils/format.h" +#include "td/utils/misc.h" #include #include diff --git a/td/telegram/files/FileStatsWorker.cpp b/td/telegram/files/FileStatsWorker.cpp index 2b757d0d0..c46554cb3 100644 --- a/td/telegram/files/FileStatsWorker.cpp +++ b/td/telegram/files/FileStatsWorker.cpp @@ -31,7 +31,6 @@ #include #include -#include namespace td { namespace { diff --git a/td/telegram/net/NetStatsManager.cpp b/td/telegram/net/NetStatsManager.cpp index e6ff6aeec..2c4dadff3 100644 --- a/td/telegram/net/NetStatsManager.cpp +++ b/td/telegram/net/NetStatsManager.cpp @@ -257,7 +257,7 @@ std::shared_ptr NetStatsManager::get_media_stats_callback() co std::vector> NetStatsManager::get_file_stats_callbacks() const { auto result = transform(files_stats_, [](auto &stat) { return stat.stats.get_callback(); }); for (int32 i = 0; i < MAX_FILE_TYPE; i++) { - int32 main_file_type = static_cast(get_main_file_type(static_cast(i))); + int32 main_file_type = static_cast(get_main_file_type(static_cast(i))); if (main_file_type != i) { result[i] = result[main_file_type]; } diff --git a/tddb/td/db/TQueue.cpp b/tddb/td/db/TQueue.cpp index 058219335..450a1af74 100644 --- a/tddb/td/db/TQueue.cpp +++ b/tddb/td/db/TQueue.cpp @@ -11,7 +11,6 @@ #include "td/db/binlog/BinlogHelper.h" #include "td/db/binlog/BinlogInterface.h" -#include "td/utils/format.h" #include "td/utils/misc.h" #include "td/utils/port/Clocks.h" #include "td/utils/Random.h" @@ -226,7 +225,7 @@ class TQueueImpl : public TQueue { return 0; } - td::MutableSpan span; + MutableSpan span; return do_get(queue_id, q, q.events.front().event_id, true, Time::now(), span); } diff --git a/tddb/td/db/binlog/BinlogHelper.h b/tddb/td/db/binlog/BinlogHelper.h index 763ff54ff..6d387941b 100644 --- a/tddb/td/db/binlog/BinlogHelper.h +++ b/tddb/td/db/binlog/BinlogHelper.h @@ -16,12 +16,13 @@ namespace td { -inline uint64 binlog_add(BinlogInterface *binlog_ptr, int32 type, const Storer &storer, Promise<> promise = Promise<>()) { +inline uint64 binlog_add(BinlogInterface *binlog_ptr, int32 type, const Storer &storer, + Promise<> promise = Promise<>()) { return binlog_ptr->add(type, storer, std::move(promise)); } inline uint64 binlog_rewrite(BinlogInterface *binlog_ptr, uint64 logevent_id, int32 type, const Storer &storer, - Promise<> promise = Promise<>()) { + Promise<> promise = Promise<>()) { return binlog_ptr->rewrite(logevent_id, type, storer, std::move(promise)); } diff --git a/tdnet/td/net/HttpQuery.cpp b/tdnet/td/net/HttpQuery.cpp index cbc75d68e..c826e021d 100644 --- a/tdnet/td/net/HttpQuery.cpp +++ b/tdnet/td/net/HttpQuery.cpp @@ -24,8 +24,8 @@ MutableSlice HttpQuery::get_arg(Slice key) const { return it == args_.end() ? MutableSlice() : it->second; } -td::vector> HttpQuery::get_args() const { - td::vector> res; +vector> HttpQuery::get_args() const { + vector> res; res.reserve(args_.size()); for (auto &it : args_) { res.emplace_back(it.first.str(), it.second.str()); diff --git a/tdnet/td/net/HttpQuery.h b/tdnet/td/net/HttpQuery.h index da53e2e4f..fe89905d4 100644 --- a/tdnet/td/net/HttpQuery.h +++ b/tdnet/td/net/HttpQuery.h @@ -21,23 +21,23 @@ class HttpQuery { public: enum class Type : int8 { Empty, Get, Post, Response }; - td::vector container_; + vector container_; Type type_ = Type::Empty; int32 code_ = 0; MutableSlice url_path_; - td::vector> args_; + vector> args_; MutableSlice reason_; bool keep_alive_ = true; - td::vector> headers_; - td::vector files_; + vector> headers_; + vector files_; MutableSlice content_; Slice get_header(Slice key) const; MutableSlice get_arg(Slice key) const; - td::vector> get_args() const; + vector> get_args() const; int get_retry_after() const; }; diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index 4f7928d47..9bd14a866 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -1,14 +1,6 @@ cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) -option(TDUTILS_MIME_TYPE "Generate mime types conversion (gperf is required)" ON) - -if (WIN32) - if (WINGETOPT_FOUND) - set(TD_HAVE_GETOPT 1) - endif() -else() - set(TD_HAVE_GETOPT 1) -endif() +option(TDUTILS_MIME_TYPE "Generate mime types conversion; requires gperf" ON) if (NOT DEFINED CMAKE_INSTALL_LIBDIR) set(CMAKE_INSTALL_LIBDIR "lib") @@ -69,7 +61,6 @@ set(TDUTILS_SOURCE td/utils/port/Stat.cpp td/utils/port/StdStreams.cpp td/utils/port/thread_local.cpp - td/utils/port/user.cpp td/utils/port/UdpSocketFd.cpp td/utils/port/uname.cpp td/utils/port/user.cpp @@ -115,10 +106,10 @@ set(TDUTILS_SOURCE td/utils/StackAllocator.cpp td/utils/Status.cpp td/utils/StringBuilder.cpp + td/utils/tests.cpp td/utils/Time.cpp td/utils/Timer.cpp td/utils/TsFileLog.cpp - td/utils/tests.cpp td/utils/tl_parsers.cpp td/utils/translit.cpp td/utils/TsFileLog.cpp @@ -150,7 +141,6 @@ set(TDUTILS_SOURCE td/utils/port/StdStreams.h td/utils/port/thread.h td/utils/port/thread_local.h - td/utils/port/user.h td/utils/port/UdpSocketFd.h td/utils/port/uname.h td/utils/port/user.h @@ -174,6 +164,7 @@ set(TDUTILS_SOURCE td/utils/AesCtrByteFlow.h td/utils/as.h + td/utils/AtomicRead.h td/utils/base64.h td/utils/benchmark.h td/utils/BigNum.h @@ -283,6 +274,7 @@ set(TDUTILS_TEST_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/test/ConcurrentHashMap.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/crypto.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/Enumerator.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/EpochBasedMemoryReclamation.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/filesystem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/gzip.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/HazardPointers.cpp @@ -337,11 +329,7 @@ if (CRC32C_FOUND) target_link_libraries(tdutils PRIVATE crc32c) endif() if (ABSL_FOUND) - target_link_libraries_system(tdutils absl::flat_hash_map absl::flat_hash_set absl::hash) -endif() - -if (WIN32 AND WINGETOPT_FOUND) - target_link_libraries(tdutils PRIVATE wingetopt) + target_link_libraries(tdutils SYSTEM PUBLIC absl::flat_hash_map absl::flat_hash_set absl::hash) endif() if (ANDROID) diff --git a/tdutils/td/utils/AtomicRead.h b/tdutils/td/utils/AtomicRead.h new file mode 100644 index 000000000..80af707dc --- /dev/null +++ b/tdutils/td/utils/AtomicRead.h @@ -0,0 +1,79 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/port/thread.h" + +#include +#include +#include + +namespace td { + +template +class AtomicRead { + public: + void read(T &dest) const { + while (true) { + static_assert(std::is_trivially_copyable::value, "T must be trivially copyable"); + auto version_before = version.load(); + if (version_before % 2 == 0) { + std::memcpy(&dest, &value, sizeof(dest)); + auto version_after = version.load(); + if (version_before == version_after) { + break; + } + } + td::this_thread::yield(); + } + } + + struct Write { + explicit Write(AtomicRead *read) { + read->do_lock(); + ptr.reset(read); + } + struct Destructor { + void operator()(AtomicRead *read) const { + read->do_unlock(); + } + }; + T &operator*() { + return value(); + } + T *operator->() { + return &value(); + } + T &value() { + CHECK(ptr); + return ptr->value; + } + + private: + std::unique_ptr ptr; + }; + + Write lock() { + return Write(this); + } + + private: + std::atomic version{0}; + T value; + + void do_lock() { + bool is_locked = ++version % 2 == 1; + CHECK(is_locked); + } + void do_unlock() { + bool is_unlocked = ++version % 2 == 0; + CHECK(is_unlocked); + } +}; + +} // namespace td diff --git a/tdutils/td/utils/BigNum.cpp b/tdutils/td/utils/BigNum.cpp index f83cd3268..7b3e2ed82 100644 --- a/tdutils/td/utils/BigNum.cpp +++ b/tdutils/td/utils/BigNum.cpp @@ -213,13 +213,13 @@ string BigNum::to_le_binary(int exact_size) const { } else { CHECK(exact_size >= num_size); } - string res(exact_size, '\0'); + string result(exact_size, '\0'); #if defined(OPENSSL_IS_BORINGSSL) - BN_bn2le_padded(MutableSlice(res).ubegin(), exact_size, impl_->big_num); + BN_bn2le_padded(MutableSlice(result).ubegin(), exact_size, impl_->big_num); #else - BN_bn2lebinpad(impl_->big_num, MutableSlice(res).ubegin(), exact_size); + BN_bn2lebinpad(impl_->big_num, MutableSlice(result).ubegin(), exact_size); #endif - return res; + return result; #else string result = to_binary(exact_size); std::reverse(result.begin(), result.end()); diff --git a/tdutils/td/utils/BufferedFd.h b/tdutils/td/utils/BufferedFd.h index 32ed3d7e6..d8ad078ca 100644 --- a/tdutils/td/utils/BufferedFd.h +++ b/tdutils/td/utils/BufferedFd.h @@ -72,10 +72,11 @@ class BufferedFd : public BufferedFdBase { ~BufferedFd(); void close(); - size_t left_unread() { + + size_t left_unread() const { return input_reader_.size(); } - size_t left_unwritten() { + size_t left_unwritten() const { return output_reader_.size(); } diff --git a/tdutils/td/utils/CancellationToken.h b/tdutils/td/utils/CancellationToken.h index 54277feb7..884a0ffb9 100644 --- a/tdutils/td/utils/CancellationToken.h +++ b/tdutils/td/utils/CancellationToken.h @@ -20,7 +20,7 @@ struct RawCancellationToken { class CancellationToken { public: explicit operator bool() const { - // Empty CancellationToken is never cancelled + // empty CancellationToken is never cancelled if (!token_) { return false; } diff --git a/tdutils/td/utils/ConcurrentHashTable.h b/tdutils/td/utils/ConcurrentHashTable.h index 30ae4dea4..c350bb0fa 100644 --- a/tdutils/td/utils/ConcurrentHashTable.h +++ b/tdutils/td/utils/ConcurrentHashTable.h @@ -317,6 +317,6 @@ class ConcurrentHashMap { }; template -td::HazardPointers::HashMap> ConcurrentHashMap::hp_(64); +HazardPointers::HashMap> ConcurrentHashMap::hp_(64); } // namespace td diff --git a/tdutils/td/utils/FileLog.cpp b/tdutils/td/utils/FileLog.cpp index 9f4639af6..d37cb3d52 100644 --- a/tdutils/td/utils/FileLog.cpp +++ b/tdutils/td/utils/FileLog.cpp @@ -13,8 +13,6 @@ #include "td/utils/port/StdStreams.h" #include "td/utils/Slice.h" -#include - namespace td { Status FileLog::init(string path, int64 rotate_threshold, bool redirect_stderr) { @@ -104,7 +102,7 @@ void FileLog::lazy_rotate() { void FileLog::do_rotate() { want_rotate_ = false; - td::ScopedDisableLog disable_log; // to ensure that nothing will be printed to the closed log + ScopedDisableLog disable_log; // to ensure that nothing will be printed to the closed log CHECK(!path_.empty()); fd_.close(); auto r_fd = FileFd::open(path_, FileFd::Create | FileFd::Truncate | FileFd::Write); @@ -118,7 +116,7 @@ void FileLog::do_rotate() { size_ = 0; } -Result> FileLog::create(string path, int64 rotate_threshold, bool redirect_stderr) { +Result> FileLog::create(string path, int64 rotate_threshold, bool redirect_stderr) { auto l = make_unique(); TRY_STATUS(l->init(std::move(path), rotate_threshold, redirect_stderr)); return std::move(l); diff --git a/tdutils/td/utils/FileLog.h b/tdutils/td/utils/FileLog.h index 6a0d78907..fe34244d1 100644 --- a/tdutils/td/utils/FileLog.h +++ b/tdutils/td/utils/FileLog.h @@ -12,14 +12,16 @@ #include "td/utils/Slice.h" #include "td/utils/Status.h" +#include + namespace td { class FileLog : public LogInterface { static constexpr int64 DEFAULT_ROTATE_THRESHOLD = 10 * (1 << 20); public: - static Result> create(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD, - bool redirect_stderr = true); + static Result> create(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD, + bool redirect_stderr = true); Status init(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD, bool redirect_stderr = true); Slice get_path() const; @@ -42,7 +44,7 @@ class FileLog : public LogInterface { int64 size_ = 0; int64 rotate_threshold_ = 0; bool redirect_stderr_ = false; - std::atomic want_rotate_{}; + std::atomic want_rotate_{false}; void do_rotate(); }; diff --git a/tdutils/td/utils/Heap.h b/tdutils/td/utils/Heap.h index 19f44ad75..86f494888 100644 --- a/tdutils/td/utils/Heap.h +++ b/tdutils/td/utils/Heap.h @@ -37,7 +37,7 @@ class KHeap { return array_[0].key_; } - HeapNode *top() const { + const HeapNode *top() const { return array_[0].node_; } diff --git a/tdutils/td/utils/JsonBuilder.h b/tdutils/td/utils/JsonBuilder.h index 963dad99c..aa87c7cdd 100644 --- a/tdutils/td/utils/JsonBuilder.h +++ b/tdutils/td/utils/JsonBuilder.h @@ -350,10 +350,7 @@ class JsonArrayScope : public JsonScope { } void leave() { jb_->dec_offset(); - if (jb_->is_pretty()) { - *sb_ << "\n"; - jb_->print_offset(); - } + jb_->print_offset(); *sb_ << "]"; } template @@ -372,10 +369,7 @@ class JsonArrayScope : public JsonScope { } else { is_first_ = true; } - if (jb_->is_pretty()) { - *sb_ << "\n"; - jb_->print_offset(); - } + jb_->print_offset(); return jb_->enter_value(); } @@ -397,10 +391,7 @@ class JsonObjectScope : public JsonScope { } void leave() { jb_->dec_offset(); - if (jb_->is_pretty()) { - *sb_ << "\n"; - jb_->print_offset(); - } + jb_->print_offset(); *sb_ << "}"; } template @@ -411,10 +402,7 @@ class JsonObjectScope : public JsonScope { } else { is_first_ = true; } - if (jb_->is_pretty()) { - *sb_ << "\n"; - jb_->print_offset(); - } + jb_->print_offset(); jb_->enter_value() << key; if (jb_->is_pretty()) { *sb_ << " : "; diff --git a/tdutils/td/utils/MpmcWaiter.h b/tdutils/td/utils/MpmcWaiter.h index df1e70120..657a85d5b 100644 --- a/tdutils/td/utils/MpmcWaiter.h +++ b/tdutils/td/utils/MpmcWaiter.h @@ -10,8 +10,8 @@ #include "td/utils/logging.h" #include "td/utils/port/thread.h" -#include #include +#include #include #include @@ -149,7 +149,7 @@ class MpmcSleepyWaiter { } void unpark() { - //TODO: try unlock guard before notify_all + //TODO: try to unlock guard before notify_all std::unique_lock guard(mutex_); unpark_flag_ = true; condition_variable_.notify_all(); @@ -182,10 +182,10 @@ class MpmcSleepyWaiter { // // When worker found nothing it may try to call wait. // This may put it in a Sleep for some time. - // After wait return worker will be in Search state again. + // After wait returns worker will be in Search state again. // // Suppose worker found a work and ready to process it. - // Than it may call stop_wait. This will cause transition from + // Then it may call stop_wait. This will cause transition from // Search to Work state. // // Main invariant: @@ -200,7 +200,7 @@ class MpmcSleepyWaiter { VLOG(waiter) << "Init slot " << worker_id; } - int VERBOSITY_NAME(waiter) = VERBOSITY_NAME(DEBUG) + 10; + static constexpr int VERBOSITY_NAME(waiter) = VERBOSITY_NAME(DEBUG) + 10; void wait(Slot &slot) { if (slot.state_ == Slot::State::Work) { VLOG(waiter) << "Work -> Search"; @@ -225,10 +225,10 @@ class MpmcSleepyWaiter { } sleepers_.push_back(&slot); LOG_CHECK(slot.unpark_flag_ == false) << slot.worker_id; - VLOG(waiter) << "add to sleepers " << slot.worker_id; + VLOG(waiter) << "Add to sleepers " << slot.worker_id; //guard.unlock(); if (should_search) { - VLOG(waiter) << "Search -> Search once then Sleep "; + VLOG(waiter) << "Search -> Search once, then Sleep "; return; } VLOG(waiter) << "Search -> Sleep " << state_view.searching_count << " " << state_view.parked_count; @@ -247,22 +247,22 @@ class MpmcSleepyWaiter { return; } if (slot.state_ == Slot::State::Sleep) { - VLOG(waiter) << "Search once then Sleep -> Work/Search " << slot.worker_id; + VLOG(waiter) << "Search once, then Sleep -> Work/Search " << slot.worker_id; slot.state_ = Slot::State::Work; std::unique_lock guard(sleepers_mutex_); auto it = std::find(sleepers_.begin(), sleepers_.end(), &slot); if (it != sleepers_.end()) { sleepers_.erase(it); - VLOG(waiter) << "remove from sleepers " << slot.worker_id; + VLOG(waiter) << "Remove from sleepers " << slot.worker_id; state_.fetch_sub((1 << PARKING_SHIFT) - 1); guard.unlock(); } else { guard.unlock(); - VLOG(waiter) << "not in sleepers" << slot.worker_id; + VLOG(waiter) << "Not in sleepers" << slot.worker_id; CHECK(slot.cancel_park()); } } - VLOG(waiter) << "Search once then Sleep -> Work " << slot.worker_id; + VLOG(waiter) << "Search once, then Sleep -> Work " << slot.worker_id; slot.state_ = Slot::State::Search; auto state_view = StateView(state_.fetch_sub(1)); CHECK(state_view.searching_count != 0); @@ -313,19 +313,19 @@ class MpmcSleepyWaiter { } private: - static constexpr td::int32 PARKING_SHIFT = 16; + static constexpr int32 PARKING_SHIFT = 16; struct StateView { - td::int32 parked_count; - td::int32 searching_count; + int32 parked_count; + int32 searching_count; explicit StateView(int32 x) { parked_count = x >> PARKING_SHIFT; searching_count = x & ((1 << PARKING_SHIFT) - 1); } }; - std::atomic state_{0}; + std::atomic state_{0}; std::mutex sleepers_mutex_; - std::vector sleepers_; + vector sleepers_; bool closed_ = false; }; diff --git a/tdutils/td/utils/OptionParser.cpp b/tdutils/td/utils/OptionParser.cpp index ec1f1aaaf..f688d3b2b 100644 --- a/tdutils/td/utils/OptionParser.cpp +++ b/tdutils/td/utils/OptionParser.cpp @@ -4,10 +4,9 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #include "td/utils/OptionParser.h" -#include "td/utils/misc.h" +#include "td/utils/logging.h" #include #include @@ -21,8 +20,8 @@ void OptionParser::set_description(string description) { void OptionParser::add_option(Option::Type type, char short_key, Slice long_key, Slice description, std::function callback) { for (auto &option : options_) { - if (option.short_key == short_key || (!long_key.empty() && long_key == option.long_key)) { - LOG(ERROR) << "Ignore duplicated option '" << short_key << "' '" << long_key << "'"; + if ((short_key != '\0' && option.short_key == short_key) || (!long_key.empty() && long_key == option.long_key)) { + LOG(ERROR) << "Ignore duplicated option '" << (short_key == '\0' ? '-' : short_key) << "' '" << long_key << "'"; } } options_.push_back(Option{type, short_key, long_key.str(), description.str(), std::move(callback)}); diff --git a/tdutils/td/utils/PathView.cpp b/tdutils/td/utils/PathView.cpp index 867372051..124a9f6fd 100644 --- a/tdutils/td/utils/PathView.cpp +++ b/tdutils/td/utils/PathView.cpp @@ -4,9 +4,9 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #include "td/utils/PathView.h" +#include "td/utils/common.h" #include "td/utils/misc.h" namespace td { @@ -26,6 +26,18 @@ PathView::PathView(Slice path) : path_(path) { } } +Slice PathView::parent_dir_noslash() const { + if (last_slash_ < 0) { + return Slice("."); + } + if (last_slash_ == 0) { + static char buf[1]; + buf[0] = TD_DIR_SLASH; + return Slice(buf, 1); + } + return path_.substr(0, last_slash_); +} + Slice PathView::relative(Slice path, Slice dir, bool force) { if (begins_with(path, dir)) { path.remove_prefix(dir.size()); diff --git a/tdutils/td/utils/PathView.h b/tdutils/td/utils/PathView.h index 52f5e6788..027650ade 100644 --- a/tdutils/td/utils/PathView.h +++ b/tdutils/td/utils/PathView.h @@ -28,9 +28,7 @@ class PathView { Slice parent_dir() const { return path_.substr(0, last_slash_ + 1); } - Slice parent_dir_noslash() const { - return last_slash_ <= 0 ? td::Slice(".") : path_.substr(0, last_slash_); - } + Slice parent_dir_noslash() const; Slice extension() const { if (last_dot_ == static_cast(path_.size())) { diff --git a/tdutils/td/utils/Random.cpp b/tdutils/td/utils/Random.cpp index 45fb2c2c5..1cda18e65 100644 --- a/tdutils/td/utils/Random.cpp +++ b/tdutils/td/utils/Random.cpp @@ -40,7 +40,7 @@ void Random::secure_bytes(unsigned char *ptr, size_t size) { generation = 0; } if (ptr == nullptr) { - td::MutableSlice(buf, BUF_SIZE).fill_zero_secure(); + MutableSlice(buf, BUF_SIZE).fill_zero_secure(); buf_pos = BUF_SIZE; return; } @@ -145,10 +145,7 @@ int Random::fast(int min, int max) { double Random::fast(double min, double max) { DCHECK(min <= max); - return min + - fast_uint32() * 1.0 / - (static_cast(std::numeric_limits::max()) - std::numeric_limits::min()) * - (max - min); + return min + fast_uint32() * 1.0 / std::numeric_limits::max() * (max - min); } Random::Xorshift128plus::Xorshift128plus(uint64 seed) { diff --git a/tdutils/td/utils/Random.h b/tdutils/td/utils/Random.h index d7e604b17..41a7d60e6 100644 --- a/tdutils/td/utils/Random.h +++ b/tdutils/td/utils/Random.h @@ -10,6 +10,8 @@ #include "td/utils/Slice.h" #include "td/utils/Span.h" +#include + namespace td { class Random { @@ -55,10 +57,11 @@ class Random { }; template -void random_shuffle(td::MutableSpan v, R &rnd) { - for (std::size_t i = 1; i < v.size(); i++) { - auto pos = static_cast(rnd() % (i + 1)); - std::swap(v[i], v[pos]); +void random_shuffle(MutableSpan v, R &rnd) { + for (size_t i = 1; i < v.size(); i++) { + auto pos = static_cast(rnd()) % (i + 1); + using std::swap; + swap(v[i], v[pos]); } } diff --git a/tdutils/td/utils/SharedObjectPool.h b/tdutils/td/utils/SharedObjectPool.h index b054cbcb2..b548f9418 100644 --- a/tdutils/td/utils/SharedObjectPool.h +++ b/tdutils/td/utils/SharedObjectPool.h @@ -37,7 +37,9 @@ class AtomicRefCnt { }; template -class SharedPtrRaw : public DeleterT, private MpscLinkQueueImpl::Node { +class SharedPtrRaw + : public DeleterT + , private MpscLinkQueueImpl::Node { public: explicit SharedPtrRaw(DeleterT deleter) : DeleterT(std::move(deleter)), ref_cnt_{0}, option_magic_(Magic) { } diff --git a/tdutils/td/utils/SharedSlice.cpp b/tdutils/td/utils/SharedSlice.cpp index 9f5cb6249..02b682a53 100644 --- a/tdutils/td/utils/SharedSlice.cpp +++ b/tdutils/td/utils/SharedSlice.cpp @@ -4,7 +4,6 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #include "td/utils/SharedSlice.h" #include "td/utils/buffer.h" diff --git a/tdutils/td/utils/SharedSlice.h b/tdutils/td/utils/SharedSlice.h index fe69405f0..284b561a2 100644 --- a/tdutils/td/utils/SharedSlice.h +++ b/tdutils/td/utils/SharedSlice.h @@ -4,9 +4,9 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #pragma once +#include "td/utils/common.h" #include "td/utils/Slice.h" #include diff --git a/tdutils/td/utils/Slice.cpp b/tdutils/td/utils/Slice.cpp index 285870a9f..e3ae1acee 100644 --- a/tdutils/td/utils/Slice.cpp +++ b/tdutils/td/utils/Slice.cpp @@ -4,7 +4,6 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #include "td/utils/Slice.h" #if TD_HAVE_OPENSSL diff --git a/tdutils/td/utils/Span.h b/tdutils/td/utils/Span.h index 7e93a9e64..caa7a009b 100644 --- a/tdutils/td/utils/Span.h +++ b/tdutils/td/utils/Span.h @@ -105,6 +105,7 @@ class SpanImpl { InnerT *end() const { return data_ + size_; } + std::reverse_iterator rbegin() const { return std::reverse_iterator(end()); } @@ -149,7 +150,7 @@ Span span(const T *ptr, size_t size) { return Span(ptr, size); } template -Span span(const std::vector &vec) { +Span span(const vector &vec) { return Span(vec); } @@ -158,17 +159,17 @@ MutableSpan mutable_span(T *ptr, size_t size) { return MutableSpan(ptr, size); } template -MutableSpan mutable_span(std::vector &vec) { +MutableSpan mutable_span(vector &vec) { return MutableSpan(vec); } template Span span_one(const T &value) { - return td::Span(&value, 1); + return Span(&value, 1); } template MutableSpan mutable_span_one(T &value) { - return td::MutableSpan(&value, 1); + return MutableSpan(&value, 1); } template diff --git a/tdutils/td/utils/Status.h b/tdutils/td/utils/Status.h index 0023eb6ef..5b8dd8dbe 100644 --- a/tdutils/td/utils/Status.h +++ b/tdutils/td/utils/Status.h @@ -318,10 +318,7 @@ class Status { return std::move(*this); } - Auto move_as_ok() { - UNREACHABLE(); - return {}; - } + Status move_as_ok() = delete; Status move_as_error_prefix(const Status &status) const TD_WARN_UNUSED_RESULT { return status.move_as_error_suffix(message()); @@ -533,13 +530,13 @@ class Result { } Status move_as_error_prefix(const Status &prefix) TD_WARN_UNUSED_RESULT { SCOPE_EXIT { - status_ = Status::Error<-5>(); + status_ = Status::Error<-6>(); }; return status_.move_as_error_prefix(prefix); } Status move_as_error_suffix(Slice suffix) TD_WARN_UNUSED_RESULT { SCOPE_EXIT { - status_ = Status::Error<-5>(); + status_ = Status::Error<-7>(); }; return status_.move_as_error_suffix(suffix); } @@ -571,7 +568,7 @@ class Result { } template - td::Result()(std::declval()))> move_map(F &&f) { + Result()(std::declval()))> move_map(F &&f) { if (is_error()) { return move_as_error(); } diff --git a/tdutils/td/utils/StealingQueue.h b/tdutils/td/utils/StealingQueue.h new file mode 100644 index 000000000..cad94aa6b --- /dev/null +++ b/tdutils/td/utils/StealingQueue.h @@ -0,0 +1,125 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/misc.h" + +#include +#include + +namespace td { + +template +class StealingQueue { + public: + static_assert(N > 0 && (N & (N - 1)) == 0, ""); + + // tries to put a value + // returns if succeeded + // only owner is allowed to to do this + template + void local_push(T value, F &&overflow_f) { + while (true) { + auto tail = tail_.load(std::memory_order_relaxed); + auto head = head_.load(); // TODO: memory order + + if (static_cast(tail - head) < N) { + buf_[tail & MASK].store(value, std::memory_order_relaxed); + tail_.store(tail + 1, std::memory_order_release); + return; + } + + // queue is full + // TODO: batch insert into global queue? + auto n = N / 2 + 1; + auto new_head = head + n; + if (!head_.compare_exchange_strong(head, new_head)) { + continue; + } + + for (size_t i = 0; i < n; i++) { + overflow_f(buf_[(i + head) & MASK].load(std::memory_order_relaxed)); + } + overflow_f(value); + + return; + } + } + + // tries to pop a value + // returns if succeeded + // only owner is allowed to do this + bool local_pop(T &value) { + auto tail = tail_.load(std::memory_order_relaxed); + auto head = head_.load(); + + if (head == tail) { + return false; + } + + value = buf_[head & MASK].load(std::memory_order_relaxed); + return head_.compare_exchange_strong(head, head + 1); + } + + bool steal(T &value, StealingQueue &other) { + while (true) { + auto tail = tail_.load(std::memory_order_relaxed); + auto head = head_.load(); // TODO: memory order + + auto other_head = other.head_.load(); + auto other_tail = other.tail_.load(std::memory_order_acquire); + + if (other_tail < other_head) { + continue; + } + size_t n = narrow_cast(other_tail - other_head); + if (n > N) { + continue; + } + n -= n / 2; + n = td::min(n, static_cast(head + N - tail)); + if (n == 0) { + return false; + } + + for (size_t i = 0; i < n; i++) { + buf_[(i + tail) & MASK].store(other.buf_[(i + other_head) & MASK].load(std::memory_order_relaxed), + std::memory_order_relaxed); + } + + if (!other.head_.compare_exchange_strong(other_head, other_head + n)) { + continue; + } + + n--; + value = buf_[(tail + n) & MASK].load(std::memory_order_relaxed); + tail_.store(tail + n, std::memory_order_release); + return true; + } + } + + StealingQueue() { + for (auto &x : buf_) { +// workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=64658 +#if TD_GCC && GCC_VERSION <= 40902 + x = T(); +#else + std::atomic_init(&x, T()); +#endif + } + std::atomic_thread_fence(std::memory_order_seq_cst); + } + + private: + std::atomic head_{0}; + std::atomic tail_{0}; + static constexpr size_t MASK{N - 1}; + std::array, N> buf_; +}; + +} // namespace td diff --git a/tdutils/td/utils/ThreadLocalStorage.h b/tdutils/td/utils/ThreadLocalStorage.h index 4bbe8614d..79dc82ede 100644 --- a/tdutils/td/utils/ThreadLocalStorage.h +++ b/tdutils/td/utils/ThreadLocalStorage.h @@ -4,7 +4,6 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #pragma once #include "td/utils/common.h" @@ -14,6 +13,7 @@ #include namespace td { + template class ThreadLocalStorage { public: @@ -51,4 +51,5 @@ class ThreadLocalStorage { return nodes_[thread_id]; } }; + } // namespace td diff --git a/tdutils/td/utils/ThreadSafeCounter.h b/tdutils/td/utils/ThreadSafeCounter.h index 0b6cb114f..b3d94a99e 100644 --- a/tdutils/td/utils/ThreadSafeCounter.h +++ b/tdutils/td/utils/ThreadSafeCounter.h @@ -4,7 +4,6 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #pragma once #include "td/utils/common.h" @@ -17,6 +16,7 @@ #include namespace td { + template class ThreadSafeMultiCounter { public: @@ -28,11 +28,11 @@ class ThreadSafeMultiCounter { int64 sum(size_t index) const { CHECK(index < N); int64 res = 0; - tls_.for_each([&](auto &value) { res += value[index].load(std::memory_order_relaxed); }); + tls_.for_each([&res](auto &value) { res += value[index].load(std::memory_order_relaxed); }); return res; } void clear() { - tls_.for_each([&](auto &value) { + tls_.for_each([](auto &value) { for (auto &x : value) { x = 0; } diff --git a/tdutils/td/utils/Time.cpp b/tdutils/td/utils/Time.cpp index 57dbc9ec8..6aa825b3a 100644 --- a/tdutils/td/utils/Time.cpp +++ b/tdutils/td/utils/Time.cpp @@ -6,17 +6,17 @@ // #include "td/utils/Time.h" -#include #include +#include namespace td { bool operator==(Timestamp a, Timestamp b) { return std::abs(a.at() - b.at()) < 1e-6; } -namespace { -std::atomic time_diff; -} + +static std::atomic time_diff; + double Time::now() { return now_unadjusted() + time_diff.load(std::memory_order_relaxed); } @@ -26,9 +26,8 @@ double Time::now_unadjusted() { } void Time::jump_in_future(double at) { - auto old_time_diff = time_diff.load(); - while (true) { + auto old_time_diff = time_diff.load(); auto diff = at - now(); if (diff < 0) { return; diff --git a/tdutils/td/utils/Time.h b/tdutils/td/utils/Time.h index 296720518..da82c0c2b 100644 --- a/tdutils/td/utils/Time.h +++ b/tdutils/td/utils/Time.h @@ -62,11 +62,11 @@ class Timestamp { return Timestamp{timeout - Clocks::system() + Time::now()}; } - static Timestamp in(double timeout, td::Timestamp now = td::Timestamp::now_cached()) { + static Timestamp in(double timeout, Timestamp now = now_cached()) { return Timestamp{now.at() + timeout}; } - bool is_in_past(td::Timestamp now) const { + bool is_in_past(Timestamp now) const { return at_ <= now.at(); } bool is_in_past() const { diff --git a/tdutils/td/utils/TimedStat.h b/tdutils/td/utils/TimedStat.h index bbd4f16b2..6969f50e3 100644 --- a/tdutils/td/utils/TimedStat.h +++ b/tdutils/td/utils/TimedStat.h @@ -9,6 +9,7 @@ #include "td/utils/common.h" #include "td/utils/optional.h" +#include #include namespace td { @@ -69,27 +70,28 @@ class TimedStat { } }; +namespace detail { template struct MinMaxStat { - public: using Event = T; void on_event(Event event) { if (!best_ || Cmp()(event, best_.value())) { best_ = event; } } - td::optional get_stat() const { + optional get_stat() const { return best_.copy(); } private: - td::optional best_; + optional best_; }; +} // namespace detail template -using MinStat = MinMaxStat>; +using MinStat = detail::MinMaxStat>; template -using MaxStat = MinMaxStat>; +using MaxStat = detail::MinMaxStat>; } // namespace td diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index 318a8a584..5dc839338 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -12,11 +12,9 @@ namespace td { -Timer::Timer(bool is_paused) : is_paused_(is_paused) { - if (is_paused_) { - start_time_ = 0; - } else { - start_time_ = Time::now(); +Timer::Timer(bool is_paused) { + if (!is_paused) { + resume(); } } @@ -45,7 +43,7 @@ double Timer::elapsed() const { } StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer) { - return string_builder << format::as_time(timer.elapsed()); + return string_builder << " in " << format::as_time(timer.elapsed()); } PerfWarningTimer::PerfWarningTimer(string name, double max_duration) diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index aa60890e5..3c11231dd 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -15,19 +15,19 @@ class Timer { Timer() : Timer(false) { } explicit Timer(bool is_paused); - Timer(const Timer &other) = default; - Timer &operator=(const Timer &other) = default; double elapsed() const; + void pause(); + void resume(); private: friend StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer); double elapsed_{0}; - double start_time_; - bool is_paused_{false}; + double start_time_{0}; + bool is_paused_{true}; }; class PerfWarningTimer { diff --git a/tdutils/td/utils/TsFileLog.cpp b/tdutils/td/utils/TsFileLog.cpp index 1d98d1b14..e8f75c964 100644 --- a/tdutils/td/utils/TsFileLog.cpp +++ b/tdutils/td/utils/TsFileLog.cpp @@ -4,7 +4,6 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #include "td/utils/TsFileLog.h" #include "td/utils/common.h" @@ -14,17 +13,20 @@ #include "td/utils/Slice.h" #include +#include #include +#include namespace td { + namespace detail { class TsFileLog : public LogInterface { public: - Status init(string path, td::int64 rotate_threshold, bool redirect_stderr) { + Status init(string path, int64 rotate_threshold, bool redirect_stderr) { path_ = std::move(path); rotate_threshold_ = rotate_threshold; redirect_stderr_ = redirect_stderr; - for (int i = 0; i < (int)logs_.size(); i++) { + for (size_t i = 0; i < logs_.size(); i++) { logs_[i].id = i; } return init_info(&logs_[0]); @@ -49,18 +51,23 @@ class TsFileLog : public LogInterface { struct Info { FileLog log; std::atomic is_inited{false}; - int id; + size_t id; }; - static constexpr int MAX_THREAD_ID = 128; - td::int64 rotate_threshold_; + + static constexpr size_t MAX_THREAD_ID = 128; + int64 rotate_threshold_; bool redirect_stderr_; std::string path_; std::array logs_; + std::mutex init_mutex_; LogInterface *get_current_logger() { auto *info = get_current_info(); if (!info->is_inited.load(std::memory_order_relaxed)) { - CHECK(init_info(info).is_ok()); + std::unique_lock lock(init_mutex_); + if (!info->is_inited.load(std::memory_order_relaxed)) { + init_info(info).ensure(); + } } return &info->log; } @@ -75,7 +82,7 @@ class TsFileLog : public LogInterface { return Status::OK(); } - string get_path(Info *info) { + string get_path(const Info *info) const { if (info->id == 0) { return path_; } @@ -84,7 +91,7 @@ class TsFileLog : public LogInterface { void rotate() override { for (auto &info : logs_) { - if (info.is_inited.load(std::memory_order_consume)) { + if (info.is_inited.load(std::memory_order_acquire)) { info.log.lazy_rotate(); } } @@ -92,9 +99,10 @@ class TsFileLog : public LogInterface { }; } // namespace detail -Result> TsFileLog::create(string path, td::int64 rotate_threshold, bool redirect_stderr) { - auto res = td::make_unique(); +Result> TsFileLog::create(string path, int64 rotate_threshold, bool redirect_stderr) { + auto res = make_unique(); TRY_STATUS(res->init(path, rotate_threshold, redirect_stderr)); return std::move(res); } + } // namespace td diff --git a/tdutils/td/utils/TsFileLog.h b/tdutils/td/utils/TsFileLog.h index 563e5d811..de005a457 100644 --- a/tdutils/td/utils/TsFileLog.h +++ b/tdutils/td/utils/TsFileLog.h @@ -4,7 +4,6 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #pragma once #include "td/utils/common.h" @@ -12,11 +11,13 @@ #include "td/utils/Status.h" namespace td { + class TsFileLog { static constexpr int64 DEFAULT_ROTATE_THRESHOLD = 10 * (1 << 20); public: - static Result> create(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD, - bool redirect_stderr = true); + static Result> create(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD, + bool redirect_stderr = true); }; + } // namespace td diff --git a/tdutils/td/utils/TsList.h b/tdutils/td/utils/TsList.h index fde1ec891..c398489cc 100644 --- a/tdutils/td/utils/TsList.h +++ b/tdutils/td/utils/TsList.h @@ -4,7 +4,6 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #pragma once #include "td/utils/common.h" diff --git a/tdutils/td/utils/VectorQueue.h b/tdutils/td/utils/VectorQueue.h index c71ffeae5..13af425db 100644 --- a/tdutils/td/utils/VectorQueue.h +++ b/tdutils/td/utils/VectorQueue.h @@ -20,42 +20,51 @@ class VectorQueue { void push(S &&s) { vector_.emplace_back(std::forward(s)); } + template void emplace(Args &&... args) { vector_.emplace_back(std::forward(args)...); } + T pop() { try_shrink(); return std::move(vector_[read_pos_++]); } + void pop_n(size_t n) { read_pos_ += n; try_shrink(); } + + const T &front() const { + return vector_[read_pos_]; + } T &front() { return vector_[read_pos_]; } + + const T &back() const { + return vector_.back(); + } T &back() { return vector_.back(); } - const T &front() const { - return vector_[read_pos_]; - } - const T &back() const { - return vector_.back(); - } + bool empty() const { return size() == 0; } + size_t size() const { return vector_.size() - read_pos_; } + const T *data() const { return vector_.data() + read_pos_; } T *data() { return vector_.data() + read_pos_; } + Span as_span() const { return {data(), size()}; } diff --git a/tdutils/td/utils/base64.cpp b/tdutils/td/utils/base64.cpp index 1ede0bafa..d47401100 100644 --- a/tdutils/td/utils/base64.cpp +++ b/tdutils/td/utils/base64.cpp @@ -4,7 +4,6 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #include "td/utils/base64.h" #include "td/utils/common.h" @@ -237,11 +236,28 @@ string base64_filter(Slice input) { return res; } -static const char *const symbols32_lc = "abcdefghijklmnopqrstuvwxyz234567"; -static const char *const symbols32_uc = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; +static const char *get_base32_characters(bool upper_case) { + return upper_case ? "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" : "abcdefghijklmnopqrstuvwxyz234567"; +} + +static const unsigned char *get_base32_character_table() { + static unsigned char char_to_value[256]; + static bool is_inited = [] { + std::fill(std::begin(char_to_value), std::end(char_to_value), static_cast(32)); + auto characters_lc = get_base32_characters(false); + auto characters_uc = get_base32_characters(true); + for (unsigned char i = 0; i < 32; i++) { + char_to_value[static_cast(characters_lc[i])] = i; + char_to_value[static_cast(characters_uc[i])] = i; + } + return true; + }(); + CHECK(is_inited); + return char_to_value; +} string base32_encode(Slice input, bool upper_case) { - auto *symbols32 = (upper_case ? symbols32_uc : symbols32_lc); + auto *characters = get_base32_characters(upper_case); string base32; base32.reserve((input.size() * 8 + 4) / 5); uint32 c = 0; @@ -251,45 +267,32 @@ string base32_encode(Slice input, bool upper_case) { length += 8; while (length >= 5) { length -= 5; - base32.push_back(symbols32[(c >> length) & 31]); + base32.push_back(characters[(c >> length) & 31]); } } if (length != 0) { - base32.push_back(symbols32[(c << (5 - length)) & 31]); + base32.push_back(characters[(c << (5 - length)) & 31]); } //TODO: optional padding return base32; } -static unsigned char b32_char_to_value[256]; -static void init_base32_table() { - static bool is_inited = [] { - std::fill(std::begin(b32_char_to_value), std::end(b32_char_to_value), static_cast(32)); - for (unsigned char i = 0; i < 32; i++) { - b32_char_to_value[static_cast(symbols32_lc[i])] = i; - b32_char_to_value[static_cast(symbols32_uc[i])] = i; - } - return true; - }(); - CHECK(is_inited); -} - Result base32_decode(Slice base32) { - init_base32_table(); string res; res.reserve(base32.size() * 5 / 8); uint32 c = 0; uint32 length = 0; + auto *table = get_base32_character_table(); for (size_t i = 0; i < base32.size(); i++) { - auto value = b32_char_to_value[base32.ubegin()[i]]; + auto value = table[base32.ubegin()[i]]; if (value == 32) { return Status::Error("Wrong character in the string"); } c = (c << 5) | value; length += 5; - while (length >= 8) { + if (length >= 8) { length -= 8; - res.push_back(td::uint8((c >> length) & 255)); + res.push_back(static_cast((c >> length) & 255)); } } if ((c & ((1 << length) - 1)) != 0) { @@ -298,4 +301,5 @@ Result base32_decode(Slice base32) { //TODO: check padding return res; } + } // namespace td diff --git a/tdutils/td/utils/base64.h b/tdutils/td/utils/base64.h index 1f60b2a73..9c431b12c 100644 --- a/tdutils/td/utils/base64.h +++ b/tdutils/td/utils/base64.h @@ -4,7 +4,6 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #pragma once #include "td/utils/common.h" @@ -28,6 +27,8 @@ bool is_base64_characters(Slice input); bool is_base64url_characters(Slice input); string base64_filter(Slice input); + string base32_encode(Slice input, bool upper_case = false); Result base32_decode(Slice base32); + } // namespace td diff --git a/tdutils/td/utils/bits.h b/tdutils/td/utils/bits.h index aa355da74..baefe7bf6 100644 --- a/tdutils/td/utils/bits.h +++ b/tdutils/td/utils/bits.h @@ -271,10 +271,7 @@ inline int32 count_bits64(uint64 x) { #endif struct BitsRange { - td::uint64 bits{0}; - mutable td::int32 pos{-1}; - - explicit BitsRange(td::uint64 bits = 0) : bits{bits}, pos{-1} { + explicit BitsRange(uint64 bits = 0) : bits{bits}, pos{-1} { } BitsRange begin() const { @@ -285,9 +282,9 @@ struct BitsRange { return BitsRange{}; } - td::int32 operator*() const { + int32 operator*() const { if (pos == -1) { - pos = td::count_trailing_zeroes64(bits); + pos = count_trailing_zeroes64(bits); } return pos; } @@ -304,6 +301,10 @@ struct BitsRange { pos = -1; return *this; } + + private: + uint64 bits{0}; + mutable int32 pos{-1}; }; } // namespace td diff --git a/tdutils/td/utils/crypto.cpp b/tdutils/td/utils/crypto.cpp index 604b4fccd..15cdd01a5 100644 --- a/tdutils/td/utils/crypto.cpp +++ b/tdutils/td/utils/crypto.cpp @@ -606,7 +606,7 @@ void aes_cbc_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlic aes_cbc_xcrypt(aes_key, aes_iv, from, to, false); } -AesCbcState::AesCbcState(Slice key256, Slice iv128) : raw_{td::SecureString(key256), td::SecureString(iv128)} { +AesCbcState::AesCbcState(Slice key256, Slice iv128) : raw_{SecureString(key256), SecureString(iv128)} { CHECK(raw_.key.size() == 32); CHECK(raw_.iv.size() == 16); } diff --git a/tdutils/td/utils/crypto.h b/tdutils/td/utils/crypto.h index 7c924c383..0045f7e1a 100644 --- a/tdutils/td/utils/crypto.h +++ b/tdutils/td/utils/crypto.h @@ -10,7 +10,6 @@ #include "td/utils/common.h" #include "td/utils/SharedSlice.h" #include "td/utils/Slice.h" -#include "td/utils/SharedSlice.h" #include "td/utils/Status.h" namespace td { @@ -95,11 +94,11 @@ class AesCbcState { void encrypt(Slice from, MutableSlice to); void decrypt(Slice from, MutableSlice to); + struct Raw { SecureString key; SecureString iv; }; - const Raw &raw() const { return raw_; } diff --git a/tdutils/td/utils/filesystem.cpp b/tdutils/td/utils/filesystem.cpp index 47f047863..b65964277 100644 --- a/tdutils/td/utils/filesystem.cpp +++ b/tdutils/td/utils/filesystem.cpp @@ -33,6 +33,7 @@ template <> BufferSlice create_empty(size_t size) { return BufferSlice{size}; } + template <> SecureString create_empty(size_t size) { return SecureString{size}; @@ -45,15 +46,8 @@ Result read_file_impl(CSlice path, int64 size, int64 offset) { if (offset < 0 || offset > file_size) { return Status::Error("Failed to read file: invalid offset"); } - if (size == -1) { + if (size < 0 || size > file_size - offset) { size = file_size - offset; - } else if (size >= 0) { - if (size + offset > file_size) { - size = file_size - offset; - } - } - if (size < 0) { - return Status::Error("Failed to read file: invalid size"); } auto content = create_empty(narrow_cast(size)); TRY_RESULT(got_size, from_file.pread(as_mutable_slice(content), offset)); @@ -193,4 +187,5 @@ Status atomic_write_file(CSlice path, Slice data, CSlice path_tmp) { TRY_STATUS(write_file(path_tmp, data, options)); return rename(path_tmp, path); } + } // namespace td diff --git a/tdutils/td/utils/filesystem.h b/tdutils/td/utils/filesystem.h index b1457f38e..72a392571 100644 --- a/tdutils/td/utils/filesystem.h +++ b/tdutils/td/utils/filesystem.h @@ -9,7 +9,6 @@ #include "td/utils/buffer.h" #include "td/utils/SharedSlice.h" #include "td/utils/Slice.h" -#include "td/utils/SharedSlice.h" #include "td/utils/Status.h" namespace td { @@ -21,15 +20,15 @@ Result read_file_secure(CSlice path, int64 size = -1, int64 offset Status copy_file(CSlice from, CSlice to, int64 size = -1) TD_WARN_UNUSED_RESULT; struct WriteFileOptions { - bool need_sync = true; + bool need_sync = false; bool need_lock = true; }; Status write_file(CSlice to, Slice data, WriteFileOptions options = {}) TD_WARN_UNUSED_RESULT; string clean_filename(CSlice name); -// write file and ensure that it either fully overriden with new data, or left intact. -// Uses path_tmp to temporary storat data, than calls rename +// writes data to file and ensures that the file is either fully overriden, or is left intact +// uses path_tmp to temporary store data, then calls rename Status atomic_write_file(CSlice path, Slice data, CSlice path_tmp = {}); } // namespace td diff --git a/tdutils/td/utils/format.h b/tdutils/td/utils/format.h index ff98c07fd..f6bbc4063 100644 --- a/tdutils/td/utils/format.h +++ b/tdutils/td/utils/format.h @@ -11,9 +11,9 @@ #include "td/utils/Slice.h" #include "td/utils/StringBuilder.h" +#include #include #include -#include namespace td { namespace format { diff --git a/tdutils/td/utils/logging.cpp b/tdutils/td/utils/logging.cpp index 70214d906..ffc03bcbe 100644 --- a/tdutils/td/utils/logging.cpp +++ b/tdutils/td/utils/logging.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #if TD_ANDROID @@ -268,8 +269,8 @@ namespace { std::mutex sdl_mutex; int sdl_cnt = 0; int sdl_verbosity = 0; - } // namespace + ScopedDisableLog::ScopedDisableLog() { std::unique_lock guard(sdl_mutex); if (sdl_cnt == 0) { diff --git a/tdutils/td/utils/logging.h b/tdutils/td/utils/logging.h index 4bfab52e0..aa0eba74d 100644 --- a/tdutils/td/utils/logging.h +++ b/tdutils/td/utils/logging.h @@ -150,6 +150,9 @@ struct LogOptions { add_info = other.add_info; return *this; } + LogOptions(LogOptions &&) = default; // i.e. deleted + LogOptions &operator=(LogOptions &&) = default; // i.e. deleted + ~LogOptions() = default; }; extern LogOptions log_options; @@ -163,6 +166,10 @@ inline int get_verbosity_level() { class ScopedDisableLog { public: ScopedDisableLog(); + ScopedDisableLog(const ScopedDisableLog &) = delete; + ScopedDisableLog &operator=(const ScopedDisableLog &) = delete; + ScopedDisableLog(ScopedDisableLog &&) = delete; + ScopedDisableLog &operator=(ScopedDisableLog &&) = delete; ~ScopedDisableLog(); }; diff --git a/tdutils/td/utils/misc.h b/tdutils/td/utils/misc.h index e91c06ccd..e8f301a42 100644 --- a/tdutils/td/utils/misc.h +++ b/tdutils/td/utils/misc.h @@ -40,12 +40,11 @@ vector full_split(T s, char delimiter = ' ', size_t max_parts = std::numeric_ while (result.size() + 1 < max_parts) { auto delimiter_pos = s.find(delimiter); if (delimiter_pos == string::npos) { - result.push_back(std::move(s)); - return result; - } else { - result.push_back(s.substr(0, delimiter_pos)); - s = s.substr(delimiter_pos + 1); + break; } + + result.push_back(s.substr(0, delimiter_pos)); + s = s.substr(delimiter_pos + 1); } result.push_back(std::move(s)); return result; @@ -349,10 +348,14 @@ Result::value, T>::type> hex_to_inte auto begin = str.begin(); auto end = str.end(); while (begin != end) { - if (!is_hex_digit(*begin)) { - return Status::Error("not a hex digit"); + T digit = hex_to_int(*begin++); + if (digit == 16) { + return Status::Error("Not a hex digit"); } - integer_value = static_cast(integer_value * 16 + hex_to_int(*begin++)); + if (integer_value > std::numeric_limits::max() / 16) { + return Status::Error("Hex number overflow"); + } + integer_value = integer_value * 16 + digit; } return integer_value; } @@ -380,8 +383,8 @@ string url_encode(Slice data); namespace detail { template -struct is_same_signedness : public std::integral_constant::value == std::is_signed::value> { -}; +struct is_same_signedness + : public std::integral_constant::value == std::is_signed::value> {}; template struct safe_undeflying_type { diff --git a/tdutils/td/utils/optional.h b/tdutils/td/utils/optional.h index 639dfedbe..548ed143d 100644 --- a/tdutils/td/utils/optional.h +++ b/tdutils/td/utils/optional.h @@ -64,7 +64,7 @@ class optional { return res; } - td::optional copy() const { + optional copy() const { if (*this) { return value(); } diff --git a/tdutils/td/utils/port/StdStreams.cpp b/tdutils/td/utils/port/StdStreams.cpp index c08f7650d..49983131c 100644 --- a/tdutils/td/utils/port/StdStreams.cpp +++ b/tdutils/td/utils/port/StdStreams.cpp @@ -15,7 +15,6 @@ #include "td/utils/Slice.h" #include -#include namespace td { @@ -23,7 +22,9 @@ namespace td { template static FileFd &get_file_fd() { static FileFd result = FileFd::from_native_fd(NativeFd(id, true)); - static auto guard = ScopeExit() + [&] { result.move_as_native_fd().release(); }; + static auto guard = ScopeExit() + [&] { + result.move_as_native_fd().release(); + }; return result; } @@ -43,7 +44,9 @@ static FileFd &get_file_fd() { static auto handle = GetStdHandle(id); LOG_IF(FATAL, handle == INVALID_HANDLE_VALUE) << "Failed to GetStdHandle " << id; static FileFd result = FileFd::from_native_fd(NativeFd(handle, true)); - static auto guard = ScopeExit() + [&] { result.move_as_native_fd().release(); }; + static auto guard = ScopeExit() + [&] { + result.move_as_native_fd().release(); + }; #else static FileFd result; #endif diff --git a/tdutils/td/utils/port/detail/Epoll.cpp b/tdutils/td/utils/port/detail/Epoll.cpp index 75e419dcc..e30b87336 100644 --- a/tdutils/td/utils/port/detail/Epoll.cpp +++ b/tdutils/td/utils/port/detail/Epoll.cpp @@ -101,8 +101,6 @@ void Epoll::run(int timeout_ms) { if (event->events & EPOLLRDHUP) { event->events &= ~EPOLLRDHUP; flags = flags | PollFlags::Close(); - // flags |= Fd::Close; - // TODO } #endif if (event->events & EPOLLHUP) { diff --git a/tdutils/td/utils/port/detail/ThreadPthread.cpp b/tdutils/td/utils/port/detail/ThreadPthread.cpp index 2d7e4ef1a..882d72056 100644 --- a/tdutils/td/utils/port/detail/ThreadPthread.cpp +++ b/tdutils/td/utils/port/detail/ThreadPthread.cpp @@ -4,7 +4,6 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #include "td/utils/port/detail/ThreadPthread.h" char disable_linker_warning_about_empty_file_thread_pthread_cpp TD_UNUSED; diff --git a/tdutils/td/utils/port/detail/skip_eintr.h b/tdutils/td/utils/port/detail/skip_eintr.h index 7a1309270..3fe805ba5 100644 --- a/tdutils/td/utils/port/detail/skip_eintr.h +++ b/tdutils/td/utils/port/detail/skip_eintr.h @@ -4,7 +4,6 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #pragma once #include diff --git a/tdutils/td/utils/port/rlimit.cpp b/tdutils/td/utils/port/rlimit.cpp index 3ff2cfc60..48029137d 100644 --- a/tdutils/td/utils/port/rlimit.cpp +++ b/tdutils/td/utils/port/rlimit.cpp @@ -4,7 +4,6 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #include "td/utils/port/rlimit.h" #include "td/utils/port/config.h" @@ -20,76 +19,79 @@ namespace td { #if TD_PORT_POSIX - -namespace { - -int get_rlimit_type(ResourceLimitType rlim_type) { - switch (rlim_type) { +static int get_resource(ResourceLimitType type) { + switch (type) { case ResourceLimitType::NoFile: return RLIMIT_NOFILE; - case ResourceLimitType::Rss: - return RLIMIT_RSS; default: UNREACHABLE(); + return -1; } } - -} // namespace - -td::Status set_resource_limit(ResourceLimitType rlim_type, td::uint64 value, td::uint64 cap) { - if (cap && value > cap) { - return td::Status::Error("setrlimit(): bad argument"); - } - int resource = get_rlimit_type(rlim_type); - - struct rlimit r; - if (getrlimit(resource, &r) < 0) { - return td::Status::PosixError(errno, "failed getrlimit()"); - } - - if (cap) { - r.rlim_max = cap; - } else if (r.rlim_max < value) { - r.rlim_max = value; - } - r.rlim_cur = value; - if (setrlimit(resource, &r) < 0) { - return td::Status::PosixError(errno, "failed setrlimit()"); - } - return td::Status::OK(); -} - -td::Status set_maximize_resource_limit(ResourceLimitType rlim_type, td::uint64 value) { - int resource = get_rlimit_type(rlim_type); - - struct rlimit r; - if (getrlimit(resource, &r) < 0) { - return td::Status::PosixError(errno, "failed getrlimit()"); - } - - if (r.rlim_max < value) { - auto t = r; - t.rlim_cur = value; - t.rlim_max = value; - if (setrlimit(resource, &t) >= 0) { - return td::Status::OK(); - } - } - - r.rlim_cur = value < r.rlim_max ? value : r.rlim_max; - if (setrlimit(resource, &r) < 0) { - return td::Status::PosixError(errno, "failed setrlimit()"); - } - return td::Status::OK(); -} -#else -td::Status set_resource_limit(ResourceLimitType rlim, td::uint64 value) { - return td::Status::Error("setrlimit not implemented on WINDOWS"); -} -td::Status set_maximize_resource_limit(ResourceLimitType rlim, td::uint64 value) { - return td::Status::OK(); -} #endif -} // namespace td +Status set_resource_limit(ResourceLimitType type, uint64 value, uint64 max_value) { +#if TD_PORT_POSIX + if (max_value != 0 && value > max_value) { + return Status::Error("New resource limit value must not be bigger than max_value"); + } + int resource = get_resource(type); + + rlimit rlim; + if (getrlimit(resource, &rlim) == -1) { + return OS_ERROR("Failed to get current resource limit"); + } + + TRY_RESULT(new_value, narrow_cast_safe(value)); + TRY_RESULT(new_max_value, narrow_cast_safe(max_value)); + if (new_max_value) { + rlim.rlim_max = new_max_value; + } else if (rlim.rlim_max < new_value) { + rlim.rlim_max = new_value; + } + rlim.rlim_cur = new_value; + + if (setrlimit(resource, &rlim) < 0) { + return OS_ERROR("Failed to set resource limit"); + } + return Status::OK(); +#elif TD_PORT_WINDOWS + return Status::OK(); // Windows has no limits +#endif +} + +Status set_maximize_resource_limit(ResourceLimitType type, uint64 value) { +#if TD_PORT_POSIX + int resource = get_resource(type); + + rlimit rlim; + if (getrlimit(resource, &rlim) == -1) { + return OS_ERROR("Failed to get current resource limit"); + } + + TRY_RESULT(new_value, narrow_cast_safe(value)); + if (rlim.rlim_max < new_value) { + // trying to increase rlim_max + rlimit new_rlim; + new_rlim.rlim_cur = new_value; + new_rlim.rlim_max = new_value; + if (setrlimit(resource, &new_rlim) >= 0) { + return Status::OK(); + } + + // do not increase rlim_max if have no rights + new_value = rlim.rlim_max; + } + rlim.rlim_cur = new_value; + + if (setrlimit(resource, &rlim) < 0) { + return OS_ERROR("Failed to set resource limit"); + } + return Status::OK(); +#elif TD_PORT_WINDOWS + return Status::OK(); // Windows has no limits +#endif +} + +} // namespace td diff --git a/tdutils/td/utils/port/rlimit.h b/tdutils/td/utils/port/rlimit.h index c5ecddb45..c47773a21 100644 --- a/tdutils/td/utils/port/rlimit.h +++ b/tdutils/td/utils/port/rlimit.h @@ -4,7 +4,6 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #pragma once #include "td/utils/common.h" @@ -12,8 +11,10 @@ namespace td { -enum class ResourceLimitType { NoFile, Rss }; +enum class ResourceLimitType { NoFile }; + +Status set_resource_limit(ResourceLimitType type, uint64 value, uint64 max_value = 0); + +Status set_maximize_resource_limit(ResourceLimitType type, uint64 value); -td::Status set_resource_limit(ResourceLimitType rlim_type, td::uint64 value, td::uint64 cap = 0); -td::Status set_maximize_resource_limit(ResourceLimitType rlim, td::uint64 value); } // namespace td diff --git a/tdutils/td/utils/port/signals.cpp b/tdutils/td/utils/port/signals.cpp index 4658bc031..7fea86d1a 100644 --- a/tdutils/td/utils/port/signals.cpp +++ b/tdutils/td/utils/port/signals.cpp @@ -320,8 +320,8 @@ static void default_failure_signal_handler(int sig) { Status set_default_failure_signal_handler() { #if TD_PORT_POSIX Stdin(); // init static variables before atexit -#endif std::atexit(block_stdin); +#endif TRY_STATUS(setup_signals_alt_stack()); TRY_STATUS(set_signal_handler(SignalType::Abort, default_failure_signal_handler)); TRY_STATUS(set_signal_handler(SignalType::Error, default_failure_signal_handler)); diff --git a/tdutils/td/utils/port/uname.h b/tdutils/td/utils/port/uname.h index c0f368209..be7443436 100644 --- a/tdutils/td/utils/port/uname.h +++ b/tdutils/td/utils/port/uname.h @@ -12,4 +12,4 @@ namespace td { Slice get_operating_system_version(); -} +} // namespace td diff --git a/tdutils/td/utils/port/user.cpp b/tdutils/td/utils/port/user.cpp index 598681674..20b20ec84 100644 --- a/tdutils/td/utils/port/user.cpp +++ b/tdutils/td/utils/port/user.cpp @@ -4,12 +4,13 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #include "td/utils/port/user.h" #include "td/utils/port/config.h" #if TD_PORT_POSIX +#include "td/utils/logging.h" + #include #include #if TD_DARWIN || TD_FREEBSD || TD_NETBSD diff --git a/tdutils/td/utils/port/user.h b/tdutils/td/utils/port/user.h index ea338851a..9ac61fa8d 100644 --- a/tdutils/td/utils/port/user.h +++ b/tdutils/td/utils/port/user.h @@ -4,12 +4,13 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #pragma once #include "td/utils/Slice.h" #include "td/utils/Status.h" namespace td { + Status change_user(CSlice username, CSlice groupname = CSlice()); -} + +} // namespace td diff --git a/tdutils/td/utils/tests.cpp b/tdutils/td/utils/tests.cpp index 350e09f53..675fa341f 100644 --- a/tdutils/td/utils/tests.cpp +++ b/tdutils/td/utils/tests.cpp @@ -202,7 +202,7 @@ bool TestsRunner::run_all_step() { auto passed = Time::now() - state_.start; auto real_passed = Time::now_unadjusted() - state_.start_unadjusted; - if (real_passed + 1e-9 > passed) { + if (real_passed + 1e-6 > passed) { LOG(ERROR) << format::as_time(passed); } else { LOG(ERROR) << format::as_time(passed) << " real[" << format::as_time(real_passed) << "]"; diff --git a/tdutils/td/utils/tests.h b/tdutils/td/utils/tests.h index 6f6fb211f..ce17ae39b 100644 --- a/tdutils/td/utils/tests.h +++ b/tdutils/td/utils/tests.h @@ -13,7 +13,6 @@ #include "td/utils/port/thread.h" #include "td/utils/Random.h" #include "td/utils/Slice.h" -#include "td/utils/Span.h" #include "td/utils/Status.h" #include @@ -32,17 +31,19 @@ class RandomSteps { public: struct Step { std::function func; - td::uint32 weight; + uint32 weight; }; - RandomSteps(std::vector steps) : steps_(std::move(steps)) { - for (auto &step : steps_) { + + explicit RandomSteps(vector steps) : steps_(std::move(steps)) { + for (const auto &step : steps_) { steps_sum_ += step.weight; } } + template - void step(Random &rnd) { + void step(Random &rnd) const { auto w = rnd() % steps_sum_; - for (auto &step : steps_) { + for (const auto &step : steps_) { if (w < step.weight) { step.func(); break; @@ -52,8 +53,8 @@ class RandomSteps { } private: - std::vector steps_; - td::int32 steps_sum_ = 0; + vector steps_; + int32 steps_sum_ = 0; }; class RegressionTester { diff --git a/tdutils/td/utils/tl_helpers.h b/tdutils/td/utils/tl_helpers.h index e0dbbbb65..41803176a 100644 --- a/tdutils/td/utils/tl_helpers.h +++ b/tdutils/td/utils/tl_helpers.h @@ -11,7 +11,6 @@ #include "td/utils/misc.h" #include "td/utils/SharedSlice.h" #include "td/utils/Slice.h" -#include "td/utils/SharedSlice.h" #include "td/utils/StackAllocator.h" #include "td/utils/Status.h" #include "td/utils/tl_parsers.h" diff --git a/tdutils/td/utils/tl_storers.h b/tdutils/td/utils/tl_storers.h index da6d36c6b..46b0e3b80 100644 --- a/tdutils/td/utils/tl_storers.h +++ b/tdutils/td/utils/tl_storers.h @@ -8,12 +8,11 @@ #include "td/utils/common.h" #include "td/utils/logging.h" +#include "td/utils/SharedSlice.h" #include "td/utils/Slice.h" #include "td/utils/StorerBase.h" #include "td/utils/UInt.h" -#include "td/utils/SharedSlice.h" - #include namespace td { diff --git a/tdutils/test/List.cpp b/tdutils/test/List.cpp index fc64188a8..17a3fe184 100644 --- a/tdutils/test/List.cpp +++ b/tdutils/test/List.cpp @@ -4,7 +4,6 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #include "td/utils/common.h" #include "td/utils/List.h" #include "td/utils/MovableValue.h" diff --git a/tdutils/test/MpmcWaiter.cpp b/tdutils/test/MpmcWaiter.cpp index f6713a04e..5de025181 100644 --- a/tdutils/test/MpmcWaiter.cpp +++ b/tdutils/test/MpmcWaiter.cpp @@ -14,7 +14,7 @@ #if !TD_THREAD_UNSUPPORTED template -void test_waiter_stress_one_one() { +static void test_waiter_stress_one_one() { td::Stage run; td::Stage check; @@ -60,15 +60,17 @@ void test_waiter_stress_one_one() { thread.join(); } } + TEST(MpmcEagerWaiter, stress_one_one) { test_waiter_stress_one_one(); } + TEST(MpmcSleepyWaiter, stress_one_one) { test_waiter_stress_one_one(); } template -void test_waiter_stress() { +static void test_waiter_stress() { td::Stage run; td::Stage check; @@ -130,9 +132,11 @@ void test_waiter_stress() { thread.join(); } } + TEST(MpmcEagerWaiter, stress_multi) { test_waiter_stress(); } + TEST(MpmcSleepyWaiter, stress_multi) { test_waiter_stress(); } diff --git a/tdutils/test/OptionParser.cpp b/tdutils/test/OptionParser.cpp index 079a0bb19..82a3cba2c 100644 --- a/tdutils/test/OptionParser.cpp +++ b/tdutils/test/OptionParser.cpp @@ -4,12 +4,10 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #include "td/utils/common.h" #include "td/utils/misc.h" #include "td/utils/OptionParser.h" #include "td/utils/Slice.h" -#include "td/utils/Status.h" #include "td/utils/tests.h" TEST(OptionParser, run) { diff --git a/tdutils/test/SharedSlice.cpp b/tdutils/test/SharedSlice.cpp index 33f7af795..aa8912cc3 100644 --- a/tdutils/test/SharedSlice.cpp +++ b/tdutils/test/SharedSlice.cpp @@ -4,7 +4,6 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #include "td/utils/common.h" #include "td/utils/port/thread.h" #include "td/utils/SharedSlice.h" diff --git a/tdutils/test/StealingQueue.cpp b/tdutils/test/StealingQueue.cpp new file mode 100644 index 000000000..dc30999a1 --- /dev/null +++ b/tdutils/test/StealingQueue.cpp @@ -0,0 +1,173 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/AtomicRead.h" +#include "td/utils/benchmark.h" +#include "td/utils/MpmcQueue.h" +#include "td/utils/port/thread.h" +#include "td/utils/StealingQueue.h" +#include "td/utils/tests.h" + +#include + +TEST(StealingQueue, very_simple) { + td::StealingQueue q; + q.local_push(1, [](auto x) { UNREACHABLE(); }); + int x; + CHECK(q.local_pop(x)); + ASSERT_EQ(1, x); +} + +TEST(AtomicRead, simple) { + td::Stage run; + td::Stage check; + + std::size_t threads_n = 10; + td::vector threads; + + int x{0}; + std::atomic version{0}; + + td::int64 res = 0; + for (std::size_t i = 0; i < threads_n; i++) { + threads.emplace_back([&, id = static_cast(i)] { + for (td::uint64 round = 1; round < 10000; round++) { + run.wait(round * threads_n); + if (id == 0) { + version++; + x++; + version++; + } else { + int y = 0; + auto v1 = version.load(); + y = x; + auto v2 = version.load(); + if (v1 == v2 && v1 % 2 == 0) { + res += y; + } + } + + check.wait(round * threads_n); + } + }); + } + td::do_not_optimize_away(res); + for (auto &thread : threads) { + thread.join(); + } +} + +TEST(AtomicRead, simple2) { + td::Stage run; + td::Stage check; + + std::size_t threads_n = 10; + td::vector threads; + + struct Value { + td::uint64 value = 0; + char str[50] = "0 0 0 0"; + }; + td::AtomicRead value; + + auto to_str = [](td::uint64 i) { + return PSTRING() << i << " " << i << " " << i << " " << i; + }; + for (std::size_t i = 0; i < threads_n; i++) { + threads.emplace_back([&, id = static_cast(i)] { + for (td::uint64 round = 1; round < 10000; round++) { + run.wait(round * threads_n); + if (id == 0) { + auto x = value.lock(); + x->value = round; + auto str = to_str(round); + memcpy(x->str, str.c_str(), str.size() + 1); + } else { + Value x; + value.read(x); + LOG_CHECK(x.value == round || x.value == round - 1) << x.value << " " << round; + CHECK(x.str == to_str(x.value)); + } + check.wait(round * threads_n); + } + }); + } + for (auto &thread : threads) { + thread.join(); + } +} + +TEST(StealingQueue, simple) { + td::uint64 sum; + std::atomic got_sum; + + td::Stage run; + td::Stage check; + + std::size_t threads_n = 10; + td::vector threads; + td::vector> lq(threads_n); + td::MpmcQueue gq(threads_n); + + constexpr td::uint64 XN = 20; + td::uint64 x_sum[XN]; + x_sum[0] = 0; + x_sum[1] = 1; + for (td::uint64 i = 2; i < XN; i++) { + x_sum[i] = i + x_sum[i - 1] + x_sum[i - 2]; + } + + td::Random::Xorshift128plus rnd(123); + for (std::size_t i = 0; i < threads_n; i++) { + threads.emplace_back([&, id = static_cast(i)] { + for (td::uint64 round = 1; round < 10000; round++) { + if (id == 0) { + sum = 0; + int n = rnd() % 5; + for (int j = 0; j < n; j++) { + int x = rand() % XN; + sum += x_sum[x]; + gq.push(x, id); + } + got_sum = 0; + } + run.wait(round * threads_n); + while (got_sum.load() != sum) { + auto x = [&] { + int res; + if (lq[id].local_pop(res)) { + return res; + } + if (gq.try_pop(res, id)) { + return res; + } + if (lq[id].steal(res, lq[rand() % threads_n])) { + //LOG(ERROR) << "STEAL"; + return res; + } + return 0; + }(); + if (x == 0) { + continue; + } + //LOG(ERROR) << x << " " << got_sum.load() << " " << sum; + got_sum.fetch_add(x, std::memory_order_relaxed); + lq[id].local_push(x - 1, [&](auto y) { + //LOG(ERROR) << "OVERFLOW"; + gq.push(y, id); + }); + if (x > 1) { + lq[id].local_push(x - 2, [&](auto y) { gq.push(y, id); }); + } + } + check.wait(round * threads_n); + } + }); + } + for (auto &thread : threads) { + thread.join(); + } +} diff --git a/tdutils/test/heap.cpp b/tdutils/test/heap.cpp index 6d0ea0541..13afd1e4e 100644 --- a/tdutils/test/heap.cpp +++ b/tdutils/test/heap.cpp @@ -9,6 +9,7 @@ #include "td/utils/common.h" #include "td/utils/Heap.h" #include "td/utils/Random.h" +#include "td/utils/Span.h" #include #include @@ -25,7 +26,7 @@ TEST(Heap, sort_random_perm) { } td::Random::Xorshift128plus rnd(123); td::random_shuffle(td::as_mutable_span(v), rnd); - std::vector nodes(n); + td::vector nodes(n); td::KHeap kheap; for (int i = 0; i < n; i++) { kheap.insert(v[i], &nodes[i]); diff --git a/tdutils/test/log.cpp b/tdutils/test/log.cpp index 416f85c1f..b76f95579 100644 --- a/tdutils/test/log.cpp +++ b/tdutils/test/log.cpp @@ -4,7 +4,6 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // - #include "td/utils/benchmark.h" #include "td/utils/FileLog.h" #include "td/utils/format.h" diff --git a/tdutils/test/misc.cpp b/tdutils/test/misc.cpp index c109d2fb8..030def970 100644 --- a/tdutils/test/misc.cpp +++ b/tdutils/test/misc.cpp @@ -127,14 +127,14 @@ TEST(Misc, errno_tls_bug) { event.release(); } for (auto &event : events) { - threads.push_back(td::thread([&] { + threads.emplace_back([&] { { EventFd tmp; tmp.init(); tmp.acquire(); } event.acquire(); - })); + }); } for (auto &thread : threads) { thread.join(); @@ -154,7 +154,8 @@ TEST(Misc, get_last_argument) { } TEST(Misc, call_n_arguments) { - auto f = [](int, int) {}; + auto f = [](int, int) { + }; call_n_arguments<2>(f, 1, 3, 4); } @@ -253,10 +254,18 @@ static void test_remove_if(vector v, const T &func, vector expected) { } TEST(Misc, remove_if) { - auto odd = [](int x) { return x % 2 == 1; }; - auto even = [](int x) { return x % 2 == 0; }; - auto all = [](int x) { return true; }; - auto none = [](int x) { return false; }; + auto odd = [](int x) { + return x % 2 == 1; + }; + auto even = [](int x) { + return x % 2 == 0; + }; + auto all = [](int x) { + return true; + }; + auto none = [](int x) { + return false; + }; vector v{1, 2, 3, 4, 5, 6}; test_remove_if(v, odd, {2, 4, 6}); @@ -862,7 +871,7 @@ TEST(Misc, Bits) { TEST(Misc, BitsRange) { auto to_vec_a = [](td::uint64 x) { - std::vector bits; + td::vector bits; for (auto i : td::BitsRange(x)) { bits.push_back(i); } @@ -870,7 +879,7 @@ TEST(Misc, BitsRange) { }; auto to_vec_b = [](td::uint64 x) { - std::vector bits; + td::vector bits; td::int32 pos = 0; while (x != 0) { if ((x & 1) != 0) { @@ -882,8 +891,12 @@ TEST(Misc, BitsRange) { return bits; }; - auto do_check = [](std::vector a, std::vector b) { ASSERT_EQ(b, a); }; - auto check = [&](td::uint64 x) { do_check(to_vec_a(x), to_vec_b(x)); }; + auto do_check = [](const td::vector &a, const td::vector &b) { + ASSERT_EQ(b, a); + }; + auto check = [&](td::uint64 x) { + do_check(to_vec_a(x), to_vec_b(x)); + }; do_check(to_vec_a(21), {0, 2, 4}); for (int x = 0; x < 100; x++) { @@ -954,8 +967,12 @@ TEST(Misc, uint128) { static_cast(std::numeric_limits::min()) - 1}; #if TD_HAVE_INT128 - auto to_intrinsic = [](uint128_emulated num) { return uint128_intrinsic(num.hi(), num.lo()); }; - auto eq = [](uint128_emulated a, uint128_intrinsic b) { return a.hi() == b.hi() && a.lo() == b.lo(); }; + auto to_intrinsic = [](uint128_emulated num) { + return uint128_intrinsic(num.hi(), num.lo()); + }; + auto eq = [](uint128_emulated a, uint128_intrinsic b) { + return a.hi() == b.hi() && a.lo() == b.lo(); + }; auto ensure_eq = [&](uint128_emulated a, uint128_intrinsic b) { if (!eq(a, b)) { LOG(FATAL) << "[" << a.hi() << ";" << a.lo() << "] vs [" << b.hi() << ";" << b.lo() << "]"; diff --git a/test/main.cpp b/test/main.cpp index c7427d1b4..6999a0dcc 100644 --- a/test/main.cpp +++ b/test/main.cpp @@ -11,9 +11,6 @@ #include "td/utils/logging.h" #include "td/utils/OptionParser.h" #include "td/utils/Slice.h" -#include "td/utils/Status.h" - -#include #if TD_EMSCRIPTEN #include