tdlight-team/tdlight
tdlight-team
/
tdlight
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

tdutils: update from other project 

GitOrigin-RevId: 73c666dbdd72811b151a48504716ed4aee6af1a0
This commit is contained in:
Arseny Smirnov 2019-07-06 13:29:15 +02:00
parent a695b0823c
commit 02c31f486f
62 changed files with 1199 additions and 245 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
tdutils/CMakeLists.txt
View File

@ -1,5 +1,13 @@
cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
if (WIN32)
if (WINGETOPT_FOUND)
set(TD_HAVE_GETOPT 1)
endif()
else()
set(TD_HAVE_GETOPT 1)
endif()
if (NOT ZLIB_FOUND)
find_package(ZLIB)
endif()
@ -20,6 +28,10 @@ if (CRC32C_FOUND)
set(TD_HAVE_CRC32C 1)
endif()
if (ABSL_FOUND)
set(TD_HAVE_ABSL 1)
endif()
configure_file(td/utils/config.h.in td/utils/config.h @ONLY)
add_subdirectory(generate)
@ -39,12 +51,14 @@ set(TDUTILS_SOURCE
td/utils/port/Clocks.cpp
td/utils/port/FileFd.cpp
td/utils/port/IPAddress.cpp
td/utils/port/MemoryMapping.cpp
td/utils/port/path.cpp
td/utils/port/PollFlags.cpp
td/utils/port/ServerSocketFd.cpp
td/utils/port/signals.cpp
td/utils/port/sleep.cpp
td/utils/port/SocketFd.cpp
td/utils/port/stacktrace.cpp
td/utils/port/Stat.cpp
td/utils/port/StdStreams.cpp
td/utils/port/thread_local.cpp
@ -82,6 +96,8 @@ set(TDUTILS_SOURCE
td/utils/logging.cpp
td/utils/misc.cpp
td/utils/MimeType.cpp
td/utils/MpmcQueue.cpp
td/utils/OptionsParser.cpp
td/utils/Random.cpp
td/utils/StackAllocator.cpp
td/utils/Status.cpp
@ -89,6 +105,7 @@ set(TDUTILS_SOURCE
td/utils/tests.cpp
td/utils/Time.cpp
td/utils/Timer.cpp
td/utils/tests.cpp
td/utils/tl_parsers.cpp
td/utils/translit.cpp
td/utils/unicode.cpp
@ -101,6 +118,8 @@ set(TDUTILS_SOURCE
td/utils/port/EventFdBase.h
td/utils/port/FileFd.h
td/utils/port/IPAddress.h
td/utils/port/IoSlice.h
td/utils/port/MemoryMapping.h
td/utils/port/path.h
td/utils/port/platform.h
td/utils/port/Poll.h
@ -111,6 +130,7 @@ set(TDUTILS_SOURCE
td/utils/port/signals.h
td/utils/port/sleep.h
td/utils/port/SocketFd.h
td/utils/port/stacktrace.h
td/utils/port/Stat.h
td/utils/port/StdStreams.h
td/utils/port/thread.h
@ -149,12 +169,14 @@ set(TDUTILS_SOURCE
td/utils/check.h
td/utils/Closure.h
td/utils/common.h
td/utils/ConcurrentHashTable.h
td/utils/Container.h
td/utils/Context.h
td/utils/crypto.h
td/utils/DecTree.h
td/utils/Destructor.h
td/utils/Enumerator.h
td/utils/EpochBasedMemoryReclamation.h
td/utils/FileLog.h
td/utils/filesystem.h
td/utils/find_boundary.h
@ -210,7 +232,9 @@ set(TDUTILS_SOURCE
td/utils/tl_parsers.h
td/utils/tl_storers.h
td/utils/translit.h
td/utils/ThreadSafeCounter.h
td/utils/type_traits.h
td/utils/uint128.h
td/utils/UInt.h
td/utils/unicode.h
td/utils/unique_ptr.h
@ -222,7 +246,9 @@ set(TDUTILS_SOURCE
set(TDUTILS_TEST_SOURCE
${CMAKE_CURRENT_SOURCE_DIR}/test/buffer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/crypto.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/ConcurrentHashMap.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/Enumerator.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/EpochBasedMemoryReclamation.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/filesystem.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/gzip.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/HazardPointers.cpp
@ -271,6 +297,18 @@ endif()
if (CRC32C_FOUND)
target_link_libraries(tdutils PRIVATE crc32c)
endif()
if (ABSL_FOUND)
#target_link_libraries(tdutils PRIVATE absl::base absl::time)
target_link_libraries_system(tdutils absl::base absl::container absl::hash )
endif()
if (WIN32 AND WINGETOPT_FOUND)
target_link_libraries(tdutils PRIVATE wingetopt)
endif()
if (ANDROID)
target_link_libraries(tdutils PRIVATE log)
endif()
install(TARGETS tdutils EXPORT TdTargets
LIBRARY DESTINATION lib

22
tdutils/td/utils/BufferedFd.h
View File

@ -11,7 +11,9 @@
#include "td/utils/format.h"
#include "td/utils/logging.h"
#include "td/utils/port/detail/PollableFd.h"
#include "td/utils/port/IoSlice.h"
#include "td/utils/Slice.h"
#include "td/utils/Span.h"
#include "td/utils/Status.h"
#include <limits>
@ -103,13 +105,25 @@ Result<size_t> BufferedFdBase<FdT>::flush_read(size_t max_read) {
template <class FdT>
Result<size_t> BufferedFdBase<FdT>::flush_write() {
size_t result = 0;
// TODO: sync on demand
write_->sync_with_writer();
size_t result = 0;
while (!write_->empty() && ::td::can_write(*this)) {
Slice slice = write_->prepare_read();
TRY_RESULT(x, FdT::write(slice));
write_->confirm_read(x);
constexpr size_t buf_size = 20;
IoSlice buf[buf_size];
auto it = write_->clone();
size_t buf_i;
for (buf_i = 0; buf_i < buf_size; buf_i++) {
Slice slice = it.prepare_read();
if (slice.empty()) {
break;
}
buf[buf_i] = as_io_slice(slice);
it.confirm_read(slice.size());
}
TRY_RESULT(x, FdT::writev(Span<IoSlice>(buf, buf_i)));
write_->advance(x);
result += x;
}
return result;

2
tdutils/td/utils/BufferedReader.h
View File

@ -16,7 +16,7 @@
namespace td {
class BufferedReader {
public:
explciit BufferedReader(FileFd &file, size_t buff_size = 8152)
explicit BufferedReader(FileFd &file, size_t buff_size = 8152)
: file_(file), buff_(buff_size), begin_pos_(0), end_pos_(0) {
}

4
tdutils/td/utils/DecTree.h
View File

@ -53,7 +53,7 @@ class DecTree {
T->left_ = std::move(P.first);
T->right_ = std::move(P.second);
T->relax();
return std::move(T);
return T;
}
if (Compare()(key, Tree->key_)) {
Tree->left_ = insert_node(std::move(Tree->left_), std::move(key), std::move(value), y);
@ -151,7 +151,7 @@ class DecTree {
P.first = std::move(Tree);
return P;
}
}
} // namespace td
static unique_ptr<Node> merge_node(unique_ptr<Node> left, unique_ptr<Node> right) {
if (left == nullptr) {

18
tdutils/td/utils/Gzip.cpp
View File

@ -129,9 +129,23 @@ void Gzip::clear() {
Gzip::Gzip() : impl_(make_unique<Impl>()) {
}
Gzip::Gzip(Gzip &&other) = default;
Gzip::Gzip(Gzip &&other) : Gzip() {
swap(other);
}
Gzip &Gzip::operator=(Gzip &&other) = default;
Gzip &Gzip::operator=(Gzip &&other) {
clear();
swap(other);
return *this;
}
void Gzip::swap(Gzip &other) {
std::swap(impl_, other.impl_);
std::swap(input_size_, other.input_size_);
std::swap(output_size_, other.output_size_);
std::swap(close_input_flag_, other.close_input_flag_);
std::swap(mode_, other.mode_);
}
Gzip::~Gzip() {
clear();

2
tdutils/td/utils/Gzip.h
View File

@ -93,6 +93,8 @@ class Gzip {
void init_common();
void clear();
void swap(Gzip &other);
};
BufferSlice gzdecode(Slice s);

31
tdutils/td/utils/HazardPointers.h
View File

@ -10,10 +10,11 @@
#include <array>
#include <atomic>
#include <memory>
namespace td {
template <class T, int MaxPointersN = 1>
template <class T, int MaxPointersN = 1, class Deleter = std::default_delete<T>>
class HazardPointers {
public:
explicit HazardPointers(size_t threads_n) : threads_(threads_n) {
@ -35,12 +36,17 @@ class HazardPointers {
class Holder {
public:
T *protect(std::atomic<T *> &to_protect) {
template <class S>
S *protect(std::atomic<S *> &to_protect) {
return do_protect(hazard_ptr_, to_protect);
}
Holder(HazardPointers &hp, size_t thread_id, size_t pos) : Holder(hp.get_hazard_ptr(thread_id, pos)) {
CHECK(hazard_ptr_.load() == 0);
hazard_ptr_.store(reinterpret_cast<T *>(1));
}
Holder(const Holder &other) = delete;
Holder &operator=(const Holder &other) = delete;
Holder(Holder &&other) = default; // TODO
Holder(Holder &&other) = delete;
Holder &operator=(Holder &&other) = delete;
~Holder() {
clear();
@ -56,15 +62,11 @@ class HazardPointers {
std::atomic<T *> &hazard_ptr_;
};
Holder get_holder(size_t thread_id, size_t pos) {
return Holder(get_hazard_ptr(thread_id, pos));
}
void retire(size_t thread_id, T *ptr = nullptr) {
CHECK(thread_id < threads_.size());
auto &data = threads_[thread_id];
if (ptr) {
data.to_delete.push_back(unique_ptr<T>(ptr));
data.to_delete.push_back(std::unique_ptr<T, Deleter>(ptr));
}
for (auto it = data.to_delete.begin(); it != data.to_delete.end();) {
if (!is_protected(it->get())) {
@ -95,23 +97,24 @@ class HazardPointers {
private:
struct ThreadData {
std::array<std::atomic<T *>, MaxPointersN> hazard;
char pad[TD_CONCURRENCY_PAD - sizeof(std::array<std::atomic<T *>, MaxPointersN>)];
char pad[TD_CONCURRENCY_PAD - sizeof(hazard)];
// stupid gc
std::vector<unique_ptr<T>> to_delete;
char pad2[TD_CONCURRENCY_PAD - sizeof(std::vector<unique_ptr<T>>)];
std::vector<std::unique_ptr<T, Deleter>> to_delete;
char pad2[TD_CONCURRENCY_PAD - sizeof(to_delete)];
};
std::vector<ThreadData> threads_;
char pad2[TD_CONCURRENCY_PAD - sizeof(std::vector<ThreadData>)];
char pad2[TD_CONCURRENCY_PAD - sizeof(threads_)];
static T *do_protect(std::atomic<T *> &hazard_ptr, std::atomic<T *> &to_protect) {
template <class S>
static S *do_protect(std::atomic<T *> &hazard_ptr, std::atomic<S *> &to_protect) {
T *saved = nullptr;
T *to_save;
while ((to_save = to_protect.load()) != saved) {
hazard_ptr.store(to_save);
saved = to_save;
}
return saved;
return static_cast<S *>(saved);
}
static void do_clear(std::atomic<T *> &hazard_ptr) {

9
tdutils/td/utils/Heap.h
View File

@ -71,7 +71,14 @@ class KHeap {
}
template <class F>
void for_each(F &f) const {
void for_each(F &&f) const {
for (auto &it : array_) {
f(it.key_, it.node_);
}
}
template <class F>
void for_each(F &&f) {
for (auto &it : array_) {
f(it.key_, it.node_);
}

17
tdutils/td/utils/MpmcQueue.h
View File

@ -65,12 +65,12 @@ struct MpmcStat {
}
};
} // namespace detail
//detail::MpmcStat stat_;
extern detail::MpmcStat stat_;
template <class T>
class OneValue {
public:
bool set_value(T &value) {
inline bool set_value(T &value) {
value_ = std::move(value);
int state = Empty;
if (state_.compare_exchange_strong(state, Value, std::memory_order_acq_rel)) {
@ -79,7 +79,7 @@ class OneValue {
value = std::move(value_);
return false;
}
bool get_value(T &value) {
inline bool get_value(T &value) {
auto old_state = state_.exchange(Taken, std::memory_order_acq_rel);
if (old_state == Value) {
value = std::move(value_);
@ -149,6 +149,11 @@ class MpmcQueueBlock {
//returns Ok, Empty or Closed
PopStatus try_pop(T &value) {
while (true) {
// this check slows 1:1 case but prevents writer starvation in 1:N case
if (write_pos_.load(std::memory_order_relaxed) <= read_pos_.load(std::memory_order_relaxed) &&
read_pos_.load(std::memory_order_relaxed) < nodes_.size()) {
return PopStatus::Empty;
}
auto read_pos = read_pos_.fetch_add(1, std::memory_order_relaxed);
if (read_pos >= nodes_.size()) {
return PopStatus::Closed;
@ -217,7 +222,7 @@ class MpmcQueueOld {
delete to_delete;
}
//stat_.dump();
//stat_ = MpmcStat();
//stat_ = detail::MpmcStat();
}
size_t hazard_pointers_to_delele_size_unsafe() const {
@ -231,7 +236,7 @@ class MpmcQueueOld {
using PopStatus = typename MpmcQueueBlock<T>::PopStatus;
void push(T value, size_t thread_id) {
auto hazard_ptr_holder = hazard_pointers_.get_holder(thread_id, 0);
typename decltype(hazard_pointers_)::Holder hazard_ptr_holder(hazard_pointers_, thread_id, 0);
while (true) {
auto node = hazard_ptr_holder.protect(write_pos_);
auto status = node->block.push(value);
@ -263,7 +268,7 @@ class MpmcQueueOld {
}
bool try_pop(T &value, size_t thread_id) {
auto hazard_ptr_holder = hazard_pointers_.get_holder(thread_id, 0);
typename decltype(hazard_pointers_)::Holder hazard_ptr_holder(hazard_pointers_, thread_id, 0);
while (true) {
auto node = hazard_ptr_holder.protect(read_pos_);
auto status = node->block.try_pop(value);

4
tdutils/td/utils/MpmcWaiter.h
View File

@ -8,6 +8,7 @@
#include "td/utils/common.h"
#include "td/utils/port/thread.h"
#include "td/utils/logging.h"
#include <atomic>
#include <condition_variable>
@ -25,7 +26,7 @@ class MpmcWaiter {
auto state = state_.load(std::memory_order_relaxed);
if (!State::has_worker(state)) {
auto new_state = State::with_worker(state, worker_id);
if (state_.compare_exchange_strong(state, new_state)) {
if (state_.compare_exchange_strong(state, new_state, std::memory_order_acq_rel)) {
td::this_thread::yield();
return yields + 1;
}
@ -62,6 +63,7 @@ class MpmcWaiter {
}
void notify() {
std::atomic_thread_fence(std::memory_order_seq_cst);
if (state_.load(std::memory_order_acquire) == State::awake()) {
return;
}

112
tdutils/td/utils/OptionsParser.h
View File

@ -14,11 +14,6 @@
#include <functional>
#include <string>
#if !TD_WINDOWS
#include <getopt.h>
#include <unistd.h>
#endif
namespace td {
class OptionsParser {
@ -33,113 +28,18 @@ class OptionsParser {
};
public:
void set_description(std::string description) {
description_ = std::move(description);
}
void set_description(std::string description);
void add_option(Option::Type type, char short_key, Slice long_key, Slice description,
std::function<Status(Slice)> callback) {
options_.push_back(Option{type, short_key, long_key.str(), description.str(), std::move(callback)});
}
std::function<Status(Slice)> callback);
void add_option(char short_key, Slice long_key, Slice description, std::function<Status(Slice)> callback) {
add_option(Option::Type::Arg, short_key, long_key, description, std::move(callback));
}
void add_option(char short_key, Slice long_key, Slice description, std::function<Status(Slice)> callback);
void add_option(char short_key, Slice long_key, Slice description, std::function<Status(void)> callback) {
// Ouch. There must be some better way
add_option(Option::Type::NoArg, short_key, long_key, description,
std::bind([](std::function<Status(void)> &func, Slice) { return func(); }, std::move(callback),
std::placeholders::_1));
}
void add_option(char short_key, Slice long_key, Slice description, std::function<Status(void)> callback);
Result<int> run(int argc, char *argv[]) TD_WARN_UNUSED_RESULT {
#if TD_WINDOWS
return -1;
#else
char buff[1024];
StringBuilder sb(MutableSlice{buff, sizeof(buff)});
for (auto &opt : options_) {
CHECK(opt.type != Option::OptionalArg);
sb << opt.short_key;
if (opt.type == Option::Arg) {
sb << ":";
}
}
if (sb.is_error()) {
return Status::Error("Can't parse options");
}
CSlice short_options = sb.as_cslice();
Result<int> run(int argc, char *argv[]) TD_WARN_UNUSED_RESULT;
vector<option> long_options;
for (auto &opt : options_) {
if (opt.long_key.empty()) {
continue;
}
option o;
o.flag = nullptr;
o.val = opt.short_key;
o.has_arg = opt.type == Option::Arg ? required_argument : no_argument;
o.name = opt.long_key.c_str();
long_options.push_back(o);
}
long_options.push_back({nullptr, 0, nullptr, 0});
while (true) {
int opt_i = getopt_long(argc, argv, short_options.c_str(), &long_options[0], nullptr);
if (opt_i == ':') {
return Status::Error("Missing argument");
}
if (opt_i == '?') {
return Status::Error("Unrecognized option");
}
if (opt_i == -1) {
break;
}
bool found = false;
for (auto &opt : options_) {
if (opt.short_key == opt_i) {
Slice arg;
if (opt.type == Option::Arg) {
arg = Slice(optarg);
}
auto status = opt.arg_callback(arg);
if (status.is_error()) {
return std::move(status);
}
found = true;
break;
}
}
if (!found) {
return Status::Error("Unknown argument");
}
}
return optind;
#endif
}
friend StringBuilder &operator<<(StringBuilder &sb, const OptionsParser &o) {
sb << o.description_ << "\n";
for (auto &opt : o.options_) {
sb << "-" << opt.short_key;
if (!opt.long_key.empty()) {
sb << "|--" << opt.long_key;
}
if (opt.type == Option::OptionalArg) {
sb << "[";
}
if (opt.type != Option::NoArg) {
sb << "<arg>";
}
if (opt.type == Option::OptionalArg) {
sb << "]";
}
sb << "\t" << opt.description;
sb << "\n";
}
return sb;
}
friend StringBuilder &operator<<(StringBuilder &sb, const OptionsParser &o);
private:
std::vector<Option> options_;

4
tdutils/td/utils/SharedObjectPool.h
View File

@ -37,9 +37,7 @@ class AtomicRefCnt {
};
template <class DataT, class DeleterT>
class SharedPtrRaw
: public DeleterT
, private MpscLinkQueueImpl::Node {
class SharedPtrRaw : public DeleterT, private MpscLinkQueueImpl::Node {
public:
explicit SharedPtrRaw(DeleterT deleter) : DeleterT(std::move(deleter)), ref_cnt_{0}, option_magic_(Magic) {
}

5
tdutils/td/utils/Slice-decl.h
View File

@ -9,6 +9,7 @@
#include "td/utils/common.h"
#include <type_traits>
#include <limits>
namespace td {
@ -57,6 +58,8 @@ class MutableSlice {
char &back();
char &operator[](size_t i);
static const size_t npos = std::numeric_limits<size_t>::max();
};
class Slice {
@ -120,6 +123,8 @@ class Slice {
char back() const;
char operator[](size_t i) const;
static const size_t npos = std::numeric_limits<size_t>::max();
};
bool operator==(const Slice &a, const Slice &b);

9
tdutils/td/utils/Slice.h
View File

@ -110,7 +110,7 @@ inline size_t MutableSlice::find(char c) const {
return pos;
}
}
return static_cast<size_t>(-1);
return npos;
}
inline size_t MutableSlice::rfind(char c) const {
@ -119,7 +119,7 @@ inline size_t MutableSlice::rfind(char c) const {
return pos;
}
}
return static_cast<size_t>(-1);
return npos;
}
inline void MutableSlice::copy_from(Slice from) {
@ -245,7 +245,7 @@ inline size_t Slice::find(char c) const {
return pos;
}
}
return static_cast<size_t>(-1);
return npos;
}
inline size_t Slice::rfind(char c) const {
@ -254,7 +254,7 @@ inline size_t Slice::rfind(char c) const {
return pos;
}
}
return static_cast<size_t>(-1);
return npos;
}
inline char Slice::back() const {
@ -303,6 +303,7 @@ inline std::size_t SliceHash::operator()(Slice slice) const {
inline Slice as_slice(Slice slice) {
return slice;
}
inline MutableSlice as_slice(MutableSlice slice) {
return slice;
}

24
tdutils/td/utils/Span.h
View File

@ -51,6 +51,18 @@ class SpanImpl {
SpanImpl copy{other};
*this = copy;
}
template <class OtherInnerT>
bool operator==(const SpanImpl<T, OtherInnerT> &other) const {
if (size() != other.size()) {
return false;
}
for (size_t i = 0; i < size(); i++) {
if ((*this)[i] != other[i]) {
return false;
}
}
return true;
}
InnerT &operator[](size_t i) {
DCHECK(i < size());
@ -74,6 +86,9 @@ class SpanImpl {
size_t size() const {
return size_;
}
bool empty() const {
return size() == 0;
}
SpanImpl &truncate(size_t size) {
CHECK(size <= size_);
@ -99,4 +114,13 @@ using Span = detail::SpanImpl<T, const T>;
template <class T>
using MutableSpan = detail::SpanImpl<T, T>;
template <class T>
auto span(const T *ptr, size_t size) {
return Span<T>(ptr, size);
}
template <class T>
auto mutable_span(T *ptr, size_t size) {
return MutableSpan<T>(ptr, size);
}
} // namespace td

1
tdutils/td/utils/SpinLock.h
View File

@ -27,6 +27,7 @@ class SpinLock {
bool next() {
cnt++;
if (cnt < 50) {
//TODO pause
return true;
} else {
td::this_thread::yield();

42
tdutils/td/utils/Status.h
View File

@ -27,7 +27,16 @@
return try_status.move_as_error(); \
} \
}
#define TRY_STATUS_PREFIX(status, prefix) \
{ \
auto try_status = (status); \
if (try_status.is_error()) { \
return try_status.move_as_error_prefix(prefix); \
} \
}
#define TRY_RESULT(name, result) TRY_RESULT_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), name, result)
#define TRY_RESULT_PREFIX(name, result, prefix) \
TRY_RESULT_PREFIX_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), name, result, prefix)
#define TRY_RESULT_IMPL(r_name, name, result) \
auto r_name = (result); \
@ -36,6 +45,13 @@
} \
auto name = r_name.move_as_ok();
#define TRY_RESULT_PREFIX_IMPL(r_name, name, result, prefix) \
auto r_name = (result); \
if (r_name.is_error()) { \
return r_name.move_as_error_prefix(prefix); \
} \
auto name = r_name.move_as_ok();
#define LOG_STATUS(status) \
{ \
auto log_status = (status); \
@ -248,6 +264,10 @@ class Status {
return std::move(*this);
}
Status move_as_error_prefix(std::string prefix) TD_WARN_UNUSED_RESULT {
return td::Status::Error(code(), prefix + message().c_str());
}
private:
struct Info {
bool static_flag : 1;
@ -277,6 +297,9 @@ class Status {
Status(bool static_flag, ErrorType error_type, int error_code, Slice message)
: Status(to_info(static_flag, error_type, error_code), message) {
if (static_flag) {
TD_LSAN_IGNORE(ptr_.get());
}
}
Status clone_static() const TD_WARN_UNUSED_RESULT {
@ -330,6 +353,10 @@ class Result {
template <class S, std::enable_if_t<!std::is_same<std::decay_t<S>, Result>::value, int> = 0>
Result(S &&x) : status_(), value_(std::forward<S>(x)) {
}
struct emplace_t {};
template <class... ArgsT>
Result(emplace_t, ArgsT &&... args) : status_(), value_(std::forward<ArgsT>(args)...) {
}
Result(Status &&status) : status_(std::move(status)) {
CHECK(status_.is_error());
}
@ -362,6 +389,14 @@ class Result {
other.status_ = Status::Error<-3>();
return *this;
}
template <class... ArgsT>
void emplace(ArgsT &&... args) {
if (status_.is_ok()) {
value_.~T();
}
new (&value_) T(std::forward<ArgsT>(args)...);
status_ = Status::OK();
}
~Result() {
if (status_.is_ok()) {
value_.~T();
@ -403,6 +438,13 @@ class Result {
};
return std::move(status_);
}
Status move_as_error_prefix(std::string prefix) TD_WARN_UNUSED_RESULT {
CHECK(status_.is_error());
SCOPE_EXIT {
status_ = Status::Error<-4>();
};
return status_.move_as_error_prefix(prefix);
}
const T &ok() const {
LOG_CHECK(status_.is_ok()) << status_;
return value_;

5
tdutils/td/utils/Time.cpp
View File

@ -10,10 +10,11 @@
namespace td {
std::atomic<double> Time::now_;
bool operator==(Timestamp a, Timestamp b) {
return std::abs(a.at() - b.at()) < 1e-6;
}
double Time::now() {
return Clocks::monotonic();
}
} // namespace td

25
tdutils/td/utils/Time.h
View File

@ -15,17 +15,21 @@ namespace td {
class Time {
public:
static double now() {
double now = Clocks::monotonic();
now_.store(now, std::memory_order_relaxed);
return now;
}
static double now();
static double now_cached() {
return now_.load(std::memory_order_relaxed);
// Temporary(?) use now in now_cached
// Problem:
// thread A: check that now() > timestamp and notifies thread B
// thread B: must see that now() > timestamp()
//
// now() and now_cached() must be monotonic
//
// if a=now[_cached]() happens before b=now[_cached] than
// a <= b
//
// As an alternative we may say that now_cached is a thread local copy of now
return now();
}
private:
static std::atomic<double> now_;
};
inline void relax_timeout_at(double *timeout, double new_timeout) {
@ -71,6 +75,9 @@ class Timestamp {
double at() const {
return at_;
}
double at_unix() {
return at_ + td::Clocks::system() - Time::now();
}
double in() const {
return at_ - Time::now_cached();

3
tdutils/td/utils/Timer.cpp
View File

@ -16,6 +16,9 @@ namespace td {
Timer::Timer() : start_time_(Time::now()) {
}
double Timer::elapsed() const {
return Time::now() - start_time_;
}
StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer) {
return string_builder << "in " << Time::now() - timer.start_time_;
}

1
tdutils/td/utils/Timer.h
View File

@ -13,6 +13,7 @@ namespace td {
class Timer {
public:
Timer();
double elapsed() const;
private:
friend StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer);

2
tdutils/td/utils/Variant.h
View File

@ -109,7 +109,7 @@ class Variant {
static constexpr int npos = -1;
Variant() {
}
Variant(Variant &&other) {
Variant(Variant &&other) noexcept {
other.visit([&](auto &&value) { this->init_empty(std::forward<decltype(value)>(value)); });
}
Variant(const Variant &other) {

4
tdutils/td/utils/VectorQueue.h
View File

@ -20,6 +20,10 @@ class VectorQueue {
void push(S &&s) {
vector_.push_back(std::forward<S>(s));
}
template <class... Args>
void emplace(Args &&... args) {
vector_.emplace_back(std::forward<Args>(args)...);
}
T pop() {
try_shrink();
return std::move(vector_[read_pos_++]);

2
tdutils/td/utils/as.h
View File

@ -28,7 +28,7 @@ class As {
}
~As() = default;
As &operator=(const T &new_value) && {
As &operator=(T new_value) && {
std::memcpy(ptr_, &new_value, sizeof(T));
return *this;
}

76
tdutils/td/utils/bits.h
View File

@ -5,8 +5,8 @@
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/check.h"
#if TD_MSVC
#include <intrin.h>
@ -19,18 +19,53 @@
#ifdef bswap64
#undef bswap64
#endif
namespace td {
int32 count_leading_zeroes32(uint32 x);
int32 count_leading_zeroes64(uint64 x);
int32 count_trailing_zeroes32(uint32 x);
int32 count_trailing_zeroes64(uint64 x);
uint32 bswap32(uint32 x);
uint64 bswap64(uint64 x);
int32 count_bits32(uint32 x);
int32 count_bits64(uint64 x);
inline int32 count_leading_zeroes32(uint32 x);
inline int32 count_leading_zeroes64(uint64 x);
inline int32 count_trailing_zeroes32(uint32 x);
inline int32 count_trailing_zeroes64(uint64 x);
inline uint32 bswap32(uint32 x);
inline uint64 bswap64(uint64 x);
inline int32 count_bits32(uint32 x);
inline int32 count_bits64(uint64 x);
inline uint32 bits_negate32(uint32 x) {
return ~x + 1;
}
inline uint64 bits_negate64(uint64 x) {
return ~x + 1;
}
inline uint32 lower_bit32(uint32 x) {
return x & bits_negate32(x);
}
inline uint64 lower_bit64(uint64 x) {
return x & bits_negate64(x);
}
//TODO: optimize
inline int32 count_leading_zeroes_non_zero32(uint32 x) {
DCHECK(x != 0);
return count_leading_zeroes32(x);
}
inline int32 count_leading_zeroes_non_zero64(uint64 x) {
DCHECK(x != 0);
return count_leading_zeroes64(x);
}
inline int32 count_trailing_zeroes_non_zero32(uint32 x) {
DCHECK(x != 0);
return count_trailing_zeroes32(x);
}
inline int32 count_trailing_zeroes_non_zero64(uint64 x) {
DCHECK(x != 0);
return count_trailing_zeroes64(x);
}
//
// Platform specific implementation
//
#if TD_MSVC
inline int32 count_leading_zeroes32(uint32 x) {
@ -84,11 +119,9 @@ inline int32 count_trailing_zeroes64(uint64 x) {
inline uint32 bswap32(uint32 x) {
return _byteswap_ulong(x);
}
inline uint64 bswap64(uint64 x) {
return _byteswap_uint64(x);
}
inline int32 count_bits32(uint32 x) {
return __popcnt(x);
}
@ -214,23 +247,4 @@ inline int32 count_bits64(uint64 x) {
}
#endif
//TODO: optimize
inline int32 count_leading_zeroes_non_zero32(uint32 x) {
DCHECK(x != 0);
return count_leading_zeroes32(x);
}
inline int32 count_leading_zeroes_non_zero64(uint64 x) {
DCHECK(x != 0);
return count_leading_zeroes64(x);
}
inline int32 count_trailing_zeroes_non_zero32(uint32 x) {
DCHECK(x != 0);
return count_trailing_zeroes32(x);
}
inline int32 count_trailing_zeroes_non_zero64(uint64 x) {
DCHECK(x != 0);
return count_trailing_zeroes64(x);
}
} // namespace td

12
tdutils/td/utils/buffer.h
View File

@ -658,6 +658,14 @@ class ChainBufferWriter {
}
return res;
}
MutableSlice prepare_append_at_least(size_t size) {
CHECK(!empty());
auto res = prepare_append_inplace();
if (res.size() < size) {
return prepare_append_alloc(size);
}
return res;
}
MutableSlice prepare_append_inplace() {
CHECK(!empty());
return writer_.prepare_append();
@ -679,9 +687,9 @@ class ChainBufferWriter {
writer_.confirm_append(size);
}
void append(Slice slice) {
void append(Slice slice, size_t hint = 0) {
while (!slice.empty()) {
auto ready = prepare_append(slice.size());
auto ready = prepare_append(td::max(slice.size(), hint));
auto shift = min(ready.size(), slice.size());
std::memcpy(ready.data(), slice.data(), shift);
confirm_append(shift);

7
tdutils/td/utils/common.h
View File

@ -64,6 +64,13 @@
#endif
// clang-format on
#if TD_USE_ASAN
#include <sanitizer/lsan_interface.h>
#define TD_LSAN_IGNORE(x) __lsan_ignore_object(x)
#else
#define TD_LSAN_IGNORE(x) (void)(x)
#endif
namespace td {
inline bool likely(bool x) {

3
tdutils/td/utils/config.h.in
View File

@ -3,3 +3,6 @@
#cmakedefine01 TD_HAVE_OPENSSL
#cmakedefine01 TD_HAVE_ZLIB
#cmakedefine01 TD_HAVE_CRC32C
#cmakedefine01 TD_HAVE_COROUTINES
#cmakedefine01 TD_HAVE_ABSL
#cmakedefine01 TD_HAVE_GETOPT

103
tdutils/td/utils/crypto.cpp
View File

@ -387,7 +387,9 @@ Sha256State &Sha256State::operator=(Sha256State &&from) = default;
Sha256State::~Sha256State() = default;
void sha256_init(Sha256State *state) {
state->impl = make_unique<Sha256StateImpl>();
if (!state->impl) {
state->impl = make_unique<Sha256StateImpl>();
}
int err = SHA256_Init(&state->impl->ctx);
LOG_IF(FATAL, err != 1);
}
@ -398,12 +400,14 @@ void sha256_update(Slice data, Sha256State *state) {
LOG_IF(FATAL, err != 1);
}
void sha256_final(Sha256State *state, MutableSlice output) {
void sha256_final(Sha256State *state, MutableSlice output, bool destroy) {
CHECK(output.size() >= 32);
CHECK(state->impl);
int err = SHA256_Final(output.ubegin(), &state->impl->ctx);
LOG_IF(FATAL, err != 1);
state->impl.reset();
if (destroy) {
state->impl.reset();
}
}
void md5(Slice input, MutableSlice output) {
@ -465,6 +469,14 @@ void hmac_sha256(Slice key, Slice message, MutableSlice dest) {
CHECK(result == dest.ubegin());
CHECK(len == dest.size());
}
void hmac_sha512(Slice key, Slice message, MutableSlice dest) {
CHECK(dest.size() == 512 / 8);
unsigned int len = 0;
auto result = HMAC(EVP_sha512(), key.ubegin(), narrow_cast<int>(key.size()), message.ubegin(),
narrow_cast<int>(message.size()), dest.ubegin(), &len);
CHECK(result == dest.ubegin());
CHECK(len == dest.size());
}
static int get_evp_pkey_type(EVP_PKEY *pkey) {
#if OPENSSL_VERSION_NUMBER < 0x10100000L
@ -639,6 +651,61 @@ uint32 crc32(Slice data) {
uint32 crc32c(Slice data) {
return crc32c::Crc32c(data.data(), data.size());
}
uint32 crc32c_extend(uint32 old_crc, Slice data) {
return crc32c::Extend(old_crc, data.ubegin(), data.size());
}
namespace {
unsigned gf32_matrix_times(uint32 *matrix, uint32 vector) {
unsigned sum = 0;
while (vector) {
if (vector & 1) {
sum ^= *matrix;
}
vector >>= 1;
matrix++;
}
return sum;
}
void gf32_matrix_square(uint32 *square, uint32 *matrix) {
int n = 0;
do {
square[n] = gf32_matrix_times(matrix, matrix[n]);
} while (++n < 32);
}
} // namespace
static uint32 power_buf_raw[1024];
uint32 crc32c_extend(uint32 old_crc, uint32 data_crc, size_t data_size) {
static uint32 *power_buf = [&] {
auto *buf = power_buf_raw;
buf[0] = 0x82F63B78UL;
for (int n = 0; n < 31; n++) {
buf[n + 1] = 1U << n;
}
for (int n = 1; n < 32; n++) {
gf32_matrix_square(buf + (n << 5), buf + ((n - 1) << 5));
}
return buf;
}();
/* degenerate case (also disallow negative lengths) */
if (data_size == 0) {
return old_crc;
}
unsigned int *p = power_buf + 64;
do {
p += 32;
if (data_size % 2 != 0) {
old_crc = gf32_matrix_times(p, old_crc);
}
data_size >>= 1;
} while (data_size != 0);
return old_crc ^ data_crc;
}
#endif
static const uint64 crc64_table[256] = {
@ -707,4 +774,34 @@ uint64 crc64(Slice data) {
return crc64_partial(data, static_cast<uint64>(-1)) ^ static_cast<uint64>(-1);
}
static unsigned short crc16_table[256] = {
0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad,
0xe1ce, 0xf1ef, 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, 0x9339, 0x8318, 0xb37b, 0xa35a,
0xd3bd, 0xc39c, 0xf3ff, 0xe3de, 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, 0xa56a, 0xb54b,
0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861,
0x2802, 0x3823, 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, 0x5af5, 0x4ad4, 0x7ab7, 0x6a96,
0x1a71, 0x0a50, 0x3a33, 0x2a12, 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, 0x6ca6, 0x7c87,
0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a,
0x9f59, 0x8f78, 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f, 0x1080, 0x00a1, 0x30c2, 0x20e3,
0x5004, 0x4025, 0x7046, 0x6067, 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, 0x02b1, 0x1290,
0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e,
0xc71d, 0xd73c, 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, 0xd94c, 0xc96d, 0xf90e, 0xe92f,
0x99c8, 0x89e9, 0xb98a, 0xa9ab, 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, 0xcb7d, 0xdb5c,
0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92,
0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83,
0x1ce0, 0x0cc1, 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, 0x6e17, 0x7e36, 0x4e55, 0x5e74,
0x2e93, 0x3eb2, 0x0ed1, 0x1ef0};
td::uint16 crc16(td::Slice data) {
unsigned crc = 0;
for (std::size_t i = 0; i < data.size(); i++) {
unsigned t = ((unsigned char)data[i] ^ (crc >> 8)) & 0xff;
crc = crc16_table[t] ^ (crc << 8);
}
return (td::uint16)crc;
}
} // namespace td

22
tdutils/td/utils/crypto.h
View File

@ -71,24 +71,35 @@ string sha512(Slice data) TD_WARN_UNUSED_RESULT;
struct Sha256StateImpl;
struct Sha256State;
void sha256_init(Sha256State *state);
void sha256_update(Slice data, Sha256State *state);
void sha256_final(Sha256State *state, MutableSlice output, bool destroy = true);
struct Sha256State {
Sha256State();
Sha256State(Sha256State &&from);
Sha256State &operator=(Sha256State &&from);
~Sha256State();
void init() {
sha256_init(this);
}
void feed(Slice data) {
sha256_update(data, this);
}
void extract(MutableSlice dest) {
sha256_final(this, dest, false);
}
unique_ptr<Sha256StateImpl> impl;
};
void sha256_init(Sha256State *state);
void sha256_update(Slice data, Sha256State *state);
void sha256_final(Sha256State *state, MutableSlice output);
void md5(Slice input, MutableSlice output);
void pbkdf2_sha256(Slice password, Slice salt, int iteration_count, MutableSlice dest);
void pbkdf2_sha512(Slice password, Slice salt, int iteration_count, MutableSlice dest);
void hmac_sha256(Slice key, Slice message, MutableSlice dest);
void hmac_sha512(Slice key, Slice message, MutableSlice dest);
// Interface may be improved
Result<BufferSlice> rsa_encrypt_pkcs1_oaep(Slice public_key, Slice data);
@ -103,8 +114,11 @@ uint32 crc32(Slice data);
#if TD_HAVE_CRC32C
uint32 crc32c(Slice data);
uint32 crc32c_extend(uint32 old_crc, Slice data);
uint32 crc32c_extend(uint32 old_crc, uint32 new_crc, size_t data_size);
#endif
uint64 crc64(Slice data);
uint16 crc16(Slice data);
} // namespace td

4
tdutils/td/utils/filesystem.cpp
View File

@ -83,7 +83,7 @@ Status write_file(CSlice to, Slice data) {
return Status::OK();
}
static std::string clean_filename_part(Slice name, int max_length) {
static string clean_filename_part(Slice name, int max_length) {
auto is_ok = [](uint32 code) {
if (code < 32) {
return false;
@ -137,7 +137,7 @@ static std::string clean_filename_part(Slice name, int max_length) {
return new_name;
}
std::string clean_filename(CSlice name) {
string clean_filename(CSlice name) {
if (!check_utf8(name)) {
return {};
}

2
tdutils/td/utils/filesystem.h
View File

@ -19,6 +19,6 @@ Status copy_file(CSlice from, CSlice to, int64 size = -1) TD_WARN_UNUSED_RESULT;
Status write_file(CSlice to, Slice data) TD_WARN_UNUSED_RESULT;
std::string clean_filename(CSlice name);
string clean_filename(CSlice name);
} // namespace td

5
tdutils/td/utils/invoke.h
View File

@ -202,4 +202,9 @@ auto call_n_arguments(F &&f, Args &&... args) {
return detail::call_n_arguments_impl(detail::IntRange<N>(), f, std::forward<Args>(args)...);
}
template <class F, class X, class = void>
struct is_callable : public std::false_type {};
template <class F, class X>
struct is_callable<F, X, decltype(std::declval<F>()(std::declval<X>()))> : public std::true_type {};
} // namespace td

6
tdutils/td/utils/logging.cpp
View File

@ -225,7 +225,11 @@ class DefaultLog : public LogInterface {
color = Slice(TC_CYAN);
break;
}
TsCerr() << color << slice << TC_EMPTY;
if (!slice.empty() && slice.back() == '\n') {
TsCerr() << color << slice.substr(0, slice.size() - 1) << TC_EMPTY << "\n";
} else {
TsCerr() << color << slice << TC_EMPTY;
}
#else
// TODO: color
TsCerr() << slice;

11
tdutils/td/utils/misc.cpp
View File

@ -169,6 +169,17 @@ bool is_zero_or_one(unsigned char c) {
} // namespace
string buffer_to_hex(Slice buffer) {
const char *hex = "0123456789ABCDEF";
std::string res(2 * buffer.size(), '\0');
for (std::size_t i = 0; i < buffer.size(); i++) {
auto c = buffer.ubegin()[i];
res[2 * i] = hex[c & 15];
res[2 * i + 1] = hex[c >> 4];
}
return res;
}
std::string zero_encode(Slice data) {
return x_encode(data, is_zero);
}

7
tdutils/td/utils/misc.h
View File

@ -16,6 +16,7 @@
#include <limits>
#include <type_traits>
#include <utility>
#include <string>
namespace td {
@ -309,8 +310,8 @@ string url_encode(Slice str);
namespace detail {
template <class T, class U>
struct is_same_signedness
: public std::integral_constant<bool, std::is_signed<T>::value == std::is_signed<U>::value> {};
struct is_same_signedness : public std::integral_constant<bool, std::is_signed<T>::value == std::is_signed<U>::value> {
};
template <class T, class Enable = void>
struct safe_undeflying_type {
@ -397,6 +398,8 @@ detail::reversion_wrapper<T> reversed(T &iterable) {
return {iterable};
}
string buffer_to_hex(Slice buffer);
string zero_encode(Slice data);
string zero_decode(Slice data);

11
tdutils/td/utils/optional.h
View File

@ -54,6 +54,17 @@ class optional {
T &operator*() {
return value();
}
T unwrap() {
CHECK(*this);
auto res = std::move(value());
impl_ = {};
return res;
}
template <class... ArgsT>
void emplace(ArgsT &&... args) {
impl_.emplace(std::forward<ArgsT>(args)...);
}
private:
Result<T> impl_;

57
tdutils/td/utils/port/FileFd.cpp
View File

@ -41,8 +41,8 @@ struct PrintFlags {
StringBuilder &operator<<(StringBuilder &sb, const PrintFlags &print_flags) {
auto flags = print_flags.flags;
if (flags &
~(FileFd::Write | FileFd::Read | FileFd::Truncate | FileFd::Create | FileFd::Append | FileFd::CreateNew)) {
if (flags & ~(FileFd::Write | FileFd::Read | FileFd::Truncate | FileFd::Create | FileFd::Append | FileFd::CreateNew |
FileFd::Direct | FileFd::WinStat)) {
return sb << "opened with invalid flags " << flags;
}
@ -75,6 +75,12 @@ StringBuilder &operator<<(StringBuilder &sb, const PrintFlags &print_flags) {
if (flags & FileFd::Truncate) {
sb << " with truncation";
}
if (flags & FileFd::Direct) {
sb << " for direct io";
}
if (flags & FileFd::WinStat) {
sb << " for stat";
}
return sb;
}
@ -96,7 +102,7 @@ FileFd::FileFd(unique_ptr<detail::FileFdImpl> impl) : impl_(std::move(impl)) {
}
Result<FileFd> FileFd::open(CSlice filepath, int32 flags, int32 mode) {
if (flags & ~(Write | Read | Truncate | Create | Append | CreateNew)) {
if (flags & ~(Write | Read | Truncate | Create | Append | CreateNew | Direct | WinStat)) {
return Status::Error(PSLICE() << "File \"" << filepath << "\" has failed to be " << PrintFlags{flags});
}
@ -131,6 +137,12 @@ Result<FileFd> FileFd::open(CSlice filepath, int32 flags, int32 mode) {
native_flags |= O_APPEND;
}
if (flags & Direct) {
#if TD_LINUX
LOG(ERROR) << "DIRECT";
native_flags |= O_DIRECT;
#endif
}
int native_fd = detail::skip_eintr([&] { return ::open(filepath.c_str(), native_flags, static_cast<mode_t>(mode)); });
if (native_fd < 0) {
return OS_ERROR(PSLICE() << "File \"" << filepath << "\" can't be " << PrintFlags{flags});
@ -173,10 +185,21 @@ Result<FileFd> FileFd::open(CSlice filepath, int32 flags, int32 mode) {
}
}
DWORD native_flags = 0;
if (flags & Direct) {
native_flags |= FILE_FLAG_WRITE_THROUGH | FILE_FLAG_NO_BUFFERING;
}
if (flags & WinStat) {
native_flags |= FILE_FLAG_BACKUP_SEMANTICS;
}
#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM)
auto handle = CreateFile(w_filepath.c_str(), desired_access, share_mode, nullptr, creation_disposition, 0, nullptr);
auto handle =
CreateFile(w_filepath.c_str(), desired_access, share_mode, nullptr, creation_disposition, native_flags, nullptr);
#else
auto handle = CreateFile2(w_filepath.c_str(), desired_access, share_mode, creation_disposition, nullptr);
CREATEFILE2_EXTENDED_PARAMETERS extended_parameters;
memset(&extended_parameters, 0, sizeof(extended_parameters));
extended_parameters.dwFileFlags = native_flags;
auto handle = CreateFile2(w_filepath.c_str(), desired_access, share_mode, creation_disposition, &extended_parameters);
#endif
if (handle == INVALID_HANDLE_VALUE) {
return OS_ERROR(PSLICE() << "File \"" << filepath << "\" can't be " << PrintFlags{flags});
@ -216,6 +239,26 @@ Result<size_t> FileFd::write(Slice slice) {
return OS_ERROR(PSLICE() << "Write to " << get_native_fd() << " has failed");
}
Result<size_t> FileFd::writev(Span<IoSlice> slices) {
#if TD_PORT_POSIX
auto native_fd = get_native_fd().fd();
TRY_RESULT(slices_size, narrow_cast_safe<int>(slices.size()));
auto bytes_written = detail::skip_eintr([&] { return ::writev(native_fd, slices.begin(), slices_size); });
bool success = bytes_written >= 0;
if (success) {
return narrow_cast<size_t>(bytes_written);
}
return OS_ERROR(PSLICE() << "Writev to " << get_native_fd() << " has failed");
#else
size_t res = 0;
for (auto slice : slices) {
TRY_RESULT(size, write(slice));
res += size;
}
return res;
#endif
}
Result<size_t> FileFd::read(MutableSlice slice) {
auto native_fd = get_native_fd().fd();
#if TD_PORT_POSIX
@ -424,7 +467,7 @@ NativeFd FileFd::move_as_native_fd() {
return res;
}
Result<int64> FileFd::get_size() {
Result<int64> FileFd::get_size() const {
TRY_RESULT(s, stat());
return s.size_;
}
@ -436,7 +479,7 @@ static uint64 filetime_to_unix_time_nsec(LONGLONG filetime) {
}
#endif
Result<Stat> FileFd::stat() {
Result<Stat> FileFd::stat() const {
CHECK(!empty());
#if TD_PORT_POSIX
return detail::fstat(get_native_fd().fd());

10
tdutils/td/utils/port/FileFd.h
View File

@ -12,7 +12,9 @@
#include "td/utils/port/detail/NativeFd.h"
#include "td/utils/port/detail/PollableFd.h"
#include "td/utils/port/Stat.h"
#include "td/utils/port/IoSlice.h"
#include "td/utils/Slice.h"
#include "td/utils/Span.h"
#include "td/utils/Status.h"
namespace td {
@ -29,12 +31,14 @@ class FileFd {
FileFd(const FileFd &) = delete;
FileFd &operator=(const FileFd &) = delete;
enum Flags : int32 { Write = 1, Read = 2, Truncate = 4, Create = 8, Append = 16, CreateNew = 32 };
enum Flags : int32 { Write = 1, Read = 2, Truncate = 4, Create = 8, Append = 16, CreateNew = 32, Direct = 64 };
enum PrivateFlags : int32 { WinStat = 128 };
static Result<FileFd> open(CSlice filepath, int32 flags, int32 mode = 0600) TD_WARN_UNUSED_RESULT;
static FileFd from_native_fd(NativeFd fd) TD_WARN_UNUSED_RESULT;
Result<size_t> write(Slice slice) TD_WARN_UNUSED_RESULT;
Result<size_t> writev(Span<IoSlice> slices) TD_WARN_UNUSED_RESULT;
Result<size_t> read(MutableSlice slice) TD_WARN_UNUSED_RESULT;
Result<size_t> pwrite(Slice slice, int64 offset) TD_WARN_UNUSED_RESULT;
@ -49,9 +53,9 @@ class FileFd {
void close();
bool empty() const;
Result<int64> get_size();
Result<int64> get_size() const;
Result<Stat> stat();
Result<Stat> stat() const;
Status sync() TD_WARN_UNUSED_RESULT;

9
tdutils/td/utils/port/IPAddress.cpp
View File

@ -401,7 +401,7 @@ Status IPAddress::init_host_port(CSlice host, CSlice port, bool prefer_ipv6) {
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
LOG(INFO) << "Try to init IP address of " << host << " with port " << port;
LOG(DEBUG + 10) << "Try to init IP address of " << host << " with port " << port;
auto err = getaddrinfo(host.c_str(), port.c_str(), &hints, &info);
if (err != 0) {
#if TD_WINDOWS
@ -466,7 +466,7 @@ Status IPAddress::init_sockaddr(sockaddr *addr, socklen_t len) {
}
is_valid_ = true;
LOG(INFO) << "Have address " << get_ip_str() << " with port " << get_port();
LOG(DEBUG + 10) << "Have address " << get_ip_str() << " with port " << get_port();
return Status::OK();
}
@ -499,6 +499,11 @@ CSlice IPAddress::ipv4_to_str(uint32 ipv4) {
return ::td::get_ip_str(AF_INET, &ipv4);
}
CSlice IPAddress::ipv6_to_str(Slice ipv6) {
CHECK(ipv6.size() == 16);
return ::td::get_ip_str(AF_INET6, ipv6.ubegin());
}
Slice IPAddress::get_ip_str() const {
if (!is_valid()) {
return Slice("0.0.0.0");

1
tdutils/td/utils/port/IPAddress.h
View File

@ -64,6 +64,7 @@ class IPAddress {
size_t get_sockaddr_len() const;
int get_address_family() const;
static CSlice ipv4_to_str(uint32 ipv4);
static CSlice ipv6_to_str(Slice ipv6);
Status init_sockaddr(sockaddr *addr);
Status init_sockaddr(sockaddr *addr, socklen_t len) TD_WARN_UNUSED_RESULT;

10
tdutils/td/utils/port/ServerSocketFd.cpp
View File

@ -109,7 +109,15 @@ class ServerSocketFdImpl : private Iocp::Callback {
VLOG(fd) << get_native_fd() << " on_read";
if (is_read_active_) {
is_read_active_ = false;
auto r_socket = SocketFd::from_native_fd(std::move(accept_socket_));
auto r_socket = [&]() -> Result<SocketFd> {
auto from = get_native_fd().socket();
auto status = setsockopt(accept_socket_.socket(), SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT,
reinterpret_cast<const char *>(&from), sizeof(from));
if (status != 0) {
return OS_SOCKET_ERROR("Failed to set SO_UPDATE_ACCEPT_CONTEXT options");
}
return SocketFd::from_native_fd(std::move(accept_socket_));
}();
VLOG(fd) << get_native_fd() << " finish accept";
if (r_socket.is_error()) {
return on_error(r_socket.move_as_error());

59
tdutils/td/utils/port/SocketFd.cpp
View File

@ -88,12 +88,33 @@ class SocketFdImpl : private Iocp::Callback {
Result<size_t> write(Slice data) {
// LOG(ERROR) << "Write: " << format::as_hex_dump<0>(data);
output_writer_.append(data);
return write_finish(data.size());
}
Result<size_t> writev(Span<IoSlice> slices) {
size_t total_size = 0;
for (auto io_slice : slices) {
total_size += as_slice(io_slice).size();
}
auto left_size = total_size;
for (auto io_slice : slices) {
auto slice = as_slice(io_slice);
output_writer_.append(slice, left_size);
left_size -= slice.size();
}
return write_finish(total_size);
}
Result<size_t> write_finish(size_t total_size) {
if (is_write_waiting_) {
auto lock = lock_.lock();
is_write_waiting_ = false;
lock.reset();
notify_iocp_write();
}
return data.size();
return total_size;
}
Result<size_t> read(MutableSlice slice) {
@ -182,6 +203,7 @@ class SocketFdImpl : private Iocp::Callback {
auto to_write = output_reader_.prepare_read();
if (to_write.empty()) {
auto lock = lock_.lock();
output_reader_.sync_with_writer();
to_write = output_reader_.prepare_read();
if (to_write.empty()) {
is_write_waiting_ = true;
@ -191,12 +213,22 @@ class SocketFdImpl : private Iocp::Callback {
if (to_write.empty()) {
return;
}
auto dest = output_reader_.prepare_read();
std::memset(&write_overlapped_, 0, sizeof(write_overlapped_));
WSABUF buf;
buf.len = narrow_cast<ULONG>(dest.size());
buf.buf = const_cast<CHAR *>(dest.data());
int status = WSASend(get_native_fd().socket(), &buf, 1, nullptr, 0, &write_overlapped_, nullptr);
constexpr size_t buf_size = 20;
WSABUF buf[buf_size];
auto it = output_reader_.clone();
size_t buf_i;
for (buf_i = 0; buf_i < buf_size; buf_i++) {
auto src = it.prepare_read();
if (src.empty()) {
break;
}
buf[buf_i].len = narrow_cast<ULONG>(src.size());
buf[buf_i].buf = const_cast<CHAR *>(src.data());
it.confirm_read(src.size());
}
int status =
WSASend(get_native_fd().socket(), buf, narrow_cast<DWORD>(buf_i), nullptr, 0, &write_overlapped_, nullptr);
if (status == 0 || check_status("Failed to write to connection")) {
inc_refcnt();
is_write_active_ = true;
@ -277,7 +309,7 @@ class SocketFdImpl : private Iocp::Callback {
}
CHECK(is_write_active_);
is_write_active_ = false;
output_reader_.confirm_read(size);
output_reader_.advance(size);
loop_write();
}
@ -348,9 +380,18 @@ class SocketFdImpl {
const NativeFd &get_native_fd() const {
return info.native_fd();
}
Result<size_t> writev(Span<IoSlice> slices) {
int native_fd = get_native_fd().socket();
auto write_res =
detail::skip_eintr([&] { return ::writev(native_fd, slices.begin(), narrow_cast<int>(slices.size())); });
return write_finish(write_res);
}
Result<size_t> write(Slice slice) {
int native_fd = get_native_fd().socket();
auto write_res = detail::skip_eintr([&] { return ::write(native_fd, slice.begin(), slice.size()); });
return write_finish(write_res);
}
Result<size_t> write_finish(ssize_t write_res) {
auto write_errno = errno;
if (write_res >= 0) {
return narrow_cast<size_t>(write_res);
@ -567,6 +608,10 @@ Result<size_t> SocketFd::write(Slice slice) {
return impl_->write(slice);
}
Result<size_t> SocketFd::writev(Span<IoSlice> slices) {
return impl_->writev(slices);
}
Result<size_t> SocketFd::read(MutableSlice slice) {
return impl_->read(slice);
}

3
tdutils/td/utils/port/SocketFd.h
View File

@ -11,7 +11,9 @@
#include "td/utils/port/detail/NativeFd.h"
#include "td/utils/port/detail/PollableFd.h"
#include "td/utils/port/IPAddress.h"
#include "td/utils/port/IoSlice.h"
#include "td/utils/Slice.h"
#include "td/utils/Span.h"
#include "td/utils/Status.h"
#include <memory>
@ -43,6 +45,7 @@ class SocketFd {
Status get_pending_error() TD_WARN_UNUSED_RESULT;
Result<size_t> write(Slice slice) TD_WARN_UNUSED_RESULT;
Result<size_t> writev(Span<IoSlice> slices) TD_WARN_UNUSED_RESULT;
Result<size_t> read(MutableSlice slice) TD_WARN_UNUSED_RESULT;
const NativeFd &get_native_fd() const;

2
tdutils/td/utils/port/Stat.cpp
View File

@ -349,7 +349,7 @@ Result<CpuStat> cpu_stat() {
namespace td {
Result<Stat> stat(CSlice path) {
TRY_RESULT(fd, FileFd::open(path, FileFd::Flags::Read));
TRY_RESULT(fd, FileFd::open(path, FileFd::Flags::Read | FileFd::PrivateFlags::WinStat));
return fd.stat();
}

35
tdutils/td/utils/port/StdStreams.cpp
View File

@ -9,6 +9,7 @@
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include "td/utils/port/detail/NativeFd.h"
#include "td/utils/port/detail/Iocp.h"
#include "td/utils/port/PollFlags.h"
#include "td/utils/port/thread.h"
#include "td/utils/Slice.h"
@ -21,6 +22,7 @@ namespace td {
template <int id>
static FileFd &get_file_fd() {
static FileFd result = FileFd::from_native_fd(NativeFd(id, true));
static auto guard = td::ScopeExit() + [&] { result.move_as_native_fd().release(); };
return result;
}
@ -40,6 +42,7 @@ static FileFd &get_file_fd() {
static auto handle = GetStdHandle(id);
LOG_IF(FATAL, handle == INVALID_HANDLE_VALUE) << "Failed to GetStdHandle " << id;
static FileFd result = FileFd::from_native_fd(NativeFd(handle, true));
static auto guard = td::ScopeExit() + [&] { result.move_as_native_fd().release(); };
#else
static FileFd result;
#endif
@ -59,10 +62,11 @@ FileFd &Stderr() {
#if TD_PORT_WINDOWS
namespace detail {
class BufferedStdinImpl {
class BufferedStdinImpl : public Iocp::Callback {
public:
#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM)
BufferedStdinImpl() : info_(NativeFd(GetStdHandle(STD_INPUT_HANDLE), true)) {
iocp_ref_ = Iocp::get()->get_ref();
read_thread_ = td::thread([this] { this->read_loop(); });
}
#else
@ -105,6 +109,8 @@ class BufferedStdinImpl {
ChainBufferReader reader_ = writer_.extract_reader();
td::thread read_thread_;
std::atomic<bool> close_flag_{false};
IocpRef iocp_ref_;
std::atomic<int> refcnt_{1};
void read_loop() {
while (!close_flag_) {
@ -115,9 +121,31 @@ class BufferedStdinImpl {
break;
}
writer_.confirm_append(r_size.ok());
info_.add_flags_from_poll(td::PollFlags::Read());
if (iocp_ref_.post(0, this, nullptr)) {
inc_refcnt();
}
}
//TODO delete
LOG(ERROR) << "Close";
if (!iocp_ref_.post(0, this, nullptr)) {
dec_refcnt();
}
}
void on_iocp(Result<size_t> r_size, WSAOVERLAPPED *overlapped) override {
info_.add_flags_from_poll(td::PollFlags::Read());
dec_refcnt();
}
bool dec_refcnt() {
if (--refcnt_ == 0) {
delete this;
LOG(ERROR) << "Delete this";
return true;
}
return false;
}
void inc_refcnt() {
CHECK(refcnt_ != 0);
refcnt_++;
}
Result<size_t> read(MutableSlice slice) {
@ -131,6 +159,7 @@ class BufferedStdinImpl {
}
};
void BufferedStdinImplDeleter::operator()(BufferedStdinImpl *impl) {
// LOG(ERROR) << "Close";
impl->close();
}
} // namespace detail

74
tdutils/td/utils/port/UdpSocketFd.cpp
View File

@ -784,6 +784,80 @@ bool UdpSocketFd::empty() const {
const NativeFd &UdpSocketFd::get_native_fd() const {
return get_poll_info().native_fd();
}
#if TD_PORT_POSIX
td::Result<td::uint32> UdpSocketFd::maximize_snd_buffer(td::uint32 max) {
socklen_t intsize = sizeof(td::uint32);
td::uint32 last_good = 0;
td::uint32 min, avg;
td::uint32 old_size;
auto socket_fd = get_native_fd().fd();
if (!max) {
max = default_udp_max_snd_buffer_size;
}
/* Start with the default size. */
if (getsockopt(socket_fd, SOL_SOCKET, SO_SNDBUF, &old_size, &intsize)) {
return td::Status::PosixError(errno, "getsockopt() failed");
}
/* Binary-search for the real maximum. */
min = last_good = old_size;
while (min <= max) {
avg = (min + max) / 2;
if (setsockopt(socket_fd, SOL_SOCKET, SO_SNDBUF, &avg, intsize) == 0) {
last_good = avg;
min = avg + 1;
} else {
max = avg - 1;
}
}
return last_good;
}
td::Result<td::uint32> UdpSocketFd::maximize_rcv_buffer(td::uint32 max) {
socklen_t intsize = sizeof(td::uint32);
td::uint32 last_good = 0;
td::uint32 min, avg;
td::uint32 old_size;
auto socket_fd = get_native_fd().fd();
if (!max) {
max = default_udp_max_rcv_buffer_size;
}
/* Start with the default size. */
if (getsockopt(socket_fd, SOL_SOCKET, SO_RCVBUF, &old_size, &intsize)) {
return td::Status::PosixError(errno, "getsockopt() failed");
}
/* Binary-search for the real maximum. */
min = last_good = old_size;
while (min <= max) {
avg = (min + max) / 2;
if (setsockopt(socket_fd, SOL_SOCKET, SO_RCVBUF, &avg, intsize) == 0) {
last_good = avg;
min = avg + 1;
} else {
max = avg - 1;
}
}
return last_good;
}
#else
td::Result<td::uint32> UdpSocketFd::maximize_snd_buffer(td::uint32 max) {
return 0;
}
td::Result<td::uint32> UdpSocketFd::maximize_rcv_buffer(td::uint32 max) {
return 0;
}
#endif
#if TD_PORT_POSIX
Status UdpSocketFd::send_message(const OutboundMessage &message, bool &is_sent) {
return impl_->send_message(message, is_sent);

5
tdutils/td/utils/port/UdpSocketFd.h
View File

@ -45,6 +45,9 @@ class UdpSocketFd {
UdpSocketFd(const UdpSocketFd &) = delete;
UdpSocketFd &operator=(const UdpSocketFd &) = delete;
td::Result<td::uint32> maximize_snd_buffer(td::uint32 max_buffer_size = 0);
td::Result<td::uint32> maximize_rcv_buffer(td::uint32 max_buffer_size = 0);
static Result<UdpSocketFd> open(const IPAddress &address) TD_WARN_UNUSED_RESULT;
PollableFdInfo &get_poll_info();
@ -81,6 +84,8 @@ class UdpSocketFd {
#endif
private:
static constexpr td::uint32 default_udp_max_snd_buffer_size = (1 << 24);
static constexpr td::uint32 default_udp_max_rcv_buffer_size = (1 << 24);
std::unique_ptr<detail::UdpSocketFdImpl, detail::UdpSocketFdImplDeleter> impl_;
explicit UdpSocketFd(unique_ptr<detail::UdpSocketFdImpl> impl);
};

2
tdutils/td/utils/port/config.h
View File

@ -40,7 +40,7 @@
#if TD_EMSCRIPTEN
#define TD_THREAD_UNSUPPORTED 1
#elif TD_TIZEN
#elif TD_TIZEN || TD_LINUX || TD_DARWIN
#define TD_THREAD_PTHREAD 1
#else
#define TD_THREAD_STL 1

36
tdutils/td/utils/port/detail/Iocp.cpp
View File

@ -25,7 +25,7 @@ void Iocp::loop() {
ULONG_PTR key = 0;
WSAOVERLAPPED *overlapped = nullptr;
BOOL ok =
GetQueuedCompletionStatus(iocp_handle_.fd(), &bytes, &key, reinterpret_cast<OVERLAPPED **>(&overlapped), 1000);
GetQueuedCompletionStatus(iocp_handle_->fd(), &bytes, &key, reinterpret_cast<OVERLAPPED **>(&overlapped), 1000);
if (bytes || key || overlapped) {
// LOG(ERROR) << "Got IOCP " << bytes << " " << key << " " << overlapped;
}
@ -58,31 +58,53 @@ void Iocp::init() {
auto error = OS_ERROR("IOCP creation failed");
LOG(FATAL) << error;
}
iocp_handle_ = NativeFd(res);
iocp_handle_ = std::make_shared<NativeFd>(res);
}
void Iocp::clear() {
iocp_handle_.close();
iocp_handle_.reset();
}
void Iocp::subscribe(const NativeFd &native_fd, Callback *callback) {
CHECK(iocp_handle_);
auto iocp_handle =
CreateIoCompletionPort(native_fd.fd(), iocp_handle_.fd(), reinterpret_cast<ULONG_PTR>(callback), 0);
CreateIoCompletionPort(native_fd.fd(), iocp_handle_->fd(), reinterpret_cast<ULONG_PTR>(callback), 0);
if (iocp_handle == nullptr) {
auto error = OS_ERROR("CreateIoCompletionPort");
LOG(FATAL) << error;
}
LOG_CHECK(iocp_handle == iocp_handle_.fd()) << iocp_handle << " " << iocp_handle_.fd();
LOG_CHECK(iocp_handle == iocp_handle_->fd()) << iocp_handle << " " << iocp_handle_->fd();
}
void Iocp::post(size_t size, Callback *callback, WSAOVERLAPPED *overlapped) {
if (PostQueuedCompletionStatus(iocp_handle_.fd(), DWORD(size), reinterpret_cast<ULONG_PTR>(callback),
IocpRef Iocp::get_ref() const {
return IocpRef(iocp_handle_);
}
namespace {
void iocp_post(NativeFd &iocp_handle, size_t size, Iocp::Callback *callback, WSAOVERLAPPED *overlapped) {
if (PostQueuedCompletionStatus(iocp_handle.fd(), DWORD(size), reinterpret_cast<ULONG_PTR>(callback),
reinterpret_cast<OVERLAPPED *>(overlapped)) == 0) {
auto error = OS_ERROR("IOCP post failed");
LOG(FATAL) << error;
}
}
} // namespace
void Iocp::post(size_t size, Callback *callback, WSAOVERLAPPED *overlapped) {
iocp_post(*iocp_handle_, size, callback, overlapped);
}
IocpRef::IocpRef(std::weak_ptr<NativeFd> iocp_handle) : iocp_handle_(std::move(iocp_handle)) {
}
bool IocpRef::post(size_t size, Iocp::Callback *callback, WSAOVERLAPPED *overlapped) {
auto iocp_handle = iocp_handle_.lock();
if (!iocp_handle) {
return false;
}
iocp_post(*iocp_handle, size, callback, overlapped);
return true;
}
} // namespace detail
} // namespace td

22
tdutils/td/utils/port/detail/Iocp.h
View File

@ -19,6 +19,7 @@
namespace td {
namespace detail {
class IocpRef;
class Iocp final : public Context<Iocp> {
public:
Iocp() = default;
@ -41,12 +42,27 @@ class Iocp final : public Context<Iocp> {
void interrupt_loop();
void clear();
IocpRef get_ref() const;
private:
NativeFd iocp_handle_;
std::vector<td::thread> workers_;
std::shared_ptr<NativeFd> iocp_handle_;
};
} // namespace detail
class IocpRef {
public:
IocpRef() = default;
IocpRef(const Iocp &) = delete;
IocpRef &operator=(const Iocp &) = delete;
IocpRef(IocpRef &&) = default;
IocpRef &operator=(IocpRef &&) = default;
IocpRef(std::weak_ptr<NativeFd> iocp_handle);
bool post(size_t size, Iocp::Callback *callback, WSAOVERLAPPED *overlapped);
private:
std::weak_ptr<NativeFd> iocp_handle_;
};
} // namespace detail
} // namespace td
#endif

8
tdutils/td/utils/port/detail/ThreadPthread.h
View File

@ -16,6 +16,7 @@
#include "td/utils/MovableValue.h"
#include "td/utils/port/detail/ThreadIdGuard.h"
#include "td/utils/port/thread_local.h"
#include "td/utils/Slice.h"
#include <tuple>
#include <type_traits>
@ -48,6 +49,13 @@ class ThreadPthread {
pthread_create(&thread_, nullptr, run_thread, func.release());
is_inited_ = true;
}
void set_name(CSlice name) {
#if defined(_GNU_SOURCE) && defined(__GLIBC_PREREQ)
#if __GLIBC_PREREQ(2, 12)
pthread_setname_np(thread_, name.c_str());
#endif
#endif
}
void join() {
if (is_inited_.get()) {
is_inited_ = false;

3
tdutils/td/utils/port/detail/ThreadStl.h
View File

@ -14,6 +14,7 @@
#include "td/utils/invoke.h"
#include "td/utils/port/detail/ThreadIdGuard.h"
#include "td/utils/port/thread_local.h"
#include "td/utils/Slice.h"
#include <thread>
#include <tuple>
@ -46,6 +47,8 @@ class ThreadStl {
void detach() {
thread_.detach();
}
void set_name(CSlice name) {
}
static unsigned hardware_concurrency() {
return std::thread::hardware_concurrency();

4
tdutils/td/utils/port/platform.h
View File

@ -105,4 +105,8 @@
#define TD_CONCURRENCY_PAD 128
#if !TD_WINDOWS && defined(__SIZEOF_INT128__)
#define TD_HAVE_INT128 1
#endif
// clang-format on

26
tdutils/td/utils/port/signals.cpp
View File

@ -7,6 +7,8 @@
#include "td/utils/port/signals.h"
#include "td/utils/port/config.h"
#include "td/utils/port/stacktrace.h"
#include "td/utils/port/StdStreams.h"
#include "td/utils/common.h"
#include "td/utils/format.h"
@ -296,4 +298,28 @@ void signal_safe_write_pointer(void *p, bool add_header) {
signal_safe_write(Slice(ptr, end), add_header);
}
static void unblock_stdin() {
#if TD_PORT_POSIX
td::Stdin().get_native_fd().set_is_blocking(true).ignore();
#endif
}
void default_failure_signal_hanler(int sig) {
td::signal_safe_write_signal_number(sig, true);
td::Stacktrace::PrintOptions options;
options.use_gdb = true;
td::Stacktrace::print_to_stderr(options);
unblock_stdin();
_Exit(EXIT_FAILURE);
}
Status set_default_failure_signal_handler() {
atexit(unblock_stdin);
TRY_STATUS(setup_signals_alt_stack());
TRY_STATUS(set_signal_handler(td::SignalType::Abort, default_failure_signal_hanler));
TRY_STATUS(td::set_signal_handler(td::SignalType::Error, default_failure_signal_hanler));
return Status::OK();
}
} // namespace td

2
tdutils/td/utils/port/signals.h
View File

@ -31,4 +31,6 @@ void signal_safe_write_signal_number(int sig, bool add_header = true);
void signal_safe_write_pointer(void *p, bool add_header = true);
Status set_default_failure_signal_handler();
} // namespace td

2
tdutils/td/utils/tests.cpp
View File

@ -6,6 +6,7 @@
//
#include "td/utils/tests.h"
#include "td/utils/base64.h"
#include "td/utils/crypto.h"
#include "td/utils/filesystem.h"
#include "td/utils/Parser.h"
@ -227,5 +228,4 @@ Status TestsRunner::verify(Slice data) {
}
return regression_tester_->verify_test(PSLICE() << name() << "_default", data);
}
} // namespace td

5
tdutils/td/utils/tests.h
View File

@ -12,6 +12,7 @@
#include "td/utils/logging.h"
#include "td/utils/port/thread.h"
#include "td/utils/Random.h"
#include "td/utils/optional.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
@ -118,7 +119,7 @@ inline string rand_string(char from, char to, int len) {
return res;
}
inline vector<string> rand_split(string str) {
inline vector<string> rand_split(Slice str) {
vector<string> res;
size_t pos = 0;
while (pos < str.size()) {
@ -128,7 +129,7 @@ inline vector<string> rand_split(string str) {
} else {
len = Random::fast(100, 200);
}
res.push_back(str.substr(pos, len));
res.push_back(str.substr(pos, len).str());
pos += len;
}
return res;

4
tdutils/test/HazardPointers.cpp
View File

@ -26,12 +26,12 @@ TEST(HazardPointers, stress) {
int thread_id = 0;
for (auto &thread : threads) {
thread = td::thread([&, thread_id] {
auto holder = hazard_pointers.get_holder(thread_id, 0);
std::remove_reference_t<decltype(hazard_pointers)>::Holder holder(hazard_pointers, thread_id, 0);
for (int i = 0; i < 1000000; i++) {
auto &node = nodes[td::Random::fast(0, threads_n - 1)];
auto *str = holder.protect(node.name_);
if (str) {
CHECK(*str == "one" || *str == "twotwo");
CHECK(*str == td::Slice("one") || *str == td::Slice("twotwo"));
}
holder.clear();
if (td::Random::fast(0, 5) == 0) {

61
tdutils/test/crypto.cpp
View File

@ -5,6 +5,7 @@
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/base64.h"
#include "td/utils/benchmark.h"
#include "td/utils/common.h"
#include "td/utils/crypto.h"
#include "td/utils/Slice.h"
@ -164,8 +165,60 @@ TEST(Crypto, crc32c) {
for (std::size_t i = 0; i < strings.size(); i++) {
ASSERT_EQ(answers[i], td::crc32c(strings[i]));
auto v = td::rand_split(strings[i]);
td::uint32 a = 0;
td::uint32 b = 0;
for (auto &x : v) {
a = td::crc32c_extend(a, x);
auto x_crc = td::crc32c(x);
b = td::crc32c_extend(b, x_crc, x.size());
}
ASSERT_EQ(answers[i], a);
ASSERT_EQ(answers[i], b);
}
}
TEST(Crypto, crc32c_benchmark) {
class Crc32cExtendBenchmark : public td::Benchmark {
public:
Crc32cExtendBenchmark(size_t chunk_size) : chunk_size_(chunk_size) {
}
std::string get_description() const override {
return PSTRING() << "Crc32c with chunk_size=" << chunk_size_;
}
void start_up_n(int n) override {
if (n > (1 << 20)) {
cnt_ = n / (1 << 20);
n = (1 << 20);
} else {
cnt_ = 1;
}
data_ = std::string(n, 'a');
}
void run(int n) override {
td::uint32 res = 0;
for (int i = 0; i < cnt_; i++) {
td::Slice data(data_);
while (!data.empty()) {
auto head = data.substr(0, chunk_size_);
data = data.substr(head.size());
res = td::crc32c_extend(res, head);
}
}
td::do_not_optimize_away(res);
}
private:
size_t chunk_size_;
std::string data_;
int cnt_;
};
bench(Crc32cExtendBenchmark(2));
bench(Crc32cExtendBenchmark(8));
bench(Crc32cExtendBenchmark(32));
bench(Crc32cExtendBenchmark(128));
bench(Crc32cExtendBenchmark(65536));
}
#endif
TEST(Crypto, crc64) {
@ -176,6 +229,14 @@ TEST(Crypto, crc64) {
}
}
TEST(Crypto, crc16) {
td::vector<td::uint16> answers{0, 9842, 25046, 37023};
for (std::size_t i = 0; i < strings.size(); i++) {
ASSERT_EQ(answers[i], td::crc16(strings[i]));
}
}
static td::Slice rsa_private_key = R"ABCD(
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDeYT5/prmLEa2Q

245
tdutils/test/misc.cpp
View File

@ -6,10 +6,14 @@
//
#include "td/utils/as.h"
#include "td/utils/base64.h"
#include "td/utils/bits.h"
#include "td/utils/BigNum.h"
#include "td/utils/bits.h"
#include "td/utils/CancellationToken.h"
#include "td/utils/common.h"
#include "td/utils/Hash.h"
#include "td/utils/HashMap.h"
#include "td/utils/HashSet.h"
#include "td/utils/HttpUrl.h"
#include "td/utils/invoke.h"
#include "td/utils/logging.h"
@ -27,6 +31,8 @@
#include "td/utils/StringBuilder.h"
#include "td/utils/tests.h"
#include "td/utils/translit.h"
#include "td/utils/Time.h"
#include "td/utils/uint128.h"
#include "td/utils/unicode.h"
#include "td/utils/utf8.h"
@ -35,6 +41,11 @@
#include <limits>
#include <locale>
#include <utility>
#include <unordered_map>
#if TD_HAVE_ABSL
#include <absl/container/flat_hash_map.h>
#endif
using namespace td;
@ -596,8 +607,6 @@ TEST(Misc, As) {
ASSERT_EQ(123, as<int>((const char *)buf));
ASSERT_EQ(123, as<int>((char *)buf));
char buf2[100];
//auto x = as<int>(buf2);
//x = 44342; //CE
as<int>(buf2) = as<int>(buf);
ASSERT_EQ(123, as<int>((const char *)buf2));
ASSERT_EQ(123, as<int>((char *)buf2));
@ -664,6 +673,238 @@ TEST(Misc, Bits) {
ASSERT_EQ(4, count_bits64((1ull << 63) | 7));
}
TEST(Misc, Time) {
Stage run;
Stage check;
Stage finish;
size_t threads_n = 3;
std::vector<thread> threads;
std::vector<std::atomic<double>> ts(threads_n);
for (size_t i = 0; i < threads_n; i++) {
threads.emplace_back([&, thread_id = i] {
for (uint64 round = 1; round < 100000; round++) {
ts[thread_id] = 0;
run.wait(round * threads_n);
ts[thread_id] = Time::now();
check.wait(round * threads_n);
for (auto &ts_ref : ts) {
auto other_ts = ts_ref.load();
if (other_ts != 0) {
ASSERT_TRUE(other_ts <= Time::now_cached());
}
}
finish.wait(round * threads_n);
}
});
}
for (auto &thread : threads) {
thread.join();
}
}
TEST(Misc, uint128) {
std::vector<uint64> parts = {0,
1,
2000,
2001,
std::numeric_limits<uint64>::max(),
std::numeric_limits<uint64>::max() - 1,
std::numeric_limits<uint32>::max(),
static_cast<uint64>(std::numeric_limits<uint32>::max()) + 1};
std::vector<int64> signed_parts = {0,
1,
2000,
2001,
-1,
-2000,
-2001,
std::numeric_limits<int64>::max(),
std::numeric_limits<int64>::max() - 1,
std::numeric_limits<int64>::min(),
std::numeric_limits<int64>::min() + 1,
std::numeric_limits<int32>::max(),
static_cast<int64>(std::numeric_limits<int32>::max()) + 1,
std::numeric_limits<int32>::max() - 1,
std::numeric_limits<int32>::min(),
std::numeric_limits<int32>::min() + 1,
static_cast<int64>(std::numeric_limits<int32>::min()) - 1};
#if TD_HAVE_INT128
auto to_intrinsic = [](uint128_emulated num) { return uint128_intrinsic(num.hi(), num.lo()); };
auto eq = [](uint128_emulated a, uint128_intrinsic b) { return a.hi() == b.hi() && a.lo() == b.lo(); };
auto ensure_eq = [&](uint128_emulated a, uint128_intrinsic b) {
if (!eq(a, b)) {
LOG(FATAL) << "[" << a.hi() << ";" << a.lo() << "] vs [" << b.hi() << ";" << b.lo() << "]";
}
};
#endif
std::vector<uint128_emulated> nums;
for (auto hi : parts) {
for (auto lo : parts) {
auto a = uint128_emulated(hi, lo);
#if TD_HAVE_INT128
auto ia = uint128_intrinsic(hi, lo);
ensure_eq(a, ia);
#endif
nums.push_back({hi, lo});
}
}
for (auto a : nums) {
#if TD_HAVE_INT128
auto ia = to_intrinsic(a);
ensure_eq(a, ia);
CHECK(a.is_zero() == ia.is_zero());
#endif
for (int i = 0; i <= 130; i++) {
#if TD_HAVE_INT128
ensure_eq(a.shl(i), ia.shl(i));
ensure_eq(a.shr(i), ia.shr(i));
#endif
}
#if TD_HAVE_INT128
for (auto b : parts) {
ensure_eq(a.mult(b), ia.mult(b));
}
for (auto b : signed_parts) {
ensure_eq(a.mult_signed(b), ia.mult_signed(b));
if (b == 0) {
continue;
}
int64 q, r;
a.divmod_signed(b, &q, &r);
int64 iq, ir;
ia.divmod_signed(b, &iq, &ir);
ASSERT_EQ(q, iq);
ASSERT_EQ(r, ir);
}
for (auto b : nums) {
auto ib = to_intrinsic(b);
//LOG(ERROR) << ia.hi() << ";" << ia.lo() << " " << ib.hi() << ";" << ib.lo();
ensure_eq(a.mult(b), ia.mult(ib));
ensure_eq(a.add(b), ia.add(ib));
ensure_eq(a.sub(b), ia.sub(ib));
if (!b.is_zero()) {
ensure_eq(a.div(b), ia.div(ib));
ensure_eq(a.mod(b), ia.mod(ib));
}
}
#endif
}
#if TD_HAVE_INT128
for (auto signed_part : signed_parts) {
auto a = uint128_emulated::from_signed(signed_part);
auto ia = uint128_intrinsic::from_signed(signed_part);
ensure_eq(a, ia);
}
#endif
}
template <template <class T> class HashT, class ValueT>
Status test_hash(const std::vector<ValueT> &values) {
for (size_t i = 0; i < values.size(); i++) {
for (size_t j = i; j < values.size(); j++) {
auto &a = values[i];
auto &b = values[j];
auto a_hash = HashT<ValueT>()(a);
auto b_hash = HashT<ValueT>()(b);
if (a == b) {
if (a_hash != b_hash) {
return Status::Error("Hash differs for same values");
}
} else {
if (a_hash == b_hash) {
return Status::Error("Hash is the same for different values");
}
}
}
}
return Status::OK();
}
class BadValue {
public:
BadValue(size_t value) : value_(value) {
}
template <class H>
friend H AbslHashValue(H hasher, const BadValue &value) {
return hasher;
}
bool operator==(const BadValue &other) const {
return value_ == other.value_;
}
private:
size_t value_;
};
class ValueA {
public:
ValueA(size_t value) : value_(value) {
}
template <class H>
friend H AbslHashValue(H hasher, ValueA value) {
return H::combine(std::move(hasher), value.value_);
}
bool operator==(const ValueA &other) const {
return value_ == other.value_;
}
private:
size_t value_;
};
class ValueB {
public:
ValueB(size_t value) : value_(value) {
}
template <class H>
friend H AbslHashValue(H hasher, ValueB value) {
return H::combine(std::move(hasher), value.value_);
}
bool operator==(const ValueB &other) const {
return value_ == other.value_;
}
private:
size_t value_;
};
template <template <class T> class HashT>
void test_hash() {
// Just check that the following compiles
AbslHashValue(Hasher(), ValueA{1});
HashT<ValueA>()(ValueA{1});
std::unordered_map<ValueA, int, HashT<ValueA>> s;
s[ValueA{1}] = 1;
HashMap<ValueA, int> su;
su[ValueA{1}] = 1;
HashSet<ValueA> su2;
su2.insert(ValueA{1});
#if TD_HAVE_ABSL
std::unordered_map<ValueA, int, absl::Hash<ValueA>> x;
absl::flat_hash_map<ValueA, int, HashT<ValueA>> sa;
sa[ValueA{1}] = 1;
#endif
test_hash<HashT, size_t>({1, 2, 3, 4, 5}).ensure();
test_hash<HashT, BadValue>({BadValue{1}, BadValue{2}}).ensure_error();
test_hash<HashT, ValueA>({ValueA{1}, ValueA{2}}).ensure();
test_hash<HashT, ValueB>({ValueB{1}, ValueB{2}}).ensure();
}
TEST(Misc, Hasher) {
test_hash<TdHash>();
#if TD_HAVE_ABSL
test_hash<AbslHash>();
#endif
}
TEST(Misc, CancellationToken) {
CancellationTokenSource source;
source.cancel();

104
tdutils/test/port.cpp
View File

@ -9,6 +9,7 @@
#include "td/utils/misc.h"
#include "td/utils/port/FileFd.h"
#include "td/utils/port/path.h"
#include "td/utils/port/signals.h"
#include "td/utils/Slice.h"
#include "td/utils/tests.h"
@ -88,3 +89,106 @@ TEST(Port, files) {
ASSERT_EQ(13u, fd.read(buf_slice.substr(0, 13)).move_as_ok());
ASSERT_STREQ("Habcd world?!", buf_slice.substr(0, 13));
}
TEST(Port, Writev) {
std::vector<IoSlice> vec;
CSlice test_file_path = "test.txt";
unlink(test_file_path).ignore();
auto fd = FileFd::open(test_file_path, FileFd::Write | FileFd::CreateNew).move_as_ok();
vec.push_back(as_io_slice("a"));
vec.push_back(as_io_slice("b"));
vec.push_back(as_io_slice("cd"));
ASSERT_EQ(4u, fd.writev(vec).move_as_ok());
vec.clear();
vec.push_back(as_io_slice("efg"));
vec.push_back(as_io_slice(""));
vec.push_back(as_io_slice("hi"));
ASSERT_EQ(5u, fd.writev(vec).move_as_ok());
fd.close();
fd = FileFd::open(test_file_path, FileFd::Read).move_as_ok();
Slice expected_content = "abcdefghi";
ASSERT_EQ(static_cast<int64>(expected_content.size()), fd.get_size().ok());
std::string content(expected_content.size(), '\0');
ASSERT_EQ(content.size(), fd.read(content).move_as_ok());
ASSERT_EQ(expected_content, content);
}
#if TD_PORT_POSIX
#include <signal.h>
#include <sys/syscall.h>
#include <unistd.h>
#include <mutex>
#include <set>
#include <algorithm>
std::mutex m;
std::vector<std::string> ptrs;
std::vector<int *> addrs;
TD_THREAD_LOCAL int thread_id;
void on_user_signal(int sig) {
int addr;
addrs[thread_id] = &addr;
char ptr[10];
snprintf(ptr, 6, "%d", thread_id);
std::unique_lock<std::mutex> guard(m);
ptrs.push_back(std::string(ptr));
}
TEST(Post, SignalsAndThread) {
setup_signals_alt_stack().ensure();
set_signal_handler(SignalType::User, on_user_signal).ensure();
std::vector<std::string> ans = {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"};
{
std::vector<td::thread> threads;
int thread_n = 10;
std::vector<Stage> stages(thread_n);
ptrs.clear();
addrs.resize(thread_n);
for (int i = 0; i < 10; i++) {
threads.emplace_back([&, i] {
setup_signals_alt_stack().ensure();
if (i != 0) {
stages[i].wait(2);
}
thread_id = i;
pthread_kill(pthread_self(), SIGUSR1);
if (i + 1 < thread_n) {
stages[i + 1].wait(2);
}
});
}
for (auto &t : threads) {
t.join();
}
CHECK(ptrs == ans);
LOG(ERROR) << ptrs;
//LOG(ERROR) << std::set<int *>(addrs.begin(), addrs.end()).size();
//LOG(ERROR) << addrs;
}
{
Stage stage;
std::vector<td::thread> threads;
int thread_n = 10;
ptrs.clear();
addrs.resize(thread_n);
for (int i = 0; i < 10; i++) {
threads.emplace_back([&, i] {
stage.wait(thread_n);
thread_id = i;
pthread_kill(pthread_self(), SIGUSR1);
//kill(pid_t(syscall(SYS_gettid)), SIGUSR1);
});
}
for (auto &t : threads) {
t.join();
}
std::sort(ptrs.begin(), ptrs.end());
CHECK(ptrs == ans);
ASSERT_EQ(10u, std::set<int *>(addrs.begin(), addrs.end()).size());
//LOG(ERROR) << addrs;
}
//ASSERT_EQ(10u, ptrs.size());
}
#endif