tdlight-team/tdlight
tdlight-team/tdlight
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix tdutils after update.

Browse Source 
GitOrigin-RevId: afc6d10dd0e2b2a7193dd2c96f07d5ca1cb11a00
This commit is contained in:
levlam 2019-07-21 21:07:07 +03:00
parent e54c0b0035
commit 635aca2924
49 changed files with 339 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
td/telegram/files/FileManager.cpp
View File

@ -2652,13 +2652,15 @@ Result<FileId> FileManager::from_persistent_id_v23(Slice binary, FileType file_t
FileData data;
data.remote_ = RemoteFileLocation(std::move(remote_location));
auto file_id =
register_file(std::move(data), FileLocationSource::FromUser, "from_persistent_id_v2", false).move_as_ok();
register_file(std::move(data), FileLocationSource::FromUser, "from_persistent_id_v23", false).move_as_ok();
return file_id;
}
Result<FileId> FileManager::from_persistent_id_v2(Slice binary, FileType file_type) {
binary.remove_suffix(1);
return from_persistent_id_v23(binary, file_type, 0);
}
Result<FileId> FileManager::from_persistent_id_v3(Slice binary, FileType file_type) {
binary.remove_suffix(1);
if (binary.empty()) {
@ -2676,6 +2678,7 @@ FileView FileManager::get_file_view(FileId file_id) const {
}
return FileView(file_node);
}
FileView FileManager::get_sync_file_view(FileId file_id) {
auto file_node = get_sync_file_node(file_id);
if (!file_node) {

8
tdutils/CMakeLists.txt
View File

@ -227,6 +227,7 @@ set(TDUTILS_SOURCE
td/utils/StorerBase.h
td/utils/StringBuilder.h
td/utils/tests.h
td/utils/ThreadSafeCounter.h
td/utils/Time.h
td/utils/TimedStat.h
td/utils/Timer.h
@ -234,10 +235,9 @@ set(TDUTILS_SOURCE
td/utils/tl_parsers.h
td/utils/tl_storers.h
td/utils/translit.h
td/utils/ThreadSafeCounter.h
td/utils/type_traits.h
td/utils/uint128.h
td/utils/UInt.h
td/utils/uint128.h
td/utils/unicode.h
td/utils/unique_ptr.h
td/utils/utf8.h
@ -247,8 +247,8 @@ set(TDUTILS_SOURCE
set(TDUTILS_TEST_SOURCE
${CMAKE_CURRENT_SOURCE_DIR}/test/buffer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/crypto.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/ConcurrentHashMap.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/crypto.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/Enumerator.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/EpochBasedMemoryReclamation.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/filesystem.cpp
@ -300,7 +300,7 @@ if (CRC32C_FOUND)
target_link_libraries(tdutils PRIVATE crc32c)
endif()
if (ABSL_FOUND)
target_link_libraries(tdutils absl::base absl::container absl::hash)
target_link_libraries(tdutils PUBLIC absl::base absl::container absl::hash)
endif()
if (WIN32 AND WINGETOPT_FOUND)

2
tdutils/td/utils/BufferedReader.h
View File

@ -14,6 +14,7 @@
#include <cstring>
namespace td {
class BufferedReader {
public:
explicit BufferedReader(FileFd &file, size_t buff_size = 8152)
@ -58,4 +59,5 @@ inline Result<size_t> BufferedReader::read(MutableSlice slice) {
begin_pos_ += left;
return left + available;
}
} // namespace td

33
tdutils/td/utils/ConcurrentHashTable.h
View File

@ -6,35 +6,37 @@
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/HazardPointers.h"
#include <mutex>
#include <atomic>
#include <condition_variable>
#include <vector>
#include <mutex>
namespace td {
// AtomicHashArray<KeyT, ValueT>
// Building block for other conurrent hash maps
// Building block for other concurrent hash maps
//
// Support one operation:
// template <class F>
// bool with_value(KeyT key, bool should_create, F &&func);
//
// Finds slot for key, and call func(value)
// Creates slot if shoul_create is true.
// Creates slot if should_create is true.
// Returns true if func was called.
//
// Concurrent calls with same key may result in concurrent calls of func(value)
// It is resposibility of caller to handle such races.
// Concurrent calls with the same key may result in concurrent calls to func(value)
// It is responsibility of the caller to handle such races.
//
// Key should already be random
// It is resposibility of caller to provide unique random key.
// One may use injective hash function, or handle collisions on some other way.
// It is responsibility of the caller to provide unique random key.
// One may use injective hash function, or handle collisions in some other way.
namespace td {
template <class KeyT, class ValueT>
class AtomicHashArray {
public:
AtomicHashArray(size_t n) : nodes_(n) {
explicit AtomicHashArray(size_t n) : nodes_(n) {
}
struct Node {
std::atomic<KeyT> key{KeyT{}};
@ -54,7 +56,7 @@ class AtomicHashArray {
bool with_value(KeyT key, bool should_create, F &&f) {
DCHECK(key != empty_key());
size_t pos = static_cast<size_t>(key) % nodes_.size();
size_t n = std::min(std::max(size_t(300), nodes_.size() / 16 + 2), nodes_.size());
size_t n = td::min(td::max(static_cast<size_t>(300), nodes_.size() / 16 + 2), nodes_.size());
for (size_t i = 0; i < n; i++) {
pos++;
@ -93,10 +95,10 @@ class AtomicHashArray {
template <class KeyT, class ValueT>
class ConcurrentHashMap {
using HashMap = AtomicHashArray<KeyT, std::atomic<ValueT>>;
static td::HazardPointers<HashMap> hp_;
static HazardPointers<HashMap> hp_;
public:
ConcurrentHashMap(size_t n = 32) {
explicit ConcurrentHashMap(size_t n = 32) {
n = 1;
hash_map_.store(make_unique<HashMap>(n).release());
}
@ -105,7 +107,7 @@ class ConcurrentHashMap {
ConcurrentHashMap(ConcurrentHashMap &&) = delete;
ConcurrentHashMap &operator=(ConcurrentHashMap &&) = delete;
~ConcurrentHashMap() {
td::unique_ptr<HashMap>(hash_map_.load());
unique_ptr<HashMap>(hash_map_.load());
}
static std::string get_name() {
@ -156,6 +158,7 @@ class ConcurrentHashMap {
do_migrate(hash_map);
}
}
ValueT find(KeyT key, ValueT value) {
typename HazardPointers<HashMap>::Holder holder(hp_, get_thread_id(), 0);
while (true) {
@ -173,6 +176,7 @@ class ConcurrentHashMap {
do_migrate(hash_map);
}
}
template <class F>
void for_each(F &&f) {
auto hash_map = hash_map_.load();
@ -312,4 +316,5 @@ class ConcurrentHashMap {
template <class KeyT, class ValueT>
td::HazardPointers<typename ConcurrentHashMap<KeyT, ValueT>::HashMap> ConcurrentHashMap<KeyT, ValueT>::hp_(64);
} // namespace td

2
tdutils/td/utils/DecTree.h
View File

@ -151,7 +151,7 @@ class DecTree {
P.first = std::move(Tree);
return P;
}
} // namespace td
}
static unique_ptr<Node> merge_node(unique_ptr<Node> left, unique_ptr<Node> right) {
if (left == nullptr) {

13
tdutils/td/utils/EpochBasedMemoryReclamation.h
View File

@ -5,12 +5,14 @@
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/port/sleep.h"
#include "td/utils/port/thread.h"
namespace td {
template <class T>
class EpochBasedMemoryReclamation {
public:
@ -18,6 +20,7 @@ class EpochBasedMemoryReclamation {
EpochBasedMemoryReclamation &operator=(const EpochBasedMemoryReclamation &other) = delete;
EpochBasedMemoryReclamation(EpochBasedMemoryReclamation &&other) = delete;
EpochBasedMemoryReclamation &operator=(EpochBasedMemoryReclamation &&other) = delete;
~EpochBasedMemoryReclamation() = default;
class Locker {
public:
@ -76,18 +79,17 @@ class EpochBasedMemoryReclamation {
size_t to_delete_size_unsafe() const {
size_t res = 0;
for (auto &thread : threads_) {
LOG(ERROR) << "---" << thread.epoch.load() / 2;
for (auto &thread_data : threads_) {
// LOG(ERROR) << "---" << thread_data.epoch.load() / 2;
for (size_t i = 0; i < MAX_BAGS; i++) {
res += thread.to_delete[i].size();
LOG(ERROR) << thread.to_delete[i].size();
res += thread_data.to_delete[i].size();
// LOG(ERROR) << thread_data.to_delete[i].size();
}
}
return res;
}
private:
friend class Locker;
static constexpr size_t MAX_BAGS = 3;
struct ThreadData {
std::atomic<int64> epoch{1};
@ -193,4 +195,5 @@ class EpochBasedMemoryReclamation {
data.to_delete[data.bag_i].push_back(unique_ptr<T>{ptr});
}
};
} // namespace td

12
tdutils/td/utils/Gzip.cpp
View File

@ -134,17 +134,19 @@ Gzip::Gzip(Gzip &&other) : Gzip() {
}
Gzip &Gzip::operator=(Gzip &&other) {
CHECK(this != &other);
clear();
swap(other);
return *this;
}
void Gzip::swap(Gzip &other) {
std::swap(impl_, other.impl_);
std::swap(input_size_, other.input_size_);
std::swap(output_size_, other.output_size_);
std::swap(close_input_flag_, other.close_input_flag_);
std::swap(mode_, other.mode_);
using std::swap;
swap(impl_, other.impl_);
swap(input_size_, other.input_size_);
swap(output_size_, other.output_size_);
swap(close_input_flag_, other.close_input_flag_);
swap(mode_, other.mode_);
}
Gzip::~Gzip() {

6
tdutils/td/utils/Hash.h
View File

@ -20,9 +20,9 @@ namespace td {
class Hasher {
public:
Hasher() = default;
Hasher(size_t init_value) : hash_(init_value) {
explicit Hasher(size_t init_value) : hash_(init_value) {
}
std::size_t finalize() {
std::size_t finalize() const {
return hash_;
}
@ -34,7 +34,7 @@ class Hasher {
template <class A, class B>
static Hasher combine(Hasher hasher, const std::pair<A, B> &value) {
hasher = AbslHashValue(std::move(hasher), value.first);
hasher = AbslHashValue(std::move(hasher), value.first);
hasher = AbslHashValue(std::move(hasher), value.second);
return hasher;
}

3
tdutils/td/utils/HashMap.h
View File

@ -13,7 +13,9 @@
#else
#include <unordered_map>
#endif
namespace td {
#if TD_HAVE_ABSL
template <class Key, class Value, class H = Hash<Key>>
using HashMap = absl::flat_hash_map<Key, Value, H>;
@ -21,4 +23,5 @@ using HashMap = absl::flat_hash_map<Key, Value, H>;
template <class Key, class Value, class H = Hash<Key>>
using HashMap = std::unordered_map<Key, Value, H>;
#endif
} // namespace td

3
tdutils/td/utils/HashSet.h
View File

@ -13,7 +13,9 @@
#else
#include <unordered_set>
#endif
namespace td {
#if TD_HAVE_ABSL
template <class Key, class H = Hash<Key>>
using HashSet = absl::flat_hash_set<Key, H>;
@ -21,4 +23,5 @@ using HashSet = absl::flat_hash_set<Key, H>;
template <class Key, class H = Hash<Key>>
using HashSet = std::unordered_set<Key, H>;
#endif
} // namespace td

22
tdutils/td/utils/HazardPointers.h
View File

@ -19,7 +19,7 @@ class HazardPointers {
public:
explicit HazardPointers(size_t threads_n) : threads_(threads_n) {
for (auto &data : threads_) {
for (auto &ptr : data.hazard) {
for (auto &ptr : data.hazard_) {
// workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=64658
#if TD_GCC && GCC_VERSION <= 40902
ptr = nullptr;
@ -66,12 +66,12 @@ class HazardPointers {
CHECK(thread_id < threads_.size());
auto &data = threads_[thread_id];
if (ptr) {
data.to_delete.push_back(std::unique_ptr<T, Deleter>(ptr));
data.to_delete_.push_back(std::unique_ptr<T, Deleter>(ptr));
}
for (auto it = data.to_delete.begin(); it != data.to_delete.end();) {
for (auto it = data.to_delete_.begin(); it != data.to_delete_.end();) {
if (!is_protected(it->get())) {
it->reset();
it = data.to_delete.erase(it);
it = data.to_delete_.erase(it);
} else {
++it;
}
@ -89,19 +89,19 @@ class HazardPointers {
size_t to_delete_size_unsafe() const {
size_t res = 0;
for (auto &thread : threads_) {
res += thread.to_delete.size();
res += thread.to_delete_.size();
}
return res;
}
private:
struct ThreadData {
std::array<std::atomic<T *>, MaxPointersN> hazard;
char pad[TD_CONCURRENCY_PAD - sizeof(hazard)];
std::array<std::atomic<T *>, MaxPointersN> hazard_;
char pad[TD_CONCURRENCY_PAD - sizeof(hazard_)];
// stupid gc
std::vector<std::unique_ptr<T, Deleter>> to_delete;
char pad2[TD_CONCURRENCY_PAD - sizeof(to_delete)];
std::vector<std::unique_ptr<T, Deleter>> to_delete_;
char pad2[TD_CONCURRENCY_PAD - sizeof(to_delete_)];
};
std::vector<ThreadData> threads_;
char pad2[TD_CONCURRENCY_PAD - sizeof(threads_)];
@ -123,7 +123,7 @@ class HazardPointers {
bool is_protected(T *ptr) {
for (auto &thread : threads_) {
for (auto &hazard_ptr : thread.hazard) {
for (auto &hazard_ptr : thread.hazard_) {
if (hazard_ptr.load() == ptr) {
return true;
}
@ -134,7 +134,7 @@ class HazardPointers {
std::atomic<T *> &get_hazard_ptr(size_t thread_id, size_t pos) {
CHECK(thread_id < threads_.size());
return threads_[thread_id].hazard[pos];
return threads_[thread_id].hazard_[pos];
}
};

9
tdutils/td/utils/MpmcQueue.cpp
View File

@ -5,6 +5,11 @@
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/MpmcQueue.h"
namespace td {
detail::MpmcStat stat_;
}
namespace detail {
MpmcStat stat_;
} // namespace detail
} // namespace td

9
tdutils/td/utils/MpmcQueue.h
View File

@ -23,6 +23,7 @@
namespace td {
namespace detail {
struct MpmcStat {
void alloc_ok(size_t thread_id) {
s(thread_id).alloc_ok_cnt++;
@ -64,13 +65,15 @@ struct MpmcStat {
return arr[thread_id];
}
};
extern MpmcStat stat_;
} // namespace detail
extern detail::MpmcStat stat_;
template <class T>
class OneValue {
public:
inline bool set_value(T &value) {
bool set_value(T &value) {
value_ = std::move(value);
int state = Empty;
if (state_.compare_exchange_strong(state, Value, std::memory_order_acq_rel)) {
@ -79,7 +82,7 @@ class OneValue {
value = std::move(value_);
return false;
}
inline bool get_value(T &value) {
bool get_value(T &value) {
auto old_state = state_.exchange(Taken, std::memory_order_acq_rel);
if (old_state == Value) {
value = std::move(value_);

2
tdutils/td/utils/MpmcWaiter.h
View File

@ -7,8 +7,8 @@
#pragma once
#include "td/utils/common.h"
#include "td/utils/port/thread.h"
#include "td/utils/logging.h"
#include "td/utils/port/thread.h"
#include <atomic>
#include <condition_variable>

1
tdutils/td/utils/OptionsParser.cpp
View File

@ -16,6 +16,7 @@
#endif
namespace td {
void OptionsParser::set_description(std::string description) {
description_ = std::move(description);
}

1
tdutils/td/utils/OptionsParser.h
View File

@ -12,7 +12,6 @@
#include "td/utils/StringBuilder.h"
#include <functional>
#include <string>
namespace td {

5
tdutils/td/utils/Slice-decl.h
View File

@ -9,7 +9,6 @@
#include "td/utils/common.h"
#include <type_traits>
#include <limits>
namespace td {
@ -59,7 +58,7 @@ class MutableSlice {
char &back();
char &operator[](size_t i);
static const size_t npos = std::numeric_limits<size_t>::max();
static const size_t npos = string::npos;
};
class Slice {
@ -124,7 +123,7 @@ class Slice {
char back() const;
char operator[](size_t i) const;
static const size_t npos = std::numeric_limits<size_t>::max();
static const size_t npos = string::npos;
};
bool operator==(const Slice &a, const Slice &b);

6
tdutils/td/utils/Span.h
View File

@ -57,7 +57,7 @@ class SpanImpl {
return false;
}
for (size_t i = 0; i < size(); i++) {
if ((*this)[i] != other[i]) {
if (!((*this)[i] == other[i])) {
return false;
}
}
@ -115,11 +115,11 @@ template <class T>
using MutableSpan = detail::SpanImpl<T, T>;
template <class T>
auto span(const T *ptr, size_t size) {
Span<T> span(const T *ptr, size_t size) {
return Span<T>(ptr, size);
}
template <class T>
auto mutable_span(T *ptr, size_t size) {
MutableSpan<T> mutable_span(T *ptr, size_t size) {
return MutableSpan<T>(ptr, size);
}

22
tdutils/td/utils/Status.h
View File

@ -27,6 +27,7 @@
return try_status.move_as_error(); \
} \
}
#define TRY_STATUS_PREFIX(status, prefix) \
{ \
auto try_status = (status); \
@ -34,7 +35,9 @@
return try_status.move_as_error_prefix(prefix); \
} \
}
#define TRY_RESULT(name, result) TRY_RESULT_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), name, result)
#define TRY_RESULT_PREFIX(name, result, prefix) \
TRY_RESULT_PREFIX_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), name, result, prefix)
@ -264,8 +267,18 @@ class Status {
return std::move(*this);
}
Status move_as_error_prefix(std::string prefix) TD_WARN_UNUSED_RESULT {
return td::Status::Error(code(), prefix + message().c_str());
Status move_as_error_prefix(Slice prefix) TD_WARN_UNUSED_RESULT {
CHECK(is_error());
Info info = get_info();
switch (info.error_type) {
case ErrorType::general:
return Error(code(), PSLICE() << prefix << message());
case ErrorType::os:
return Status(false, ErrorType::os, code(), PSLICE() << prefix << message());
default:
UNREACHABLE();
return {};
}
}
private:
@ -438,10 +451,9 @@ class Result {
};
return std::move(status_);
}
Status move_as_error_prefix(std::string prefix) TD_WARN_UNUSED_RESULT {
CHECK(status_.is_error());
Status move_as_error_prefix(Slice prefix) TD_WARN_UNUSED_RESULT {
SCOPE_EXIT {
status_ = Status::Error<-4>();
status_ = Status::Error<-5>();
};
return status_.move_as_error_prefix(prefix);
}

9
tdutils/td/utils/ThreadSafeCounter.h
View File

@ -5,14 +5,16 @@
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/int_types.h"
#include "td/utils/port/thread.h"
#include "td/utils/port/thread_local.h"
#include "td/utils/int_types.h"
#include <atomic>
#include <array>
#include <atomic>
namespace td {
class ThreadSafeCounter {
public:
void add(int64 diff) {
@ -40,8 +42,9 @@ class ThreadSafeCounter {
Node &thread_local_node() {
auto thread_id = get_thread_id();
CHECK(0 <= thread_id && static_cast<size_t>(thread_id) < nodes_.size());
CHECK(static_cast<size_t>(thread_id) < nodes_.size());
return nodes_[thread_id];
}
};
} // namespace td

6
tdutils/td/utils/Time.h
View File

@ -9,8 +9,6 @@
#include "td/utils/common.h"
#include "td/utils/port/Clocks.h"
#include <atomic>
namespace td {
class Time {
@ -75,8 +73,8 @@ class Timestamp {
double at() const {
return at_;
}
double at_unix() {
return at_ + td::Clocks::system() - Time::now();
double at_unix() const {
return at_ + Clocks::system() - Time::now();
}
double in() const {

2
tdutils/td/utils/Timer.cpp
View File

@ -19,6 +19,7 @@ Timer::Timer() : start_time_(Time::now()) {
double Timer::elapsed() const {
return Time::now() - start_time_;
}
StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer) {
return string_builder << "in " << Time::now() - timer.start_time_;
}
@ -35,6 +36,7 @@ PerfWarningTimer::PerfWarningTimer(PerfWarningTimer &&other)
PerfWarningTimer::~PerfWarningTimer() {
reset();
}
void PerfWarningTimer::reset() {
if (start_at_ == 0) {
return;

1
tdutils/td/utils/Timer.h
View File

@ -13,6 +13,7 @@ namespace td {
class Timer {
public:
Timer();
double elapsed() const;
private:

7
tdutils/td/utils/bits.h
View File

@ -5,8 +5,8 @@
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/check.h"
#if TD_MSVC
#include <intrin.h>
@ -19,7 +19,9 @@
#ifdef bswap64
#undef bswap64
#endif
namespace td {
int32 count_leading_zeroes32(uint32 x);
int32 count_leading_zeroes64(uint64 x);
int32 count_trailing_zeroes32(uint32 x);
@ -119,9 +121,11 @@ inline int32 count_trailing_zeroes64(uint64 x) {
inline uint32 bswap32(uint32 x) {
return _byteswap_ulong(x);
}
inline uint64 bswap64(uint64 x) {
return _byteswap_uint64(x);
}
inline int32 count_bits32(uint32 x) {
return __popcnt(x);
}
@ -247,4 +251,5 @@ inline int32 count_bits64(uint64 x) {
}
#endif
} // namespace td

40
tdutils/td/utils/crypto.cpp
View File

@ -469,6 +469,7 @@ void hmac_sha256(Slice key, Slice message, MutableSlice dest) {
CHECK(result == dest.ubegin());
CHECK(len == dest.size());
}
void hmac_sha512(Slice key, Slice message, MutableSlice dest) {
CHECK(dest.size() == 512 / 8);
unsigned int len = 0;
@ -657,8 +658,9 @@ uint32 crc32c_extend(uint32 old_crc, Slice data) {
}
namespace {
unsigned gf32_matrix_times(uint32 *matrix, uint32 vector) {
unsigned sum = 0;
uint32 gf32_matrix_times(const uint32 *matrix, uint32 vector) {
uint32 sum = 0;
while (vector) {
if (vector & 1) {
sum ^= *matrix;
@ -669,36 +671,36 @@ unsigned gf32_matrix_times(uint32 *matrix, uint32 vector) {
return sum;
}
void gf32_matrix_square(uint32 *square, uint32 *matrix) {
int n = 0;
do {
void gf32_matrix_square(uint32 *square, const uint32 *matrix) {
for (int n = 0; n < 32; n++) {
square[n] = gf32_matrix_times(matrix, matrix[n]);
} while (++n < 32);
}
}
} // namespace
static uint32 power_buf_raw[1024];
uint32 crc32c_extend(uint32 old_crc, uint32 data_crc, size_t data_size) {
static uint32 *power_buf = [&] {
static uint32 power_buf_raw[1024];
static const uint32 *power_buf = [&] {
auto *buf = power_buf_raw;
buf[0] = 0x82F63B78UL;
buf[0] = 0x82F63B78u;
for (int n = 0; n < 31; n++) {
buf[n + 1] = 1U << n;
buf[n + 1] = 1u << n;
}
for (int n = 1; n < 32; n++) {
gf32_matrix_square(buf + (n << 5), buf + ((n - 1) << 5));
}
return buf;
}();
/* degenerate case (also disallow negative lengths) */
if (data_size == 0) {
return old_crc;
}
unsigned int *p = power_buf + 64;
const uint32 *p = power_buf + 64;
do {
p += 32;
if (data_size % 2 != 0) {
if (data_size & 1) {
old_crc = gf32_matrix_times(p, old_crc);
}
data_size >>= 1;
@ -774,7 +776,7 @@ uint64 crc64(Slice data) {
return crc64_partial(data, static_cast<uint64>(-1)) ^ static_cast<uint64>(-1);
}
static unsigned short crc16_table[256] = {
static const uint16 crc16_table[256] = {
0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad,
0xe1ce, 0xf1ef, 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, 0x9339, 0x8318, 0xb37b, 0xa35a,
0xd3bd, 0xc39c, 0xf3ff, 0xe3de, 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, 0xa56a, 0xb54b,
@ -795,13 +797,13 @@ static unsigned short crc16_table[256] = {
0x1ce0, 0x0cc1, 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, 0x6e17, 0x7e36, 0x4e55, 0x5e74,
0x2e93, 0x3eb2, 0x0ed1, 0x1ef0};
td::uint16 crc16(td::Slice data) {
unsigned crc = 0;
for (std::size_t i = 0; i < data.size(); i++) {
unsigned t = ((unsigned char)data[i] ^ (crc >> 8)) & 0xff;
uint16 crc16(Slice data) {
uint32 crc = 0;
for (auto c : data) {
auto t = (static_cast<unsigned char>(c) ^ (crc >> 8)) & 0xff;
crc = crc16_table[t] ^ (crc << 8);
}
return (td::uint16)crc;
return static_cast<uint16>(crc);
}
} // namespace td

2
tdutils/td/utils/logging.cpp
View File

@ -226,7 +226,7 @@ class DefaultLog : public LogInterface {
break;
}
if (!slice.empty() && slice.back() == '\n') {
TsCerr() << color << slice.substr(0, slice.size() - 1) << TC_EMPTY << "\n";
TsCerr() << color << slice.substr(0, slice.size() - 1) << TC_EMPTY "\n";
} else {
TsCerr() << color << slice << TC_EMPTY;
}

2
tdutils/td/utils/misc.cpp
View File

@ -171,7 +171,7 @@ bool is_zero_or_one(unsigned char c) {
string buffer_to_hex(Slice buffer) {
const char *hex = "0123456789ABCDEF";
std::string res(2 * buffer.size(), '\0');
string res(2 * buffer.size(), '\0');
for (std::size_t i = 0; i < buffer.size(); i++) {
auto c = buffer.ubegin()[i];
res[2 * i] = hex[c & 15];

5
tdutils/td/utils/misc.h
View File

@ -16,7 +16,6 @@
#include <limits>
#include <type_traits>
#include <utility>
#include <string>
namespace td {
@ -310,8 +309,8 @@ string url_encode(Slice str);
namespace detail {
template <class T, class U>
struct is_same_signedness : public std::integral_constant<bool, std::is_signed<T>::value == std::is_signed<U>::value> {
};
struct is_same_signedness
: public std::integral_constant<bool, std::is_signed<T>::value == std::is_signed<U>::value> {};
template <class T, class Enable = void>
struct safe_undeflying_type {

6
tdutils/td/utils/port/FileFd.cpp
View File

@ -137,12 +137,12 @@ Result<FileFd> FileFd::open(CSlice filepath, int32 flags, int32 mode) {
native_flags |= O_APPEND;
}
if (flags & Direct) {
#if TD_LINUX
LOG(ERROR) << "DIRECT";
if (flags & Direct) {
native_flags |= O_DIRECT;
#endif
}
#endif
int native_fd = detail::skip_eintr([&] { return ::open(filepath.c_str(), native_flags, static_cast<mode_t>(mode)); });
if (native_fd < 0) {
return OS_ERROR(PSLICE() << "File \"" << filepath << "\" can't be " << PrintFlags{flags});

2
tdutils/td/utils/port/FileFd.h
View File

@ -11,8 +11,8 @@
#include "td/utils/common.h"
#include "td/utils/port/detail/NativeFd.h"
#include "td/utils/port/detail/PollableFd.h"
#include "td/utils/port/Stat.h"
#include "td/utils/port/IoSlice.h"
#include "td/utils/port/Stat.h"
#include "td/utils/Slice.h"
#include "td/utils/Span.h"
#include "td/utils/Status.h"

12
tdutils/td/utils/port/IoSlice.h
View File

@ -7,28 +7,36 @@
#pragma once
#include "td/utils/Slice.h"
#include "td/utils/Span.h"
#if TD_PORT_POSIX
#include <sys/uio.h>
#endif
namespace td {
#if TD_PORT_POSIX
using IoSlice = struct iovec;
inline IoSlice as_io_slice(Slice slice) {
IoSlice res;
res.iov_len = slice.size();
res.iov_base = const_cast<char *>(slice.data());
return res;
}
inline Slice as_slice(const IoSlice io_slice) {
return Slice(reinterpret_cast<const char *>(io_slice.iov_base), io_slice.iov_len);
return Slice(static_cast<const char *>(io_slice.iov_base), io_slice.iov_len);
}
#else
using IoSlice = Slice;
inline IoSlice as_io_slice(Slice slice) {
return slice;
}
#endif
} // namespace td

23
tdutils/td/utils/port/MemoryMapping.cpp
View File

@ -10,8 +10,9 @@
// TODO:
// windows,
// anonymous map
// anonymous maps,
// huge pages?
#if TD_WINDOWS
#else
#include <sys/mman.h>
@ -19,10 +20,10 @@
#endif
namespace td {
namespace detail {
class MemoryMappingImpl {
class MemoryMapping::Impl {
public:
MemoryMappingImpl(MutableSlice data, int64 offset) : data_(data), offset_(offset) {
Impl(MutableSlice data, int64 offset) : data_(data), offset_(offset) {
}
Slice as_slice() const {
return data_.substr(narrow_cast<size_t>(offset_));
@ -36,7 +37,7 @@ class MemoryMappingImpl {
int64 offset_;
};
Result<int64> get_page_size() {
static Result<int64> get_page_size() {
#if TD_WINDOWS
return Status::Error("Unimplemented");
#else
@ -50,11 +51,11 @@ Result<int64> get_page_size() {
return page_size.clone();
#endif
}
} // namespace detail
Result<MemoryMapping> MemoryMapping::create_anonymous(const MemoryMapping::Options &options) {
return Status::Error("Unsupported yet");
}
Result<MemoryMapping> MemoryMapping::create_from_file(const FileFd &file_fd, const MemoryMapping::Options &options) {
#if TD_WINDOWS
return Status::Error("Unsupported yet");
@ -76,7 +77,7 @@ Result<MemoryMapping> MemoryMapping::create_from_file(const FileFd &file_fd, con
end = begin + stat.size_;
}
TRY_RESULT(page_size, detail::get_page_size());
TRY_RESULT(page_size, get_page_size());
auto fixed_begin = begin / page_size * page_size;
auto data_offset = begin - fixed_begin;
@ -87,8 +88,7 @@ Result<MemoryMapping> MemoryMapping::create_from_file(const FileFd &file_fd, con
return OS_ERROR("mmap call failed");
}
return MemoryMapping(std::make_unique<detail::MemoryMappingImpl>(
MutableSlice(reinterpret_cast<char *>(data), data_size), data_offset));
return MemoryMapping(make_unique<Impl>(MutableSlice(static_cast<char *>(data), data_size), data_offset));
#endif
}
@ -96,14 +96,15 @@ MemoryMapping::MemoryMapping(MemoryMapping &&other) = default;
MemoryMapping &MemoryMapping::operator=(MemoryMapping &&other) = default;
MemoryMapping::~MemoryMapping() = default;
MemoryMapping::MemoryMapping(std::unique_ptr<detail::MemoryMappingImpl> impl) : impl_(std::move(impl)) {
MemoryMapping::MemoryMapping(unique_ptr<Impl> impl) : impl_(std::move(impl)) {
}
Slice MemoryMapping::as_slice() const {
return impl_->as_slice();
}
MutableSlice MemoryMapping::as_mutable_slice() {
return impl_->as_mutable_slice();
}
} // namespace td

9
tdutils/td/utils/port/MemoryMapping.h
View File

@ -12,9 +12,6 @@
#include "td/utils/Status.h"
namespace td {
namespace detail {
class MemoryMappingImpl;
}
class MemoryMapping {
public:
@ -47,7 +44,9 @@ class MemoryMapping {
~MemoryMapping();
private:
std::unique_ptr<detail::MemoryMappingImpl> impl_;
explicit MemoryMapping(std::unique_ptr<detail::MemoryMappingImpl> impl);
class Impl;
unique_ptr<Impl> impl_;
explicit MemoryMapping(unique_ptr<Impl> impl);
};
} // namespace td

2
tdutils/td/utils/port/SocketFd.h
View File

@ -10,8 +10,8 @@
#include "td/utils/port/detail/NativeFd.h"
#include "td/utils/port/detail/PollableFd.h"
#include "td/utils/port/IPAddress.h"
#include "td/utils/port/IoSlice.h"
#include "td/utils/port/IPAddress.h"
#include "td/utils/Slice.h"
#include "td/utils/Span.h"
#include "td/utils/Status.h"

2
tdutils/td/utils/port/StdStreams.cpp
View File

@ -125,7 +125,6 @@ class BufferedStdinImpl : public Iocp::Callback {
inc_refcnt();
}
}
LOG(ERROR) << "Close";
if (!iocp_ref_.post(0, this, nullptr)) {
dec_refcnt();
}
@ -138,7 +137,6 @@ class BufferedStdinImpl : public Iocp::Callback {
bool dec_refcnt() {
if (--refcnt_ == 0) {
delete this;
LOG(ERROR) << "Delete this";
return true;
}
return false;

69
tdutils/td/utils/port/UdpSocketFd.cpp
View File

@ -652,7 +652,7 @@ class UdpSocketFdImpl {
//};
struct std::array<detail::UdpSocketSendHelper, 16> helpers;
struct std::array<struct mmsghdr, 16> headers;
size_t to_send = td::min(messages.size(), headers.size());
size_t to_send = min(messages.size(), headers.size());
for (size_t i = 0; i < to_send; i++) {
helpers[i].to_native(messages[i], headers[i].msg_hdr);
headers[i].msg_len = 0;
@ -703,7 +703,7 @@ class UdpSocketFdImpl {
//};
struct std::array<detail::UdpSocketReceiveHelper, 16> helpers;
struct std::array<struct mmsghdr, 16> headers;
size_t to_receive = td::min(messages.size(), headers.size());
size_t to_receive = min(messages.size(), headers.size());
for (size_t i = 0; i < to_receive; i++) {
helpers[i].to_native(messages[i], headers[i].msg_hdr);
headers[i].msg_len = 0;
@ -786,29 +786,20 @@ const NativeFd &UdpSocketFd::get_native_fd() const {
}
#if TD_PORT_POSIX
td::Result<td::uint32> UdpSocketFd::maximize_snd_buffer(td::uint32 max) {
socklen_t intsize = sizeof(td::uint32);
td::uint32 last_good = 0;
td::uint32 min, avg;
td::uint32 old_size;
auto socket_fd = get_native_fd().fd();
if (!max) {
max = default_udp_max_snd_buffer_size;
}
static Result<uint32> maximize_buffer(int socket_fd, int optname, uint32 max) {
/* Start with the default size. */
if (getsockopt(socket_fd, SOL_SOCKET, SO_SNDBUF, &old_size, &intsize)) {
return td::Status::PosixError(errno, "getsockopt() failed");
uint32 old_size;
socklen_t intsize = sizeof(old_size);
if (getsockopt(socket_fd, SOL_SOCKET, optname, &old_size, &intsize)) {
return OS_ERROR("getsockopt() failed");
}
/* Binary-search for the real maximum. */
min = last_good = old_size;
uint32 last_good = old_size;
uint32 min = old_size;
while (min <= max) {
avg = (min + max) / 2;
if (setsockopt(socket_fd, SOL_SOCKET, SO_SNDBUF, &avg, intsize) == 0) {
uint32 avg = min + (max - min) / 2;
if (setsockopt(socket_fd, SOL_SOCKET, optname, &avg, intsize) == 0) {
last_good = avg;
min = avg + 1;
} else {
@ -818,42 +809,18 @@ td::Result<td::uint32> UdpSocketFd::maximize_snd_buffer(td::uint32 max) {
return last_good;
}
td::Result<td::uint32> UdpSocketFd::maximize_rcv_buffer(td::uint32 max) {
socklen_t intsize = sizeof(td::uint32);
td::uint32 last_good = 0;
td::uint32 min, avg;
td::uint32 old_size;
Result<uint32> UdpSocketFd::maximize_snd_buffer(uint32 max) {
return maximize_buffer(get_native_fd().fd(), SO_SNDBUF, max == 0 ? default_udp_max_snd_buffer_size : max);
}
auto socket_fd = get_native_fd().fd();
if (!max) {
max = default_udp_max_rcv_buffer_size;
}
/* Start with the default size. */
if (getsockopt(socket_fd, SOL_SOCKET, SO_RCVBUF, &old_size, &intsize)) {
return td::Status::PosixError(errno, "getsockopt() failed");
}
/* Binary-search for the real maximum. */
min = last_good = old_size;
while (min <= max) {
avg = (min + max) / 2;
if (setsockopt(socket_fd, SOL_SOCKET, SO_RCVBUF, &avg, intsize) == 0) {
last_good = avg;
min = avg + 1;
} else {
max = avg - 1;
}
}
return last_good;
Result<uint32> UdpSocketFd::maximize_rcv_buffer(uint32 max) {
return maximize_buffer(get_native_fd().fd(), SO_RCVBUF, max == 0 ? default_udp_max_rcv_buffer_size : max);
}
#else
td::Result<td::uint32> UdpSocketFd::maximize_snd_buffer(td::uint32 max) {
Result<uint32> UdpSocketFd::maximize_snd_buffer(uint32 max) {
return 0;
}
td::Result<td::uint32> UdpSocketFd::maximize_rcv_buffer(td::uint32 max) {
Result<uint32> UdpSocketFd::maximize_rcv_buffer(uint32 max) {
return 0;
}
#endif

8
tdutils/td/utils/port/UdpSocketFd.h
View File

@ -45,8 +45,8 @@ class UdpSocketFd {
UdpSocketFd(const UdpSocketFd &) = delete;
UdpSocketFd &operator=(const UdpSocketFd &) = delete;
td::Result<td::uint32> maximize_snd_buffer(td::uint32 max_buffer_size = 0);
td::Result<td::uint32> maximize_rcv_buffer(td::uint32 max_buffer_size = 0);
Result<uint32> maximize_snd_buffer(uint32 max_buffer_size = 0);
Result<uint32> maximize_rcv_buffer(uint32 max_buffer_size = 0);
static Result<UdpSocketFd> open(const IPAddress &address) TD_WARN_UNUSED_RESULT;
@ -84,8 +84,8 @@ class UdpSocketFd {
#endif
private:
static constexpr td::uint32 default_udp_max_snd_buffer_size = (1 << 24);
static constexpr td::uint32 default_udp_max_rcv_buffer_size = (1 << 24);
static constexpr uint32 default_udp_max_snd_buffer_size = (1 << 24);
static constexpr uint32 default_udp_max_rcv_buffer_size = (1 << 24);
std::unique_ptr<detail::UdpSocketFdImpl, detail::UdpSocketFdImplDeleter> impl_;
explicit UdpSocketFd(unique_ptr<detail::UdpSocketFdImpl> impl);
};

4
tdutils/td/utils/port/detail/Iocp.cpp
View File

@ -80,15 +80,13 @@ IocpRef Iocp::get_ref() const {
return IocpRef(iocp_handle_);
}
namespace {
void iocp_post(NativeFd &iocp_handle, size_t size, Iocp::Callback *callback, WSAOVERLAPPED *overlapped) {
static void iocp_post(NativeFd &iocp_handle, size_t size, Iocp::Callback *callback, WSAOVERLAPPED *overlapped) {
if (PostQueuedCompletionStatus(iocp_handle.fd(), DWORD(size), reinterpret_cast<ULONG_PTR>(callback),
reinterpret_cast<OVERLAPPED *>(overlapped)) == 0) {
auto error = OS_ERROR("IOCP post failed");
LOG(FATAL) << error;
}
}
} // namespace
void Iocp::post(size_t size, Callback *callback, WSAOVERLAPPED *overlapped) {
iocp_post(*iocp_handle_, size, callback, overlapped);

2
tdutils/td/utils/port/detail/Iocp.h
View File

@ -59,9 +59,11 @@ class IocpRef {
IocpRef(std::weak_ptr<NativeFd> iocp_handle);
bool post(size_t size, Iocp::Callback *callback, WSAOVERLAPPED *overlapped);
private:
std::weak_ptr<NativeFd> iocp_handle_;
};
} // namespace detail
} // namespace td

21
tdutils/td/utils/port/signals.cpp
View File

@ -298,27 +298,28 @@ void signal_safe_write_pointer(void *p, bool add_header) {
signal_safe_write(Slice(ptr, end), add_header);
}
static void unblock_stdin() {
static void block_stdin() {
#if TD_PORT_POSIX
td::Stdin().get_native_fd().set_is_blocking(true).ignore();
Stdin().get_native_fd().set_is_blocking(true).ignore();
#endif
}
void default_failure_signal_hanler(int sig) {
td::signal_safe_write_signal_number(sig, true);
td::Stacktrace::PrintOptions options;
static void default_failure_signal_handler(int sig) {
signal_safe_write_signal_number(sig);
Stacktrace::PrintOptions options;
options.use_gdb = true;
td::Stacktrace::print_to_stderr(options);
Stacktrace::print_to_stderr(options);
unblock_stdin();
block_stdin();
_Exit(EXIT_FAILURE);
}
Status set_default_failure_signal_handler() {
atexit(unblock_stdin);
atexit(block_stdin);
TRY_STATUS(setup_signals_alt_stack());
TRY_STATUS(set_signal_handler(td::SignalType::Abort, default_failure_signal_hanler));
TRY_STATUS(td::set_signal_handler(td::SignalType::Error, default_failure_signal_hanler));
TRY_STATUS(set_signal_handler(SignalType::Abort, default_failure_signal_handler));
TRY_STATUS(set_signal_handler(SignalType::Error, default_failure_signal_handler));
return Status::OK();
}

38
tdutils/td/utils/port/stacktrace.cpp
View File

@ -5,33 +5,41 @@
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/port/stacktrace.h"
#include "td/utils/port/signals.h"
#if !TD_WINDOWS && !TD_ANDROID && !TD_FREEBSD
#if __GLIBC__
#include <execinfo.h>
#endif
#if TD_LINUX || TD_FREEBSD
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#if TD_LINUX
#include <sys/prctl.h>
#endif
#endif
namespace td {
namespace {
void print_backtrace(void) {
#if !TD_WINDOWS && !TD_ANDROID && !TD_FREEBSD
#if __GLIBC__
void *buffer[128];
int nptrs = backtrace(buffer, 128);
td::signal_safe_write("------- Stack Backtrace -------\n", false);
signal_safe_write("------- Stack Backtrace -------\n", false);
backtrace_symbols_fd(buffer, nptrs, 2);
td::signal_safe_write("-------------------------------\n", false);
signal_safe_write("-------------------------------\n", false);
#endif
}
void print_backtrace_gdb(void) {
#if !TD_WINDOWS && !TD_DARWIN && !TD_ANDROID
char pid_buf[30], *pid_buf_begin = pid_buf + sizeof(pid_buf);
#if TD_LINUX || TD_FREEBSD
char pid_buf[30];
char *pid_buf_begin = pid_buf + sizeof(pid_buf);
pid_t pid = getpid();
*--pid_buf_begin = '\0';
do {
@ -46,23 +54,23 @@ void print_backtrace_gdb(void) {
#if TD_LINUX
if (prctl(PR_SET_DUMPABLE, 1, 0, 0, 0) < 0) {
td::signal_safe_write("Can't set dumpable\n");
signal_safe_write("Can't set dumpable\n");
return;
}
#if defined(PR_SET_PTRACER)
// We can't use td::EventFd because we are in a signal handler
// We can't use EventFd because we are in a signal handler
int fds[2];
bool need_set_ptracer = true;
if (pipe(fds) < 0) {
need_set_ptracer = false;
td::signal_safe_write("Can't create a pipe\n");
signal_safe_write("Can't create a pipe\n");
}
#endif
#endif
int child_pid = fork();
if (child_pid < 0) {
td::signal_safe_write("Can't fork() to run gdb\n");
signal_safe_write("Can't fork() to run gdb\n");
return;
}
if (!child_pid) {
@ -80,20 +88,21 @@ void print_backtrace_gdb(void) {
#if TD_LINUX && defined(PR_SET_PTRACER)
if (need_set_ptracer) {
if (prctl(PR_SET_PTRACER, child_pid, 0, 0, 0) < 0) {
td::signal_safe_write("Can't set ptracer\n");
signal_safe_write("Can't set ptracer\n");
}
if (write(fds[1], "a", 1) != 1) {
td::signal_safe_write("Can't write to pipe\n");
signal_safe_write("Can't write to pipe\n");
}
}
#endif
waitpid(child_pid, nullptr, 0);
}
} else {
td::signal_safe_write("Can't get name of executable file to pass to gdb\n");
signal_safe_write("Can't get name of executable file to pass to gdb\n");
}
#endif
}
} // namespace
void Stacktrace::print_to_stderr(const PrintOptions &options) {
@ -102,4 +111,5 @@ void Stacktrace::print_to_stderr(const PrintOptions &options) {
}
print_backtrace();
}
} // namespace td

4
tdutils/td/utils/port/stacktrace.h
View File

@ -5,13 +5,13 @@
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
namespace td {
class Stacktrace {
public:
struct PrintOptions {
bool use_gdb = false;
PrintOptions() {
}
};
static void print_to_stderr(const PrintOptions &options = PrintOptions());
};

2
tdutils/td/utils/tests.cpp
View File

@ -6,7 +6,6 @@
//
#include "td/utils/tests.h"
#include "td/utils/base64.h"
#include "td/utils/crypto.h"
#include "td/utils/filesystem.h"
#include "td/utils/Parser.h"
@ -228,4 +227,5 @@ Status TestsRunner::verify(Slice data) {
}
return regression_tester_->verify_test(PSLICE() << name() << "_default", data);
}
} // namespace td

1
tdutils/td/utils/tests.h
View File

@ -12,7 +12,6 @@
#include "td/utils/logging.h"
#include "td/utils/port/thread.h"
#include "td/utils/Random.h"
#include "td/utils/optional.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"

25
tdutils/td/utils/uint128.h
View File

@ -6,8 +6,12 @@
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/bits.h"
#include "td/utils/common.h"
#include <limits>
#include <type_traits>
namespace td {
class uint128_emulated {
@ -15,9 +19,11 @@ class uint128_emulated {
using uint128 = uint128_emulated;
uint128_emulated(uint64 hi, uint64 lo) : hi_(hi), lo_(lo) {
}
uint128_emulated(uint64 lo) : uint128_emulated(0, lo) {
template <class T, typename = std::enable_if_t<std::is_unsigned<T>::value>>
uint128_emulated(T lo) : uint128_emulated(0, lo) {
}
uint128_emulated() = default;
uint64 hi() const {
return hi_;
}
@ -40,29 +46,29 @@ class uint128_emulated {
}
uint128 shl(int cnt) const {
if (cnt >= 128) {
return uint128();
}
if (cnt == 0) {
return *this;
}
if (cnt < 64) {
return uint128((hi() << cnt) | (lo() >> (64 - cnt)), lo() << cnt);
}
if (cnt < 128) {
return uint128(lo() << (cnt - 64), 0);
}
uint128 shr(int cnt) const {
if (cnt >= 128) {
return uint128();
}
uint128 shr(int cnt) const {
if (cnt == 0) {
return *this;
}
if (cnt < 64) {
return uint128(hi() >> cnt, (lo() >> cnt) | (hi() << (64 - cnt)));
}
if (cnt < 128) {
return uint128(0, hi() >> (cnt - 64));
}
return uint128();
}
uint128 mult(uint128 other) const {
uint64 a_lo = lo() & 0xffffffff;
@ -156,7 +162,7 @@ class uint128_emulated {
uint64 lo_{0};
bool is_negative() const {
return static_cast<int64>(hi_) < 0;
return (hi_ >> 63) == 1;
}
int32 count_leading_zeroes() const {
@ -179,6 +185,7 @@ class uint128_emulated {
return res;
}
};
#if TD_HAVE_INT128
class uint128_intrinsic {
public:
@ -262,9 +269,11 @@ class uint128_intrinsic {
}
};
#endif
#if TD_HAVE_INT128
using uint128 = uint128_intrinsic;
#else
using uint128 = uint128_emulated;
#endif
} // namespace td

46
tdutils/test/ConcurrentHashMap.cpp
View File

@ -4,13 +4,18 @@
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include <cstdio>
#include "td/utils/tests.h"
#include "td/utils/benchmark.h"
#include "td/utils/SpinLock.h"
#include "td/utils/HazardPointers.h"
#include "td/utils/ConcurrentHashTable.h"
#include "td/utils/HazardPointers.h"
#include "td/utils/port/thread.h"
#include "td/utils/SpinLock.h"
#include "td/utils/tests.h"
#include <algorithm>
#include <atomic>
#include <cstdio>
#if !TD_THREAD_UNSUPPORTED
#if TD_HAVE_ABSL
#include <absl/container/flat_hash_map.h>
@ -18,7 +23,7 @@
#include <unordered_map>
#endif
#if TD_WITH_JUNCTION
#if TD_WITH_LIBCUCKOO
#include <third-party/libcuckoo/libcuckoo/cuckoohash_map.hh>
#endif
@ -27,13 +32,14 @@
#include <junction/ConcurrentMap_Linear.h>
#include <junction/ConcurrentMap_Leapfrog.h>
#endif
namespace td {
// Non resizable HashMap. Just an example
template <class KeyT, class ValueT>
class ArrayHashMap {
public:
ArrayHashMap(size_t n) : array_(n) {
explicit ArrayHashMap(size_t n) : array_(n) {
}
struct Node {
std::atomic<KeyT> key{KeyT{}};
@ -61,7 +67,7 @@ class ArrayHashMap {
template <class KeyT, class ValueT>
class ConcurrentHashMapMutex {
public:
ConcurrentHashMapMutex(size_t) {
explicit ConcurrentHashMapMutex(size_t) {
}
static std::string get_name() {
return "ConcurrentHashMapMutex";
@ -87,10 +93,11 @@ class ConcurrentHashMapMutex {
std::unordered_map<KeyT, ValueT> hash_map_;
#endif
};
template <class KeyT, class ValueT>
class ConcurrentHashMapSpinlock {
public:
ConcurrentHashMapSpinlock(size_t) {
explicit ConcurrentHashMapSpinlock(size_t) {
}
static std::string get_name() {
return "ConcurrentHashMapSpinlock";
@ -109,18 +116,19 @@ class ConcurrentHashMapSpinlock {
}
private:
td::SpinLock spinlock_;
SpinLock spinlock_;
#if TD_HAVE_ABSL
absl::flat_hash_map<KeyT, ValueT> hash_map_;
#else
std::unordered_map<KeyT, ValueT> hash_map_;
#endif
};
#if TD_WITH_LIBCUCKOO
template <class KeyT, class ValueT>
class ConcurrentHashMapLibcuckoo {
public:
ConcurrentHashMapLibcuckoo(size_t) {
explicit ConcurrentHashMapLibcuckoo(size_t) {
}
static std::string get_name() {
return "ConcurrentHashMapLibcuckoo";
@ -137,11 +145,12 @@ class ConcurrentHashMapLibcuckoo {
cuckoohash_map<KeyT, ValueT> hash_map_;
};
#endif
#if TD_WITH_JUNCTION
template <class KeyT, class ValueT>
class ConcurrentHashMapJunction {
public:
ConcurrentHashMapJunction(size_t size) : hash_map_() {
explicit ConcurrentHashMapJunction(size_t size) : hash_map_() {
}
static std::string get_name() {
return "ConcurrentHashMapJunction";
@ -152,6 +161,11 @@ class ConcurrentHashMapJunction {
ValueT find(KeyT key, ValueT default_value) {
return hash_map_.get(key);
}
ConcurrentHashMapJunction(const ConcurrentHashMapJunction &) = delete;
ConcurrentHashMapJunction &operator=(const ConcurrentHashMapJunction &) = delete;
ConcurrentHashMapJunction(ConcurrentHashMapJunction &&other) = delete;
ConcurrentHashMapJunction &operator=(ConcurrentHashMapJunction &&) = delete;
~ConcurrentHashMapJunction() {
junction::DefaultQSBR.flush();
}
@ -160,6 +174,7 @@ class ConcurrentHashMapJunction {
junction::ConcurrentMap_Leapfrog<KeyT, ValueT> hash_map_;
};
#endif
} // namespace td
template <class HashMap>
@ -169,7 +184,7 @@ class HashMapBenchmark : public td::Benchmark {
int value;
};
std::vector<Query> queries;
std::unique_ptr<HashMap> hash_map;
td::unique_ptr<HashMap> hash_map;
size_t threads_n = 16;
int mod_;
@ -177,7 +192,7 @@ class HashMapBenchmark : public td::Benchmark {
int n_;
public:
HashMapBenchmark(size_t threads_n) : threads_n(threads_n) {
explicit HashMapBenchmark(size_t threads_n) : threads_n(threads_n) {
}
std::string get_description() const override {
return hash_map->get_name();
@ -185,7 +200,7 @@ class HashMapBenchmark : public td::Benchmark {
void start_up_n(int n) override {
n *= (int)threads_n;
n_ = n;
hash_map = std::make_unique<HashMap>(n * 2);
hash_map = td::make_unique<HashMap>(n * 2);
}
void run(int n) override {
@ -217,8 +232,6 @@ class HashMapBenchmark : public td::Benchmark {
queries.clear();
hash_map.reset();
}
private:
};
template <class HashMap>
@ -240,3 +253,4 @@ TEST(ConcurrentHashMap, Benchmark) {
#endif
}
#endif

3
tdutils/test/crypto.cpp
View File

@ -178,10 +178,11 @@ TEST(Crypto, crc32c) {
ASSERT_EQ(answers[i], b);
}
}
TEST(Crypto, crc32c_benchmark) {
class Crc32cExtendBenchmark : public td::Benchmark {
public:
Crc32cExtendBenchmark(size_t chunk_size) : chunk_size_(chunk_size) {
explicit Crc32cExtendBenchmark(size_t chunk_size) : chunk_size_(chunk_size) {
}
std::string get_description() const override {
return PSTRING() << "Crc32c with chunk_size=" << chunk_size_;

21
tdutils/test/misc.cpp
View File

@ -6,7 +6,6 @@
//
#include "td/utils/as.h"
#include "td/utils/base64.h"
#include "td/utils/bits.h"
#include "td/utils/BigNum.h"
#include "td/utils/bits.h"
#include "td/utils/CancellationToken.h"
@ -30,8 +29,8 @@
#include "td/utils/Slice.h"
#include "td/utils/StringBuilder.h"
#include "td/utils/tests.h"
#include "td/utils/translit.h"
#include "td/utils/Time.h"
#include "td/utils/translit.h"
#include "td/utils/uint128.h"
#include "td/utils/unicode.h"
#include "td/utils/utf8.h"
@ -40,8 +39,8 @@
#include <clocale>
#include <limits>
#include <locale>
#include <utility>
#include <unordered_map>
#include <utility>
#if TD_HAVE_ABSL
#include <absl/container/flat_hash_map.h>
@ -673,6 +672,7 @@ TEST(Misc, Bits) {
ASSERT_EQ(4, count_bits64((1ull << 63) | 7));
}
#if !TD_THREAD_UNSUPPORTED
TEST(Misc, Time) {
Stage run;
Stage check;
@ -703,6 +703,7 @@ TEST(Misc, Time) {
thread.join();
}
}
#endif
TEST(Misc, uint128) {
std::vector<uint64> parts = {0,
@ -753,19 +754,15 @@ TEST(Misc, uint128) {
}
}
for (auto a : nums) {
#if TD_HAVE_INT128
for (auto a : nums) {
auto ia = to_intrinsic(a);
ensure_eq(a, ia);
CHECK(a.is_zero() == ia.is_zero());
#endif
for (int i = 0; i <= 130; i++) {
#if TD_HAVE_INT128
ensure_eq(a.shl(i), ia.shl(i));
ensure_eq(a.shr(i), ia.shr(i));
#endif
}
#if TD_HAVE_INT128
for (auto b : parts) {
ensure_eq(a.mult(b), ia.mult(b));
}
@ -792,10 +789,8 @@ TEST(Misc, uint128) {
ensure_eq(a.mod(b), ia.mod(ib));
}
}
#endif
}
#if TD_HAVE_INT128
for (auto signed_part : signed_parts) {
auto a = uint128_emulated::from_signed(signed_part);
auto ia = uint128_intrinsic::from_signed(signed_part);
@ -828,7 +823,7 @@ Status test_hash(const std::vector<ValueT> &values) {
class BadValue {
public:
BadValue(size_t value) : value_(value) {
explicit BadValue(size_t value) : value_(value) {
}
template <class H>
@ -845,7 +840,7 @@ class BadValue {
class ValueA {
public:
ValueA(size_t value) : value_(value) {
explicit ValueA(size_t value) : value_(value) {
}
template <class H>
friend H AbslHashValue(H hasher, ValueA value) {
@ -861,7 +856,7 @@ class ValueA {
class ValueB {
public:
ValueB(size_t value) : value_(value) {
explicit ValueB(size_t value) : value_(value) {
}
template <class H>

34
tdutils/test/port.cpp
View File

@ -36,28 +36,33 @@ TEST(Port, files) {
int cnt = 0;
const int ITER_COUNT = 1000;
for (int i = 0; i < ITER_COUNT; i++) {
walk_path(main_dir, [&](CSlice name, WalkPath::Type type) {
walk_path(main_dir,
[&](CSlice name, WalkPath::Type type) {
if (type == WalkPath::Type::NotDir) {
ASSERT_TRUE(name == fd_path || name == fd2_path);
}
cnt++;
}).ensure();
})
.ensure();
}
ASSERT_EQ((5 * 2 + 2) * ITER_COUNT, cnt);
bool was_abort = false;
walk_path(main_dir, [&](CSlice name, WalkPath::Type type) {
walk_path(main_dir,
[&](CSlice name, WalkPath::Type type) {
CHECK(!was_abort);
if (type == WalkPath::Type::EnterDir && ends_with(name, PSLICE() << TD_DIR_SLASH << "B")) {
was_abort = true;
return WalkPath::Action::Abort;
}
return WalkPath::Action::Continue;
}).ensure();
})
.ensure();
CHECK(was_abort);
cnt = 0;
bool is_first_dir = true;
walk_path(main_dir, [&](CSlice name, WalkPath::Type type) {
walk_path(main_dir,
[&](CSlice name, WalkPath::Type type) {
cnt++;
if (type == WalkPath::Type::EnterDir) {
if (is_first_dir) {
@ -67,7 +72,8 @@ TEST(Port, files) {
}
}
return WalkPath::Action::Continue;
}).ensure();
})
.ensure();
ASSERT_EQ(6, cnt);
ASSERT_EQ(0u, fd.get_size().move_as_ok());
@ -113,19 +119,21 @@ TEST(Port, Writev) {
ASSERT_EQ(expected_content, content);
}
#if TD_PORT_POSIX
#if TD_PORT_POSIX && !TD_THREAD_UNSUPPORTED
#include <signal.h>
#include <sys/syscall.h>
#include <unistd.h>
#include <algorithm>
#include <mutex>
#include <set>
#include <algorithm>
std::mutex m;
std::vector<std::string> ptrs;
std::vector<int *> addrs;
TD_THREAD_LOCAL int thread_id;
void on_user_signal(int sig) {
static std::mutex m;
static std::vector<std::string> ptrs;
static std::vector<int *> addrs;
static TD_THREAD_LOCAL int thread_id;
static void on_user_signal(int sig) {
int addr;
addrs[thread_id] = &addr;
char ptr[10];