Add missing files.
GitOrigin-RevId: 8a8503246a51483927b683ba5cd74f758e21a538
This commit is contained in:
levlam
parent
a0f6616ad3
commit
e300208960
|
|
@ -105,7 +105,6 @@ set(TDUTILS_SOURCE
|
|||
td/utils/tests.cpp
|
||||
td/utils/Time.cpp
|
||||
td/utils/Timer.cpp
|
||||
td/utils/tests.cpp
|
||||
td/utils/tl_parsers.cpp
|
||||
td/utils/translit.cpp
|
||||
td/utils/unicode.cpp
|
||||
|
|
@ -185,6 +184,9 @@ set(TDUTILS_SOURCE
|
|||
td/utils/format.h
|
||||
td/utils/Gzip.h
|
||||
td/utils/GzipByteFlow.h
|
||||
td/utils/Hash.h
|
||||
td/utils/HashMap.h
|
||||
td/utils/HashSet.h
|
||||
td/utils/HazardPointers.h
|
||||
td/utils/Heap.h
|
||||
td/utils/Hints.h
|
||||
|
|
@ -298,8 +300,7 @@ if (CRC32C_FOUND)
|
|||
target_link_libraries(tdutils PRIVATE crc32c)
|
||||
endif()
|
||||
if (ABSL_FOUND)
|
||||
#target_link_libraries(tdutils PRIVATE absl::base absl::time)
|
||||
target_link_libraries_system(tdutils absl::base absl::container absl::hash )
|
||||
target_link_libraries(tdutils absl::base absl::container absl::hash)
|
||||
endif()
|
||||
|
||||
if (WIN32 AND WINGETOPT_FOUND)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,315 @@
|
|||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#pragma once
|
||||
|
||||
#include "td/utils/HazardPointers.h"
|
||||
|
||||
#include <mutex>
|
||||
#include <atomic>
|
||||
#include <condition_variable>
|
||||
#include <vector>
|
||||
// AtomicHashArray<KeyT, ValueT>
|
||||
// Building block for other conurrent hash maps
|
||||
//
|
||||
// Support one operation:
|
||||
// template <class F>
|
||||
// bool with_value(KeyT key, bool should_create, F &&func);
|
||||
//
|
||||
// Finds slot for key, and call func(value)
|
||||
// Creates slot if shoul_create is true.
|
||||
// Returns true if func was called.
|
||||
//
|
||||
// Concurrent calls with same key may result in concurrent calls of func(value)
|
||||
// It is resposibility of caller to handle such races.
|
||||
//
|
||||
// Key should already be random
|
||||
// It is resposibility of caller to provide unique random key.
|
||||
// One may use injective hash function, or handle collisions on some other way.
|
||||
|
||||
namespace td {
|
||||
template <class KeyT, class ValueT>
|
||||
class AtomicHashArray {
|
||||
public:
|
||||
AtomicHashArray(size_t n) : nodes_(n) {
|
||||
}
|
||||
struct Node {
|
||||
std::atomic<KeyT> key{KeyT{}};
|
||||
ValueT value{};
|
||||
};
|
||||
size_t size() const {
|
||||
return nodes_.size();
|
||||
}
|
||||
Node &node_at(size_t i) {
|
||||
return nodes_[i];
|
||||
}
|
||||
static KeyT empty_key() {
|
||||
return KeyT{};
|
||||
}
|
||||
|
||||
template <class F>
|
||||
bool with_value(KeyT key, bool should_create, F &&f) {
|
||||
DCHECK(key != empty_key());
|
||||
size_t pos = static_cast<size_t>(key) % nodes_.size();
|
||||
size_t n = std::min(std::max(size_t(300), nodes_.size() / 16 + 2), nodes_.size());
|
||||
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
pos++;
|
||||
if (pos >= nodes_.size()) {
|
||||
pos = 0;
|
||||
}
|
||||
auto &node = nodes_[pos];
|
||||
while (true) {
|
||||
auto node_key = node.key.load(std::memory_order_acquire);
|
||||
if (node_key == empty_key()) {
|
||||
if (!should_create) {
|
||||
return false;
|
||||
}
|
||||
KeyT expected_key = empty_key();
|
||||
if (node.key.compare_exchange_strong(expected_key, key, std::memory_order_relaxed,
|
||||
std::memory_order_relaxed)) {
|
||||
f(node.value);
|
||||
return true;
|
||||
}
|
||||
} else if (node_key == key) {
|
||||
f(node.value);
|
||||
return true;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<Node> nodes_;
|
||||
};
|
||||
|
||||
// Simple concurrent hash map with multiple limitations
|
||||
template <class KeyT, class ValueT>
|
||||
class ConcurrentHashMap {
|
||||
using HashMap = AtomicHashArray<KeyT, std::atomic<ValueT>>;
|
||||
static td::HazardPointers<HashMap> hp_;
|
||||
|
||||
public:
|
||||
ConcurrentHashMap(size_t n = 32) {
|
||||
n = 1;
|
||||
hash_map_.store(make_unique<HashMap>(n).release());
|
||||
}
|
||||
ConcurrentHashMap(const ConcurrentHashMap &) = delete;
|
||||
ConcurrentHashMap &operator=(const ConcurrentHashMap &) = delete;
|
||||
ConcurrentHashMap(ConcurrentHashMap &&) = delete;
|
||||
ConcurrentHashMap &operator=(ConcurrentHashMap &&) = delete;
|
||||
~ConcurrentHashMap() {
|
||||
td::unique_ptr<HashMap>(hash_map_.load());
|
||||
}
|
||||
|
||||
static std::string get_name() {
|
||||
return "ConcurrrentHashMap";
|
||||
}
|
||||
|
||||
static KeyT empty_key() {
|
||||
return KeyT{};
|
||||
}
|
||||
static ValueT empty_value() {
|
||||
return ValueT{};
|
||||
}
|
||||
static ValueT migrate_value() {
|
||||
return (ValueT)(1); // c-style convertion because reinterpret_cast<int>(1) is CE in MSVC
|
||||
}
|
||||
|
||||
ValueT insert(KeyT key, ValueT value) {
|
||||
CHECK(key != empty_key());
|
||||
CHECK(value != migrate_value());
|
||||
typename HazardPointers<HashMap>::Holder holder(hp_, get_thread_id(), 0);
|
||||
while (true) {
|
||||
auto hash_map = holder.protect(hash_map_);
|
||||
if (!hash_map) {
|
||||
do_migrate(nullptr);
|
||||
continue;
|
||||
}
|
||||
|
||||
bool ok = false;
|
||||
ValueT inserted_value;
|
||||
hash_map->with_value(key, true, [&](auto &node_value) {
|
||||
ValueT expected_value = this->empty_value();
|
||||
if (node_value.compare_exchange_strong(expected_value, value, std::memory_order_release,
|
||||
std::memory_order_acquire)) {
|
||||
ok = true;
|
||||
inserted_value = value;
|
||||
} else {
|
||||
if (expected_value == this->migrate_value()) {
|
||||
ok = false;
|
||||
} else {
|
||||
ok = true;
|
||||
inserted_value = expected_value;
|
||||
}
|
||||
}
|
||||
});
|
||||
if (ok) {
|
||||
return inserted_value;
|
||||
}
|
||||
do_migrate(hash_map);
|
||||
}
|
||||
}
|
||||
ValueT find(KeyT key, ValueT value) {
|
||||
typename HazardPointers<HashMap>::Holder holder(hp_, get_thread_id(), 0);
|
||||
while (true) {
|
||||
auto hash_map = holder.protect(hash_map_);
|
||||
if (!hash_map) {
|
||||
do_migrate(nullptr);
|
||||
continue;
|
||||
}
|
||||
|
||||
bool has_value = hash_map->with_value(
|
||||
key, false, [&](auto &node_value) { value = node_value.load(std::memory_order_acquire); });
|
||||
if (!has_value || value != migrate_value()) {
|
||||
return value;
|
||||
}
|
||||
do_migrate(hash_map);
|
||||
}
|
||||
}
|
||||
template <class F>
|
||||
void for_each(F &&f) {
|
||||
auto hash_map = hash_map_.load();
|
||||
CHECK(hash_map);
|
||||
auto size = hash_map->size();
|
||||
for (size_t i = 0; i < size; i++) {
|
||||
auto &node = hash_map->node_at(i);
|
||||
auto key = node.key.load(std::memory_order_relaxed);
|
||||
auto value = node.value.load(std::memory_order_relaxed);
|
||||
|
||||
if (key != empty_key()) {
|
||||
CHECK(value != migrate_value());
|
||||
if (value != empty_value()) {
|
||||
f(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
// use no padding intentionally
|
||||
std::atomic<HashMap *> hash_map_{nullptr};
|
||||
|
||||
std::mutex migrate_mutex_;
|
||||
std::condition_variable migrate_cv_;
|
||||
|
||||
int migrate_cnt_{0};
|
||||
int migrate_generation_{0};
|
||||
HashMap *migrate_from_hash_map_{nullptr};
|
||||
HashMap *migrate_to_hash_map_{nullptr};
|
||||
struct Task {
|
||||
size_t begin;
|
||||
size_t end;
|
||||
bool empty() const {
|
||||
return begin >= end;
|
||||
}
|
||||
size_t size() const {
|
||||
if (empty()) {
|
||||
return 0;
|
||||
}
|
||||
return end - begin;
|
||||
}
|
||||
};
|
||||
|
||||
struct TaskCreator {
|
||||
size_t chunk_size;
|
||||
size_t size;
|
||||
std::atomic<size_t> pos{0};
|
||||
Task create() {
|
||||
auto i = pos++;
|
||||
auto begin = i * chunk_size;
|
||||
auto end = begin + chunk_size;
|
||||
if (end > size) {
|
||||
end = size;
|
||||
}
|
||||
return {begin, end};
|
||||
}
|
||||
};
|
||||
TaskCreator task_creator;
|
||||
|
||||
void do_migrate(HashMap *ptr) {
|
||||
//LOG(ERROR) << "do migrate: " << ptr;
|
||||
std::unique_lock<std::mutex> lock(migrate_mutex_);
|
||||
if (hash_map_.load() != ptr) {
|
||||
return;
|
||||
}
|
||||
init_migrate();
|
||||
CHECK(!ptr || migrate_from_hash_map_ == ptr);
|
||||
migrate_cnt_++;
|
||||
auto migrate_generation = migrate_generation_;
|
||||
lock.unlock();
|
||||
|
||||
run_migrate();
|
||||
|
||||
lock.lock();
|
||||
migrate_cnt_--;
|
||||
if (migrate_cnt_ == 0) {
|
||||
finish_migrate();
|
||||
}
|
||||
migrate_cv_.wait(lock, [&] { return migrate_generation_ != migrate_generation; });
|
||||
}
|
||||
|
||||
void finish_migrate() {
|
||||
//LOG(ERROR) << "finish_migrate";
|
||||
hash_map_.store(migrate_to_hash_map_);
|
||||
hp_.retire(get_thread_id(), migrate_from_hash_map_);
|
||||
migrate_from_hash_map_ = nullptr;
|
||||
migrate_to_hash_map_ = nullptr;
|
||||
migrate_generation_++;
|
||||
migrate_cv_.notify_all();
|
||||
}
|
||||
|
||||
void init_migrate() {
|
||||
if (migrate_from_hash_map_ != nullptr) {
|
||||
return;
|
||||
}
|
||||
//LOG(ERROR) << "init_migrate";
|
||||
CHECK(migrate_cnt_ == 0);
|
||||
migrate_generation_++;
|
||||
migrate_from_hash_map_ = hash_map_.exchange(nullptr);
|
||||
auto new_size = migrate_from_hash_map_->size() * 2;
|
||||
migrate_to_hash_map_ = make_unique<HashMap>(new_size).release();
|
||||
task_creator.chunk_size = 100;
|
||||
task_creator.size = migrate_from_hash_map_->size();
|
||||
task_creator.pos = 0;
|
||||
}
|
||||
|
||||
void run_migrate() {
|
||||
//LOG(ERROR) << "run_migrate";
|
||||
size_t cnt = 0;
|
||||
while (true) {
|
||||
auto task = task_creator.create();
|
||||
cnt += task.size();
|
||||
if (task.empty()) {
|
||||
break;
|
||||
}
|
||||
run_task(task);
|
||||
}
|
||||
//LOG(ERROR) << "run_migrate " << cnt;
|
||||
}
|
||||
|
||||
void run_task(Task task) {
|
||||
for (auto i = task.begin; i < task.end; i++) {
|
||||
auto &node = migrate_from_hash_map_->node_at(i);
|
||||
auto old_value = node.value.exchange(migrate_value(), std::memory_order_acq_rel);
|
||||
if (old_value == 0) {
|
||||
continue;
|
||||
}
|
||||
auto node_key = node.key.load(std::memory_order_relaxed);
|
||||
//LOG(ERROR) << node_key << " " << node_key;
|
||||
auto ok = migrate_to_hash_map_->with_value(
|
||||
node_key, true, [&](auto &node_value) { node_value.store(old_value, std::memory_order_relaxed); });
|
||||
LOG_CHECK(ok) << "migration overflow";
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <class KeyT, class ValueT>
|
||||
td::HazardPointers<typename ConcurrentHashMap<KeyT, ValueT>::HashMap> ConcurrentHashMap<KeyT, ValueT>::hp_(64);
|
||||
} // namespace td
|
||||
|
|
@ -0,0 +1,196 @@
|
|||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#pragma once
|
||||
#include "td/utils/common.h"
|
||||
#include "td/utils/logging.h"
|
||||
#include "td/utils/port/sleep.h"
|
||||
#include "td/utils/port/thread.h"
|
||||
|
||||
namespace td {
|
||||
template <class T>
|
||||
class EpochBasedMemoryReclamation {
|
||||
public:
|
||||
EpochBasedMemoryReclamation(const EpochBasedMemoryReclamation &other) = delete;
|
||||
EpochBasedMemoryReclamation &operator=(const EpochBasedMemoryReclamation &other) = delete;
|
||||
EpochBasedMemoryReclamation(EpochBasedMemoryReclamation &&other) = delete;
|
||||
EpochBasedMemoryReclamation &operator=(EpochBasedMemoryReclamation &&other) = delete;
|
||||
|
||||
class Locker {
|
||||
public:
|
||||
Locker(size_t thread_id, EpochBasedMemoryReclamation *ebmr) : thread_id_(thread_id), ebmr_(ebmr) {
|
||||
}
|
||||
Locker(const Locker &other) = delete;
|
||||
Locker &operator=(const Locker &other) = delete;
|
||||
Locker(Locker &&other) = default;
|
||||
Locker &operator=(Locker &&other) = delete;
|
||||
|
||||
~Locker() {
|
||||
if (ebmr_) {
|
||||
retire_sync();
|
||||
unlock();
|
||||
ebmr_.release();
|
||||
}
|
||||
}
|
||||
void lock() {
|
||||
DCHECK(ebmr_);
|
||||
ebmr_->lock(thread_id_);
|
||||
}
|
||||
void unlock() {
|
||||
DCHECK(ebmr_);
|
||||
ebmr_->unlock(thread_id_);
|
||||
}
|
||||
|
||||
void retire_sync() {
|
||||
ebmr_->retire_sync(thread_id_);
|
||||
}
|
||||
|
||||
void retire() {
|
||||
ebmr_->retire(thread_id_);
|
||||
}
|
||||
|
||||
void retire(T *ptr) {
|
||||
ebmr_->retire(thread_id_, ptr);
|
||||
}
|
||||
|
||||
private:
|
||||
size_t thread_id_;
|
||||
struct Never {
|
||||
template <class S>
|
||||
void operator()(S *) const {
|
||||
UNREACHABLE();
|
||||
}
|
||||
};
|
||||
std::unique_ptr<EpochBasedMemoryReclamation, Never> ebmr_;
|
||||
};
|
||||
|
||||
explicit EpochBasedMemoryReclamation(size_t threads_n) : threads_(threads_n) {
|
||||
}
|
||||
|
||||
Locker get_locker(size_t thread_id) {
|
||||
return Locker{thread_id, this};
|
||||
}
|
||||
|
||||
size_t to_delete_size_unsafe() const {
|
||||
size_t res = 0;
|
||||
for (auto &thread : threads_) {
|
||||
LOG(ERROR) << "---" << thread.epoch.load() / 2;
|
||||
for (size_t i = 0; i < MAX_BAGS; i++) {
|
||||
res += thread.to_delete[i].size();
|
||||
LOG(ERROR) << thread.to_delete[i].size();
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
private:
|
||||
friend class Locker;
|
||||
static constexpr size_t MAX_BAGS = 3;
|
||||
struct ThreadData {
|
||||
std::atomic<int64> epoch{1};
|
||||
char pad[TD_CONCURRENCY_PAD - sizeof(epoch)];
|
||||
|
||||
size_t to_skip{0};
|
||||
size_t checked_thread_i{0};
|
||||
size_t bag_i{0};
|
||||
std::vector<unique_ptr<T>> to_delete[MAX_BAGS];
|
||||
char pad2[TD_CONCURRENCY_PAD - sizeof(to_delete)];
|
||||
|
||||
void rotate_bags() {
|
||||
bag_i = (bag_i + 1) % MAX_BAGS;
|
||||
to_delete[bag_i].clear();
|
||||
}
|
||||
|
||||
void set_epoch(int64 new_epoch) {
|
||||
//LOG(ERROR) << new_epoch;
|
||||
if (epoch.load(std::memory_order_relaxed) / 2 != new_epoch) {
|
||||
checked_thread_i = 0;
|
||||
to_skip = 0;
|
||||
rotate_bags();
|
||||
}
|
||||
epoch = new_epoch * 2;
|
||||
}
|
||||
|
||||
void idle() {
|
||||
epoch.store(epoch.load(std::memory_order_relaxed) | 1);
|
||||
}
|
||||
|
||||
size_t undeleted() const {
|
||||
size_t res = 0;
|
||||
for (size_t i = 0; i < MAX_BAGS; i++) {
|
||||
res += to_delete[i].size();
|
||||
}
|
||||
return res;
|
||||
}
|
||||
};
|
||||
std::vector<ThreadData> threads_;
|
||||
char pad[TD_CONCURRENCY_PAD - sizeof(threads_)];
|
||||
|
||||
std::atomic<int64> epoch_{1};
|
||||
char pad2[TD_CONCURRENCY_PAD - sizeof(epoch_)];
|
||||
|
||||
void lock(size_t thread_id) {
|
||||
auto &data = threads_[thread_id];
|
||||
auto epoch = epoch_.load();
|
||||
data.set_epoch(epoch);
|
||||
|
||||
if (data.to_skip == 0) {
|
||||
data.to_skip = 30;
|
||||
step_check(data);
|
||||
} else {
|
||||
data.to_skip--;
|
||||
}
|
||||
}
|
||||
|
||||
void unlock(size_t thread_id) {
|
||||
//LOG(ERROR) << "UNLOCK";
|
||||
auto &data = threads_[thread_id];
|
||||
data.idle();
|
||||
}
|
||||
|
||||
bool step_check(ThreadData &data) {
|
||||
auto epoch = data.epoch.load(std::memory_order_relaxed) / 2;
|
||||
auto checked_thread_epoch = threads_[data.checked_thread_i].epoch.load();
|
||||
if (checked_thread_epoch % 2 == 1 || checked_thread_epoch / 2 == epoch) {
|
||||
data.checked_thread_i++;
|
||||
if (data.checked_thread_i == threads_.size()) {
|
||||
if (epoch_.compare_exchange_strong(epoch, epoch + 1)) {
|
||||
data.set_epoch(epoch + 1);
|
||||
} else {
|
||||
data.set_epoch(epoch);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void retire_sync(size_t thread_id) {
|
||||
auto &data = threads_[thread_id];
|
||||
|
||||
while (true) {
|
||||
retire(thread_id);
|
||||
data.idle();
|
||||
if (data.undeleted() == 0) {
|
||||
break;
|
||||
}
|
||||
usleep_for(1000);
|
||||
}
|
||||
}
|
||||
|
||||
void retire(size_t thread_id) {
|
||||
auto &data = threads_[thread_id];
|
||||
data.set_epoch(epoch_.load());
|
||||
while (step_check(data) && data.undeleted() != 0) {
|
||||
}
|
||||
}
|
||||
|
||||
void retire(size_t thread_id, T *ptr) {
|
||||
auto &data = threads_[thread_id];
|
||||
data.to_delete[data.bag_i].push_back(unique_ptr<T>{ptr});
|
||||
}
|
||||
};
|
||||
} // namespace td
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#pragma once
|
||||
|
||||
#include "td/utils/common.h"
|
||||
|
||||
#if TD_HAVE_ABSL
|
||||
#include <absl/hash/hash.h>
|
||||
#endif
|
||||
|
||||
namespace td {
|
||||
// A simple wrapper for absl::flat_hash_map, std::unordered_map and probably some our implementaion of hash map in
|
||||
// the future
|
||||
|
||||
// We will introduce out own Hashing utility like an absl one.
|
||||
class Hasher {
|
||||
public:
|
||||
Hasher() = default;
|
||||
Hasher(size_t init_value) : hash_(init_value) {
|
||||
}
|
||||
std::size_t finalize() {
|
||||
return hash_;
|
||||
}
|
||||
|
||||
static Hasher combine(Hasher hasher, size_t value) {
|
||||
hasher.hash_ ^= value;
|
||||
return hasher;
|
||||
}
|
||||
|
||||
template <class A, class B>
|
||||
static Hasher combine(Hasher hasher, const std::pair<A, B> &value) {
|
||||
hasher = AbslHashValue(std::move(hasher), value.first);
|
||||
hasher = AbslHashValue(std::move(hasher), value.first);
|
||||
return hasher;
|
||||
}
|
||||
|
||||
private:
|
||||
std::size_t hash_{0};
|
||||
};
|
||||
|
||||
template <class IgnoreT>
|
||||
class TdHash {
|
||||
public:
|
||||
template <class T>
|
||||
std::size_t operator()(const T &value) const noexcept {
|
||||
return AbslHashValue(Hasher(), value).finalize();
|
||||
}
|
||||
};
|
||||
|
||||
#if TD_HAVE_ABSL
|
||||
template <class T>
|
||||
using AbslHash = absl::Hash<T>;
|
||||
#endif
|
||||
|
||||
// default hash implementations
|
||||
template <class H, class T>
|
||||
decltype(H::combine(std::declval<H>(), std::declval<T>())) AbslHashValue(H hasher, const T &value) {
|
||||
return H::combine(std::move(hasher), value);
|
||||
}
|
||||
|
||||
#if TD_HAVE_ABSL
|
||||
template <class T>
|
||||
using Hash = AbslHash<T>;
|
||||
#else
|
||||
template <class T>
|
||||
using Hash = TdHash<T>;
|
||||
#endif
|
||||
|
||||
} // namespace td
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#pragma once
|
||||
|
||||
#include "td/utils/Hash.h"
|
||||
|
||||
#if TD_HAVE_ABSL
|
||||
#include <absl/container/flat_hash_map.h>
|
||||
#else
|
||||
#include <unordered_map>
|
||||
#endif
|
||||
namespace td {
|
||||
#if TD_HAVE_ABSL
|
||||
template <class Key, class Value, class H = Hash<Key>>
|
||||
using HashMap = absl::flat_hash_map<Key, Value, H>;
|
||||
#else
|
||||
template <class Key, class Value, class H = Hash<Key>>
|
||||
using HashMap = std::unordered_map<Key, Value, H>;
|
||||
#endif
|
||||
} // namespace td
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#pragma once
|
||||
|
||||
#include "td/utils/Hash.h"
|
||||
|
||||
#if TD_HAVE_ABSL
|
||||
#include <absl/container/flat_hash_set.h>
|
||||
#else
|
||||
#include <unordered_set>
|
||||
#endif
|
||||
namespace td {
|
||||
#if TD_HAVE_ABSL
|
||||
template <class Key, class H = Hash<Key>>
|
||||
using HashSet = absl::flat_hash_set<Key, H>;
|
||||
#else
|
||||
template <class Key, class H = Hash<Key>>
|
||||
using HashSet = std::unordered_set<Key, H>;
|
||||
#endif
|
||||
} // namespace td
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#include "td/utils/MpmcQueue.h"
|
||||
namespace td {
|
||||
detail::MpmcStat stat_;
|
||||
}
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#include "td/utils/OptionsParser.h"
|
||||
|
||||
#if TD_HAVE_GETOPT
|
||||
#include "getopt.h"
|
||||
#endif
|
||||
|
||||
#if !TD_WINDOWS
|
||||
#include <getopt.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
namespace td {
|
||||
void OptionsParser::set_description(std::string description) {
|
||||
description_ = std::move(description);
|
||||
}
|
||||
|
||||
void OptionsParser::add_option(Option::Type type, char short_key, Slice long_key, Slice description,
|
||||
std::function<Status(Slice)> callback) {
|
||||
options_.push_back(Option{type, short_key, long_key.str(), description.str(), std::move(callback)});
|
||||
}
|
||||
|
||||
void OptionsParser::add_option(char short_key, Slice long_key, Slice description,
|
||||
std::function<Status(Slice)> callback) {
|
||||
add_option(Option::Type::Arg, short_key, long_key, description, std::move(callback));
|
||||
}
|
||||
|
||||
void OptionsParser::add_option(char short_key, Slice long_key, Slice description,
|
||||
std::function<Status(void)> callback) {
|
||||
// Ouch. There must be some better way
|
||||
add_option(Option::Type::NoArg, short_key, long_key, description,
|
||||
std::bind([](std::function<Status(void)> &func, Slice) { return func(); }, std::move(callback),
|
||||
std::placeholders::_1));
|
||||
}
|
||||
|
||||
Result<int> OptionsParser::run(int argc, char *argv[]) {
|
||||
#if TD_HAVE_GETOPT
|
||||
char buff[1024];
|
||||
StringBuilder sb(MutableSlice{buff, sizeof(buff)});
|
||||
for (auto &opt : options_) {
|
||||
CHECK(opt.type != Option::OptionalArg);
|
||||
sb << opt.short_key;
|
||||
if (opt.type == Option::Arg) {
|
||||
sb << ":";
|
||||
}
|
||||
}
|
||||
if (sb.is_error()) {
|
||||
return Status::Error("Can't parse options");
|
||||
}
|
||||
CSlice short_options = sb.as_cslice();
|
||||
|
||||
vector<option> long_options;
|
||||
for (auto &opt : options_) {
|
||||
if (opt.long_key.empty()) {
|
||||
continue;
|
||||
}
|
||||
option o;
|
||||
o.flag = nullptr;
|
||||
o.val = opt.short_key;
|
||||
o.has_arg = opt.type == Option::Arg ? required_argument : no_argument;
|
||||
o.name = opt.long_key.c_str();
|
||||
long_options.push_back(o);
|
||||
}
|
||||
long_options.push_back({nullptr, 0, nullptr, 0});
|
||||
|
||||
while (true) {
|
||||
int opt_i = getopt_long(argc, argv, short_options.c_str(), &long_options[0], nullptr);
|
||||
if (opt_i == ':') {
|
||||
return Status::Error("Missing argument");
|
||||
}
|
||||
if (opt_i == '?') {
|
||||
return Status::Error("Unrecognized option");
|
||||
}
|
||||
if (opt_i == -1) {
|
||||
break;
|
||||
}
|
||||
bool found = false;
|
||||
for (auto &opt : options_) {
|
||||
if (opt.short_key == opt_i) {
|
||||
Slice arg;
|
||||
if (opt.type == Option::Arg) {
|
||||
arg = Slice(optarg);
|
||||
}
|
||||
auto status = opt.arg_callback(arg);
|
||||
if (status.is_error()) {
|
||||
return std::move(status);
|
||||
}
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!found) {
|
||||
return Status::Error("Unknown argument");
|
||||
}
|
||||
}
|
||||
return optind;
|
||||
#else
|
||||
return -1;
|
||||
#endif
|
||||
}
|
||||
|
||||
StringBuilder &operator<<(StringBuilder &sb, const OptionsParser &o) {
|
||||
sb << o.description_ << "\n";
|
||||
for (auto &opt : o.options_) {
|
||||
sb << "-" << opt.short_key;
|
||||
if (!opt.long_key.empty()) {
|
||||
sb << "|--" << opt.long_key;
|
||||
}
|
||||
if (opt.type == OptionsParser::Option::OptionalArg) {
|
||||
sb << "[";
|
||||
}
|
||||
if (opt.type != OptionsParser::Option::NoArg) {
|
||||
sb << "<arg>";
|
||||
}
|
||||
if (opt.type == OptionsParser::Option::OptionalArg) {
|
||||
sb << "]";
|
||||
}
|
||||
sb << "\t" << opt.description;
|
||||
sb << "\n";
|
||||
}
|
||||
return sb;
|
||||
}
|
||||
|
||||
} // namespace td
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#pragma once
|
||||
#include "td/utils/port/thread.h"
|
||||
#include "td/utils/port/thread_local.h"
|
||||
#include "td/utils/int_types.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <array>
|
||||
|
||||
namespace td {
|
||||
class ThreadSafeCounter {
|
||||
public:
|
||||
void add(int64 diff) {
|
||||
auto &node = thread_local_node();
|
||||
node.count_.store(node.count_.load(std::memory_order_relaxed) + diff, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
int64 sum() const {
|
||||
int n = max_thread_id_.load();
|
||||
int64 res = 0;
|
||||
for (int i = 0; i < n; i++) {
|
||||
res += nodes_[i].count_.load();
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
private:
|
||||
struct Node {
|
||||
std::atomic<int64> count_{0};
|
||||
char padding[128];
|
||||
};
|
||||
static constexpr int MAX_THREAD_ID = 128;
|
||||
std::atomic<int> max_thread_id_{MAX_THREAD_ID};
|
||||
std::array<Node, MAX_THREAD_ID> nodes_;
|
||||
|
||||
Node &thread_local_node() {
|
||||
auto thread_id = get_thread_id();
|
||||
CHECK(0 <= thread_id && static_cast<size_t>(thread_id) < nodes_.size());
|
||||
return nodes_[thread_id];
|
||||
}
|
||||
};
|
||||
} // namespace td
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#pragma once
|
||||
|
||||
#include "td/utils/Slice.h"
|
||||
#include "td/utils/Span.h"
|
||||
|
||||
#if TD_PORT_POSIX
|
||||
#include <sys/uio.h>
|
||||
#endif
|
||||
|
||||
namespace td {
|
||||
#if TD_PORT_POSIX
|
||||
using IoSlice = struct iovec;
|
||||
inline IoSlice as_io_slice(Slice slice) {
|
||||
IoSlice res;
|
||||
res.iov_len = slice.size();
|
||||
res.iov_base = const_cast<char *>(slice.data());
|
||||
return res;
|
||||
}
|
||||
inline Slice as_slice(const IoSlice io_slice) {
|
||||
return Slice(reinterpret_cast<const char *>(io_slice.iov_base), io_slice.iov_len);
|
||||
}
|
||||
#else
|
||||
using IoSlice = Slice;
|
||||
inline IoSlice as_io_slice(Slice slice) {
|
||||
return slice;
|
||||
}
|
||||
#endif
|
||||
} // namespace td
|
||||
|
|
@ -0,0 +1,109 @@
|
|||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#include "td/utils/port/MemoryMapping.h"
|
||||
|
||||
#include "td/utils/misc.h"
|
||||
|
||||
// TODO:
|
||||
// windows,
|
||||
// anonymous map
|
||||
// huge pages?
|
||||
#if TD_WINDOWS
|
||||
#else
|
||||
#include <sys/mman.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
namespace td {
|
||||
namespace detail {
|
||||
class MemoryMappingImpl {
|
||||
public:
|
||||
MemoryMappingImpl(MutableSlice data, int64 offset) : data_(data), offset_(offset) {
|
||||
}
|
||||
Slice as_slice() const {
|
||||
return data_.substr(narrow_cast<size_t>(offset_));
|
||||
}
|
||||
MutableSlice as_mutable_slice() const {
|
||||
return {};
|
||||
}
|
||||
|
||||
private:
|
||||
MutableSlice data_;
|
||||
int64 offset_;
|
||||
};
|
||||
|
||||
Result<int64> get_page_size() {
|
||||
#if TD_WINDOWS
|
||||
return Status::Error("Unimplemented");
|
||||
#else
|
||||
static Result<int64> page_size = []() -> Result<int64> {
|
||||
auto page_size = sysconf(_SC_PAGESIZE);
|
||||
if (page_size < 0) {
|
||||
return OS_ERROR("Can't load page size from sysconf");
|
||||
}
|
||||
return page_size;
|
||||
}();
|
||||
return page_size.clone();
|
||||
#endif
|
||||
}
|
||||
} // namespace detail
|
||||
|
||||
Result<MemoryMapping> MemoryMapping::create_anonymous(const MemoryMapping::Options &options) {
|
||||
return Status::Error("Unsupported yet");
|
||||
}
|
||||
Result<MemoryMapping> MemoryMapping::create_from_file(const FileFd &file_fd, const MemoryMapping::Options &options) {
|
||||
#if TD_WINDOWS
|
||||
return Status::Error("Unsupported yet");
|
||||
#else
|
||||
if (file_fd.empty()) {
|
||||
return Status::Error("Can't create memory mapping: file is empty");
|
||||
}
|
||||
TRY_RESULT(stat, file_fd.stat());
|
||||
auto fd = file_fd.get_native_fd().fd();
|
||||
auto begin = options.offset;
|
||||
if (begin < 0) {
|
||||
return Status::Error(PSLICE() << "Can't create memory mapping: negative offset " << options.offset);
|
||||
}
|
||||
|
||||
int64 end;
|
||||
if (options.size < 0) {
|
||||
end = stat.size_;
|
||||
} else {
|
||||
end = begin + stat.size_;
|
||||
}
|
||||
|
||||
TRY_RESULT(page_size, detail::get_page_size());
|
||||
auto fixed_begin = begin / page_size * page_size;
|
||||
|
||||
auto data_offset = begin - fixed_begin;
|
||||
auto data_size = end - fixed_begin;
|
||||
|
||||
void *data = mmap(nullptr, data_size, PROT_READ, MAP_PRIVATE, fd, fixed_begin);
|
||||
if (data == MAP_FAILED) {
|
||||
return OS_ERROR("mmap call failed");
|
||||
}
|
||||
|
||||
return MemoryMapping(std::make_unique<detail::MemoryMappingImpl>(
|
||||
MutableSlice(reinterpret_cast<char *>(data), data_size), data_offset));
|
||||
#endif
|
||||
}
|
||||
|
||||
MemoryMapping::MemoryMapping(MemoryMapping &&other) = default;
|
||||
MemoryMapping &MemoryMapping::operator=(MemoryMapping &&other) = default;
|
||||
MemoryMapping::~MemoryMapping() = default;
|
||||
|
||||
MemoryMapping::MemoryMapping(std::unique_ptr<detail::MemoryMappingImpl> impl) : impl_(std::move(impl)) {
|
||||
}
|
||||
Slice MemoryMapping::as_slice() const {
|
||||
return impl_->as_slice();
|
||||
}
|
||||
MutableSlice MemoryMapping::as_mutable_slice() {
|
||||
return impl_->as_mutable_slice();
|
||||
}
|
||||
|
||||
} // namespace td
|
||||
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#pragma once
|
||||
|
||||
#include "td/utils/common.h"
|
||||
#include "td/utils/port/FileFd.h"
|
||||
#include "td/utils/Slice.h"
|
||||
#include "td/utils/Status.h"
|
||||
|
||||
namespace td {
|
||||
namespace detail {
|
||||
class MemoryMappingImpl;
|
||||
}
|
||||
|
||||
class MemoryMapping {
|
||||
public:
|
||||
struct Options {
|
||||
int64 offset{0};
|
||||
int64 size{-1};
|
||||
|
||||
Options() {
|
||||
}
|
||||
Options &with_offset(int64 new_offset) {
|
||||
offset = new_offset;
|
||||
return *this;
|
||||
}
|
||||
Options &with_size(int64 new_size) {
|
||||
size = new_size;
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
|
||||
static Result<MemoryMapping> create_anonymous(const Options &options = {});
|
||||
static Result<MemoryMapping> create_from_file(const FileFd &file, const Options &options = {});
|
||||
|
||||
Slice as_slice() const;
|
||||
MutableSlice as_mutable_slice(); // returns empty slice if memory is read-only
|
||||
|
||||
MemoryMapping(const MemoryMapping &other) = delete;
|
||||
const MemoryMapping &operator=(const MemoryMapping &other) = delete;
|
||||
MemoryMapping(MemoryMapping &&other);
|
||||
MemoryMapping &operator=(MemoryMapping &&other);
|
||||
~MemoryMapping();
|
||||
|
||||
private:
|
||||
std::unique_ptr<detail::MemoryMappingImpl> impl_;
|
||||
explicit MemoryMapping(std::unique_ptr<detail::MemoryMappingImpl> impl);
|
||||
};
|
||||
} // namespace td
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#include "td/utils/port/stacktrace.h"
|
||||
#include "td/utils/port/signals.h"
|
||||
|
||||
#if !TD_WINDOWS && !TD_ANDROID && !TD_FREEBSD
|
||||
#include <execinfo.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#if TD_LINUX
|
||||
#include <sys/prctl.h>
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
namespace td {
|
||||
namespace {
|
||||
void print_backtrace(void) {
|
||||
#if !TD_WINDOWS && !TD_ANDROID && !TD_FREEBSD
|
||||
void *buffer[128];
|
||||
int nptrs = backtrace(buffer, 128);
|
||||
td::signal_safe_write("------- Stack Backtrace -------\n", false);
|
||||
backtrace_symbols_fd(buffer, nptrs, 2);
|
||||
td::signal_safe_write("-------------------------------\n", false);
|
||||
#endif
|
||||
}
|
||||
void print_backtrace_gdb(void) {
|
||||
#if !TD_WINDOWS && !TD_DARWIN && !TD_ANDROID
|
||||
char pid_buf[30], *pid_buf_begin = pid_buf + sizeof(pid_buf);
|
||||
pid_t pid = getpid();
|
||||
*--pid_buf_begin = '\0';
|
||||
do {
|
||||
*--pid_buf_begin = static_cast<char>(pid % 10 + '0');
|
||||
pid /= 10;
|
||||
} while (pid > 0);
|
||||
|
||||
char name_buf[512];
|
||||
ssize_t res = readlink("/proc/self/exe", name_buf, 511); // TODO works only under Linux
|
||||
if (res >= 0) {
|
||||
name_buf[res] = 0;
|
||||
|
||||
#if TD_LINUX
|
||||
if (prctl(PR_SET_DUMPABLE, 1, 0, 0, 0) < 0) {
|
||||
td::signal_safe_write("Can't set dumpable\n");
|
||||
return;
|
||||
}
|
||||
#if defined(PR_SET_PTRACER)
|
||||
// We can't use td::EventFd because we are in a signal handler
|
||||
int fds[2];
|
||||
bool need_set_ptracer = true;
|
||||
if (pipe(fds) < 0) {
|
||||
need_set_ptracer = false;
|
||||
td::signal_safe_write("Can't create a pipe\n");
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
int child_pid = fork();
|
||||
if (child_pid < 0) {
|
||||
td::signal_safe_write("Can't fork() to run gdb\n");
|
||||
return;
|
||||
}
|
||||
if (!child_pid) {
|
||||
#if TD_LINUX && defined(PR_SET_PTRACER)
|
||||
if (need_set_ptracer) {
|
||||
char c;
|
||||
read(fds[0], &c, 1);
|
||||
}
|
||||
#endif
|
||||
dup2(2, 1); // redirect output to stderr
|
||||
execlp("gdb", "gdb", "--batch", "-n", "-ex", "thread", "-ex", "thread apply all bt full", name_buf, pid_buf_begin,
|
||||
NULL);
|
||||
return;
|
||||
} else {
|
||||
#if TD_LINUX && defined(PR_SET_PTRACER)
|
||||
if (need_set_ptracer) {
|
||||
if (prctl(PR_SET_PTRACER, child_pid, 0, 0, 0) < 0) {
|
||||
td::signal_safe_write("Can't set ptracer\n");
|
||||
}
|
||||
if (write(fds[1], "a", 1) != 1) {
|
||||
td::signal_safe_write("Can't write to pipe\n");
|
||||
}
|
||||
}
|
||||
#endif
|
||||
waitpid(child_pid, nullptr, 0);
|
||||
}
|
||||
} else {
|
||||
td::signal_safe_write("Can't get name of executable file to pass to gdb\n");
|
||||
}
|
||||
#endif
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void Stacktrace::print_to_stderr(const PrintOptions &options) {
|
||||
if (options.use_gdb) {
|
||||
print_backtrace_gdb();
|
||||
}
|
||||
print_backtrace();
|
||||
}
|
||||
} // namespace td
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#pragma once
|
||||
namespace td {
|
||||
class Stacktrace {
|
||||
public:
|
||||
struct PrintOptions {
|
||||
bool use_gdb = false;
|
||||
PrintOptions() {
|
||||
}
|
||||
};
|
||||
static void print_to_stderr(const PrintOptions &options = PrintOptions());
|
||||
};
|
||||
|
||||
} // namespace td
|
||||
|
|
@ -0,0 +1,270 @@
|
|||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#pragma once
|
||||
|
||||
#include "td/utils/common.h"
|
||||
#include "td/utils/bits.h"
|
||||
namespace td {
|
||||
|
||||
class uint128_emulated {
|
||||
public:
|
||||
using uint128 = uint128_emulated;
|
||||
uint128_emulated(uint64 hi, uint64 lo) : hi_(hi), lo_(lo) {
|
||||
}
|
||||
uint128_emulated(uint64 lo) : uint128_emulated(0, lo) {
|
||||
}
|
||||
uint128_emulated() = default;
|
||||
uint64 hi() const {
|
||||
return hi_;
|
||||
}
|
||||
uint64 lo() const {
|
||||
return lo_;
|
||||
}
|
||||
static uint128 from_signed(int64 x) {
|
||||
if (x >= 0) {
|
||||
return uint128(0, x);
|
||||
}
|
||||
return uint128(std::numeric_limits<uint64>::max(), static_cast<uint64>(x));
|
||||
}
|
||||
|
||||
uint128 add(uint128 other) const {
|
||||
uint128 res(other.hi() + hi(), other.lo() + lo());
|
||||
if (res.lo() < lo()) {
|
||||
res.hi_++;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
uint128 shl(int cnt) const {
|
||||
if (cnt >= 128) {
|
||||
return uint128();
|
||||
}
|
||||
if (cnt == 0) {
|
||||
return *this;
|
||||
}
|
||||
if (cnt < 64) {
|
||||
return uint128((hi() << cnt) | (lo() >> (64 - cnt)), lo() << cnt);
|
||||
}
|
||||
return uint128(lo() << (cnt - 64), 0);
|
||||
}
|
||||
uint128 shr(int cnt) const {
|
||||
if (cnt >= 128) {
|
||||
return uint128();
|
||||
}
|
||||
if (cnt == 0) {
|
||||
return *this;
|
||||
}
|
||||
if (cnt < 64) {
|
||||
return uint128(hi() >> cnt, (lo() >> cnt) | (hi() << (64 - cnt)));
|
||||
}
|
||||
return uint128(0, hi() >> (cnt - 64));
|
||||
}
|
||||
|
||||
uint128 mult(uint128 other) const {
|
||||
uint64 a_lo = lo() & 0xffffffff;
|
||||
uint64 a_hi = lo() >> 32;
|
||||
uint64 b_lo = other.lo() & 0xffffffff;
|
||||
uint64 b_hi = other.lo() >> 32;
|
||||
uint128 res(lo() * other.hi() + hi() * other.lo() + a_hi * b_hi, a_lo * b_lo);
|
||||
uint128 add1(0, a_lo * b_hi);
|
||||
uint128 add2(0, a_hi * b_lo);
|
||||
return res.add(add1.shl(32)).add(add2.shl(32));
|
||||
}
|
||||
uint128 mult(uint64 other) const {
|
||||
return mult(uint128(0, other));
|
||||
}
|
||||
uint128 mult_signed(int64 other) const {
|
||||
return mult(uint128::from_signed(other));
|
||||
}
|
||||
bool is_zero() const {
|
||||
return lo() == 0 && hi() == 0;
|
||||
}
|
||||
uint128 sub(uint128 other) const {
|
||||
uint32 carry = 0;
|
||||
if (other.lo() > lo()) {
|
||||
carry = 1;
|
||||
}
|
||||
return uint128(hi() - other.hi() - carry, lo() - other.lo());
|
||||
}
|
||||
void divmod(uint128 other, uint128 *div_res, uint128 *mod_res) const {
|
||||
CHECK(!other.is_zero());
|
||||
|
||||
auto from = *this;
|
||||
auto ctz = from.count_leading_zeroes();
|
||||
auto other_ctz = other.count_leading_zeroes();
|
||||
if (ctz > other_ctz) {
|
||||
*div_res = uint128();
|
||||
*mod_res = from;
|
||||
return;
|
||||
}
|
||||
auto shift = other_ctz - ctz;
|
||||
auto res = uint128();
|
||||
for (int i = shift; i >= 0; i--) {
|
||||
auto sub = other.shl(i);
|
||||
res = res.shl(1);
|
||||
if (from.greater_or_equal(sub)) {
|
||||
from = from.sub(sub);
|
||||
res = res.set_lower_bit();
|
||||
}
|
||||
}
|
||||
|
||||
*div_res = res;
|
||||
*mod_res = from;
|
||||
}
|
||||
uint128 div(uint128 other) const {
|
||||
uint128 a, b;
|
||||
divmod(other, &a, &b);
|
||||
return a;
|
||||
}
|
||||
uint128 mod(uint128 other) const {
|
||||
uint128 a, b;
|
||||
divmod(other, &a, &b);
|
||||
return b;
|
||||
}
|
||||
|
||||
void divmod_signed(int64 y, int64 *quot, int64 *rem) const {
|
||||
CHECK(y != 0);
|
||||
auto x = *this;
|
||||
int x_sgn = x.is_negative();
|
||||
int y_sgn = y < 0;
|
||||
if (x_sgn) {
|
||||
x = x.negate();
|
||||
}
|
||||
uint128 uy = from_signed(y);
|
||||
if (uy.is_negative()) {
|
||||
uy = uy.negate();
|
||||
}
|
||||
|
||||
uint128 t_quot, t_mod;
|
||||
x.divmod(uy, &t_quot, &t_mod);
|
||||
*quot = t_quot.lo();
|
||||
*rem = t_mod.lo();
|
||||
if (x_sgn != y_sgn) {
|
||||
*quot = -*quot;
|
||||
}
|
||||
if (x_sgn) {
|
||||
*rem = -*rem;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
uint64 hi_{0};
|
||||
uint64 lo_{0};
|
||||
|
||||
bool is_negative() const {
|
||||
return static_cast<int64>(hi_) < 0;
|
||||
}
|
||||
|
||||
int32 count_leading_zeroes() const {
|
||||
if (hi() == 0) {
|
||||
return 64 + count_leading_zeroes64(lo());
|
||||
}
|
||||
return count_leading_zeroes64(hi());
|
||||
}
|
||||
uint128 set_lower_bit() const {
|
||||
return uint128(hi(), lo() | 1);
|
||||
}
|
||||
bool greater_or_equal(uint128 other) const {
|
||||
return hi() > other.hi() || (hi() == other.hi() && lo() >= other.lo());
|
||||
}
|
||||
uint128 negate() const {
|
||||
uint128 res(~hi(), ~lo() + 1);
|
||||
if (res.lo() == 0) {
|
||||
return uint128(res.hi() + 1, 0);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
};
|
||||
#if TD_HAVE_INT128
|
||||
class uint128_intrinsic {
|
||||
public:
|
||||
using ValueT = unsigned __int128;
|
||||
using uint128 = uint128_intrinsic;
|
||||
explicit uint128_intrinsic(ValueT value) : value_(value) {
|
||||
}
|
||||
uint128_intrinsic(uint64 hi, uint64 lo) : value_((ValueT(hi) << 64) | lo) {
|
||||
}
|
||||
uint128_intrinsic() = default;
|
||||
|
||||
static uint128 from_signed(int64 x) {
|
||||
return uint128(static_cast<ValueT>(x));
|
||||
}
|
||||
uint64 hi() const {
|
||||
return uint64(value() >> 64);
|
||||
}
|
||||
uint64 lo() const {
|
||||
return uint64(value() & std::numeric_limits<uint64>::max());
|
||||
}
|
||||
|
||||
uint128 add(uint128 other) const {
|
||||
return uint128(value() + other.value());
|
||||
}
|
||||
uint128 sub(uint128 other) const {
|
||||
return uint128(value() - other.value());
|
||||
}
|
||||
|
||||
uint128 shl(int cnt) const {
|
||||
if (cnt >= 128) {
|
||||
return uint128();
|
||||
}
|
||||
return uint128(value() << cnt);
|
||||
}
|
||||
|
||||
uint128 shr(int cnt) const {
|
||||
if (cnt >= 128) {
|
||||
return uint128();
|
||||
}
|
||||
return uint128(value() >> cnt);
|
||||
}
|
||||
|
||||
uint128 mult(uint128 other) const {
|
||||
return uint128(value() * other.value());
|
||||
}
|
||||
uint128 mult(uint64 other) const {
|
||||
return uint128(value() * other);
|
||||
}
|
||||
uint128 mult_signed(int64 other) const {
|
||||
return uint128(value() * other);
|
||||
}
|
||||
bool is_zero() const {
|
||||
return value() == 0;
|
||||
}
|
||||
void divmod(uint128 other, uint128 *div_res, uint128 *mod_res) const {
|
||||
CHECK(!other.is_zero());
|
||||
*div_res = uint128(value() / other.value());
|
||||
*mod_res = uint128(value() % other.value());
|
||||
}
|
||||
uint128 div(uint128 other) const {
|
||||
CHECK(!other.is_zero());
|
||||
return uint128(value() / other.value());
|
||||
}
|
||||
uint128 mod(uint128 other) const {
|
||||
CHECK(!other.is_zero());
|
||||
return uint128(value() % other.value());
|
||||
}
|
||||
void divmod_signed(int64 y, int64 *quot, int64 *rem) const {
|
||||
CHECK(y != 0);
|
||||
*quot = (int64)(signed_value() / y);
|
||||
*rem = (int64)(signed_value() % y);
|
||||
}
|
||||
|
||||
private:
|
||||
unsigned __int128 value_{0};
|
||||
ValueT value() const {
|
||||
return value_;
|
||||
}
|
||||
__int128 signed_value() const {
|
||||
return static_cast<__int128>(value());
|
||||
}
|
||||
};
|
||||
#endif
|
||||
#if TD_HAVE_INT128
|
||||
using uint128 = uint128_intrinsic;
|
||||
#else
|
||||
using uint128 = uint128_emulated;
|
||||
#endif
|
||||
} // namespace td
|
||||
|
|
@ -0,0 +1,242 @@
|
|||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#include <cstdio>
|
||||
#include "td/utils/tests.h"
|
||||
#include "td/utils/benchmark.h"
|
||||
#include "td/utils/SpinLock.h"
|
||||
#include "td/utils/HazardPointers.h"
|
||||
#include "td/utils/ConcurrentHashTable.h"
|
||||
#include <algorithm>
|
||||
|
||||
#if TD_HAVE_ABSL
|
||||
#include <absl/container/flat_hash_map.h>
|
||||
#else
|
||||
#include <unordered_map>
|
||||
#endif
|
||||
|
||||
#if TD_WITH_JUNCTION
|
||||
#include <third-party/libcuckoo/libcuckoo/cuckoohash_map.hh>
|
||||
#endif
|
||||
|
||||
#if TD_WITH_JUNCTION
|
||||
#include <junction/ConcurrentMap_Grampa.h>
|
||||
#include <junction/ConcurrentMap_Linear.h>
|
||||
#include <junction/ConcurrentMap_Leapfrog.h>
|
||||
#endif
|
||||
namespace td {
|
||||
|
||||
// Non resizable HashMap. Just an example
|
||||
template <class KeyT, class ValueT>
|
||||
class ArrayHashMap {
|
||||
public:
|
||||
ArrayHashMap(size_t n) : array_(n) {
|
||||
}
|
||||
struct Node {
|
||||
std::atomic<KeyT> key{KeyT{}};
|
||||
std::atomic<ValueT> value{};
|
||||
};
|
||||
static std::string get_name() {
|
||||
return "ArrayHashMap";
|
||||
}
|
||||
KeyT empty_key() const {
|
||||
return KeyT{};
|
||||
}
|
||||
|
||||
void insert(KeyT key, ValueT value) {
|
||||
array_.with_value(key, true, [&](auto &node_value) { node_value.store(value, std::memory_order_release); });
|
||||
}
|
||||
ValueT find(KeyT key, ValueT value) {
|
||||
array_.with_value(key, false, [&](auto &node_value) { value = node_value.load(std::memory_order_acquire); });
|
||||
return value;
|
||||
}
|
||||
|
||||
private:
|
||||
AtomicHashArray<KeyT, std::atomic<ValueT>> array_;
|
||||
};
|
||||
|
||||
template <class KeyT, class ValueT>
|
||||
class ConcurrentHashMapMutex {
|
||||
public:
|
||||
ConcurrentHashMapMutex(size_t) {
|
||||
}
|
||||
static std::string get_name() {
|
||||
return "ConcurrentHashMapMutex";
|
||||
}
|
||||
void insert(KeyT key, ValueT value) {
|
||||
std::unique_lock<std::mutex> lock(mutex_);
|
||||
hash_map_.emplace(key, value);
|
||||
}
|
||||
ValueT find(KeyT key, ValueT default_value) {
|
||||
std::unique_lock<std::mutex> lock(mutex_);
|
||||
auto it = hash_map_.find(key);
|
||||
if (it == hash_map_.end()) {
|
||||
return default_value;
|
||||
}
|
||||
return it->second;
|
||||
}
|
||||
|
||||
private:
|
||||
std::mutex mutex_;
|
||||
#if TD_HAVE_ABSL
|
||||
absl::flat_hash_map<KeyT, ValueT> hash_map_;
|
||||
#else
|
||||
std::unordered_map<KeyT, ValueT> hash_map_;
|
||||
#endif
|
||||
};
|
||||
template <class KeyT, class ValueT>
|
||||
class ConcurrentHashMapSpinlock {
|
||||
public:
|
||||
ConcurrentHashMapSpinlock(size_t) {
|
||||
}
|
||||
static std::string get_name() {
|
||||
return "ConcurrentHashMapSpinlock";
|
||||
}
|
||||
void insert(KeyT key, ValueT value) {
|
||||
auto guard = spinlock_.lock();
|
||||
hash_map_.emplace(key, value);
|
||||
}
|
||||
ValueT find(KeyT key, ValueT default_value) {
|
||||
auto guard = spinlock_.lock();
|
||||
auto it = hash_map_.find(key);
|
||||
if (it == hash_map_.end()) {
|
||||
return default_value;
|
||||
}
|
||||
return it->second;
|
||||
}
|
||||
|
||||
private:
|
||||
td::SpinLock spinlock_;
|
||||
#if TD_HAVE_ABSL
|
||||
absl::flat_hash_map<KeyT, ValueT> hash_map_;
|
||||
#else
|
||||
std::unordered_map<KeyT, ValueT> hash_map_;
|
||||
#endif
|
||||
};
|
||||
#if TD_WITH_LIBCUCKOO
|
||||
template <class KeyT, class ValueT>
|
||||
class ConcurrentHashMapLibcuckoo {
|
||||
public:
|
||||
ConcurrentHashMapLibcuckoo(size_t) {
|
||||
}
|
||||
static std::string get_name() {
|
||||
return "ConcurrentHashMapLibcuckoo";
|
||||
}
|
||||
void insert(KeyT key, ValueT value) {
|
||||
hash_map_.insert(key, value);
|
||||
}
|
||||
ValueT find(KeyT key, ValueT default_value) {
|
||||
hash_map_.find(key, default_value);
|
||||
return default_value;
|
||||
}
|
||||
|
||||
private:
|
||||
cuckoohash_map<KeyT, ValueT> hash_map_;
|
||||
};
|
||||
#endif
|
||||
#if TD_WITH_JUNCTION
|
||||
template <class KeyT, class ValueT>
|
||||
class ConcurrentHashMapJunction {
|
||||
public:
|
||||
ConcurrentHashMapJunction(size_t size) : hash_map_() {
|
||||
}
|
||||
static std::string get_name() {
|
||||
return "ConcurrentHashMapJunction";
|
||||
}
|
||||
void insert(KeyT key, ValueT value) {
|
||||
hash_map_.assign(key, value);
|
||||
}
|
||||
ValueT find(KeyT key, ValueT default_value) {
|
||||
return hash_map_.get(key);
|
||||
}
|
||||
~ConcurrentHashMapJunction() {
|
||||
junction::DefaultQSBR.flush();
|
||||
}
|
||||
|
||||
private:
|
||||
junction::ConcurrentMap_Leapfrog<KeyT, ValueT> hash_map_;
|
||||
};
|
||||
#endif
|
||||
} // namespace td
|
||||
|
||||
template <class HashMap>
|
||||
class HashMapBenchmark : public td::Benchmark {
|
||||
struct Query {
|
||||
int key;
|
||||
int value;
|
||||
};
|
||||
std::vector<Query> queries;
|
||||
std::unique_ptr<HashMap> hash_map;
|
||||
|
||||
size_t threads_n = 16;
|
||||
int mod_;
|
||||
constexpr static size_t mul_ = 7273; //1000000000 + 7;
|
||||
int n_;
|
||||
|
||||
public:
|
||||
HashMapBenchmark(size_t threads_n) : threads_n(threads_n) {
|
||||
}
|
||||
std::string get_description() const override {
|
||||
return hash_map->get_name();
|
||||
}
|
||||
void start_up_n(int n) override {
|
||||
n *= (int)threads_n;
|
||||
n_ = n;
|
||||
hash_map = std::make_unique<HashMap>(n * 2);
|
||||
}
|
||||
|
||||
void run(int n) override {
|
||||
n = n_;
|
||||
std::vector<td::thread> threads;
|
||||
|
||||
for (size_t i = 0; i < threads_n; i++) {
|
||||
size_t l = n * i / threads_n;
|
||||
size_t r = n * (i + 1) / threads_n;
|
||||
threads.emplace_back([l, r, this] {
|
||||
for (size_t i = l; i < r; i++) {
|
||||
auto x = int((i + 1) * mul_ % n_) + 3;
|
||||
auto y = int(i + 2);
|
||||
hash_map->insert(x, y);
|
||||
}
|
||||
});
|
||||
}
|
||||
for (auto &thread : threads) {
|
||||
thread.join();
|
||||
}
|
||||
}
|
||||
|
||||
void tear_down() override {
|
||||
for (int i = 0; i < n_; i++) {
|
||||
auto x = int((i + 1) * mul_ % n_) + 3;
|
||||
auto y = int(i + 2);
|
||||
ASSERT_EQ(y, hash_map->find(x, -1));
|
||||
}
|
||||
queries.clear();
|
||||
hash_map.reset();
|
||||
}
|
||||
|
||||
private:
|
||||
};
|
||||
|
||||
template <class HashMap>
|
||||
void bench_hash_map() {
|
||||
td::bench(HashMapBenchmark<HashMap>(16));
|
||||
td::bench(HashMapBenchmark<HashMap>(1));
|
||||
}
|
||||
|
||||
TEST(ConcurrentHashMap, Benchmark) {
|
||||
bench_hash_map<td::ConcurrentHashMap<int, int>>();
|
||||
bench_hash_map<td::ArrayHashMap<int, int>>();
|
||||
bench_hash_map<td::ConcurrentHashMapSpinlock<int, int>>();
|
||||
bench_hash_map<td::ConcurrentHashMapMutex<int, int>>();
|
||||
#if TD_WITH_LIBCUCKOO
|
||||
bench_hash_map<td::ConcurrentHashMapLibcuckoo<int, int>>();
|
||||
#endif
|
||||
#if TD_WITH_JUNCTION
|
||||
bench_hash_map<td::ConcurrentHashMapJunction<int, int>>();
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#include "td/utils/EpochBasedMemoryReclamation.h"
|
||||
#include "td/utils/logging.h"
|
||||
#include "td/utils/port/thread.h"
|
||||
#include "td/utils/Random.h"
|
||||
#include "td/utils/tests.h"
|
||||
|
||||
#include <atomic>
|
||||
|
||||
#if !TD_THREAD_UNSUPPORTED
|
||||
TEST(EpochBaseMemoryReclamation, stress) {
|
||||
struct Node {
|
||||
std::atomic<std::string *> name_;
|
||||
char pad[64];
|
||||
};
|
||||
|
||||
int threads_n = 10;
|
||||
std::vector<Node> nodes(threads_n);
|
||||
td::EpochBasedMemoryReclamation<std::string> ebmr(threads_n + 1);
|
||||
auto locker = ebmr.get_locker(threads_n);
|
||||
locker.lock();
|
||||
locker.unlock();
|
||||
std::vector<td::thread> threads(threads_n);
|
||||
int thread_id = 0;
|
||||
for (auto &thread : threads) {
|
||||
thread = td::thread([&, thread_id] {
|
||||
auto locker = ebmr.get_locker(thread_id);
|
||||
locker.lock();
|
||||
for (int i = 0; i < 1000000; i++) {
|
||||
auto &node = nodes[td::Random::fast(0, threads_n - 1)];
|
||||
auto *str = node.name_.load(std::memory_order_acquire);
|
||||
if (str) {
|
||||
CHECK(*str == "one" || *str == "twotwo");
|
||||
}
|
||||
if ((i + 1) % 100 == 0) {
|
||||
locker.retire();
|
||||
}
|
||||
if (td::Random::fast(0, 5) == 0) {
|
||||
std::string *new_str = new std::string(td::Random::fast(0, 1) == 0 ? "one" : "twotwo");
|
||||
if (node.name_.compare_exchange_strong(str, new_str, std::memory_order_acq_rel)) {
|
||||
locker.retire(str);
|
||||
} else {
|
||||
delete new_str;
|
||||
}
|
||||
}
|
||||
}
|
||||
locker.retire_sync();
|
||||
locker.unlock();
|
||||
});
|
||||
thread_id++;
|
||||
}
|
||||
for (auto &thread : threads) {
|
||||
thread.join();
|
||||
}
|
||||
LOG(ERROR) << "Undeleted pointers: " << ebmr.to_delete_size_unsafe();
|
||||
//CHECK(static_cast<int>(ebmr.to_delete_size_unsafe()) <= threads_n * threads_n);
|
||||
for (int i = 0; i < threads_n; i++) {
|
||||
ebmr.get_locker(i).retire_sync();
|
||||
}
|
||||
CHECK(ebmr.to_delete_size_unsafe() == 0);
|
||||
}
|
||||
#endif //!TD_THREAD_UNSUPPORTED
|
||||
Loading…
Reference in New Issue