tdlight-team/tdlight
tdlight-team/tdlight
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Move control block back to FlatHashTable itself to avoid cache line sharing between control blocks of all big tables.

Browse Source
This commit is contained in:
levlam 2022-03-09 15:17:29 +03:00
parent 88ce431f39
commit 5cda813bd9
1 changed files with 49 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
tdutils/td/utils/FlatHashMapLinear.h
View File

@ -11,7 +11,6 @@
#include "td/utils/Random.h" #include "td/utils/Random.h"
#include <cstddef> #include <cstddef>
#include <cstdlib>
#include <functional> #include <functional>
#include <initializer_list> #include <initializer_list>
#include <iterator> #include <iterator>
@ -161,70 +160,37 @@ struct SetNode {
template <class NodeT, class HashT, class EqT> template <class NodeT, class HashT, class EqT>
class FlatHashTable { class FlatHashTable {
struct FlatHashTableInner {
uint32 used_node_count_;
uint32 bucket_count_mask_;
uint32 bucket_count_;
uint32 begin_bucket_;
NodeT nodes_[1];
};
static constexpr size_t OFFSET = 4 * sizeof(uint32);
static constexpr uint32 INVALID_BUCKET = 0xFFFFFFFF; static constexpr uint32 INVALID_BUCKET = 0xFFFFFFFF;
static inline FlatHashTableInner *get_inner(NodeT *nodes) { void allocate_nodes(uint32 size) {
DCHECK(nodes != nullptr);
return reinterpret_cast<FlatHashTableInner *>(reinterpret_cast<char *>(nodes) - OFFSET);
}
static NodeT *allocate_nodes(uint32 size) {
DCHECK(size >= 8); DCHECK(size >= 8);
DCHECK((size & (size - 1)) == 0); DCHECK((size & (size - 1)) == 0);
CHECK(size <= min(static_cast<uint32>(1) << 29, static_cast<uint32>((0x7FFFFFFF - OFFSET) / sizeof(NodeT)))); CHECK(size <= min(static_cast<uint32>(1) << 29, static_cast<uint32>(0x7FFFFFFF / sizeof(NodeT))));
auto inner = static_cast<FlatHashTableInner *>(std::malloc(OFFSET + sizeof(NodeT) * size)); nodes_ = new NodeT[size];
NodeT *nodes = &inner->nodes_[0]; // used_node_count_ = 0;
for (uint32 i = 0; i < size; i++) { bucket_count_mask_ = size - 1;
new (nodes + i) NodeT(); bucket_count_ = size;
} begin_bucket_ = INVALID_BUCKET;
// inner->used_node_count_ = 0;
inner->bucket_count_mask_ = size - 1;
inner->bucket_count_ = size;
inner->begin_bucket_ = INVALID_BUCKET;
return nodes;
} }
static void clear_nodes(NodeT *nodes) { static void clear_nodes(NodeT *nodes) {
auto inner = get_inner(nodes); delete[] nodes;
auto size = inner->bucket_count_;
for (uint32 i = 0; i < size; i++) {
nodes[i].~NodeT();
}
std::free(inner);
}
inline FlatHashTableInner *get_inner() {
return get_inner(nodes_);
}
inline const FlatHashTableInner *get_inner() const {
DCHECK(nodes_ != nullptr);
return get_inner(const_cast<NodeT *>(nodes_));
} }
inline uint32 &used_node_count() { inline uint32 &used_node_count() {
return get_inner()->used_node_count_; return used_node_count_;
} }
inline uint32 get_used_node_count() const { inline uint32 get_used_node_count() const {
return get_inner()->used_node_count_; return used_node_count_;
} }
inline uint32 get_bucket_count_mask() const { inline uint32 get_bucket_count_mask() const {
return get_inner()->bucket_count_mask_; return bucket_count_mask_;
} }
inline uint32 get_bucket_count() const { inline uint32 get_bucket_count() const {
return get_inner()->bucket_count_; return bucket_count_;
} }
public: public:
@ -351,21 +317,34 @@ class FlatHashTable {
used_node_count() = used_nodes; used_node_count() = used_nodes;
} }
FlatHashTable(FlatHashTable &&other) noexcept : nodes_(other.nodes_) { FlatHashTable(FlatHashTable &&other) noexcept
other.nodes_ = nullptr; : nodes_(other.nodes_)
, used_node_count_(other.used_node_count_)
, bucket_count_mask_(other.bucket_count_mask_)
, bucket_count_(other.bucket_count_)
, begin_bucket_(other.begin_bucket_) {
other.drop();
} }
void operator=(FlatHashTable &&other) noexcept { void operator=(FlatHashTable &&other) noexcept {
clear(); clear();
nodes_ = other.nodes_; nodes_ = other.nodes_;
other.nodes_ = nullptr; used_node_count_ = other.used_node_count_;
bucket_count_mask_ = other.bucket_count_mask_;
bucket_count_ = other.bucket_count_;
begin_bucket_ = other.begin_bucket_;
other.drop();
} }
void swap(FlatHashTable &other) noexcept { void swap(FlatHashTable &other) noexcept {
std::swap(nodes_, other.nodes_); std::swap(nodes_, other.nodes_);
std::swap(used_node_count_, other.used_node_count_);
std::swap(bucket_count_mask_, other.bucket_count_mask_);
std::swap(bucket_count_, other.bucket_count_);
std::swap(begin_bucket_, other.begin_bucket_);
} }
~FlatHashTable() = default; ~FlatHashTable() = default;
uint32 bucket_count() const { uint32 bucket_count() const {
return unlikely(nodes_ == nullptr) ? 0 : get_bucket_count(); return get_bucket_count();
} }
Iterator find(const KeyT &key) { Iterator find(const KeyT &key) {
@ -390,18 +369,18 @@ class FlatHashTable {
} }
size_t size() const { size_t size() const {
return unlikely(nodes_ == nullptr) ? 0 : get_used_node_count(); return get_used_node_count();
} }
bool empty() const { bool empty() const {
return unlikely(nodes_ == nullptr) || get_used_node_count() == 0; return get_used_node_count() == 0;
} }
Iterator begin() { Iterator begin() {
if (empty()) { if (empty()) {
return end(); return end();
} }
auto &begin_bucket = get_inner()->begin_bucket_; auto &begin_bucket = begin_bucket_;
if (begin_bucket == INVALID_BUCKET) { if (begin_bucket == INVALID_BUCKET) {
begin_bucket = Random::fast_uint32() & get_bucket_count_mask(); begin_bucket = Random::fast_uint32() & get_bucket_count_mask();
while (nodes_[begin_bucket].empty()) { while (nodes_[begin_bucket].empty()) {
@ -483,7 +462,7 @@ class FlatHashTable {
void clear() { void clear() {
if (nodes_ != nullptr) { if (nodes_ != nullptr) {
clear_nodes(nodes_); clear_nodes(nodes_);
nodes_ = nullptr; drop();
} }
} }
@ -530,6 +509,18 @@ class FlatHashTable {
private: private:
NodeT *nodes_ = nullptr; NodeT *nodes_ = nullptr;
uint32 used_node_count_ = 0;
uint32 bucket_count_mask_ = 0;
uint32 bucket_count_ = 0;
uint32 begin_bucket_ = 0;
void drop() {
nodes_ = nullptr;
used_node_count_ = 0;
bucket_count_mask_ = 0;
bucket_count_ = 0;
begin_bucket_ = 0;
}
void assign(const FlatHashTable &other) { void assign(const FlatHashTable &other) {
if (other.size() == 0) { if (other.size() == 0) {
@ -581,7 +572,7 @@ class FlatHashTable {
void resize(uint32 new_size) { void resize(uint32 new_size) {
if (unlikely(nodes_ == nullptr)) { if (unlikely(nodes_ == nullptr)) {
nodes_ = allocate_nodes(new_size); allocate_nodes(new_size);
used_node_count() = 0; used_node_count() = 0;
return; return;
} }
@ -589,7 +580,7 @@ class FlatHashTable {
auto old_nodes = nodes_; auto old_nodes = nodes_;
uint32 old_size = get_used_node_count(); uint32 old_size = get_used_node_count();
uint32 old_bucket_count = get_bucket_count(); uint32 old_bucket_count = get_bucket_count();
nodes_ = allocate_nodes(new_size); allocate_nodes(new_size);
used_node_count() = old_size; used_node_count() = old_size;
auto old_nodes_end = old_nodes + old_bucket_count; auto old_nodes_end = old_nodes + old_bucket_count;
@ -647,12 +638,13 @@ class FlatHashTable {
} }
inline void invalidate_iterators() { inline void invalidate_iterators() {
get_inner()->begin_bucket_ = INVALID_BUCKET; begin_bucket_ = INVALID_BUCKET;
} }
}; };
template <class KeyT, class ValueT, class HashT = std::hash<KeyT>, class EqT = std::equal_to<KeyT>> template <class KeyT, class ValueT, class HashT = std::hash<KeyT>, class EqT = std::equal_to<KeyT>>
using FlatHashMapImpl = FlatHashTable<MapNode<KeyT, ValueT>, HashT, EqT>; using FlatHashMapImpl = FlatHashTable<MapNode<KeyT, ValueT>, HashT, EqT>;
template <class KeyT, class HashT = std::hash<KeyT>, class EqT = std::equal_to<KeyT>> template <class KeyT, class HashT = std::hash<KeyT>, class EqT = std::equal_to<KeyT>>
using FlatHashSetImpl = FlatHashTable<SetNode<KeyT>, HashT, EqT>; using FlatHashSetImpl = FlatHashTable<SetNode<KeyT>, HashT, EqT>;