tdlight-team/tdlight
tdlight-team
/
tdlight
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Move control block back to FlatHashTable itself to avoid cache line sharing between control blocks of all big tables.

This commit is contained in:
levlam 2022-03-09 15:17:29 +03:00
parent 88ce431f39
commit 5cda813bd9
1 changed files with 49 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
tdutils/td/utils/FlatHashMapLinear.h
View File

@ -11,7 +11,6 @@
#include "td/utils/Random.h"
#include <cstddef>
#include <cstdlib>
#include <functional>
#include <initializer_list>
#include <iterator>
@ -161,70 +160,37 @@ struct SetNode {
template <class NodeT, class HashT, class EqT>
class FlatHashTable {
struct FlatHashTableInner {
uint32 used_node_count_;
uint32 bucket_count_mask_;
uint32 bucket_count_;
uint32 begin_bucket_;
NodeT nodes_[1];
};
static constexpr size_t OFFSET = 4 * sizeof(uint32);
static constexpr uint32 INVALID_BUCKET = 0xFFFFFFFF;
static inline FlatHashTableInner *get_inner(NodeT *nodes) {
DCHECK(nodes != nullptr);
return reinterpret_cast<FlatHashTableInner *>(reinterpret_cast<char *>(nodes) - OFFSET);
}
static NodeT *allocate_nodes(uint32 size) {
void allocate_nodes(uint32 size) {
DCHECK(size >= 8);
DCHECK((size & (size - 1)) == 0);
CHECK(size <= min(static_cast<uint32>(1) << 29, static_cast<uint32>((0x7FFFFFFF - OFFSET) / sizeof(NodeT))));
auto inner = static_cast<FlatHashTableInner *>(std::malloc(OFFSET + sizeof(NodeT) * size));
NodeT *nodes = &inner->nodes_[0];
for (uint32 i = 0; i < size; i++) {
new (nodes + i) NodeT();
}
// inner->used_node_count_ = 0;
inner->bucket_count_mask_ = size - 1;
inner->bucket_count_ = size;
inner->begin_bucket_ = INVALID_BUCKET;
return nodes;
CHECK(size <= min(static_cast<uint32>(1) << 29, static_cast<uint32>(0x7FFFFFFF / sizeof(NodeT))));
nodes_ = new NodeT[size];
// used_node_count_ = 0;
bucket_count_mask_ = size - 1;
bucket_count_ = size;
begin_bucket_ = INVALID_BUCKET;
}
static void clear_nodes(NodeT *nodes) {
auto inner = get_inner(nodes);
auto size = inner->bucket_count_;
for (uint32 i = 0; i < size; i++) {
nodes[i].~NodeT();
}
std::free(inner);
}
inline FlatHashTableInner *get_inner() {
return get_inner(nodes_);
}
inline const FlatHashTableInner *get_inner() const {
DCHECK(nodes_ != nullptr);
return get_inner(const_cast<NodeT *>(nodes_));
delete[] nodes;
}
inline uint32 &used_node_count() {
return get_inner()->used_node_count_;
return used_node_count_;
}
inline uint32 get_used_node_count() const {
return get_inner()->used_node_count_;
return used_node_count_;
}
inline uint32 get_bucket_count_mask() const {
return get_inner()->bucket_count_mask_;
return bucket_count_mask_;
}
inline uint32 get_bucket_count() const {
return get_inner()->bucket_count_;
return bucket_count_;
}
public:
@ -351,21 +317,34 @@ class FlatHashTable {
used_node_count() = used_nodes;
}
FlatHashTable(FlatHashTable &&other) noexcept : nodes_(other.nodes_) {
other.nodes_ = nullptr;
FlatHashTable(FlatHashTable &&other) noexcept
: nodes_(other.nodes_)
, used_node_count_(other.used_node_count_)
, bucket_count_mask_(other.bucket_count_mask_)
, bucket_count_(other.bucket_count_)
, begin_bucket_(other.begin_bucket_) {
other.drop();
}
void operator=(FlatHashTable &&other) noexcept {
clear();
nodes_ = other.nodes_;
other.nodes_ = nullptr;
used_node_count_ = other.used_node_count_;
bucket_count_mask_ = other.bucket_count_mask_;
bucket_count_ = other.bucket_count_;
begin_bucket_ = other.begin_bucket_;
other.drop();
}
void swap(FlatHashTable &other) noexcept {
std::swap(nodes_, other.nodes_);
std::swap(used_node_count_, other.used_node_count_);
std::swap(bucket_count_mask_, other.bucket_count_mask_);
std::swap(bucket_count_, other.bucket_count_);
std::swap(begin_bucket_, other.begin_bucket_);
}
~FlatHashTable() = default;
uint32 bucket_count() const {
return unlikely(nodes_ == nullptr) ? 0 : get_bucket_count();
return get_bucket_count();
}
Iterator find(const KeyT &key) {
@ -390,18 +369,18 @@ class FlatHashTable {
}
size_t size() const {
return unlikely(nodes_ == nullptr) ? 0 : get_used_node_count();
return get_used_node_count();
}
bool empty() const {
return unlikely(nodes_ == nullptr) || get_used_node_count() == 0;
return get_used_node_count() == 0;
}
Iterator begin() {
if (empty()) {
return end();
}
auto &begin_bucket = get_inner()->begin_bucket_;
auto &begin_bucket = begin_bucket_;
if (begin_bucket == INVALID_BUCKET) {
begin_bucket = Random::fast_uint32() & get_bucket_count_mask();
while (nodes_[begin_bucket].empty()) {
@ -483,7 +462,7 @@ class FlatHashTable {
void clear() {
if (nodes_ != nullptr) {
clear_nodes(nodes_);
nodes_ = nullptr;
drop();
}
}
@ -530,6 +509,18 @@ class FlatHashTable {
private:
NodeT *nodes_ = nullptr;
uint32 used_node_count_ = 0;
uint32 bucket_count_mask_ = 0;
uint32 bucket_count_ = 0;
uint32 begin_bucket_ = 0;
void drop() {
nodes_ = nullptr;
used_node_count_ = 0;
bucket_count_mask_ = 0;
bucket_count_ = 0;
begin_bucket_ = 0;
}
void assign(const FlatHashTable &other) {
if (other.size() == 0) {
@ -581,7 +572,7 @@ class FlatHashTable {
void resize(uint32 new_size) {
if (unlikely(nodes_ == nullptr)) {
nodes_ = allocate_nodes(new_size);
allocate_nodes(new_size);
used_node_count() = 0;
return;
}
@ -589,7 +580,7 @@ class FlatHashTable {
auto old_nodes = nodes_;
uint32 old_size = get_used_node_count();
uint32 old_bucket_count = get_bucket_count();
nodes_ = allocate_nodes(new_size);
allocate_nodes(new_size);
used_node_count() = old_size;
auto old_nodes_end = old_nodes + old_bucket_count;
@ -647,12 +638,13 @@ class FlatHashTable {
}
inline void invalidate_iterators() {
get_inner()->begin_bucket_ = INVALID_BUCKET;
begin_bucket_ = INVALID_BUCKET;
}
};
template <class KeyT, class ValueT, class HashT = std::hash<KeyT>, class EqT = std::equal_to<KeyT>>
using FlatHashMapImpl = FlatHashTable<MapNode<KeyT, ValueT>, HashT, EqT>;
template <class KeyT, class HashT = std::hash<KeyT>, class EqT = std::equal_to<KeyT>>
using FlatHashSetImpl = FlatHashTable<SetNode<KeyT>, HashT, EqT>;