andreacavalli/tdlib-fork
Archived
1
0
Fork 0
Code Pull Requests Releases Activity

change file and node from vector to hashmap

Browse Source
This commit is contained in:
andrew (from workstation) 2020-05-23 12:39:32 +02:00
parent b1bd49ac46
commit 7ed66c190b
2 changed files with 97 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

199
td/telegram/files/FileManager.cpp
View File

@ -48,6 +48,8 @@
#include <tuple> #include <tuple>
#include <utility> #include <utility>
#define FILE_TTL 20
namespace td { namespace td {
namespace { namespace {
constexpr int64 MAX_FILE_SIZE = 1500 * (1 << 20) /* 1500MB */; constexpr int64 MAX_FILE_SIZE = 1500 * (1 << 20) /* 1500MB */;
@ -982,8 +984,6 @@ bool FileManager::try_fix_partial_local_location(FileNodePtr node) {
FileManager::FileIdInfo *FileManager::get_file_id_info(FileId file_id) { FileManager::FileIdInfo *FileManager::get_file_id_info(FileId file_id) {
file_id.set_time(); file_id.set_time();
LOG_CHECK(0 <= file_id.get() && file_id.get() < static_cast<int32>(file_id_info_.size()))
<< file_id << " " << file_id_info_.size();
return &file_id_info_[file_id.get()]; return &file_id_info_[file_id.get()];
} }
@ -1019,7 +1019,7 @@ void FileManager::try_forget_file_id(FileId file_id) {
bool is_removed = td::remove(file_node->file_ids_, file_id); bool is_removed = td::remove(file_node->file_ids_, file_id);
CHECK(is_removed); CHECK(is_removed);
*info = FileIdInfo(); *info = FileIdInfo();
empty_file_ids_.push_back(file_id.get()); file_id_info_.erase(file_id.get());
} }
FileId FileManager::register_empty(FileType type) { FileId FileManager::register_empty(FileType type) {
@ -1805,7 +1805,7 @@ void FileManager::flush_to_pmc(FileNodePtr node, bool new_remote, bool new_local
} }
FileNode *FileManager::get_file_node_raw(FileId file_id, FileNodeId *file_node_id) { FileNode *FileManager::get_file_node_raw(FileId file_id, FileNodeId *file_node_id) {
if (file_id.get() <= 0 || file_id.get() >= static_cast<int32>(file_id_info_.size())) { if (file_id.get() <= 0) {
return nullptr; return nullptr;
} }
FileNodeId node_id = file_id_info_[file_id.get()].node_id_; FileNodeId node_id = file_id_info_[file_id.get()].node_id_;
@ -3192,30 +3192,16 @@ string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::Inp
} }
FileId FileManager::next_file_id() { FileId FileManager::next_file_id() {
if (!empty_file_ids_.empty()) { auto id = file_id_seqno++;
auto res = empty_file_ids_.back();
empty_file_ids_.pop_back();
auto ret = FileId{res, 0};
ret.set_time();
return ret;
}
auto id = (int32) file_id_info_.size();
FileId res(static_cast<int32>(id), 0); FileId res(static_cast<int32>(id), 0);
file_id_info_.push_back({}); file_id_info_[id] = {};
res.set_time(); res.set_time();
return res; return res;
} }
FileManager::FileNodeId FileManager::next_file_node_id() { FileManager::FileNodeId FileManager::next_file_node_id() {
if (!empty_node_ids_.empty()) { auto res = static_cast<FileNodeId>(file_node_seqno++);
auto res = empty_node_ids_.back(); file_nodes_[res] = nullptr;
empty_node_ids_.pop_back();
return static_cast<FileNodeId>(res);
}
auto res = static_cast<FileNodeId>(file_nodes_.size());
file_nodes_.emplace_back(nullptr);
return res; return res;
} }
@ -3735,102 +3721,91 @@ void FileManager::destroy_query(int32 file_id) {
void FileManager::memory_cleanup() { void FileManager::memory_cleanup() {
is_closed_ = true; /* DESTROY OLD file_id_info_ */
auto time = std::time(nullptr); {
auto it = file_id_info_.begin();
auto time = std::time(nullptr);
std::vector<int32> file_to_be_deleted = {};
for (unsigned int i = 0; i < file_id_info_.size(); i++) { while (it != file_id_info_.end()) {
auto main_node_id = file_id_info_[i].node_id_; if (it->second.node_id_ != 0) {
if (main_node_id != 0) { auto &node = file_nodes_[it->second.node_id_];
auto &node = file_nodes_[main_node_id];
if (node != nullptr && if (node != nullptr && time - node->main_file_id_.get_time() > FILE_TTL) {
node->download_priority_ == 0 && auto can_reset = node->download_priority_ == 0;
((int32) i) == node->main_file_id_.fast_get()) { can_reset &= node->generate_download_priority_ == 0;
if (time - node->main_file_id_.get_time() > 120 /* MAIN FILE TTL */) { can_reset &= node->download_id_ == 0;
node->main_file_id_.reset_time();
for (auto &file_id : node->file_ids_) { if (can_reset) {
/* DESTROY ASSOCIATED QUERIES */ auto file_ids_it = node->file_ids_.begin();
destroy_query(file_id.fast_get());
/* DESTROY ASSOCIATED NODE */ while (file_ids_it != node->file_ids_.end() && can_reset) {
auto &ref_node_id = file_id_info_[file_id.fast_get()].node_id_; auto &file = file_id_info_[file_ids_it->fast_get()];
if (ref_node_id != 0 && ref_node_id != main_node_id) { can_reset &= file.download_priority_ == 0;
auto &ref_node = file_nodes_[ref_node_id]; can_reset &= time - file_ids_it->get_time() > FILE_TTL;
if (ref_node != nullptr) { ++file_ids_it;
context_->destroy_file_source(file_id);
ref_node.reset();
}
empty_node_ids_.push_back(ref_node_id);
file_nodes_[ref_node_id] = nullptr;
} }
/* DESTROY ASSOCIATED FILE */
empty_file_ids_.push_back(file_id.fast_get());
file_id_info_[file_id.fast_get()] = FileIdInfo();
} }
/* DESTROY MAIN QUERY */ if (can_reset) {
destroy_query(i); node->main_file_id_.reset_time();
/* DESTROY FILE REFERENCE */ for (auto &file_id : node->file_ids_) {
context_->destroy_file_source(node->main_file_id_); file_id.reset_time();
/* DESTROY MAIN FILE */ /* DESTROY ASSOCIATED QUERIES */
empty_file_ids_.push_back(i); destroy_query(file_id.fast_get());
file_id_info_[i] = FileIdInfo();
/* DESTROY MAIN NODE */ /* DESTROY ASSOCIATED LATE */
node.reset(); file_to_be_deleted.push_back(file_id.fast_get());
file_nodes_[main_node_id] = nullptr;
empty_node_ids_.push_back(main_node_id); /* DESTROY REF NODE */
auto ref_node_id = file_id_info_[file_id.fast_get()].node_id_;
if (it->second.node_id_ != ref_node_id) {
file_nodes_[ref_node_id].reset();
file_nodes_.erase(ref_node_id);
}
}
/* DESTROY MAIN QUERY */
destroy_query(it->first);
/* DESTROY FILE REFERENCE */
context_->destroy_file_source(node->main_file_id_);
/* DESTROY MAIN NODE */
node.reset();
file_nodes_.erase(it->first);
/* DESTROY MAIN FILE LATE */
file_to_be_deleted.push_back(it->first);
}
} }
} }
++it;
}
for (auto file_id : file_to_be_deleted) {
file_id_info_.erase(file_id);
} }
} }
/* DESTROY INVALID FILES */ /* DESTROY INVALID FILES */
for (unsigned int i = 0; i < file_id_info_.size(); i++) { {
auto &file = file_id_info_[i]; auto it = file_id_info_.begin();
while (it != file_id_info_.end()) {
if (file.node_id_ != 0) { if (it->second.node_id_ != 0) {
auto &node = file_nodes_[file.node_id_]; auto &node = file_nodes_[it->second.node_id_];
if (node == nullptr) {
if (node == nullptr) { destroy_query(it->first);
destroy_query(i); context_->destroy_file_source({it->first, 0});
context_->destroy_file_source({(int32) i, 0}); file_id_info_.erase(it++);
empty_file_ids_.push_back(i); } else {
file_id_info_[i] = FileIdInfo(); ++it;
}
}
}
/* DESTROY INVALID NODES */
for (unsigned int i = 0; i < file_nodes_.size(); i++) {
auto &node = file_nodes_[i];
if (node != nullptr) {
auto invalid = file_id_info_[node->main_file_id_.fast_get()].node_id_ != ((int32) i);
if (!invalid) {
for (auto &file : node->file_ids_) {
if (file_id_info_[file.fast_get()].node_id_ != ((int32) i)) {
invalid = true;
break;
}
} }
} } else {
++it;
if (invalid) {
for (auto &file : node->file_ids_) {
destroy_query(file.fast_get());
context_->destroy_file_source(file);
empty_file_ids_.push_back(file.fast_get());
file_id_info_[file.fast_get()] = FileIdInfo();
}
empty_node_ids_.push_back(i);
file_nodes_[i] = nullptr;
} }
} }
} }
@ -3889,13 +3864,23 @@ void FileManager::memory_cleanup() {
} }
} }
file_hash_to_file_id_.rehash(0); /* DESTROY NULL file_id_info_ */
file_id_info_.shrink_to_fit(); {
empty_file_ids_.shrink_to_fit(); auto it = file_id_info_.begin();
empty_node_ids_.shrink_to_fit(); while (it != file_id_info_.end()) {
if (&it->second == nullptr || file_nodes_[it->second.node_id_] == nullptr) {
file_id_info_.erase(it++);
} else {
++it;
}
}
}
LOG(ERROR) << empty_file_ids_.size() << " empty ids and " << queries_container_.size() << " running queries"; file_nodes_.rehash(0);
is_closed_ = false; file_hash_to_file_id_.rehash(0);
file_id_info_.rehash(0);
LOG(ERROR) << "registered ids: " << file_id_info_.size() << " registered nodes: " << file_nodes_.size();
} }
void FileManager::tear_down() { void FileManager::tear_down() {

9
td/telegram/files/FileManager.h
View File

@ -554,16 +554,17 @@ class FileManager : public FileLoadManager::Callback {
}; };
Enumerator<RemoteInfo> remote_location_info_; Enumerator<RemoteInfo> remote_location_info_;
FileNodeId file_node_seqno = 0;
int32 file_id_seqno = 0;
std::unordered_map<string, FileId> file_hash_to_file_id_; std::unordered_map<string, FileId> file_hash_to_file_id_;
std::map<FullLocalFileLocation, FileId> local_location_to_file_id_; std::map<FullLocalFileLocation, FileId> local_location_to_file_id_;
std::map<FullGenerateFileLocation, FileId> generate_location_to_file_id_; std::map<FullGenerateFileLocation, FileId> generate_location_to_file_id_;
std::map<FileDbId, int32> pmc_id_to_file_node_id_; std::map<FileDbId, int32> pmc_id_to_file_node_id_;
std::vector<FileIdInfo> file_id_info_; std::unordered_map<int32, FileIdInfo> file_id_info_;
vector<int32> empty_file_ids_; std::unordered_map<FileNodeId, unique_ptr<FileNode>> file_nodes_;
vector<int32> empty_node_ids_;
vector<unique_ptr<FileNode>> file_nodes_;
ActorOwn<FileLoadManager> file_load_manager_; ActorOwn<FileLoadManager> file_load_manager_;
ActorOwn<FileGenerateManager> file_generate_manager_; ActorOwn<FileGenerateManager> file_generate_manager_;