change file and node from vector to hashmap
This commit is contained in:
andrew (from workstation)
parent
b1bd49ac46
commit
7ed66c190b
@ -48,6 +48,8 @@
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
|
||||
#define FILE_TTL 20
|
||||
|
||||
namespace td {
|
||||
namespace {
|
||||
constexpr int64 MAX_FILE_SIZE = 1500 * (1 << 20) /* 1500MB */;
|
||||
@ -982,8 +984,6 @@ bool FileManager::try_fix_partial_local_location(FileNodePtr node) {
|
||||
|
||||
FileManager::FileIdInfo *FileManager::get_file_id_info(FileId file_id) {
|
||||
file_id.set_time();
|
||||
LOG_CHECK(0 <= file_id.get() && file_id.get() < static_cast<int32>(file_id_info_.size()))
|
||||
<< file_id << " " << file_id_info_.size();
|
||||
return &file_id_info_[file_id.get()];
|
||||
}
|
||||
|
||||
@ -1019,7 +1019,7 @@ void FileManager::try_forget_file_id(FileId file_id) {
|
||||
bool is_removed = td::remove(file_node->file_ids_, file_id);
|
||||
CHECK(is_removed);
|
||||
*info = FileIdInfo();
|
||||
empty_file_ids_.push_back(file_id.get());
|
||||
file_id_info_.erase(file_id.get());
|
||||
}
|
||||
|
||||
FileId FileManager::register_empty(FileType type) {
|
||||
@ -1805,7 +1805,7 @@ void FileManager::flush_to_pmc(FileNodePtr node, bool new_remote, bool new_local
|
||||
}
|
||||
|
||||
FileNode *FileManager::get_file_node_raw(FileId file_id, FileNodeId *file_node_id) {
|
||||
if (file_id.get() <= 0 || file_id.get() >= static_cast<int32>(file_id_info_.size())) {
|
||||
if (file_id.get() <= 0) {
|
||||
return nullptr;
|
||||
}
|
||||
FileNodeId node_id = file_id_info_[file_id.get()].node_id_;
|
||||
@ -3192,30 +3192,16 @@ string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::Inp
|
||||
}
|
||||
|
||||
FileId FileManager::next_file_id() {
|
||||
if (!empty_file_ids_.empty()) {
|
||||
auto res = empty_file_ids_.back();
|
||||
empty_file_ids_.pop_back();
|
||||
auto ret = FileId{res, 0};
|
||||
ret.set_time();
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto id = (int32) file_id_info_.size();
|
||||
auto id = file_id_seqno++;
|
||||
FileId res(static_cast<int32>(id), 0);
|
||||
file_id_info_.push_back({});
|
||||
file_id_info_[id] = {};
|
||||
res.set_time();
|
||||
return res;
|
||||
}
|
||||
|
||||
FileManager::FileNodeId FileManager::next_file_node_id() {
|
||||
if (!empty_node_ids_.empty()) {
|
||||
auto res = empty_node_ids_.back();
|
||||
empty_node_ids_.pop_back();
|
||||
return static_cast<FileNodeId>(res);
|
||||
}
|
||||
|
||||
auto res = static_cast<FileNodeId>(file_nodes_.size());
|
||||
file_nodes_.emplace_back(nullptr);
|
||||
auto res = static_cast<FileNodeId>(file_node_seqno++);
|
||||
file_nodes_[res] = nullptr;
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -3735,102 +3721,91 @@ void FileManager::destroy_query(int32 file_id) {
|
||||
|
||||
|
||||
void FileManager::memory_cleanup() {
|
||||
is_closed_ = true;
|
||||
auto time = std::time(nullptr);
|
||||
/* DESTROY OLD file_id_info_ */
|
||||
{
|
||||
auto it = file_id_info_.begin();
|
||||
auto time = std::time(nullptr);
|
||||
std::vector<int32> file_to_be_deleted = {};
|
||||
|
||||
for (unsigned int i = 0; i < file_id_info_.size(); i++) {
|
||||
auto main_node_id = file_id_info_[i].node_id_;
|
||||
if (main_node_id != 0) {
|
||||
auto &node = file_nodes_[main_node_id];
|
||||
while (it != file_id_info_.end()) {
|
||||
if (it->second.node_id_ != 0) {
|
||||
auto &node = file_nodes_[it->second.node_id_];
|
||||
|
||||
if (node != nullptr &&
|
||||
node->download_priority_ == 0 &&
|
||||
((int32) i) == node->main_file_id_.fast_get()) {
|
||||
if (time - node->main_file_id_.get_time() > 120 /* MAIN FILE TTL */) {
|
||||
node->main_file_id_.reset_time();
|
||||
if (node != nullptr && time - node->main_file_id_.get_time() > FILE_TTL) {
|
||||
auto can_reset = node->download_priority_ == 0;
|
||||
can_reset &= node->generate_download_priority_ == 0;
|
||||
can_reset &= node->download_id_ == 0;
|
||||
|
||||
for (auto &file_id : node->file_ids_) {
|
||||
/* DESTROY ASSOCIATED QUERIES */
|
||||
destroy_query(file_id.fast_get());
|
||||
if (can_reset) {
|
||||
auto file_ids_it = node->file_ids_.begin();
|
||||
|
||||
/* DESTROY ASSOCIATED NODE */
|
||||
auto &ref_node_id = file_id_info_[file_id.fast_get()].node_id_;
|
||||
if (ref_node_id != 0 && ref_node_id != main_node_id) {
|
||||
auto &ref_node = file_nodes_[ref_node_id];
|
||||
if (ref_node != nullptr) {
|
||||
context_->destroy_file_source(file_id);
|
||||
ref_node.reset();
|
||||
}
|
||||
empty_node_ids_.push_back(ref_node_id);
|
||||
file_nodes_[ref_node_id] = nullptr;
|
||||
while (file_ids_it != node->file_ids_.end() && can_reset) {
|
||||
auto &file = file_id_info_[file_ids_it->fast_get()];
|
||||
can_reset &= file.download_priority_ == 0;
|
||||
can_reset &= time - file_ids_it->get_time() > FILE_TTL;
|
||||
++file_ids_it;
|
||||
}
|
||||
|
||||
/* DESTROY ASSOCIATED FILE */
|
||||
empty_file_ids_.push_back(file_id.fast_get());
|
||||
file_id_info_[file_id.fast_get()] = FileIdInfo();
|
||||
}
|
||||
|
||||
/* DESTROY MAIN QUERY */
|
||||
destroy_query(i);
|
||||
if (can_reset) {
|
||||
node->main_file_id_.reset_time();
|
||||
|
||||
/* DESTROY FILE REFERENCE */
|
||||
context_->destroy_file_source(node->main_file_id_);
|
||||
for (auto &file_id : node->file_ids_) {
|
||||
file_id.reset_time();
|
||||
|
||||
/* DESTROY MAIN FILE */
|
||||
empty_file_ids_.push_back(i);
|
||||
file_id_info_[i] = FileIdInfo();
|
||||
/* DESTROY ASSOCIATED QUERIES */
|
||||
destroy_query(file_id.fast_get());
|
||||
|
||||
/* DESTROY MAIN NODE */
|
||||
node.reset();
|
||||
file_nodes_[main_node_id] = nullptr;
|
||||
empty_node_ids_.push_back(main_node_id);
|
||||
/* DESTROY ASSOCIATED LATE */
|
||||
file_to_be_deleted.push_back(file_id.fast_get());
|
||||
|
||||
/* DESTROY REF NODE */
|
||||
auto ref_node_id = file_id_info_[file_id.fast_get()].node_id_;
|
||||
if (it->second.node_id_ != ref_node_id) {
|
||||
file_nodes_[ref_node_id].reset();
|
||||
file_nodes_.erase(ref_node_id);
|
||||
}
|
||||
}
|
||||
|
||||
/* DESTROY MAIN QUERY */
|
||||
destroy_query(it->first);
|
||||
|
||||
/* DESTROY FILE REFERENCE */
|
||||
context_->destroy_file_source(node->main_file_id_);
|
||||
|
||||
/* DESTROY MAIN NODE */
|
||||
node.reset();
|
||||
file_nodes_.erase(it->first);
|
||||
|
||||
/* DESTROY MAIN FILE LATE */
|
||||
file_to_be_deleted.push_back(it->first);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
++it;
|
||||
}
|
||||
|
||||
for (auto file_id : file_to_be_deleted) {
|
||||
file_id_info_.erase(file_id);
|
||||
}
|
||||
}
|
||||
|
||||
/* DESTROY INVALID FILES */
|
||||
for (unsigned int i = 0; i < file_id_info_.size(); i++) {
|
||||
auto &file = file_id_info_[i];
|
||||
|
||||
if (file.node_id_ != 0) {
|
||||
auto &node = file_nodes_[file.node_id_];
|
||||
|
||||
if (node == nullptr) {
|
||||
destroy_query(i);
|
||||
context_->destroy_file_source({(int32) i, 0});
|
||||
empty_file_ids_.push_back(i);
|
||||
file_id_info_[i] = FileIdInfo();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* DESTROY INVALID NODES */
|
||||
for (unsigned int i = 0; i < file_nodes_.size(); i++) {
|
||||
auto &node = file_nodes_[i];
|
||||
|
||||
if (node != nullptr) {
|
||||
auto invalid = file_id_info_[node->main_file_id_.fast_get()].node_id_ != ((int32) i);
|
||||
|
||||
if (!invalid) {
|
||||
for (auto &file : node->file_ids_) {
|
||||
if (file_id_info_[file.fast_get()].node_id_ != ((int32) i)) {
|
||||
invalid = true;
|
||||
break;
|
||||
}
|
||||
{
|
||||
auto it = file_id_info_.begin();
|
||||
while (it != file_id_info_.end()) {
|
||||
if (it->second.node_id_ != 0) {
|
||||
auto &node = file_nodes_[it->second.node_id_];
|
||||
if (node == nullptr) {
|
||||
destroy_query(it->first);
|
||||
context_->destroy_file_source({it->first, 0});
|
||||
file_id_info_.erase(it++);
|
||||
} else {
|
||||
++it;
|
||||
}
|
||||
}
|
||||
|
||||
if (invalid) {
|
||||
for (auto &file : node->file_ids_) {
|
||||
destroy_query(file.fast_get());
|
||||
context_->destroy_file_source(file);
|
||||
empty_file_ids_.push_back(file.fast_get());
|
||||
file_id_info_[file.fast_get()] = FileIdInfo();
|
||||
}
|
||||
|
||||
empty_node_ids_.push_back(i);
|
||||
file_nodes_[i] = nullptr;
|
||||
} else {
|
||||
++it;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -3889,13 +3864,23 @@ void FileManager::memory_cleanup() {
|
||||
}
|
||||
}
|
||||
|
||||
file_hash_to_file_id_.rehash(0);
|
||||
file_id_info_.shrink_to_fit();
|
||||
empty_file_ids_.shrink_to_fit();
|
||||
empty_node_ids_.shrink_to_fit();
|
||||
/* DESTROY NULL file_id_info_ */
|
||||
{
|
||||
auto it = file_id_info_.begin();
|
||||
while (it != file_id_info_.end()) {
|
||||
if (&it->second == nullptr || file_nodes_[it->second.node_id_] == nullptr) {
|
||||
file_id_info_.erase(it++);
|
||||
} else {
|
||||
++it;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
LOG(ERROR) << empty_file_ids_.size() << " empty ids and " << queries_container_.size() << " running queries";
|
||||
is_closed_ = false;
|
||||
file_nodes_.rehash(0);
|
||||
file_hash_to_file_id_.rehash(0);
|
||||
file_id_info_.rehash(0);
|
||||
|
||||
LOG(ERROR) << "registered ids: " << file_id_info_.size() << " registered nodes: " << file_nodes_.size();
|
||||
}
|
||||
|
||||
void FileManager::tear_down() {
|
||||
|
||||
@ -554,16 +554,17 @@ class FileManager : public FileLoadManager::Callback {
|
||||
};
|
||||
Enumerator<RemoteInfo> remote_location_info_;
|
||||
|
||||
FileNodeId file_node_seqno = 0;
|
||||
int32 file_id_seqno = 0;
|
||||
|
||||
std::unordered_map<string, FileId> file_hash_to_file_id_;
|
||||
|
||||
std::map<FullLocalFileLocation, FileId> local_location_to_file_id_;
|
||||
std::map<FullGenerateFileLocation, FileId> generate_location_to_file_id_;
|
||||
std::map<FileDbId, int32> pmc_id_to_file_node_id_;
|
||||
|
||||
std::vector<FileIdInfo> file_id_info_;
|
||||
vector<int32> empty_file_ids_;
|
||||
vector<int32> empty_node_ids_;
|
||||
vector<unique_ptr<FileNode>> file_nodes_;
|
||||
std::unordered_map<int32, FileIdInfo> file_id_info_;
|
||||
std::unordered_map<FileNodeId, unique_ptr<FileNode>> file_nodes_;
|
||||
ActorOwn<FileLoadManager> file_load_manager_;
|
||||
ActorOwn<FileGenerateManager> file_generate_manager_;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user