Added TTL options to keep the access hashes of recent chats.

This commit is contained in:
Andrea Cavalli 2020-08-28 19:46:35 +02:00
parent 496281f4ca
commit 33c250f884
7 changed files with 145 additions and 34 deletions

View File

@ -9,6 +9,7 @@
#include "td/utils/common.h" #include "td/utils/common.h"
#include "td/utils/StringBuilder.h" #include "td/utils/StringBuilder.h"
#include <ctime>
#include <functional> #include <functional>
#include <type_traits> #include <type_traits>
@ -16,6 +17,7 @@ namespace td {
class ChannelId { class ChannelId {
int32 id = 0; int32 id = 0;
int64 time_ = INT64_MAX;
public: public:
ChannelId() = default; ChannelId() = default;
@ -33,6 +35,18 @@ class ChannelId {
return id; return id;
} }
void set_time() {
time_ = std::time(nullptr);
}
int64 get_time() const {
return time_;
}
void reset_time() {
time_ = INT64_MAX;
}
bool operator==(const ChannelId &other) const { bool operator==(const ChannelId &other) const {
return id == other.id; return id == other.id;
} }

View File

@ -9,6 +9,7 @@
#include "td/utils/common.h" #include "td/utils/common.h"
#include "td/utils/StringBuilder.h" #include "td/utils/StringBuilder.h"
#include <ctime>
#include <functional> #include <functional>
#include <type_traits> #include <type_traits>
@ -16,6 +17,7 @@ namespace td {
class ChatId { class ChatId {
int32 id = 0; int32 id = 0;
int64 time_ = INT64_MAX;
public: public:
ChatId() = default; ChatId() = default;
@ -33,6 +35,18 @@ class ChatId {
return id; return id;
} }
void set_time() {
time_ = std::time(nullptr);
}
int64 get_time() const {
return time_;
}
void reset_time() {
time_ = INT64_MAX;
}
bool operator==(const ChatId &other) const { bool operator==(const ChatId &other) const {
return id == other.id; return id == other.id;
} }

View File

@ -12248,6 +12248,8 @@ Result<BotData> ContactsManager::get_bot_data(UserId user_id) const {
return Status::Error(5, "Bot is inaccessible"); return Status::Error(5, "Bot is inaccessible");
} }
user_id.set_time(); // update last access time of UserId
BotData bot_data; BotData bot_data;
bot_data.username = bot->username; bot_data.username = bot->username;
bot_data.can_join_groups = bot->can_join_groups; bot_data.can_join_groups = bot->can_join_groups;
@ -12272,6 +12274,7 @@ const ContactsManager::User *ContactsManager::get_user(UserId user_id) const {
if (p == users_.end()) { if (p == users_.end()) {
return nullptr; return nullptr;
} else { } else {
user_id.set_time(); // update last access time of UserId
return p->second.get(); return p->second.get();
} }
} }
@ -12281,6 +12284,7 @@ ContactsManager::User *ContactsManager::get_user(UserId user_id) {
if (p == users_.end()) { if (p == users_.end()) {
return nullptr; return nullptr;
} else { } else {
user_id.set_time(); // update last access time of UserId
return p->second.get(); return p->second.get();
} }
} }
@ -12641,6 +12645,7 @@ const ContactsManager::Chat *ContactsManager::get_chat(ChatId chat_id) const {
if (p == chats_.end()) { if (p == chats_.end()) {
return nullptr; return nullptr;
} else { } else {
chat_id.set_time(); // update last access time of ChatId
return p->second.get(); return p->second.get();
} }
} }
@ -12650,6 +12655,7 @@ ContactsManager::Chat *ContactsManager::get_chat(ChatId chat_id) {
if (p == chats_.end()) { if (p == chats_.end()) {
return nullptr; return nullptr;
} else { } else {
chat_id.set_time(); // update last access time of ChatId
return p->second.get(); return p->second.get();
} }
} }
@ -12941,6 +12947,7 @@ const ContactsManager::Channel *ContactsManager::get_channel(ChannelId channel_i
if (p == channels_.end()) { if (p == channels_.end()) {
return nullptr; return nullptr;
} else { } else {
channel_id.set_time(); // update last access time of ChannelId
return p->second.get(); return p->second.get();
} }
} }
@ -12950,6 +12957,7 @@ ContactsManager::Channel *ContactsManager::get_channel(ChannelId channel_id) {
if (p == channels_.end()) { if (p == channels_.end()) {
return nullptr; return nullptr;
} else { } else {
channel_id.set_time(); // update last access time of ChannelId
return p->second.get(); return p->second.get();
} }
} }
@ -14476,7 +14484,27 @@ void ContactsManager::get_current_state(vector<td_api::object_ptr<td_api::Update
void ContactsManager::memory_cleanup() { void ContactsManager::memory_cleanup() {
std::lock_guard<std::shared_timed_mutex> writerLock(memory_cleanup_mutex); std::lock_guard<std::shared_timed_mutex> writerLock(memory_cleanup_mutex);
users_.clear();
auto user_ttl = !G()->shared_config().get_option_integer("delete_user_reference_after_seconds", 3600);
auto chat_ttl = !G()->shared_config().get_option_integer("delete_chat_reference_after_seconds", 3600);
/* DESTROY INVALID USERS */
{
auto it = users_.begin();
while (it != users_.end()) {
auto &user = it->second;
auto user_id = it->first;
auto is_invalid = it->first.get_time() > user_ttl;
if (is_invalid) {
user_id.reset_time();
it = users_.erase(it);
} else {
it++;
}
}
}
users_.rehash(0); users_.rehash(0);
users_full_.clear(); users_full_.clear();
users_full_.rehash(0); users_full_.rehash(0);
@ -14492,7 +14520,25 @@ void ContactsManager::memory_cleanup() {
user_profile_photo_file_source_ids_.rehash(0); user_profile_photo_file_source_ids_.rehash(0);
my_photo_file_id_.clear(); my_photo_file_id_.clear();
my_photo_file_id_.rehash(0); my_photo_file_id_.rehash(0);
chats_.clear();
/* DESTROY INVALID CHATS */
{
auto it = chats_.begin();
while (it != chats_.end()) {
auto &chat = it->second;
auto chat_id = it->first;
auto is_invalid = it->first.get_time() > chat_ttl;
if (is_invalid) {
chat_id.reset_time();
it = chats_.erase(it);
} else {
it++;
}
}
}
chats_.rehash(0); chats_.rehash(0);
chats_full_.clear(); chats_full_.clear();
chats_full_.rehash(0); chats_full_.rehash(0);
@ -14502,7 +14548,25 @@ void ContactsManager::memory_cleanup() {
chat_full_file_source_ids_.rehash(0); chat_full_file_source_ids_.rehash(0);
min_channels_.clear(); min_channels_.clear();
min_channels_.rehash(0); min_channels_.rehash(0);
channels_.clear();
/* DESTROY INVALID CHANNELS */
{
auto it = channels_.begin();
while (it != channels_.end()) {
auto &channel = it->second;
auto channel_id = it->first;
auto is_invalid = it->first.get_time() > chat_ttl;
if (is_invalid) {
channel_id.reset_time();
it = channels_.erase(it);
} else {
it++;
}
}
}
channels_.rehash(0); channels_.rehash(0);
channels_full_.clear(); channels_full_.clear();
channels_full_.rehash(0); channels_full_.rehash(0);
@ -14510,12 +14574,12 @@ void ContactsManager::memory_cleanup() {
unknown_channels_.rehash(0); unknown_channels_.rehash(0);
channel_full_file_source_ids_.clear(); channel_full_file_source_ids_.clear();
channel_full_file_source_ids_.rehash(0); channel_full_file_source_ids_.rehash(0);
secret_chats_.clear(); //secret_chats_.clear();
secret_chats_.rehash(0); //secret_chats_.rehash(0);
unknown_secret_chats_.clear(); //unknown_secret_chats_.clear();
unknown_secret_chats_.rehash(0); //unknown_secret_chats_.rehash(0);
secret_chats_with_user_.clear(); //secret_chats_with_user_.clear();
secret_chats_with_user_.rehash(0); //secret_chats_with_user_.rehash(0);
dialog_invite_links_.clear(); dialog_invite_links_.clear();
dialog_invite_links_.rehash(0); dialog_invite_links_.rehash(0);
invite_link_infos_.clear(); invite_link_infos_.clear();
@ -14542,10 +14606,10 @@ void ContactsManager::memory_cleanup() {
loaded_from_database_channels_.rehash(0); loaded_from_database_channels_.rehash(0);
unavailable_channel_fulls_.clear(); unavailable_channel_fulls_.clear();
unavailable_channel_fulls_.rehash(0); unavailable_channel_fulls_.rehash(0);
load_secret_chat_from_database_queries_.clear(); //load_secret_chat_from_database_queries_.clear();
load_secret_chat_from_database_queries_.rehash(0); //load_secret_chat_from_database_queries_.rehash(0);
loaded_from_database_secret_chats_.clear(); //loaded_from_database_secret_chats_.clear();
loaded_from_database_secret_chats_.rehash(0); //loaded_from_database_secret_chats_.rehash(0);
dialog_administrators_.clear(); dialog_administrators_.clear();
dialog_administrators_.rehash(0); dialog_administrators_.rehash(0);
uploaded_profile_photos_.clear(); uploaded_profile_photos_.clear();

View File

@ -7061,6 +7061,15 @@ void Td::on_request(uint64 id, td_api::setOption &request) {
if (set_boolean_option("disable_notifications")) { if (set_boolean_option("disable_notifications")) {
return; return;
} }
if (set_integer_option("delete_file_reference_after_seconds")) {
return;
}
if (set_integer_option("delete_user_reference_after_seconds")) {
return;
}
if (set_integer_option("delete_chat_reference_after_seconds")) {
return;
}
// End custom-patches // End custom-patches
if (set_boolean_option("disable_persistent_network_statistics")) { if (set_boolean_option("disable_persistent_network_statistics")) {
return; return;

View File

@ -9,6 +9,7 @@
#include "td/utils/common.h" #include "td/utils/common.h"
#include "td/utils/StringBuilder.h" #include "td/utils/StringBuilder.h"
#include <ctime>
#include <functional> #include <functional>
#include <type_traits> #include <type_traits>
@ -16,6 +17,7 @@ namespace td {
class UserId { class UserId {
int32 id = 0; int32 id = 0;
int64 time_ = INT64_MAX;
public: public:
UserId() = default; UserId() = default;
@ -33,6 +35,18 @@ class UserId {
return id; return id;
} }
void set_time() {
time_ = std::time(nullptr);
}
int64 get_time() const {
return time_;
}
void reset_time() {
time_ = INT64_MAX;
}
bool operator==(const UserId &other) const { bool operator==(const UserId &other) const {
return id == other.id; return id == other.id;
} }

View File

@ -36,10 +36,6 @@ class FileId {
return id > 0; return id > 0;
} }
int32 fast_get() const {
return id;
}
int32 get() const { int32 get() const {
return id; return id;
} }

View File

@ -49,8 +49,6 @@
#include <unordered_set> #include <unordered_set>
#include <utility> #include <utility>
#define FILE_TTL 30
namespace td { namespace td {
namespace { namespace {
constexpr int64 MAX_FILE_SIZE = 2000 * (1 << 20) /* 2000MB */; constexpr int64 MAX_FILE_SIZE = 2000 * (1 << 20) /* 2000MB */;
@ -1031,7 +1029,7 @@ bool FileManager::try_fix_partial_local_location(FileNodePtr node) {
} }
FileManager::FileIdInfo *FileManager::get_file_id_info(FileId file_id) { FileManager::FileIdInfo *FileManager::get_file_id_info(FileId file_id) {
file_id.set_time(); file_id.set_time(); // update last access time of FileId
return &file_id_info_.at(file_id.get()); return &file_id_info_.at(file_id.get());
} }
@ -1074,7 +1072,7 @@ void FileManager::try_forget_file_id(FileId file_id) {
*info = FileIdInfo(); *info = FileIdInfo();
file_id_info_.erase(file_id.get()); file_id_info_.erase(file_id.get());
// Start custom-patches // Start custom-patches
file_id.reset_time(); file_id.reset_time(); // delete last access time of FileId
destroy_query(file_id.get()); destroy_query(file_id.get());
context_->destroy_file_source(file_id); context_->destroy_file_source(file_id);
// End custom-patches // End custom-patches
@ -3306,7 +3304,7 @@ FileId FileManager::next_file_id() {
auto id = file_id_seqno++; auto id = file_id_seqno++;
FileId res(static_cast<int32>(id), 0); FileId res(static_cast<int32>(id), 0);
file_id_info_[id] = {}; file_id_info_[id] = {};
res.set_time(); res.set_time(); // update last access time of FileId
return res; return res;
} }
@ -3846,7 +3844,7 @@ void FileManager::hangup() {
void FileManager::destroy_query(int32 file_id) { void FileManager::destroy_query(int32 file_id) {
for (auto &query_id : queries_container_.ids()) { for (auto &query_id : queries_container_.ids()) {
auto query = queries_container_.get(query_id); auto query = queries_container_.get(query_id);
if (query != nullptr && file_id == query->file_id_.fast_get()) { if (query != nullptr && file_id == query->file_id_.get()) {
on_error(query_id, Status::Error(400, "FILE_DOWNLOAD_RESTART")); on_error(query_id, Status::Error(400, "FILE_DOWNLOAD_RESTART"));
} }
} }
@ -3860,6 +3858,8 @@ void FileManager::memory_cleanup() {
std::unordered_set<int32> file_to_be_deleted = {}; std::unordered_set<int32> file_to_be_deleted = {};
auto file_ttl = !G()->shared_config().get_option_integer("delete_file_reference_after_seconds", 30);
/* DESTROY OLD file_id_info_ */ /* DESTROY OLD file_id_info_ */
if (true) { if (true) {
auto it = file_id_info_.begin(); auto it = file_id_info_.begin();
@ -3870,7 +3870,7 @@ void FileManager::memory_cleanup() {
if (find_node != file_nodes_.end()) { if (find_node != file_nodes_.end()) {
auto &node = find_node->second; auto &node = find_node->second;
if (time - node->main_file_id_.get_time() > FILE_TTL) { if (time - node->main_file_id_.get_time() > file_ttl) {
auto can_reset = node->download_priority_ == 0; auto can_reset = node->download_priority_ == 0;
can_reset &= node->generate_download_priority_ == 0; can_reset &= node->generate_download_priority_ == 0;
can_reset &= node->download_id_ == 0; can_reset &= node->download_id_ == 0;
@ -3879,27 +3879,27 @@ void FileManager::memory_cleanup() {
auto file_ids_it = node->file_ids_.begin(); auto file_ids_it = node->file_ids_.begin();
while (file_ids_it != node->file_ids_.end() && can_reset) { while (file_ids_it != node->file_ids_.end() && can_reset) {
auto find_file = file_id_info_.find(file_ids_it->fast_get()); auto find_file = file_id_info_.find(file_ids_it->get());
if (find_file != file_id_info_.end()) { if (find_file != file_id_info_.end()) {
auto &file = find_file->second; auto &file = find_file->second;
can_reset &= file.download_priority_ == 0; can_reset &= file.download_priority_ == 0;
can_reset &= time - file_ids_it->get_time() > FILE_TTL; can_reset &= time - file_ids_it->get_time() > file_ttl;
} }
file_ids_it++; file_ids_it++;
} }
} }
if (can_reset) { if (can_reset) {
node->main_file_id_.reset_time(); node->main_file_id_.reset_time(); // delete last access time of FileId
for (auto &file_id : node->file_ids_) { for (auto &file_id : node->file_ids_) {
file_id.reset_time(); file_id.reset_time(); // delete last access time of FileId
/* DESTROY ASSOCIATED QUERIES */ /* DESTROY ASSOCIATED QUERIES */
destroy_query(file_id.fast_get()); destroy_query(file_id.get());
/* DESTROY ASSOCIATED LATE */ /* DESTROY ASSOCIATED LATE */
file_to_be_deleted.insert(file_id.fast_get()); file_to_be_deleted.insert(file_id.get());
} }
/* DESTROY MAIN QUERY */ /* DESTROY MAIN QUERY */
@ -3995,7 +3995,7 @@ void FileManager::memory_cleanup() {
while (it != file_hash_to_file_id_.end()) { while (it != file_hash_to_file_id_.end()) {
auto is_invalid = false; auto is_invalid = false;
auto find_file = file_id_info_.find(it->second.fast_get()); auto find_file = file_id_info_.find(it->second.get());
if (find_file != file_id_info_.end()) { if (find_file != file_id_info_.end()) {
auto &file = find_file->second; auto &file = find_file->second;
auto find_file_node = file_nodes_.find(file.node_id_); auto find_file_node = file_nodes_.find(file.node_id_);
@ -4021,7 +4021,7 @@ void FileManager::memory_cleanup() {
while (it != local_location_to_file_id_.end()) { while (it != local_location_to_file_id_.end()) {
auto is_invalid = false; auto is_invalid = false;
auto find_file = file_id_info_.find(it->second.fast_get()); auto find_file = file_id_info_.find(it->second.get());
if (find_file != file_id_info_.end()) { if (find_file != file_id_info_.end()) {
auto &file = find_file->second; auto &file = find_file->second;
auto find_file_node = file_nodes_.find(file.node_id_); auto find_file_node = file_nodes_.find(file.node_id_);
@ -4047,7 +4047,7 @@ void FileManager::memory_cleanup() {
while (it != generate_location_to_file_id_.end()) { while (it != generate_location_to_file_id_.end()) {
auto is_invalid = false; auto is_invalid = false;
auto find_file = file_id_info_.find(it->second.fast_get()); auto find_file = file_id_info_.find(it->second.get());
if (find_file != file_id_info_.end()) { if (find_file != file_id_info_.end()) {
auto &file = find_file->second; auto &file = find_file->second;
auto find_file_node = file_nodes_.find(file.node_id_); auto find_file_node = file_nodes_.find(file.node_id_);
@ -4086,7 +4086,7 @@ void FileManager::memory_cleanup() {
while (it != old_remote_info.end()) { while (it != old_remote_info.end()) {
auto is_invalid = false; auto is_invalid = false;
auto find_file = file_id_info_.find(it->second.file_id_.fast_get()); auto find_file = file_id_info_.find(it->second.file_id_.get());
if (find_file != file_id_info_.end()) { if (find_file != file_id_info_.end()) {
auto &file = find_file->second; auto &file = find_file->second;
auto find_file_node = file_nodes_.find(file.node_id_); auto find_file_node = file_nodes_.find(file.node_id_);