andreacavalli
/
tdlib-fork
Archived
1
0
Fork 0
Code Pull Requests Releases Activity

Releasing memory

This commit is contained in:
Andrea Cavalli 2020-05-19 12:42:58 +02:00
parent 4e021b397a
commit 78e43eadaf
11 changed files with 111 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
td/telegram/ContactsManager.cpp
View File

@ -13611,27 +13611,50 @@ void ContactsManager::get_current_state(vector<td_api::object_ptr<td_api::Update
void ContactsManager::memory_cleanup() {
users_full_.clear();
users_full_.rehash(0);
bot_infos_.clear();
bot_infos_.rehash(0);
user_photos_.clear();
user_photos_.rehash(0);
user_profile_photo_file_source_ids_.clear();
user_profile_photo_file_source_ids_.rehash(0);
chat_photo_file_source_ids_.clear();
chat_photo_file_source_ids_.rehash(0);
channels_full_.clear();
channels_full_.rehash(0);
channel_photo_file_source_ids_.clear();
channel_photo_file_source_ids_.rehash(0);
secret_chats_.clear();
secret_chats_.rehash(0);
secret_chats_with_user_.clear();
secret_chats_with_user_.rehash(0);
chat_invite_links_.clear();
chat_invite_links_.rehash(0);
channel_invite_links_.clear();
channel_invite_links_.rehash(0);
invite_link_infos_.clear();
invite_link_infos_.rehash(0);
load_user_from_database_queries_.clear();
load_user_from_database_queries_.rehash(0);
load_chat_from_database_queries_.clear();
load_chat_from_database_queries_.rehash(0);
load_channel_from_database_queries_.clear();
load_channel_from_database_queries_.rehash(0);
load_secret_chat_from_database_queries_.clear();
load_secret_chat_from_database_queries_.rehash(0);
dialog_administrators_.clear();
dialog_administrators_.rehash(0);
uploaded_profile_photos_.clear();
uploaded_profile_photos_.rehash(0);
imported_contacts_.clear();
imported_contacts_.rehash(0);
received_channel_participant_.clear();
received_channel_participant_.rehash(0);
received_channel_participants_.clear();
received_channel_participants_.rehash(0);
cached_channel_participants_.clear();
cached_channel_participants_.rehash(0);
linked_channel_ids_.clear();
linked_channel_ids_.rehash(0);
}
} // namespace td

1
td/telegram/DocumentsManager.cpp
View File

@ -667,6 +667,7 @@ bool DocumentsManager::merge_documents(FileId new_id, FileId old_id, bool can_de
void DocumentsManager::memory_cleanup() {
documents_.clear();
documents_.rehash(0);
}
string DocumentsManager::get_document_search_text(FileId file_id) const {

13
td/telegram/StickersManager.cpp
View File

@ -6209,18 +6209,31 @@ void StickersManager::get_current_state(vector<td_api::object_ptr<td_api::Update
void StickersManager::memory_cleanup() {
stickers_.clear();
stickers_.rehash(0);
sticker_sets_.clear();
sticker_sets_.rehash(0);
short_name_to_sticker_set_id_.clear();
short_name_to_sticker_set_id_.rehash(0);
attached_sticker_sets_.clear();
attached_sticker_sets_.rehash(0);
found_stickers_.clear();
found_stickers_.rehash(0);
found_sticker_sets_.clear();
found_sticker_sets_.rehash(0);
special_sticker_sets_.clear();
special_sticker_sets_.rehash(0);
sticker_set_load_requests_.clear();
sticker_set_load_requests_.rehash(0);
emoji_language_codes_.clear();
emoji_language_codes_.rehash(0);
emoji_language_code_versions_.clear();
emoji_language_code_versions_.rehash(0);
emoji_language_code_last_difference_times_.clear();
emoji_language_code_last_difference_times_.rehash(0);
emoji_suggestions_urls_.clear();
emoji_suggestions_urls_.rehash(0);
dice_messages_.clear();
dice_messages_.rehash(0);
}
} // namespace td

2
td/telegram/Td.cpp
View File

@ -5127,6 +5127,8 @@ void Td::on_request(uint64 id, td_api::optimizeStorage &request) {
videos_manager_->memory_cleanup();
file_manager_->memory_cleanup();
malloc_trim(0);
std::vector<FileType> file_types;
for (auto &file_type : request.file_types_) {
if (file_type == nullptr) {

1
td/telegram/Td.h
View File

@ -30,6 +30,7 @@
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include <malloc.h>
#include <memory>
#include <unordered_map>
#include <unordered_set>

1
td/telegram/VideoNotesManager.cpp
View File

@ -248,6 +248,7 @@ tl_object_ptr<telegram_api::InputMedia> VideoNotesManager::get_input_media(
void VideoNotesManager::memory_cleanup() {
video_notes_.clear();
video_notes_.rehash(0);
}
} // namespace td

1
td/telegram/VideosManager.cpp
View File

@ -302,6 +302,7 @@ string VideosManager::get_video_search_text(FileId file_id) const {
}
void VideosManager::memory_cleanup() {
videos_.clear();
videos_.rehash(0);
}
} // namespace td

67
td/telegram/WebPagesManager.cpp
View File

@ -667,7 +667,9 @@ void WebPagesManager::on_get_web_page_instant_view_view_count(WebPageId web_page
}
auto *instant_view = &web_pages_[web_page_id]->instant_view;
CHECK(!instant_view->is_empty);
if (instant_view->is_empty) {
return;
}
if (instant_view->view_count >= view_count) {
return;
}
@ -705,7 +707,9 @@ void WebPagesManager::register_web_page(WebPageId web_page_id, FullMessageId ful
LOG(INFO) << "Register " << web_page_id << " from " << full_message_id << " from " << source;
bool is_inserted = web_page_messages_[web_page_id].insert(full_message_id).second;
LOG_CHECK(is_inserted) << source << " " << web_page_id << " " << full_message_id;
if (!is_inserted) {
return;
}
if (!td_->auth_manager_->is_bot() && !have_web_page_force(web_page_id)) {
LOG(INFO) << "Waiting for " << web_page_id << " needed in " << full_message_id;
@ -721,7 +725,9 @@ void WebPagesManager::unregister_web_page(WebPageId web_page_id, FullMessageId f
LOG(INFO) << "Unregister " << web_page_id << " from " << full_message_id << " from " << source;
auto &message_ids = web_page_messages_[web_page_id];
auto is_deleted = message_ids.erase(full_message_id);
LOG_CHECK(is_deleted) << source << " " << web_page_id << " " << full_message_id;
if (!is_deleted) {
return;
}
if (message_ids.empty()) {
web_page_messages_.erase(web_page_id);
@ -767,9 +773,13 @@ void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const st
void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const string &url, WebPageId web_page_id,
Promise<Unit> &&promise) {
CHECK(web_page_id == WebPageId() || have_web_page(web_page_id));
if (!(web_page_id == WebPageId() || have_web_page(web_page_id))) {
return;
}
CHECK(got_web_page_previews_.find(request_id) == got_web_page_previews_.end());
if (!(got_web_page_previews_.find(request_id) == got_web_page_previews_.end())) {
return;
}
got_web_page_previews_[request_id] = web_page_id;
if (web_page_id.is_valid() && !url.empty()) {
@ -782,7 +792,9 @@ void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const st
void WebPagesManager::on_get_web_page_preview_fail(int64 request_id, const string &url, Status error,
Promise<Unit> &&promise) {
LOG(INFO) << "Clean up getting of web page preview with url \"" << url << '"';
CHECK(error.is_error());
if (!error.is_error()) {
return;
}
promise.set_error(std::move(error));
}
@ -915,7 +927,9 @@ void WebPagesManager::load_web_page_instant_view(WebPageId web_page_id, bool for
void WebPagesManager::reload_web_page_instant_view(WebPageId web_page_id) {
LOG(INFO) << "Reload " << web_page_id << " instant view";
const WebPage *web_page = get_web_page(web_page_id);
CHECK(web_page != nullptr && !web_page->instant_view.is_empty);
if (!(web_page != nullptr && !web_page->instant_view.is_empty)) {
return;
}
auto promise = PromiseCreator::lambda([web_page_id](Result<> result) {
send_closure(G()->web_pages_manager(), &WebPagesManager::update_web_page_instant_view_load_requests, web_page_id,
@ -931,7 +945,9 @@ void WebPagesManager::reload_web_page_instant_view(WebPageId web_page_id) {
}
void WebPagesManager::on_load_web_page_instant_view_from_database(WebPageId web_page_id, string value) {
CHECK(G()->parameters().use_message_db);
if(!(G()->parameters().use_message_db)) {
return;
}
LOG(INFO) << "Successfully loaded " << web_page_id << " instant view of size " << value.size() << " from database";
// G()->td_db()->get_sqlite_pmc()->erase(get_web_page_instant_view_database_key(web_page_id), Auto());
// return;
@ -1122,7 +1138,9 @@ void WebPagesManager::on_load_web_page_id_by_url_from_database(const string &url
void WebPagesManager::on_load_web_page_by_url_from_database(WebPageId web_page_id, const string &url,
Promise<Unit> &&promise, Result<> result) {
if (result.is_error()) {
CHECK(G()->close_flag());
if(!(G()->close_flag())) {
return;
}
promise.set_error(Status::Error(500, "Request aborted"));
return;
}
@ -1200,7 +1218,9 @@ void WebPagesManager::on_web_page_changed(WebPageId web_page_id, bool have_web_p
for (auto full_message_id : it->second) {
full_message_ids.push_back(full_message_id);
}
CHECK(!full_message_ids.empty());
if (full_message_ids.empty()) {
return;
}
for (auto full_message_id : full_message_ids) {
if (!have_web_page) {
td_->messages_manager_->delete_pending_message_web_page(full_message_id);
@ -1209,9 +1229,13 @@ void WebPagesManager::on_web_page_changed(WebPageId web_page_id, bool have_web_p
}
}
if (have_web_page) {
CHECK(web_page_messages_[web_page_id].size() == full_message_ids.size());
if (!(web_page_messages_[web_page_id].size() == full_message_ids.size())) {
return;
}
} else {
CHECK(web_page_messages_.count(web_page_id) == 0);
if (!(web_page_messages_.count(web_page_id) == 0)) {
return;
}
}
}
auto get_it = pending_get_web_pages_.find(web_page_id);
@ -1497,7 +1521,9 @@ void WebPagesManager::on_load_web_page_from_database(WebPageId web_page_id, stri
vector<Promise<Unit>> promises;
if (it != load_web_page_from_database_queries_.end()) {
promises = std::move(it->second);
CHECK(!promises.empty());
if (promises.empty()) {
return;
}
load_web_page_from_database_queries_.erase(it);
}
@ -1598,10 +1624,15 @@ vector<FileId> WebPagesManager::get_web_page_file_ids(const WebPage *web_page) c
}
void WebPagesManager::memory_cleanup() {
// web_pages_.clear();
// web_page_messages_.clear();
// got_web_page_previews_.clear();
// url_to_web_page_id_.clear();
// url_to_file_source_id_.clear();
web_pages_.clear();
web_pages_.rehash(0);
web_page_messages_.clear();
web_page_messages_.rehash(0);
got_web_page_previews_.clear();
got_web_page_previews_.rehash(0);
url_to_web_page_id_.clear();
url_to_web_page_id_.rehash(0);
url_to_file_source_id_.clear();
url_to_file_source_id_.rehash(0);
}
} // namespace td

22
td/telegram/files/FileManager.cpp
View File

@ -2946,7 +2946,13 @@ FileId FileManager::next_file_id() {
}
FileManager::FileNodeId FileManager::next_file_node_id() {
FileNodeId res = static_cast<FileNodeId>(file_nodes_.size());
if (!empty_node_ids_.empty()) {
auto res = empty_node_ids_.back();
empty_node_ids_.pop_back();
return static_cast<FileNodeId>(res);
}
auto res = static_cast<FileNodeId>(file_nodes_.size());
file_nodes_.emplace_back(nullptr);
return res;
}
@ -3456,7 +3462,7 @@ void FileManager::hangup() {
}
void FileManager::memory_cleanup() {
if (file_id_info_.size() > 1000 && empty_file_ids_.size() < 200) {
if (file_id_info_.size() > 1000 || empty_file_ids_.size() < 200) {
is_closed_ = true;
auto time = std::time(nullptr);
@ -3466,6 +3472,7 @@ void FileManager::memory_cleanup() {
auto &node = file_nodes_[main_node_id];
if (node != nullptr && ((int32) i) == node->main_file_id_.get()) {
if (time - file_id_insert_time_[i] > 60 * 5 /* MAIN FILE TTL */) {
file_id_insert_time_[i] = INT64_MAX;
for (auto &file_id : node->file_ids_) {
/* DESTROY ASSOCIATED QUERIES */
for (auto &query_id : queries_container_.ids()) {
@ -3476,8 +3483,8 @@ void FileManager::memory_cleanup() {
}
/* DESTROY ASSOCIATED FILE */
empty_file_ids_.push_back(file_id.get());
file_id_info_[file_id.get()] = FileIdInfo();
empty_file_ids_.push_back(file_id.get());
}
/* DESTROY MAIN QUERY */
@ -3489,17 +3496,22 @@ void FileManager::memory_cleanup() {
}
/* DESTROY MAIN FILE */
file_id_insert_time_[i] = INT64_MAX;
empty_file_ids_.push_back(i);
file_id_info_[i] = FileIdInfo();
/* DESTROY MAIN NODE */
node.reset();
file_nodes_[main_node_id] = nullptr;
empty_node_ids_.push_back(main_node_id);
}
}
}
}
file_id_info_.shrink_to_fit();
empty_file_ids_.shrink_to_fit();
empty_node_ids_.shrink_to_fit();
LOG(INFO) << empty_file_ids_.size() << " empty ids and " << queries_container_.size() << " running queries";
is_closed_ = false;
}
@ -3508,4 +3520,4 @@ void FileManager::memory_cleanup() {
void FileManager::tear_down() {
parent_.reset();
}
} // namespace tds
} // namespace td

3
td/telegram/files/FileManager.h
View File

@ -557,8 +557,9 @@ class FileManager : public FileLoadManager::Callback {
std::map<FileDbId, int32> pmc_id_to_file_node_id_;
std::unordered_map<int32, int64> file_id_insert_time_;
vector<FileIdInfo> file_id_info_;
std::vector<FileIdInfo> file_id_info_;
vector<int32> empty_file_ids_;
vector<int32> empty_node_ids_;
vector<unique_ptr<FileNode>> file_nodes_;
ActorOwn<FileLoadManager> file_load_manager_;
ActorOwn<FileGenerateManager> file_generate_manager_;

1
tdutils/td/utils/common.h
View File

@ -48,6 +48,7 @@
#include <string>
#include <vector>
#include <unordered_map>
#define TD_DEBUG