From 78e43eadafe6a17dae8146710d5998a8c377ecb6 Mon Sep 17 00:00:00 2001
From: Andrea Cavalli <andrea@warp.ovh>
Date: Tue, 19 May 2020 12:42:58 +0200
Subject: [PATCH] Releasing memory

---
 td/telegram/ContactsManager.cpp   | 23 +++++++++++
 td/telegram/DocumentsManager.cpp  |  1 +
 td/telegram/StickersManager.cpp   | 13 ++++++
 td/telegram/Td.cpp                |  2 +
 td/telegram/Td.h                  |  1 +
 td/telegram/VideoNotesManager.cpp |  1 +
 td/telegram/VideosManager.cpp     |  1 +
 td/telegram/WebPagesManager.cpp   | 67 ++++++++++++++++++++++---------
 td/telegram/files/FileManager.cpp | 22 +++++++---
 td/telegram/files/FileManager.h   |  3 +-
 tdutils/td/utils/common.h         |  1 +
 11 files changed, 111 insertions(+), 24 deletions(-)

diff --git a/td/telegram/ContactsManager.cpp b/td/telegram/ContactsManager.cpp
index 4931dafe..f1d0b4c5 100644
--- a/td/telegram/ContactsManager.cpp
+++ b/td/telegram/ContactsManager.cpp
@@ -13611,27 +13611,50 @@ void ContactsManager::get_current_state(vector<td_api::object_ptr<td_api::Update
 
 void ContactsManager::memory_cleanup() {
     users_full_.clear();
+    users_full_.rehash(0);
     bot_infos_.clear();
+    bot_infos_.rehash(0);
     user_photos_.clear();
+    user_photos_.rehash(0);
     user_profile_photo_file_source_ids_.clear();
+    user_profile_photo_file_source_ids_.rehash(0);
     chat_photo_file_source_ids_.clear();
+    chat_photo_file_source_ids_.rehash(0);
     channels_full_.clear();
+    channels_full_.rehash(0);
     channel_photo_file_source_ids_.clear();
+    channel_photo_file_source_ids_.rehash(0);
     secret_chats_.clear();
+    secret_chats_.rehash(0);
     secret_chats_with_user_.clear();
+    secret_chats_with_user_.rehash(0);
     chat_invite_links_.clear();
+    chat_invite_links_.rehash(0);
     channel_invite_links_.clear();
+    channel_invite_links_.rehash(0);
     invite_link_infos_.clear();
+    invite_link_infos_.rehash(0);
     load_user_from_database_queries_.clear();
+    load_user_from_database_queries_.rehash(0);
     load_chat_from_database_queries_.clear();
+    load_chat_from_database_queries_.rehash(0);
     load_channel_from_database_queries_.clear();
+    load_channel_from_database_queries_.rehash(0);
     load_secret_chat_from_database_queries_.clear();
+    load_secret_chat_from_database_queries_.rehash(0);
     dialog_administrators_.clear();
+    dialog_administrators_.rehash(0);
     uploaded_profile_photos_.clear();
+    uploaded_profile_photos_.rehash(0);
     imported_contacts_.clear();
+    imported_contacts_.rehash(0);
     received_channel_participant_.clear();
+    received_channel_participant_.rehash(0);
     received_channel_participants_.clear();
+    received_channel_participants_.rehash(0);
     cached_channel_participants_.clear();
+    cached_channel_participants_.rehash(0);
     linked_channel_ids_.clear();
+    linked_channel_ids_.rehash(0);
 }
 }  // namespace td
diff --git a/td/telegram/DocumentsManager.cpp b/td/telegram/DocumentsManager.cpp
index d66a29d0..ede248db 100644
--- a/td/telegram/DocumentsManager.cpp
+++ b/td/telegram/DocumentsManager.cpp
@@ -667,6 +667,7 @@ bool DocumentsManager::merge_documents(FileId new_id, FileId old_id, bool can_de
 
 void DocumentsManager::memory_cleanup() {
   documents_.clear();
+  documents_.rehash(0);
 }
 
 string DocumentsManager::get_document_search_text(FileId file_id) const {
diff --git a/td/telegram/StickersManager.cpp b/td/telegram/StickersManager.cpp
index ba9feda5..79111baf 100644
--- a/td/telegram/StickersManager.cpp
+++ b/td/telegram/StickersManager.cpp
@@ -6209,18 +6209,31 @@ void StickersManager::get_current_state(vector<td_api::object_ptr<td_api::Update
 
 void StickersManager::memory_cleanup() {
     stickers_.clear();
+    stickers_.rehash(0);
     sticker_sets_.clear();
+    sticker_sets_.rehash(0);
     short_name_to_sticker_set_id_.clear();
+    short_name_to_sticker_set_id_.rehash(0);
     attached_sticker_sets_.clear();
+    attached_sticker_sets_.rehash(0);
     found_stickers_.clear();
+    found_stickers_.rehash(0);
     found_sticker_sets_.clear();
+    found_sticker_sets_.rehash(0);
     special_sticker_sets_.clear();
+    special_sticker_sets_.rehash(0);
     sticker_set_load_requests_.clear();
+    sticker_set_load_requests_.rehash(0);
     emoji_language_codes_.clear();
+    emoji_language_codes_.rehash(0);
     emoji_language_code_versions_.clear();
+    emoji_language_code_versions_.rehash(0);
     emoji_language_code_last_difference_times_.clear();
+    emoji_language_code_last_difference_times_.rehash(0);
     emoji_suggestions_urls_.clear();
+    emoji_suggestions_urls_.rehash(0);
     dice_messages_.clear();
+    dice_messages_.rehash(0);
 }
 
 }  // namespace td
diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp
index 91763aca..c4493a33 100644
--- a/td/telegram/Td.cpp
+++ b/td/telegram/Td.cpp
@@ -5127,6 +5127,8 @@ void Td::on_request(uint64 id, td_api::optimizeStorage &request) {
   videos_manager_->memory_cleanup();
   file_manager_->memory_cleanup();
 
+  malloc_trim(0);
+
   std::vector<FileType> file_types;
   for (auto &file_type : request.file_types_) {
     if (file_type == nullptr) {
diff --git a/td/telegram/Td.h b/td/telegram/Td.h
index 6b6873ac..6fd8cb71 100644
--- a/td/telegram/Td.h
+++ b/td/telegram/Td.h
@@ -30,6 +30,7 @@
 #include "td/utils/Slice.h"
 #include "td/utils/Status.h"
 
+#include <malloc.h>
 #include <memory>
 #include <unordered_map>
 #include <unordered_set>
diff --git a/td/telegram/VideoNotesManager.cpp b/td/telegram/VideoNotesManager.cpp
index 79aa0105..351aa2fc 100644
--- a/td/telegram/VideoNotesManager.cpp
+++ b/td/telegram/VideoNotesManager.cpp
@@ -248,6 +248,7 @@ tl_object_ptr<telegram_api::InputMedia> VideoNotesManager::get_input_media(
 
 void VideoNotesManager::memory_cleanup() {
   video_notes_.clear();
+  video_notes_.rehash(0);
 }
 
 }  // namespace td
diff --git a/td/telegram/VideosManager.cpp b/td/telegram/VideosManager.cpp
index f20c1744..00c58809 100644
--- a/td/telegram/VideosManager.cpp
+++ b/td/telegram/VideosManager.cpp
@@ -302,6 +302,7 @@ string VideosManager::get_video_search_text(FileId file_id) const {
 }
 void VideosManager::memory_cleanup() {
   videos_.clear();
+  videos_.rehash(0);
 }
 
 }  // namespace td
diff --git a/td/telegram/WebPagesManager.cpp b/td/telegram/WebPagesManager.cpp
index 56fa05b2..01c096ac 100644
--- a/td/telegram/WebPagesManager.cpp
+++ b/td/telegram/WebPagesManager.cpp
@@ -667,7 +667,9 @@ void WebPagesManager::on_get_web_page_instant_view_view_count(WebPageId web_page
   }
 
   auto *instant_view = &web_pages_[web_page_id]->instant_view;
-  CHECK(!instant_view->is_empty);
+  if (instant_view->is_empty) {
+    return;
+  }
   if (instant_view->view_count >= view_count) {
     return;
   }
@@ -705,7 +707,9 @@ void WebPagesManager::register_web_page(WebPageId web_page_id, FullMessageId ful
 
   LOG(INFO) << "Register " << web_page_id << " from " << full_message_id << " from " << source;
   bool is_inserted = web_page_messages_[web_page_id].insert(full_message_id).second;
-  LOG_CHECK(is_inserted) << source << " " << web_page_id << " " << full_message_id;
+  if (!is_inserted) {
+    return;
+  }
 
   if (!td_->auth_manager_->is_bot() && !have_web_page_force(web_page_id)) {
     LOG(INFO) << "Waiting for " << web_page_id << " needed in " << full_message_id;
@@ -721,7 +725,9 @@ void WebPagesManager::unregister_web_page(WebPageId web_page_id, FullMessageId f
   LOG(INFO) << "Unregister " << web_page_id << " from " << full_message_id << " from " << source;
   auto &message_ids = web_page_messages_[web_page_id];
   auto is_deleted = message_ids.erase(full_message_id);
-  LOG_CHECK(is_deleted) << source << " " << web_page_id << " " << full_message_id;
+  if (!is_deleted) {
+    return;
+  }
 
   if (message_ids.empty()) {
     web_page_messages_.erase(web_page_id);
@@ -767,9 +773,13 @@ void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const st
 
 void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const string &url, WebPageId web_page_id,
                                                       Promise<Unit> &&promise) {
-  CHECK(web_page_id == WebPageId() || have_web_page(web_page_id));
+  if (!(web_page_id == WebPageId() || have_web_page(web_page_id))) {
+    return;
+  }
 
-  CHECK(got_web_page_previews_.find(request_id) == got_web_page_previews_.end());
+  if (!(got_web_page_previews_.find(request_id) == got_web_page_previews_.end())) {
+    return;
+  }
   got_web_page_previews_[request_id] = web_page_id;
 
   if (web_page_id.is_valid() && !url.empty()) {
@@ -782,7 +792,9 @@ void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const st
 void WebPagesManager::on_get_web_page_preview_fail(int64 request_id, const string &url, Status error,
                                                    Promise<Unit> &&promise) {
   LOG(INFO) << "Clean up getting of web page preview with url \"" << url << '"';
-  CHECK(error.is_error());
+  if (!error.is_error()) {
+    return;
+  }
   promise.set_error(std::move(error));
 }
 
@@ -915,7 +927,9 @@ void WebPagesManager::load_web_page_instant_view(WebPageId web_page_id, bool for
 void WebPagesManager::reload_web_page_instant_view(WebPageId web_page_id) {
   LOG(INFO) << "Reload " << web_page_id << " instant view";
   const WebPage *web_page = get_web_page(web_page_id);
-  CHECK(web_page != nullptr && !web_page->instant_view.is_empty);
+  if (!(web_page != nullptr && !web_page->instant_view.is_empty)) {
+    return;
+  }
 
   auto promise = PromiseCreator::lambda([web_page_id](Result<> result) {
     send_closure(G()->web_pages_manager(), &WebPagesManager::update_web_page_instant_view_load_requests, web_page_id,
@@ -931,7 +945,9 @@ void WebPagesManager::reload_web_page_instant_view(WebPageId web_page_id) {
 }
 
 void WebPagesManager::on_load_web_page_instant_view_from_database(WebPageId web_page_id, string value) {
-  CHECK(G()->parameters().use_message_db);
+  if(!(G()->parameters().use_message_db)) {
+    return;
+  }
   LOG(INFO) << "Successfully loaded " << web_page_id << " instant view of size " << value.size() << " from database";
   //  G()->td_db()->get_sqlite_pmc()->erase(get_web_page_instant_view_database_key(web_page_id), Auto());
   //  return;
@@ -1122,7 +1138,9 @@ void WebPagesManager::on_load_web_page_id_by_url_from_database(const string &url
 void WebPagesManager::on_load_web_page_by_url_from_database(WebPageId web_page_id, const string &url,
                                                             Promise<Unit> &&promise, Result<> result) {
   if (result.is_error()) {
-    CHECK(G()->close_flag());
+    if(!(G()->close_flag())) {
+      return;
+    }
     promise.set_error(Status::Error(500, "Request aborted"));
     return;
   }
@@ -1200,7 +1218,9 @@ void WebPagesManager::on_web_page_changed(WebPageId web_page_id, bool have_web_p
     for (auto full_message_id : it->second) {
       full_message_ids.push_back(full_message_id);
     }
-    CHECK(!full_message_ids.empty());
+    if (full_message_ids.empty()) {
+      return;
+    }
     for (auto full_message_id : full_message_ids) {
       if (!have_web_page) {
         td_->messages_manager_->delete_pending_message_web_page(full_message_id);
@@ -1209,9 +1229,13 @@ void WebPagesManager::on_web_page_changed(WebPageId web_page_id, bool have_web_p
       }
     }
     if (have_web_page) {
-      CHECK(web_page_messages_[web_page_id].size() == full_message_ids.size());
+      if (!(web_page_messages_[web_page_id].size() == full_message_ids.size())) {
+        return;
+      }
     } else {
-      CHECK(web_page_messages_.count(web_page_id) == 0);
+      if (!(web_page_messages_.count(web_page_id) == 0)) {
+        return;
+      }
     }
   }
   auto get_it = pending_get_web_pages_.find(web_page_id);
@@ -1497,7 +1521,9 @@ void WebPagesManager::on_load_web_page_from_database(WebPageId web_page_id, stri
   vector<Promise<Unit>> promises;
   if (it != load_web_page_from_database_queries_.end()) {
     promises = std::move(it->second);
-    CHECK(!promises.empty());
+    if (promises.empty()) {
+      return;
+    }
     load_web_page_from_database_queries_.erase(it);
   }
 
@@ -1598,10 +1624,15 @@ vector<FileId> WebPagesManager::get_web_page_file_ids(const WebPage *web_page) c
 }
 
 void WebPagesManager::memory_cleanup() {
-//    web_pages_.clear();
-//    web_page_messages_.clear();
-//    got_web_page_previews_.clear();
-//    url_to_web_page_id_.clear();
-//    url_to_file_source_id_.clear();
+    web_pages_.clear();
+    web_pages_.rehash(0);
+    web_page_messages_.clear();
+    web_page_messages_.rehash(0);
+    got_web_page_previews_.clear();
+    got_web_page_previews_.rehash(0);
+    url_to_web_page_id_.clear();
+    url_to_web_page_id_.rehash(0);
+    url_to_file_source_id_.clear();
+    url_to_file_source_id_.rehash(0);
 }
 }  // namespace td
diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp
index 6a2e5968..6397b8e4 100644
--- a/td/telegram/files/FileManager.cpp
+++ b/td/telegram/files/FileManager.cpp
@@ -2946,7 +2946,13 @@ FileId FileManager::next_file_id() {
 }
 
 FileManager::FileNodeId FileManager::next_file_node_id() {
-  FileNodeId res = static_cast<FileNodeId>(file_nodes_.size());
+  if (!empty_node_ids_.empty()) {
+    auto res = empty_node_ids_.back();
+    empty_node_ids_.pop_back();
+    return static_cast<FileNodeId>(res);
+  }
+
+  auto res = static_cast<FileNodeId>(file_nodes_.size());
   file_nodes_.emplace_back(nullptr);
   return res;
 }
@@ -3456,7 +3462,7 @@ void FileManager::hangup() {
 }
 
 void FileManager::memory_cleanup() {
-  if (file_id_info_.size() > 1000 && empty_file_ids_.size() < 200) {
+  if (file_id_info_.size() > 1000 || empty_file_ids_.size() < 200) {
     is_closed_ = true;
     auto time = std::time(nullptr);
 
@@ -3466,6 +3472,7 @@ void FileManager::memory_cleanup() {
         auto &node = file_nodes_[main_node_id];
         if (node != nullptr && ((int32) i) == node->main_file_id_.get()) {
           if (time - file_id_insert_time_[i] > 60 * 5 /* MAIN FILE TTL */) {
+            file_id_insert_time_[i] = INT64_MAX;
             for (auto &file_id : node->file_ids_) {
               /* DESTROY ASSOCIATED QUERIES */
               for (auto &query_id : queries_container_.ids()) {
@@ -3476,8 +3483,8 @@ void FileManager::memory_cleanup() {
               }
 
               /* DESTROY ASSOCIATED FILE */
-              empty_file_ids_.push_back(file_id.get());
               file_id_info_[file_id.get()] = FileIdInfo();
+              empty_file_ids_.push_back(file_id.get());
             }
 
             /* DESTROY MAIN QUERY */
@@ -3489,17 +3496,22 @@ void FileManager::memory_cleanup() {
             }
 
             /* DESTROY MAIN FILE */
-            file_id_insert_time_[i] = INT64_MAX;
             empty_file_ids_.push_back(i);
             file_id_info_[i] = FileIdInfo();
 
             /* DESTROY MAIN NODE */
+            node.reset();
             file_nodes_[main_node_id] = nullptr;
+            empty_node_ids_.push_back(main_node_id);
           }
         }
       }
     }
 
+    file_id_info_.shrink_to_fit();
+    empty_file_ids_.shrink_to_fit();
+    empty_node_ids_.shrink_to_fit();
+
     LOG(INFO) << empty_file_ids_.size() << " empty ids and " <<  queries_container_.size() << " running queries";
     is_closed_ = false;
   }
@@ -3508,4 +3520,4 @@ void FileManager::memory_cleanup() {
 void FileManager::tear_down() {
   parent_.reset();
 }
-}  // namespace tds
+}  // namespace td
diff --git a/td/telegram/files/FileManager.h b/td/telegram/files/FileManager.h
index e9b6c4b1..d7497443 100644
--- a/td/telegram/files/FileManager.h
+++ b/td/telegram/files/FileManager.h
@@ -557,8 +557,9 @@ class FileManager : public FileLoadManager::Callback {
   std::map<FileDbId, int32> pmc_id_to_file_node_id_;
   std::unordered_map<int32, int64> file_id_insert_time_;
 
-  vector<FileIdInfo> file_id_info_;
+  std::vector<FileIdInfo> file_id_info_;
   vector<int32> empty_file_ids_;
+  vector<int32> empty_node_ids_;
   vector<unique_ptr<FileNode>> file_nodes_;
   ActorOwn<FileLoadManager> file_load_manager_;
   ActorOwn<FileGenerateManager> file_generate_manager_;
diff --git a/tdutils/td/utils/common.h b/tdutils/td/utils/common.h
index ade9785e..e7cce76d 100644
--- a/tdutils/td/utils/common.h
+++ b/tdutils/td/utils/common.h
@@ -48,6 +48,7 @@
 
 #include <string>
 #include <vector>
+#include <unordered_map>
 
 #define TD_DEBUG