fix sigfault in "DESTROY NULL file_id_info_"

GitOrigin-RevId: 5ff0dc048c53abbc27d931b39374738ba9e25531

CHANGELOG.md

@@ -716,7 +716,7 @@ Changes in 1.2.0:
 * Added method `searchInstalledStickerSets` to search by title and name for installed sticker sets.
 * Added methods for handling connected websites: `getConnectedWebsites`, `disconnectWebsite` and
   `disconnectAllWebsites`.
-* Added method `getCountryCode`, which uses current user IP to identify their country.
+* Added method `getCountryCode`, which uses current user IP address to identify their country.
 * Added option `t_me_url`.
 * Fixed `BlackBerry` spelling in `deviceTokenBlackBerryPush`.
 * Fixed return type of `getChatMessageByDate` method, which is `Message` and not `Messages`.

CMakeLists.txt

@@ -356,7 +356,6 @@ set(TL_DOTNET_SCHEME_SOURCE
 
 set(TDLIB_SOURCE
   td/mtproto/AuthData.cpp
-  td/mtproto/crypto.cpp
   td/mtproto/DhHandshake.cpp
   td/mtproto/Handshake.cpp
   td/mtproto/HandshakeActor.cpp
@@ -367,6 +366,7 @@ set(TDLIB_SOURCE
   td/mtproto/PingConnection.cpp
   td/mtproto/ProxySecret.cpp
   td/mtproto/RawConnection.cpp
+  td/mtproto/RSA.cpp
   td/mtproto/SessionConnection.cpp
   td/mtproto/TcpTransport.cpp
   td/mtproto/TlsInit.cpp
@@ -494,7 +494,6 @@ set(TDLIB_SOURCE
 
   td/mtproto/AuthData.h
   td/mtproto/AuthKey.h
-  td/mtproto/crypto.h
   td/mtproto/CryptoStorer.h
   td/mtproto/DhHandshake.h
   td/mtproto/Handshake.h
@@ -511,6 +510,7 @@ set(TDLIB_SOURCE
   td/mtproto/ProxySecret.h
   td/mtproto/Query.h
   td/mtproto/RawConnection.h
+  td/mtproto/RSA.h
   td/mtproto/SessionConnection.h
   td/mtproto/TcpTransport.h
   td/mtproto/TlsInit.h

README.md

@@ -52,7 +52,6 @@ for a list of all available `TDLib` [methods](https://core.telegram.org/tdlib/do
 * gperf (build only)
 * CMake (3.0.2+, build only)
 * PHP (optional, for documentation generation)
-* Doxygen (optional, for documentation generation)
 
 <a name="building"></a>
 ## Building

benchmark/wget.cpp

@@ -14,21 +14,22 @@
 #include "td/utils/logging.h"
 #include "td/utils/Status.h"
 
-#include <string>
-
 int main(int argc, char *argv[]) {
   SET_VERBOSITY_LEVEL(VERBOSITY_NAME(DEBUG));
   td::VERBOSITY_NAME(fd) = VERBOSITY_NAME(INFO);
 
-  std::string url = (argc > 1 ? argv[1] : "https://telegram.org");
+  td::string url = (argc > 1 ? argv[1] : "https://telegram.org");
   auto timeout = 10;
   auto ttl = 3;
-  auto prefer_ipv6 = (argc > 2 && std::string(argv[2]) == "-6");
+  auto prefer_ipv6 = (argc > 2 && td::string(argv[2]) == "-6");
   auto scheduler = td::make_unique<td::ConcurrentScheduler>();
   scheduler->init(0);
   scheduler
       ->create_actor_unsafe<td::Wget>(0, "Client",
                                       td::PromiseCreator::lambda([](td::Result<td::unique_ptr<td::HttpQuery>> res) {
+                                        if (res.is_error()) {
+                                          LOG(FATAL) << res.error();
+                                        }
                                         LOG(ERROR) << *res.ok();
                                         td::Scheduler::instance()->finish();
                                       }),

example/README.md

@@ -162,9 +162,14 @@ See [erl-tdlib](https://github.com/lattenwald/erl-tdlib) for an example of TDLib
 <a name="php"></a>
 ## Using TDLib in PHP projects
 
-TDLib can be used from the PHP programming language by wrapping its functionality in a PHP extension.
+If you use modern PHP >= 7.4, you can use TDLib via a PHP FFI extension. For example, take a look at [ffi-tdlib](https://github.com/aurimasniekis/php-ffi-tdlib) - an FFI-based TDLib wrapper.
+
+See also [tdlib-schema](https://github.com/aurimasniekis/php-tdlib-schema) - a generator for TDLib API classes.
+
+For older PHP versions you can use TDLib by wrapping its functionality in a PHP extension.
 
 See [phptdlib](https://github.com/yaroslavche/phptdlib), [tdlib](https://github.com/aurimasniekis/php-ext-tdlib) or [PIF-TDPony](https://github.com/danog/pif-tdpony) for examples of such extensions which provide access to TDLib from PHP.
+
 See [tdlib-bundle](https://github.com/yaroslavche/tdlib-bundle) – a Symfony bundle based on [phptdlib](https://github.com/yaroslavche/phptdlib).
 
 <a name="lua"></a>

sqlite/CMakeLists.txt

@@ -42,6 +42,9 @@ if (GCC OR CLANG)
   if (CLANG)
     target_compile_options(tdsqlite PRIVATE -Wno-parentheses-equality -Wno-unused-value)
   endif()
+  if (GCC AND NOT (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 10.0))
+    target_compile_options(tdsqlite PRIVATE -Wno-return-local-addr -Wno-stringop-overflow)
+  endif()
 elseif (MSVC)
   target_compile_options(tdsqlite PRIVATE /wd4996)
 endif()

td/generate/scheme/td_api.tl

@@ -1857,17 +1857,14 @@ httpUrl url:string = HttpUrl;
 
 //@class InputInlineQueryResult @description Represents a single result of an inline query; for bots only
 
-//@description Represents a link to an animated GIF @id Unique identifier of the query result @title Title of the query result @thumbnail_url URL of the static result thumbnail (JPEG or GIF), if it exists
-//@gif_url The URL of the GIF-file (file size must not exceed 1MB) @gif_duration Duration of the GIF, in seconds @gif_width Width of the GIF @gif_height Height of the GIF
+//@description Represents a link to an animated GIF or an animated (i.e. without sound) H.264/MPEG-4 AVC video
+//@id Unique identifier of the query result @title Title of the query result
+//@thumbnail_url URL of the result thumbnail (JPEG, GIF, or MPEG4), if it exists @thumbnail_mime_type MIME type of the video thumbnail. If non-empty, must be one of "image/jpeg", "image/gif" and "video/mp4"
+//@video_url The URL of the video file (file size must not exceed 1MB) @video_mime_type MIME type of the video file. Must be one of "image/gif" and "video/mp4"
+//@video_duration Duration of the video, in seconds @video_width Width of the video @video_height Height of the video
 //@reply_markup The message reply markup. Must be of type replyMarkupInlineKeyboard or null
 //@input_message_content The content of the message to be sent. Must be one of the following types: InputMessageText, InputMessageAnimation, InputMessageLocation, InputMessageVenue or InputMessageContact
-inputInlineQueryResultAnimatedGif id:string title:string thumbnail_url:string gif_url:string gif_duration:int32 gif_width:int32 gif_height:int32 reply_markup:ReplyMarkup input_message_content:InputMessageContent = InputInlineQueryResult;
-
-//@description Represents a link to an animated (i.e. without sound) H.264/MPEG-4 AVC video @id Unique identifier of the query result @title Title of the result @thumbnail_url URL of the static result thumbnail (JPEG or GIF), if it exists
-//@mpeg4_url The URL of the MPEG4-file (file size must not exceed 1MB) @mpeg4_duration Duration of the video, in seconds @mpeg4_width Width of the video @mpeg4_height Height of the video
-//@reply_markup The message reply markup. Must be of type replyMarkupInlineKeyboard or null
-//@input_message_content The content of the message to be sent. Must be one of the following types: InputMessageText, InputMessageAnimation, InputMessageLocation, InputMessageVenue or InputMessageContact
-inputInlineQueryResultAnimatedMpeg4 id:string title:string thumbnail_url:string mpeg4_url:string mpeg4_duration:int32 mpeg4_width:int32 mpeg4_height:int32 reply_markup:ReplyMarkup input_message_content:InputMessageContent = InputInlineQueryResult;
+inputInlineQueryResultAnimation id:string title:string thumbnail_url:string thumbnail_mime_type:string video_url:string video_mime_type:string video_duration:int32 video_width:int32 video_height:int32 reply_markup:ReplyMarkup input_message_content:InputMessageContent = InputInlineQueryResult;
 
 //@description Represents a link to an article or web page @id Unique identifier of the query result @url URL of the result, if it exists @hide_url True, if the URL must be not shown @title Title of the result
 //@param_description A short description of the result @thumbnail_url URL of the result thumbnail, if it exists @thumbnail_width Thumbnail width, if known @thumbnail_height Thumbnail height, if known
@@ -4386,7 +4383,7 @@ answerCustomQuery custom_query_id:int64 data:string = Ok;
 setAlarm seconds:double = Ok;
 
 
-//@description Uses current user IP to found their country. Returns two-letter ISO 3166-1 alpha-2 country code. Can be called before authorization
+//@description Uses current user IP address to found their country. Returns two-letter ISO 3166-1 alpha-2 country code. Can be called before authorization
 getCountryCode = Text;
 
 //@description Returns the default text for invitation messages to be used as a placeholder when the current user invites friends to Telegram

td/mtproto/Handshake.cpp

@@ -6,7 +6,6 @@
 //
 #include "td/mtproto/Handshake.h"
 
-#include "td/mtproto/crypto.h"
 #include "td/mtproto/KDF.h"
 #include "td/mtproto/utils.h"
 

td/mtproto/Handshake.h

@@ -7,8 +7,8 @@
 #pragma once
 
 #include "td/mtproto/AuthKey.h"
-#include "td/mtproto/crypto.h"
 #include "td/mtproto/DhHandshake.h"
+#include "td/mtproto/RSA.h"
 
 #include "td/utils/buffer.h"
 #include "td/utils/Slice.h"

td/mtproto/RSA.cpp

@@ -4,7 +4,7 @@
 // Distributed under the Boost Software License, Version 1.0. (See accompanying
 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 //
-#include "td/mtproto/crypto.h"
+#include "td/mtproto/RSA.h"
 
 #include "td/mtproto/mtproto_api.h"
 
@@ -25,7 +25,6 @@
 
 namespace td {
 
-/*** RSA ***/
 RSA::RSA(BigNum n, BigNum e) : n_(std::move(n)), e_(std::move(e)) {
 }
 

td/mtproto/RSA.h

@@ -15,7 +15,6 @@
 
 namespace td {
 
-/*** RSA ***/
 class RSA {
  public:
   RSA clone() const;
@@ -33,7 +32,6 @@ class RSA {
   BigNum e_;
 };
 
-/*** PublicRsaKeyInterface ***/
 class PublicRsaKeyInterface {
  public:
   virtual ~PublicRsaKeyInterface() = default;

td/telegram/AnimationsManager.cpp

@@ -516,6 +516,9 @@ void AnimationsManager::load_saved_animations(Promise<Unit> &&promise) {
 }
 
 void AnimationsManager::on_load_saved_animations_from_database(const string &value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (value.empty()) {
     LOG(INFO) << "Saved animations aren't found in database";
     reload_saved_animations(true);
@@ -843,5 +846,9 @@ void AnimationsManager::get_current_state(vector<td_api::object_ptr<td_api::Upda
     updates.push_back(get_update_saved_animations_object());
   }
 }
+void AnimationsManager::memory_cleanup() {
+  animations_.clear();
+  animations_.rehash(0);
+}
 
 }  // namespace td

td/telegram/AnimationsManager.h

@@ -31,6 +31,8 @@ class AnimationsManager : public Actor {
  public:
   AnimationsManager(Td *td, ActorShared<> parent);
 
+  void memory_cleanup();
+
   int32 get_animation_duration(FileId file_id) const;
 
   tl_object_ptr<td_api::animation> get_animation_object(FileId file_id, const char *source);

td/telegram/AudiosManager.cpp

@@ -274,5 +274,9 @@ tl_object_ptr<telegram_api::InputMedia> AudiosManager::get_input_media(
 
   return nullptr;
 }
+void AudiosManager::memory_cleanup() {
+  audios_.clear();
+  audios_.rehash(0);
+}
 
 }  // namespace td

td/telegram/AudiosManager.h

@@ -26,6 +26,8 @@ class AudiosManager {
  public:
   explicit AudiosManager(Td *td);
 
+  void memory_cleanup();
+
   int32 get_audio_duration(FileId file_id) const;
 
   tl_object_ptr<td_api::audio> get_audio_object(FileId file_id);

td/telegram/BackgroundManager.cpp

@@ -475,6 +475,10 @@ BackgroundId BackgroundManager::search_background(const string &name, Promise<Un
 }
 
 void BackgroundManager::on_load_background_from_database(string name, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   auto promises_it = being_loaded_from_database_backgrounds_.find(name);
   CHECK(promises_it != being_loaded_from_database_backgrounds_.end());
   auto promises = std::move(promises_it->second);

td/telegram/BackgroundType.cpp

@@ -136,7 +136,7 @@ StringBuilder &operator<<(StringBuilder &string_builder, const BackgroundType &t
 
 Result<BackgroundType> get_background_type(const td_api::BackgroundType *type) {
   if (type == nullptr) {
-    return Status::Error(400, "Type must not be empty");
+    return Status::Error(400, "Type must be non-empty");
   }
 
   BackgroundType result;
@@ -149,7 +149,7 @@ Result<BackgroundType> get_background_type(const td_api::BackgroundType *type) {
     case td_api::backgroundTypePattern::ID: {
       auto pattern = static_cast<const td_api::backgroundTypePattern *>(type);
       if (pattern->fill_ == nullptr) {
-        return Status::Error(400, "Fill info must not be empty");
+        return Status::Error(400, "Fill info must be non-empty");
       }
       result = BackgroundType(pattern->is_moving_, get_background_fill(pattern->fill_.get()), pattern->intensity_);
       break;
@@ -157,7 +157,7 @@ Result<BackgroundType> get_background_type(const td_api::BackgroundType *type) {
     case td_api::backgroundTypeFill::ID: {
       auto fill = static_cast<const td_api::backgroundTypeFill *>(type);
       if (fill->fill_ == nullptr) {
-        return Status::Error(400, "Fill info must not be empty");
+        return Status::Error(400, "Fill info must be non-empty");
       }
       result = BackgroundType(get_background_fill(fill->fill_.get()));
       break;

td/telegram/ConfigManager.cpp

@@ -27,8 +27,8 @@
 
 #include "td/mtproto/AuthData.h"
 #include "td/mtproto/AuthKey.h"
-#include "td/mtproto/crypto.h"
 #include "td/mtproto/RawConnection.h"
+#include "td/mtproto/RSA.h"
 #include "td/mtproto/TransportType.h"
 
 #include "td/net/HttpQuery.h"
@@ -1426,7 +1426,6 @@ void ConfigManager::process_app_config(tl_object_ptr<telegram_api::JSONValue> &c
   }
 
   if (!dice_emojis.empty()) {
-    shared_config.set_option_string("dice_emojis", implode(dice_emojis, '\x01'));
     vector<string> dice_success_values(dice_emojis.size());
     for (auto &it : dice_emoji_success_value) {
       if (dice_emoji_index.find(it.first) == dice_emoji_index.end()) {
@@ -1436,6 +1435,7 @@ void ConfigManager::process_app_config(tl_object_ptr<telegram_api::JSONValue> &c
       dice_success_values[dice_emoji_index[it.first]] = it.second;
     }
     shared_config.set_option_string("dice_success_values", implode(dice_success_values, ','));
+    shared_config.set_option_string("dice_emojis", implode(dice_emojis, '\x01'));
   }
 }
 

td/telegram/ContactsManager.cpp

@@ -4724,6 +4724,10 @@ void ContactsManager::load_imported_contacts(Promise<Unit> &&promise) {
 }
 
 void ContactsManager::on_load_imported_contacts_from_database(string value) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   CHECK(!are_imported_contacts_loaded_);
   if (need_clear_imported_contacts_) {
     need_clear_imported_contacts_ = false;
@@ -6679,6 +6683,9 @@ void ContactsManager::on_deleted_contacts(const vector<UserId> &deleted_contact_
 }
 
 void ContactsManager::save_next_contacts_sync_date() {
+  if (G()->close_flag()) {
+    return;
+  }
   if (!G()->parameters().use_chat_info_db) {
     return;
   }
@@ -6775,6 +6782,9 @@ void ContactsManager::on_get_contacts_failed(Status error) {
 }
 
 void ContactsManager::on_load_contacts_from_database(string value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (value.empty()) {
     reload_contacts(true);
     return;
@@ -7167,6 +7177,10 @@ void ContactsManager::save_user_to_database_impl(User *u, UserId user_id, string
 }
 
 void ContactsManager::on_save_user_to_database(UserId user_id, bool success) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   User *u = get_user(user_id);
   CHECK(u != nullptr);
   LOG_CHECK(u->is_being_saved) << user_id << " " << u->is_saved << " " << u->is_status_saved << " "
@@ -7219,6 +7233,10 @@ void ContactsManager::load_user_from_database_impl(UserId user_id, Promise<Unit>
 }
 
 void ContactsManager::on_load_user_from_database(UserId user_id, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   if (!loaded_from_database_users_.insert(user_id).second) {
     return;
   }
@@ -7433,6 +7451,10 @@ void ContactsManager::save_chat_to_database_impl(Chat *c, ChatId chat_id, string
 }
 
 void ContactsManager::on_save_chat_to_database(ChatId chat_id, bool success) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   Chat *c = get_chat(chat_id);
   CHECK(c != nullptr);
   CHECK(c->is_being_saved);
@@ -7479,6 +7501,10 @@ void ContactsManager::load_chat_from_database_impl(ChatId chat_id, Promise<Unit>
 }
 
 void ContactsManager::on_load_chat_from_database(ChatId chat_id, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   if (!loaded_from_database_chats_.insert(chat_id).second) {
     return;
   }
@@ -7662,6 +7688,10 @@ void ContactsManager::save_channel_to_database_impl(Channel *c, ChannelId channe
 }
 
 void ContactsManager::on_save_channel_to_database(ChannelId channel_id, bool success) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   Channel *c = get_channel(channel_id);
   CHECK(c != nullptr);
   CHECK(c->is_being_saved);
@@ -7708,6 +7738,10 @@ void ContactsManager::load_channel_from_database_impl(ChannelId channel_id, Prom
 }
 
 void ContactsManager::on_load_channel_from_database(ChannelId channel_id, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   if (!loaded_from_database_channels_.insert(channel_id).second) {
     return;
   }
@@ -7894,6 +7928,10 @@ void ContactsManager::save_secret_chat_to_database_impl(SecretChat *c, SecretCha
 }
 
 void ContactsManager::on_save_secret_chat_to_database(SecretChatId secret_chat_id, bool success) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   SecretChat *c = get_secret_chat(secret_chat_id);
   CHECK(c != nullptr);
   CHECK(c->is_being_saved);
@@ -7941,6 +7979,10 @@ void ContactsManager::load_secret_chat_from_database_impl(SecretChatId secret_ch
 }
 
 void ContactsManager::on_load_secret_chat_from_database(SecretChatId secret_chat_id, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   if (!loaded_from_database_secret_chats_.insert(secret_chat_id).second) {
     return;
   }
@@ -8856,7 +8898,7 @@ void ContactsManager::on_get_user_photos(UserId user_id, int32 offset, int32 lim
     total_count = min_total_count;
   }
   LOG_IF(ERROR, limit < photo_count) << "Requested not more than " << limit << " photos, but " << photo_count
-                                     << " returned";
+                                     << " received";
 
   User *u = get_user(user_id);
   if (u == nullptr) {
@@ -8926,11 +8968,8 @@ void ContactsManager::on_get_user_photos(UserId user_id, int32 offset, int32 lim
   }
 
   auto known_photo_count = narrow_cast<int32>(user_photos->photos.size());
-  CHECK(user_photos->count >= known_photo_count);
   if (user_photos->offset + known_photo_count > user_photos->count) {
-    LOG(ERROR) << "Fix total photo count of " << user_id << " from " << user_photos->count << " to "
-               << user_photos->offset << " + " << known_photo_count;
-    user_photos->count = user_photos->offset + known_photo_count;
+    user_photos->photos.resize(user_photos->count - user_photos->offset);
   }
 }
 
@@ -12737,7 +12776,7 @@ void ContactsManager::load_dialog_administrators(DialogId dialog_id, Promise<Uni
 
 void ContactsManager::on_load_dialog_administrators_from_database(DialogId dialog_id, string value,
                                                                   Promise<Unit> &&promise) {
-  if (value.empty()) {
+  if (value.empty() || G()->close_flag()) {
     promise.set_value(Unit());
     return;
   }

td/telegram/DocumentsManager.cpp

@@ -138,7 +138,7 @@ Document DocumentsManager::on_get_document(RemoteDocument remote_document, Dialo
   FileType file_type = FileType::Document;
   Slice default_extension;
   bool supports_streaming = false;
-  bool has_webp_thumbnail = false;
+  PhotoFormat thumbnail_format = PhotoFormat::Jpeg;
   if (type_attributes == 1 || default_document_type != Document::Type::General) {  // not a general document
     if (animated != nullptr || default_document_type == Document::Type::Animation) {
       document_type = Document::Type::Animation;
@@ -166,7 +166,9 @@ Document DocumentsManager::on_get_document(RemoteDocument remote_document, Dialo
       default_extension = Slice("webp");
       owner_dialog_id = DialogId();
       file_name.clear();
-      has_webp_thumbnail = td_->stickers_manager_->has_webp_thumbnail(sticker);
+      if (td_->stickers_manager_->has_webp_thumbnail(sticker) && remote_document.secret_file == nullptr) {
+        thumbnail_format = PhotoFormat::Webp;
+      }
     } else if (video != nullptr || default_document_type == Document::Type::Video ||
                default_document_type == Document::Type::VideoNote) {
       bool is_video_note = default_document_type == Document::Type::VideoNote;
@@ -193,7 +195,6 @@ Document DocumentsManager::on_get_document(RemoteDocument remote_document, Dialo
                  << ", has_stickers = " << has_stickers;
   }
 
-  bool has_png_thumbnail = false;
   if (is_background) {
     if (document_type != Document::Type::General) {
       LOG(ERROR) << "Receive background of type " << document_type;
@@ -202,7 +203,7 @@ Document DocumentsManager::on_get_document(RemoteDocument remote_document, Dialo
     file_type = FileType::Background;
     if (is_pattern) {
       default_extension = Slice("png");
-      has_png_thumbnail = true;
+      thumbnail_format = PhotoFormat::Png;
     } else {
       default_extension = Slice("jpg");
     }
@@ -261,9 +262,9 @@ Document DocumentsManager::on_get_document(RemoteDocument remote_document, Dialo
 
     if (document_type != Document::Type::VoiceNote) {
       for (auto &thumb : document->thumbs_) {
-        auto photo_size = get_photo_size(td_->file_manager_.get(), {FileType::Thumbnail, 0}, id, access_hash,
-                                         file_reference, DcId::create(dc_id), owner_dialog_id, std::move(thumb),
-                                         has_webp_thumbnail, has_png_thumbnail);
+        auto photo_size =
+            get_photo_size(td_->file_manager_.get(), {FileType::Thumbnail, 0}, id, access_hash, file_reference,
+                           DcId::create(dc_id), owner_dialog_id, std::move(thumb), thumbnail_format);
         if (photo_size.get_offset() == 0) {
           thumbnail = std::move(photo_size.get<0>());
         } else {

td/telegram/FileReferenceManager.cpp

@@ -364,4 +364,11 @@ void FileReferenceManager::reload_photo(PhotoSizeSource source, Promise<Unit> pr
   }
 }
 
+void FileReferenceManager::memory_cleanup(FileId file_id) {
+  auto &node = nodes_[file_id];
+  node.query.reset();
+  node.file_source_ids.reset_position();
+  nodes_.erase(file_id);
+}
+
 }  // namespace td

td/telegram/FileReferenceManager.h

@@ -38,6 +38,8 @@ class FileReferenceManager : public Actor {
   static bool is_file_reference_error(const Status &error);
   static size_t get_file_reference_error_pos(const Status &error);
 
+  void memory_cleanup(FileId file_id);
+
   FileSourceId create_message_file_source(FullMessageId full_message_id);
   FileSourceId create_user_photo_file_source(UserId user_id, int64 photo_id);
   FileSourceId create_chat_photo_file_source(ChatId chat_id);

td/telegram/HashtagHints.cpp

@@ -86,6 +86,10 @@ void HashtagHints::hashtag_used_impl(const string &hashtag) {
 }
 
 void HashtagHints::from_db(Result<string> data, bool dummy) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   sync_with_db_ = true;
   if (data.is_error() || data.ok().empty()) {
     return;

td/telegram/InlineQueriesManager.cpp

@@ -341,7 +341,7 @@ void InlineQueriesManager::answer_inline_query(int64 inline_query_id, bool is_pe
   bool force_vertical = false;
   for (auto &input_result : input_results) {
     if (input_result == nullptr) {
-      return promise.set_error(Status::Error(400, "Inline query result must not be empty"));
+      return promise.set_error(Status::Error(400, "Inline query result must be non-empty"));
     }
 
     string id;
@@ -350,6 +350,7 @@ void InlineQueriesManager::answer_inline_query(int64 inline_query_id, bool is_pe
     string title;
     string description;
     string thumbnail_url;
+    string thumbnail_type = "image/jpeg";
     string content_url;
     string content_type;
     int32 thumbnail_width = 0;
@@ -361,42 +362,28 @@ void InlineQueriesManager::answer_inline_query(int64 inline_query_id, bool is_pe
     FileType file_type = FileType::Temp;
     Result<tl_object_ptr<telegram_api::InputBotInlineMessage>> r_inline_message = Status::Error(500, "Uninited");
     switch (input_result->get_id()) {
-      case td_api::inputInlineQueryResultAnimatedGif::ID: {
-        auto animated_gif = move_tl_object_as<td_api::inputInlineQueryResultAnimatedGif>(input_result);
+      case td_api::inputInlineQueryResultAnimation::ID: {
+        auto animation = move_tl_object_as<td_api::inputInlineQueryResultAnimation>(input_result);
         type = "gif";
-        id = std::move(animated_gif->id_);
-        title = std::move(animated_gif->title_);
-        thumbnail_url = std::move(animated_gif->thumbnail_url_);
-        content_url = std::move(animated_gif->gif_url_);
-        content_type = "image/gif";
-        // duration = animated_gif->gif_duration_;
-        width = animated_gif->gif_width_;
-        height = animated_gif->gif_height_;
+        id = std::move(animation->id_);
+        title = std::move(animation->title_);
+        thumbnail_url = std::move(animation->thumbnail_url_);
+        if (!animation->thumbnail_mime_type_.empty()) {
+          thumbnail_type = std::move(animation->thumbnail_mime_type_);
+        }
+        content_url = std::move(animation->video_url_);
+        content_type = std::move(animation->video_mime_type_);
+        if (content_type != "image/gif" && content_type != "video/mp4") {
+          return promise.set_error(Status::Error(400, "Wrong animation MIME type specified"));
+        }
+        duration = animation->video_duration_;
+        width = animation->video_width_;
+        height = animation->video_height_;
         is_gallery = true;
 
         file_type = FileType::Animation;
-        r_inline_message =
-            get_inline_message(std::move(animated_gif->input_message_content_), std::move(animated_gif->reply_markup_),
-                               td_api::inputMessageAnimation::ID);
-        break;
-      }
-      case td_api::inputInlineQueryResultAnimatedMpeg4::ID: {
-        auto animated_mpeg4 = move_tl_object_as<td_api::inputInlineQueryResultAnimatedMpeg4>(input_result);
-        type = "gif";
-        id = std::move(animated_mpeg4->id_);
-        title = std::move(animated_mpeg4->title_);
-        thumbnail_url = std::move(animated_mpeg4->thumbnail_url_);
-        content_url = std::move(animated_mpeg4->mpeg4_url_);
-        content_type = "video/mp4";
-        duration = animated_mpeg4->mpeg4_duration_;
-        width = animated_mpeg4->mpeg4_width_;
-        height = animated_mpeg4->mpeg4_height_;
-        is_gallery = true;
-
-        file_type = FileType::Animation;
-        r_inline_message =
-            get_inline_message(std::move(animated_mpeg4->input_message_content_),
-                               std::move(animated_mpeg4->reply_markup_), td_api::inputMessageAnimation::ID);
+        r_inline_message = get_inline_message(std::move(animation->input_message_content_),
+                                              std::move(animation->reply_markup_), td_api::inputMessageAnimation::ID);
         break;
       }
       case td_api::inputInlineQueryResultArticle::ID: {
@@ -701,7 +688,8 @@ void InlineQueriesManager::answer_inline_query(int64 inline_query_id, bool is_pe
         attributes.push_back(
             make_tl_object<telegram_api::documentAttributeImageSize>(thumbnail_width, thumbnail_height));
       }
-      thumbnail = make_tl_object<telegram_api::inputWebDocument>(thumbnail_url, 0, "image/jpeg", std::move(attributes));
+      thumbnail =
+          make_tl_object<telegram_api::inputWebDocument>(thumbnail_url, 0, thumbnail_type, std::move(attributes));
     }
     tl_object_ptr<telegram_api::inputWebDocument> content;
     if (!content_url.empty() || !content_type.empty()) {

td/telegram/LanguagePackManager.cpp

@@ -1579,7 +1579,7 @@ Result<LanguagePackManager::LanguageInfo> LanguagePackManager::get_language_info
 Result<LanguagePackManager::LanguageInfo> LanguagePackManager::get_language_info(
     td_api::languagePackInfo *language_pack_info) {
   if (language_pack_info == nullptr) {
-    return Status::Error(400, "Language pack info must not be empty");
+    return Status::Error(400, "Language pack info must be non-empty");
   }
 
   if (!clean_input_string(language_pack_info->id_)) {

td/telegram/Logging.cpp

@@ -47,7 +47,7 @@ static const std::map<Slice, int *> log_tags{
 
 Status Logging::set_current_stream(td_api::object_ptr<td_api::LogStream> stream) {
   if (stream == nullptr) {
-    return Status::Error("Log stream must not be empty");
+    return Status::Error("Log stream must be non-empty");
   }
 
   std::lock_guard<std::mutex> lock(logging_mutex);

td/telegram/MessageContent.cpp

@@ -663,7 +663,7 @@ class MessageDice : public MessageContent {
     if (dice_value < 0) {
       return false;
     }
-    if (emoji == "DEFAULT_EMOJI" || emoji == "🎯") {
+    if (emoji == DEFAULT_EMOJI || emoji == "🎯") {
       return dice_value <= 6;
     }
     return dice_value <= 1000;
@@ -1778,7 +1778,7 @@ static Result<InputMessageContent> create_input_message_content(
       int32 correct_option_id = -1;
       FormattedText explanation;
       if (input_poll->type_ == nullptr) {
-        return Status::Error(400, "Poll type must not be empty");
+        return Status::Error(400, "Poll type must be non-empty");
       }
       switch (input_poll->type_->get_id()) {
         case td_api::pollTypeRegular::ID: {

td/telegram/MessageEntity.cpp

@@ -422,7 +422,7 @@ static vector<Slice> match_bank_card_numbers(Slice str) {
   const unsigned char *end = str.uend();
   const unsigned char *ptr = begin;
 
-  // '/(?<=^|[^+_\pL\d-])[\d -]{13,}([^_\pL\d-]|$)/'
+  // '/(?<=^|[^+_\pL\d-.,])[\d -]{13,}([^_\pL\d-]|$)/'
 
   while (true) {
     while (ptr != end && !is_digit(*ptr)) {
@@ -435,7 +435,7 @@ static vector<Slice> match_bank_card_numbers(Slice str) {
       uint32 prev;
       next_utf8_unsafe(prev_utf8_unsafe(ptr), &prev, "match_bank_card_numbers");
 
-      if (prev == '+' || prev == '-' || prev == '_' ||
+      if (prev == '.' || prev == ',' || prev == '+' || prev == '-' || prev == '_' ||
           get_unicode_simple_category(prev) == UnicodeSimpleCategory::Letter) {
         while (ptr != end && (is_digit(*ptr) || *ptr == ' ' || *ptr == '-')) {
           ptr++;
@@ -1965,10 +1965,12 @@ static vector<Slice> find_text_url_entities_v3(Slice text) {
 }
 
 // entities must be valid for the text
-static FormattedText parse_text_url_entities_v3(Slice text, vector<MessageEntity> entities) {
+static FormattedText parse_text_url_entities_v3(Slice text, const vector<MessageEntity> &entities) {
   // continuous entities can't intersect TextUrl entities,
   // so try to find new TextUrl entities only between the predetermined continuous entities
 
+  Slice debug_initial_text = text;
+
   FormattedText result;
   int32 result_text_utf16_length = 0;
   vector<MessageEntity> part_entities;
@@ -2109,7 +2111,7 @@ static FormattedText parse_text_url_entities_v3(Slice text, vector<MessageEntity
         splittable_entities.clear();
       } else {
         CHECK(pos == splittable_entities.size() - 1);
-        CHECK(!text.empty());
+        LOG_CHECK(!text.empty()) << '"' << debug_initial_text << "\" " << entities;
         splittable_entities[0] = std::move(splittable_entities.back());
         splittable_entities.resize(1);
       }
@@ -2118,7 +2120,7 @@ static FormattedText parse_text_url_entities_v3(Slice text, vector<MessageEntity
     part_begin = part_end;
   };
 
-  for (auto &entity : entities) {
+  for (const auto &entity : entities) {
     if (is_splittable_entity(entity.type)) {
       auto index = get_splittable_entity_type_index(entity.type);
       part_splittable_entities[index].push_back(entity);
@@ -2207,7 +2209,7 @@ static FormattedText parse_markdown_v3_without_pre(Slice text, vector<MessageEnt
 
   FormattedText parsed_text_url_text;
   if (text.find('[') != string::npos) {
-    parsed_text_url_text = parse_text_url_entities_v3(text, std::move(entities));
+    parsed_text_url_text = parse_text_url_entities_v3(text, entities);
     text = parsed_text_url_text.text;
     entities = std::move(parsed_text_url_text.entities);
   }
@@ -3543,6 +3545,25 @@ static std::pair<size_t, int32> remove_invalid_entities(const string &text, vect
       break;
     }
 
+    if (!nested_entities_stack.empty() && nested_entities_stack.back()->offset == utf16_offset &&
+        (text[pos] == '\n' || text[pos] == ' ')) {
+      // entities was fixed, so there can't be more than one splittable entity of each type, one blockquote and
+      // one continuous entity for the given offset
+      for (size_t i = nested_entities_stack.size(); i > 0; i--) {
+        auto *entity = nested_entities_stack[i - 1];
+        if (entity->offset != utf16_offset || entity->type == MessageEntity::Type::TextUrl ||
+            entity->type == MessageEntity::Type::MentionName || is_pre_entity(entity->type)) {
+          break;
+        }
+        entity->offset++;
+        entity->length--;
+        if (entity->length == 0) {
+          CHECK(i == nested_entities_stack.size());
+          nested_entities_stack.pop_back();
+        }
+      }
+    }
+
     auto c = static_cast<unsigned char>(text[pos]);
     switch (c) {
       case '\n':

td/telegram/MessagesManager.cpp

@@ -4946,8 +4946,9 @@ void MessagesManager::save_dialog_to_database(DialogId dialog_id) {
   };
   add_group_key(d->message_notification_group);
   add_group_key(d->mention_notification_group);
+  auto fixed_folder_id = d->folder_id == FolderId::archive() ? FolderId::archive() : FolderId::main();
   G()->td_db()->get_dialog_db_async()->add_dialog(
-      dialog_id, d->folder_id, d->is_folder_id_inited ? d->order : 0, get_dialog_database_value(d),
+      dialog_id, fixed_folder_id, d->is_folder_id_inited ? d->order : 0, get_dialog_database_value(d),
       std::move(changed_group_keys), PromiseCreator::lambda([dialog_id, can_reuse_notification_group](Result<> result) {
         send_closure(G()->messages_manager(), &MessagesManager::on_save_dialog_to_database, dialog_id,
                      can_reuse_notification_group, result.is_ok());
@@ -4957,7 +4958,7 @@ void MessagesManager::save_dialog_to_database(DialogId dialog_id) {
 void MessagesManager::on_save_dialog_to_database(DialogId dialog_id, bool can_reuse_notification_group, bool success) {
   LOG(INFO) << "Successfully saved " << dialog_id << " to database";
 
-  if (success && can_reuse_notification_group) {
+  if (success && can_reuse_notification_group && !G()->close_flag()) {
     auto d = get_dialog(dialog_id);
     CHECK(d != nullptr);
     try_reuse_notification_group(d->message_notification_group);
@@ -7058,7 +7059,7 @@ void MessagesManager::report_dialog(DialogId dialog_id, const tl_object_ptr<td_a
   }
 
   if (reason == nullptr) {
-    return promise.set_error(Status::Error(3, "Reason must not be empty"));
+    return promise.set_error(Status::Error(3, "Reason must be non-empty"));
   }
 
   Dialog *user_d = d;
@@ -7885,7 +7886,7 @@ void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_
   CHECK(offset < 0 || from_the_end);
   CHECK(!from_message_id.is_scheduled());
 
-  // it is likely that there is no more history messages on the server
+  // it is likely that there are no more history messages on the server
   bool have_full_history = from_the_end && narrow_cast<int32>(messages.size()) < limit;
   Dialog *d = get_dialog(dialog_id);
 
@@ -7940,7 +7941,7 @@ void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_
     MessageId first_received_message_id = get_message_id(messages.back(), false);
     if (first_received_message_id >= from_message_id && d->first_database_message_id.is_valid() &&
         first_received_message_id >= d->first_database_message_id) {
-      // it is likely that there is no more history messages on the server
+      // it is likely that there are no more history messages on the server
       have_full_history = true;
     }
   }
@@ -7958,6 +7959,46 @@ void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_
     prev_have_full_history = d->have_full_history;
   }
 
+  if (from_the_end && d != nullptr) {
+    auto last_server_message_id = get_message_id(messages[0], false);
+    // delete all server messages with ID > last_server_message_id
+    vector<MessageId> message_ids;
+    find_newer_messages(d->messages.get(), last_server_message_id, message_ids);
+    if (!message_ids.empty()) {
+      bool need_update_dialog_pos = false;
+      vector<int64> deleted_message_ids;
+      for (auto message_id : message_ids) {
+        CHECK(message_id > last_server_message_id);
+        if (message_id.is_server()) {
+          auto message = delete_message(d, message_id, true, &need_update_dialog_pos, "on_get_gistory 1");
+          if (message != nullptr) {
+            deleted_message_ids.push_back(message->message_id.get());
+          }
+        }
+      }
+      if (need_update_dialog_pos) {
+        send_update_chat_last_message(d, "on_get_gistory 2");
+      }
+
+      if (!deleted_message_ids.empty()) {
+        send_update_delete_messages(dialog_id, std::move(deleted_message_ids), true, false);
+
+        message_ids.clear();
+        find_newer_messages(d->messages.get(), last_server_message_id, message_ids);
+      }
+
+      // connect all messages with ID > last_server_message_id
+      for (size_t i = 0; i + 1 < message_ids.size(); i++) {
+        auto m = get_message(d, message_ids[i]);
+        if (m == nullptr) { continue; }
+        if (!m->have_next) {
+          m->have_next = true;
+          attach_message_to_next(d, message_ids[i], "on_get_history 3");
+        }
+      }
+    }
+  }
+
   for (auto &message : messages) {
     if (!have_next && from_the_end && d != nullptr && get_message_id(message, false) < d->last_message_id) {
       // last message in the dialog should be attached to the next message if there is some
@@ -8169,18 +8210,20 @@ void MessagesManager::on_get_dialog_messages_search_result(DialogId dialog_id, c
 
     MessageId first_added_message_id;
     if (messages.empty()) {
-      // messages may be empty because there is no more messages or they can't be found due to global limit
-      // anyway pretend that there is no more messages
+      // messages may be empty because there are no more messages or they can't be found due to global limit
+      // anyway pretend that there are no more messages
       first_added_message_id = MessageId::min();
     }
 
     auto &result = it->second.second;
     CHECK(result.empty());
+    int32 added_message_count = 0;
     for (auto &message : messages) {
       auto new_full_message_id =
           on_get_message(std::move(message), false, false, false, false, false, "search call messages");
       if (new_full_message_id != FullMessageId()) {
         result.push_back(new_full_message_id);
+        added_message_count++;
       }
 
       auto message_id = new_full_message_id.get_message_id();
@@ -8188,6 +8231,11 @@ void MessagesManager::on_get_dialog_messages_search_result(DialogId dialog_id, c
         first_added_message_id = message_id;
       }
     }
+    if (total_count < added_message_count) {
+      LOG(ERROR) << "Receive total_count = " << total_count << ", but added " << added_message_count
+                 << " messages out of " << messages.size();
+      total_count = added_message_count;
+    }
     if (G()->parameters().use_message_db) {
       bool update_state = false;
 
@@ -8224,7 +8272,7 @@ void MessagesManager::on_get_dialog_messages_search_result(DialogId dialog_id, c
   CHECK(result.empty());
   MessageId first_added_message_id;
   if (messages.empty()) {
-    // messages may be empty because there is no more messages or they can't be found due to global limit
+    // messages may be empty because there are no more messages or they can't be found due to global limit
     // anyway pretend that there is no more messages
     first_added_message_id = MessageId::min();
   }
@@ -9066,18 +9114,18 @@ void MessagesManager::find_old_messages(const Message *m, MessageId max_message_
   }
 }
 
-void MessagesManager::find_new_messages(const Message *m, MessageId min_message_id, vector<MessageId> &message_ids) {
+void MessagesManager::find_newer_messages(const Message *m, MessageId min_message_id, vector<MessageId> &message_ids) {
   if (m == nullptr) {
     return;
   }
 
   if (m->message_id > min_message_id) {
-    find_new_messages(m->left.get(), min_message_id, message_ids);
+    find_newer_messages(m->left.get(), min_message_id, message_ids);
 
     message_ids.push_back(m->message_id);
   }
 
-  find_new_messages(m->right.get(), min_message_id, message_ids);
+  find_newer_messages(m->right.get(), min_message_id, message_ids);
 }
 
 void MessagesManager::find_unloadable_messages(const Dialog *d, int32 unload_before_date, const Message *m,
@@ -9829,6 +9877,10 @@ void MessagesManager::repair_secret_chat_total_count(FolderId folder_id) {
 }
 
 void MessagesManager::on_get_secret_chat_total_count(FolderId folder_id, int32 total_count) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   CHECK(!td_->auth_manager_->is_bot());
   auto &list = get_dialog_list(folder_id);
   CHECK(total_count >= 0);
@@ -10793,6 +10845,10 @@ void MessagesManager::ttl_db_loop(double server_now) {
 
 void MessagesManager::ttl_db_on_result(Result<std::pair<std::vector<std::pair<DialogId, BufferSlice>>, int32>> r_result,
                                        bool dummy) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   auto result = r_result.move_as_ok();
   ttl_db_has_query_ = false;
   ttl_db_expires_from_ = ttl_db_expires_till_;
@@ -11587,6 +11643,7 @@ FullMessageId MessagesManager::on_get_message(MessageInfo &&message_info, bool f
     }
 
     if (message_id.is_scheduled()) {
+      CHECK(message_id.is_scheduled_server());
       auto dialog_it = update_scheduled_message_ids_.find(dialog_id);
       CHECK(dialog_it != update_scheduled_message_ids_.end());
       dialog_it->second.erase(message_id.get_scheduled_server_message_id());
@@ -11757,7 +11814,7 @@ void MessagesManager::set_dialog_last_new_message_id(Dialog *d, MessageId last_n
     invalidate_message_indexes(d);
 
     vector<MessageId> to_delete_message_ids;
-    find_new_messages(d->messages.get(), last_new_message_id, to_delete_message_ids);
+    find_newer_messages(d->messages.get(), last_new_message_id, to_delete_message_ids);
     td::remove_if(to_delete_message_ids, [](MessageId message_id) { return message_id.is_yet_unsent(); });
     if (!to_delete_message_ids.empty()) {
       LOG(WARNING) << "Delete " << format::as_array(to_delete_message_ids) << " because of received last new "
@@ -13323,6 +13380,9 @@ void MessagesManager::load_dialog_list_from_database(FolderId folder_id, int32 l
 
 void MessagesManager::on_get_dialogs_from_database(FolderId folder_id, int32 limit, DialogDbGetDialogsResult &&dialogs,
                                                    Promise<Unit> &&promise) {
+  if (G()->close_flag()) {
+    return promise.set_error(Status::Error(500, "Request aborted"));
+  }
   CHECK(!td_->auth_manager_->is_bot());
   auto &list = get_dialog_list(folder_id);
   LOG(INFO) << "Receive " << dialogs.dialogs.size() << " from expected " << limit << " chats in " << folder_id
@@ -13975,7 +14035,7 @@ void MessagesManager::get_messages_from_server(vector<FullMessageId> &&message_i
     auto dialog_id = full_message_id.get_dialog_id();
     auto message_id = full_message_id.get_message_id();
     if (!message_id.is_valid() || !message_id.is_server()) {
-      if (message_id.is_valid_scheduled()) {
+      if (message_id.is_valid_scheduled() && message_id.is_scheduled_server()) {
         scheduled_message_ids[dialog_id].push_back(message_id.get_scheduled_server_message_id().get());
       }
       continue;
@@ -15571,11 +15631,15 @@ td_api::object_ptr<td_api::ChatActionBar> MessagesManager::get_chat_action_bar_o
   return nullptr;
 }
 
-td_api::object_ptr<td_api::chat> MessagesManager::get_chat_object(const Dialog *d) const {
+td_api::object_ptr<td_api::chat> MessagesManager::get_chat_object(const Dialog *d, int64 real_order) const {
   CHECK(d != nullptr);
 
   auto chat_source = is_dialog_sponsored(d) ? sponsored_dialog_source_.get_chat_source_object() : nullptr;
 
+  if (real_order == DEFAULT_ORDER) {
+    real_order = d->order;
+  }
+
   bool can_delete_for_self = false;
   bool can_delete_for_all_users = false;
   if (chat_source != nullptr) {
@@ -15588,7 +15652,7 @@ td_api::object_ptr<td_api::chat> MessagesManager::get_chat_object(const Dialog *
         // can't delete
         break;
     }
-  } else if (!td_->auth_manager_->is_bot() && d->order != DEFAULT_ORDER) {
+  } else if (!td_->auth_manager_->is_bot() && real_order != DEFAULT_ORDER) {
     switch (d->dialog_id.get_type()) {
       case DialogType::User:
         can_delete_for_self = true;
@@ -16592,6 +16656,9 @@ vector<FullMessageId> MessagesManager::get_active_live_location_messages(Promise
 }
 
 void MessagesManager::on_load_active_live_location_full_message_ids_from_database(string value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (value.empty()) {
     LOG(INFO) << "Active live location messages aren't found in the database";
     on_load_active_live_location_messages_finished();
@@ -16882,6 +16949,9 @@ void MessagesManager::on_search_dialog_messages_db_result(int64 random_id, Dialo
                                                           SearchMessagesFilter filter_type, int32 offset, int32 limit,
                                                           Result<std::vector<BufferSlice>> r_messages,
                                                           Promise<> promise) {
+  if (G()->close_flag()) {
+    return promise.set_error(Status::Error(500, "Request aborted"));
+  }
   if (r_messages.is_error()) {
     LOG(ERROR) << r_messages.error();
     if (first_db_message_id != MessageId::min() && dialog_id.get_type() != DialogType::SecretChat &&
@@ -16919,8 +16989,9 @@ void MessagesManager::on_search_dialog_messages_db_result(int64 random_id, Dialo
   int32 result_size = narrow_cast<int32>(res.size());
   bool from_the_end =
       from_message_id == MessageId::max() || (offset < 0 && (result_size == 0 || res[0] < from_message_id));
-  if (message_count < result_size || (message_count > result_size && from_the_end &&
-                                      first_db_message_id == MessageId::min() && result_size < limit + offset)) {
+  if ((message_count != -1 && message_count < result_size) ||
+      (message_count > result_size && from_the_end && first_db_message_id == MessageId::min() &&
+       result_size < limit + offset)) {
     LOG(INFO) << "Fix found message count in " << dialog_id << " from " << message_count << " to " << result_size;
     message_count = result_size;
     if (filter_type == SearchMessagesFilter::UnreadMention) {
@@ -16997,7 +17068,10 @@ std::pair<int64, vector<FullMessageId>> MessagesManager::offline_search_messages
 }
 
 void MessagesManager::on_messages_db_fts_result(Result<MessagesDbFtsResult> result, int64 random_id,
-                                                Promise<> &&promise) {
+                                                Promise<Unit> &&promise) {
+  if (G()->close_flag()) {
+    result = Status::Error(500, "Request aborted");
+  }
   if (result.is_error()) {
     found_fts_messages_.erase(random_id);
     return promise.set_error(result.move_as_error());
@@ -17026,6 +17100,9 @@ void MessagesManager::on_messages_db_calls_result(Result<MessagesDbCallsResult>
                                                   MessageId first_db_message_id, SearchMessagesFilter filter,
                                                   Promise<> &&promise) {
   CHECK(!first_db_message_id.is_scheduled());
+  if (G()->close_flag()) {
+    result = Status::Error(500, "Request aborted");
+  }
   if (result.is_error()) {
     found_call_messages_.erase(random_id);
     return promise.set_error(result.move_as_error());
@@ -17177,6 +17254,9 @@ MessageId MessagesManager::find_message_by_date(const Message *m, int32 date) {
 
 void MessagesManager::on_get_dialog_message_by_date_from_database(DialogId dialog_id, int32 date, int64 random_id,
                                                                   Result<BufferSlice> result, Promise<Unit> promise) {
+  if (G()->close_flag()) {
+    return promise.set_error(Status::Error(500, "Request aborted"));
+  }
   Dialog *d = get_dialog(dialog_id);
   CHECK(d != nullptr);
   if (result.is_ok()) {
@@ -17408,6 +17488,10 @@ void MessagesManager::on_get_history_from_database(DialogId dialog_id, MessageId
   CHECK(offset < 0 || from_the_end);
   CHECK(!from_message_id.is_scheduled());
 
+  if (G()->close_flag()) {
+    return promise.set_error(Status::Error(500, "Request aborted"));
+  }
+
   if (!have_input_peer(dialog_id, AccessRights::Read)) {
     LOG(WARNING) << "Ignore result of get_history_from_database in " << dialog_id;
     promise.set_value(Unit());
@@ -17592,13 +17676,13 @@ void MessagesManager::on_get_history_from_database(DialogId dialog_id, MessageId
 void MessagesManager::get_history_from_the_end(DialogId dialog_id, bool from_database, bool only_local,
                                                Promise<Unit> &&promise) {
   CHECK(dialog_id.is_valid());
+  if (G()->close_flag()) {
+    return promise.set_error(Status::Error(500, "Request aborted"));
+  }
   if (!have_input_peer(dialog_id, AccessRights::Read)) {
     // can't get history in dialogs without read access
     return promise.set_value(Unit());
   }
-  if (G()->close_flag()) {
-    return promise.set_error(Status::Error(500, "Request aborted"));
-  }
   int32 limit = MAX_GET_HISTORY;
   if (from_database && G()->parameters().use_message_db) {
     if (!promise) {
@@ -17799,8 +17883,20 @@ void MessagesManager::load_dialog_scheduled_messages(DialogId dialog_id, bool fr
 }
 
 void MessagesManager::on_get_scheduled_messages_from_database(DialogId dialog_id, vector<BufferSlice> &&messages) {
+  if (G()->close_flag()) {
+    auto it = load_scheduled_messages_from_database_queries_.find(dialog_id);
+    if (it == load_scheduled_messages_from_database_queries_.end()) { return; }
+    if (it->second.empty()) { return; }
+    auto promises = std::move(it->second);
+    load_scheduled_messages_from_database_queries_.erase(it);
+
+    for (auto &promise : promises) {
+      promise.set_error(Status::Error(500, "Request aborted"));
+    }
+    return;
+  }
   auto d = get_dialog(dialog_id);
-  CHECK(d != nullptr);
+  if (d == nullptr) { return; }
   d->has_loaded_scheduled_messages_from_database = true;
 
   LOG(INFO) << "Receive " << messages.size() << " scheduled messages from database in " << dialog_id;
@@ -17970,12 +18066,13 @@ tl_object_ptr<td_api::message> MessagesManager::get_message_object(DialogId dial
   auto live_location_date = m->is_failed_to_send ? 0 : m->date;
   auto date = is_scheduled ? 0 : m->date;
   auto edit_date = m->hide_edit_date ? 0 : m->edit_date;
+  auto views = m->message_id.is_scheduled() || (m->message_id.is_local() && m->forward_info == nullptr) ? 0 : m->views;
   return make_tl_object<td_api::message>(
       m->message_id.get(), td_->contacts_manager_->get_user_id_object(m->sender_user_id, "sender_user_id"),
       dialog_id.get(), std::move(sending_state), std::move(scheduling_state), is_outgoing, can_be_edited,
       can_be_forwarded, can_delete_for_self, can_delete_for_all_users, m->is_channel_post, contains_unread_mention,
       date, edit_date, get_message_forward_info_object(m->forward_info), reply_to_message_id, ttl, ttl_expires_in,
-      td_->contacts_manager_->get_user_id_object(m->via_bot_user_id, "via_bot_user_id"), m->author_signature, m->views,
+      td_->contacts_manager_->get_user_id_object(m->via_bot_user_id, "via_bot_user_id"), m->author_signature, views,
       media_album_id, get_restriction_reason_description(m->restriction_reasons),
       get_message_content_object(m->content.get(), td_, live_location_date, m->is_content_secret),
       get_reply_markup_object(m->reply_markup));
@@ -22212,6 +22309,9 @@ void MessagesManager::on_get_message_notifications_from_database(DialogId dialog
                                                                  NotificationId initial_from_notification_id,
                                                                  int32 limit, Result<vector<BufferSlice>> result,
                                                                  Promise<vector<Notification>> promise) {
+  if (G()->close_flag()) {
+    result = Status::Error(500, "Request aborted");
+  }
   if (result.is_error()) {
     return promise.set_error(result.move_as_error());
   }
@@ -22398,7 +22498,7 @@ void MessagesManager::remove_message_notifications_by_message_ids(DialogId dialo
 
 void MessagesManager::do_remove_message_notification(DialogId dialog_id, bool from_mentions,
                                                      NotificationId notification_id, vector<BufferSlice> result) {
-  if (result.empty()) {
+  if (result.empty() || G()->close_flag()) {
     return;
   }
   CHECK(result.size() == 1);
@@ -22890,10 +22990,10 @@ void MessagesManager::send_update_delete_messages(DialogId dialog_id, vector<int
       make_tl_object<td_api::updateDeleteMessages>(dialog_id.get(), std::move(message_ids), is_permanent, from_cache));
 }
 
-void MessagesManager::send_update_new_chat(Dialog *d) {
+void MessagesManager::send_update_new_chat(Dialog *d, int64 real_order) {
   CHECK(d != nullptr);
   CHECK(d->messages == nullptr);
-  auto chat_object = get_chat_object(d);
+  auto chat_object = get_chat_object(d, real_order);
   bool has_action_bar = chat_object->action_bar_ != nullptr;
   d->last_sent_has_scheduled_messages = chat_object->has_scheduled_messages_;
   send_closure(G()->td(), &Td::send_update, make_tl_object<td_api::updateNewChat>(std::move(chat_object)));
@@ -23729,6 +23829,9 @@ void MessagesManager::fail_send_message(FullMessageId full_message_id, int error
   } else {
     CHECK(message->message_id.is_valid());
   }
+  if (message->forward_info == nullptr && message->views == 1) {
+    message->views = 0;
+  }
   message->is_failed_to_send = true;
   message->send_error_code = error_code;
   message->send_error_message = error_message;
@@ -24057,6 +24160,9 @@ void MessagesManager::set_dialog_has_scheduled_server_messages(Dialog *d, bool h
 
 void MessagesManager::set_dialog_has_scheduled_database_messages(DialogId dialog_id,
                                                                  bool has_scheduled_database_messages) {
+  if (G()->close_flag()) {
+    return;
+  }
   return set_dialog_has_scheduled_database_messages_impl(get_dialog(dialog_id), has_scheduled_database_messages);
 }
 
@@ -24657,7 +24763,7 @@ bool MessagesManager::is_dialog_action_unneded(DialogId dialog_id) const {
 void MessagesManager::send_dialog_action(DialogId dialog_id, const tl_object_ptr<td_api::ChatAction> &action,
                                          Promise<Unit> &&promise) {
   if (action == nullptr) {
-    return promise.set_error(Status::Error(5, "Action must not be empty"));
+    return promise.set_error(Status::Error(5, "Action must be non-empty"));
   }
 
   if (!have_dialog_force(dialog_id)) {
@@ -25219,7 +25325,7 @@ void MessagesManager::set_dialog_permissions(DialogId dialog_id,
   }
 
   if (permissions == nullptr) {
-    return promise.set_error(Status::Error(3, "New permissions must not be empty"));
+    return promise.set_error(Status::Error(3, "New permissions must be non-empty"));
   }
 
   switch (dialog_id.get_type()) {
@@ -26141,8 +26247,10 @@ MessagesManager::Message *MessagesManager::on_get_message_from_database(DialogId
       return nullptr;
     }
 
-    // can succeed in private and group chats
-    get_message_from_server({dialog_id, m->message_id}, Auto());
+    if (m->message_id.is_valid() && m->message_id.is_any_server() &&
+        (dialog_id.get_type() == DialogType::User || dialog_id.get_type() == DialogType::Chat)) {
+      get_message_from_server({dialog_id, m->message_id}, Auto());
+    }
 
     force_create_dialog(dialog_id, source);
     d = get_dialog_force(dialog_id);
@@ -26373,7 +26481,7 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, uniq
     return nullptr;
   }
 
-  if (*need_update && message_id <= d->last_new_message_id) {
+  if (*need_update && message_id <= d->last_new_message_id && !td_->auth_manager_->is_bot()) {
     *need_update = false;
   }
 
@@ -26485,7 +26593,7 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, uniq
     // in get_message_notification_group_force
     get_dialog_notification_group_id(d->dialog_id, get_notification_group_info(d, message.get()));
   }
-  if (*need_update || (!d->last_new_message_id.is_valid() && !message_id.is_yet_unsent() && !message->from_database)) {
+  if (*need_update || (!d->last_new_message_id.is_valid() && !message_id.is_yet_unsent())) {
     auto pinned_message_id = get_message_content_pinned_message_id(message->content.get());
     if (pinned_message_id.is_valid() && pinned_message_id < message_id &&
         have_message_force({dialog_id, pinned_message_id}, "preload pinned message")) {
@@ -26493,7 +26601,7 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, uniq
     }
 
     if (d->pinned_message_notification_message_id.is_valid() &&
-        d->pinned_message_notification_message_id < message_id &&
+        d->pinned_message_notification_message_id != message_id &&
         have_message_force({dialog_id, d->pinned_message_notification_message_id},
                            "preload previously pinned message")) {
       LOG(INFO) << "Preloaded previously pinned " << d->pinned_message_notification_message_id << " from database";
@@ -27082,7 +27190,7 @@ void MessagesManager::delete_all_dialog_messages_from_database(Dialog *d, Messag
   }
   if (d->pinned_message_notification_message_id.is_valid() &&
       d->pinned_message_notification_message_id <= max_message_id) {
-    remove_dialog_pinned_message_notification(d, "delete_all_dialog_messages_from_database");
+    remove_dialog_pinned_message_notification(d, source);
   }
   remove_message_dialog_notifications(d, max_message_id, false, source);
   remove_message_dialog_notifications(d, max_message_id, true, source);
@@ -27995,7 +28103,7 @@ MessagesManager::Dialog *MessagesManager::add_new_dialog(unique_ptr<Dialog> &&d,
 
   fix_dialog_action_bar(dialog);
 
-  send_update_new_chat(dialog);
+  send_update_new_chat(dialog, order);
 
   fix_new_dialog(dialog, std::move(last_database_message), last_database_message_id, order, last_clear_history_date,
                  last_clear_history_message_id, is_loaded_from_database);
@@ -28225,8 +28333,6 @@ void MessagesManager::fix_new_dialog(Dialog *d, unique_ptr<Message> &&last_datab
       UNREACHABLE();
   }
 
-  update_dialogs_hints(d);
-
   if (d->delete_last_message_date != 0) {
     if (d->last_message_id.is_valid()) {
       LOG(ERROR) << "Last " << d->deleted_last_message_id << " in " << dialog_id << " was deleted at "
@@ -28240,19 +28346,6 @@ void MessagesManager::fix_new_dialog(Dialog *d, unique_ptr<Message> &&last_datab
     }
   }
 
-  if (need_get_history && !td_->auth_manager_->is_bot() && have_input_peer(dialog_id, AccessRights::Read) &&
-      (d->order != DEFAULT_ORDER || is_dialog_sponsored(d))) {
-    get_history_from_the_end(dialog_id, true, false, Auto());
-  }
-
-  if (d->need_repair_server_unread_count && need_unread_counter(d->order)) {
-    CHECK(dialog_type != DialogType::SecretChat);
-    repair_server_unread_count(dialog_id, d->server_unread_count);
-  }
-  if (d->need_repair_channel_server_unread_count) {
-    repair_channel_server_unread_count(d);
-  }
-
   if (!G()->parameters().use_message_db) {
     d->has_loaded_scheduled_messages_from_database = true;
   }
@@ -28263,6 +28356,19 @@ void MessagesManager::fix_new_dialog(Dialog *d, unique_ptr<Message> &&last_datab
     LOG(ERROR) << dialog_id << " has order " << d->order << " instead of saved to database order " << order;
   }
 
+  // must be after update_dialog_pos, because uses d->order
+  if (need_get_history && !td_->auth_manager_->is_bot() && dialog_id != being_added_dialog_id_ &&
+      have_input_peer(dialog_id, AccessRights::Read) && (d->order != DEFAULT_ORDER || is_dialog_sponsored(d))) {
+    get_history_from_the_end(dialog_id, true, false, Auto());
+  }
+  if (d->need_repair_server_unread_count && need_unread_counter(d->order)) {
+    CHECK(dialog_type != DialogType::SecretChat);
+    repair_server_unread_count(dialog_id, d->server_unread_count);
+  }
+  if (d->need_repair_channel_server_unread_count) {
+    repair_channel_server_unread_count(d);
+  }
+
   LOG(INFO) << "Loaded " << dialog_id << " with last new " << d->last_new_message_id << ", first database "
             << d->first_database_message_id << ", last database " << d->last_database_message_id << ", last "
             << d->last_message_id << " with order " << d->order << " and pinned order " << d->pinned_order;
@@ -28507,7 +28613,7 @@ void MessagesManager::update_dialog_pos(Dialog *d, const char *source, bool need
       }
     }
     if (new_order == DEFAULT_ORDER && !d->is_empty) {
-      LOG(INFO) << "There is no known messages in the chat, just leave it where it is";
+      LOG(INFO) << "There are no known messages in the chat, just leave it where it is";
       new_order = d->order;
     }
   }
@@ -29931,6 +30037,7 @@ void MessagesManager::on_binlog_events(vector<BinlogEvent> &&events) {
         }
 
         for (auto message_id : log_event.message_ids_) {
+          CHECK(message_id.is_scheduled_server());
           d->deleted_scheduled_server_message_ids.insert(message_id.get_scheduled_server_message_id());
         }
 

td/telegram/MessagesManager.h

@@ -1680,7 +1680,7 @@ class MessagesManager : public Actor {
 
   static void find_old_messages(const Message *m, MessageId max_message_id, vector<MessageId> &message_ids);
 
-  static void find_new_messages(const Message *m, MessageId min_message_id, vector<MessageId> &message_ids);
+  static void find_newer_messages(const Message *m, MessageId min_message_id, vector<MessageId> &message_ids);
 
   void find_unloadable_messages(const Dialog *d, int32 unload_before_date, const Message *m,
                                 vector<MessageId> &message_ids, int32 &left_to_unload) const;
@@ -1910,7 +1910,7 @@ class MessagesManager : public Actor {
   void send_update_delete_messages(DialogId dialog_id, vector<int64> &&message_ids, bool is_permanent,
                                    bool from_cache) const;
 
-  void send_update_new_chat(Dialog *d);
+  void send_update_new_chat(Dialog *d, int64 real_order);
 
   void send_update_chat_draft_message(const Dialog *d);
 
@@ -2092,7 +2092,7 @@ class MessagesManager : public Actor {
 
   td_api::object_ptr<td_api::ChatActionBar> get_chat_action_bar_object(const Dialog *d) const;
 
-  td_api::object_ptr<td_api::chat> get_chat_object(const Dialog *d) const;
+  td_api::object_ptr<td_api::chat> get_chat_object(const Dialog *d, int64 real_order = DEFAULT_ORDER) const;
 
   bool have_dialog_info(DialogId dialog_id) const;
   bool have_dialog_info_force(DialogId dialog_id) const;
@@ -2225,7 +2225,7 @@ class MessagesManager : public Actor {
   void on_messages_db_fts_result(Result<MessagesDbFtsResult> result, int64 random_id, Promise<> &&promise);
 
   void on_messages_db_calls_result(Result<MessagesDbCallsResult> result, int64 random_id, MessageId first_db_message_id,
-                                   SearchMessagesFilter filter, Promise<> &&promise);
+                                   SearchMessagesFilter filter, Promise<Unit> &&promise);
 
   void on_load_active_live_location_full_message_ids_from_database(string value);
 

td/telegram/NotificationSettings.cpp

@@ -131,7 +131,7 @@ static int32 get_mute_until(int32 mute_for) {
 Result<DialogNotificationSettings> get_dialog_notification_settings(
     td_api::object_ptr<td_api::chatNotificationSettings> &&notification_settings, bool old_silent_send_message) {
   if (notification_settings == nullptr) {
-    return Status::Error(400, "New notification settings must not be empty");
+    return Status::Error(400, "New notification settings must be non-empty");
   }
   if (!clean_input_string(notification_settings->sound_)) {
     return Status::Error(400, "Notification settings sound must be encoded in UTF-8");
@@ -155,7 +155,7 @@ Result<DialogNotificationSettings> get_dialog_notification_settings(
 Result<ScopeNotificationSettings> get_scope_notification_settings(
     td_api::object_ptr<td_api::scopeNotificationSettings> &&notification_settings) {
   if (notification_settings == nullptr) {
-    return Status::Error(400, "New notification settings must not be empty");
+    return Status::Error(400, "New notification settings must be non-empty");
   }
   if (!clean_input_string(notification_settings->sound_)) {
     return Status::Error(400, "Notification settings sound must be encoded in UTF-8");

td/telegram/Payments.cpp

@@ -647,7 +647,7 @@ static Status check_postal_code(string &postal_code) {
 
 Result<Address> get_address(td_api::object_ptr<td_api::address> &&address) {
   if (address == nullptr) {
-    return Status::Error(400, "Address must not be empty");
+    return Status::Error(400, "Address must be non-empty");
   }
   TRY_STATUS(check_country_code(address->country_code_));
   TRY_STATUS(check_state(address->state_));
@@ -770,7 +770,7 @@ void answer_shipping_query(int64 shipping_query_id, vector<tl_object_ptr<td_api:
   vector<tl_object_ptr<telegram_api::shippingOption>> options;
   for (auto &option : shipping_options) {
     if (option == nullptr) {
-      return promise.set_error(Status::Error(400, "Shipping option must not be empty"));
+      return promise.set_error(Status::Error(400, "Shipping option must be non-empty"));
     }
     if (!clean_input_string(option->id_)) {
       return promise.set_error(Status::Error(400, "Shipping option id must be encoded in UTF-8"));
@@ -782,7 +782,7 @@ void answer_shipping_query(int64 shipping_query_id, vector<tl_object_ptr<td_api:
     vector<tl_object_ptr<telegram_api::labeledPrice>> prices;
     for (auto &price_part : option->price_parts_) {
       if (price_part == nullptr) {
-        return promise.set_error(Status::Error(400, "Shipping option price part must not be empty"));
+        return promise.set_error(Status::Error(400, "Shipping option price part must be non-empty"));
       }
       if (!clean_input_string(price_part->label_)) {
         return promise.set_error(Status::Error(400, "Shipping option price part label must be encoded in UTF-8"));

td/telegram/Photo.cpp

@@ -95,18 +95,32 @@ td_api::object_ptr<td_api::minithumbnail> get_minithumbnail_object(const string
   return nullptr;
 }
 
+static StringBuilder &operator<<(StringBuilder &string_builder, PhotoFormat format) {
+  switch (format) {
+    case PhotoFormat::Jpeg:
+      return string_builder << "jpg";
+    case PhotoFormat::Png:
+      return string_builder << "png";
+    case PhotoFormat::Webp:
+      return string_builder << "webp";
+    case PhotoFormat::Tgs:
+      return string_builder << "tgs";
+    default:
+      UNREACHABLE();
+      return string_builder;
+  }
+}
+
 static FileId register_photo(FileManager *file_manager, const PhotoSizeSource &source, int64 id, int64 access_hash,
                              std::string file_reference,
                              tl_object_ptr<telegram_api::fileLocationToBeDeprecated> &&location,
-                             DialogId owner_dialog_id, int32 file_size, DcId dc_id, bool is_webp = false,
-                             bool is_png = false) {
+                             DialogId owner_dialog_id, int32 file_size, DcId dc_id, PhotoFormat format) {
   int32 local_id = location->local_id_;
   int64 volume_id = location->volume_id_;
-  LOG(DEBUG) << "Receive " << (is_webp ? "webp" : (is_png ? "png" : "jpeg")) << " photo of type "
-             << source.get_file_type() << " in [" << dc_id << "," << volume_id << "," << local_id << "]. Id: (" << id
-             << ", " << access_hash << ")";
-  auto suggested_name = PSTRING() << static_cast<uint64>(volume_id) << "_" << static_cast<uint64>(local_id)
-                                  << (is_webp ? ".webp" : (is_png ? ".png" : ".jpg"));
+  LOG(DEBUG) << "Receive " << format << " photo of type " << source.get_file_type() << " in [" << dc_id << ","
+             << volume_id << "," << local_id << "]. Id: (" << id << ", " << access_hash << ")";
+  auto suggested_name = PSTRING() << static_cast<uint64>(volume_id) << "_" << static_cast<uint64>(local_id) << '.'
+                                  << format;
   auto file_location_source = owner_dialog_id.get_type() == DialogType::SecretChat ? FileLocationSource::FromUser
                                                                                    : FileLocationSource::FromServer;
   return file_manager->register_remote(
@@ -127,10 +141,12 @@ ProfilePhoto get_profile_photo(FileManager *file_manager, UserId user_id, int64
 
       auto dc_id = DcId::create(profile_photo->dc_id_);
       result.id = profile_photo->photo_id_;
-      result.small_file_id = register_photo(file_manager, {DialogId(user_id), user_access_hash, false}, result.id, 0,
-                                            "", std::move(profile_photo->photo_small_), DialogId(), 0, dc_id);
-      result.big_file_id = register_photo(file_manager, {DialogId(user_id), user_access_hash, true}, result.id, 0, "",
-                                          std::move(profile_photo->photo_big_), DialogId(), 0, dc_id);
+      result.small_file_id =
+          register_photo(file_manager, {DialogId(user_id), user_access_hash, false}, result.id, 0, "",
+                         std::move(profile_photo->photo_small_), DialogId(), 0, dc_id, PhotoFormat::Jpeg);
+      result.big_file_id =
+          register_photo(file_manager, {DialogId(user_id), user_access_hash, true}, result.id, 0, "",
+                         std::move(profile_photo->photo_big_), DialogId(), 0, dc_id, PhotoFormat::Jpeg);
       break;
     }
     default:
@@ -176,10 +192,11 @@ DialogPhoto get_dialog_photo(FileManager *file_manager, DialogId dialog_id, int6
       auto chat_photo = move_tl_object_as<telegram_api::chatPhoto>(chat_photo_ptr);
 
       auto dc_id = DcId::create(chat_photo->dc_id_);
-      result.small_file_id = register_photo(file_manager, {dialog_id, dialog_access_hash, false}, 0, 0, "",
-                                            std::move(chat_photo->photo_small_), DialogId(), 0, dc_id);
+      result.small_file_id =
+          register_photo(file_manager, {dialog_id, dialog_access_hash, false}, 0, 0, "",
+                         std::move(chat_photo->photo_small_), DialogId(), 0, dc_id, PhotoFormat::Jpeg);
       result.big_file_id = register_photo(file_manager, {dialog_id, dialog_access_hash, true}, 0, 0, "",
-                                          std::move(chat_photo->photo_big_), DialogId(), 0, dc_id);
+                                          std::move(chat_photo->photo_big_), DialogId(), 0, dc_id, PhotoFormat::Jpeg);
 
       break;
     }
@@ -269,7 +286,7 @@ PhotoSize get_secret_thumbnail_photo_size(FileManager *file_manager, BufferSlice
 Variant<PhotoSize, string> get_photo_size(FileManager *file_manager, PhotoSizeSource source, int64 id,
                                           int64 access_hash, std::string file_reference, DcId dc_id,
                                           DialogId owner_dialog_id, tl_object_ptr<telegram_api::PhotoSize> &&size_ptr,
-                                          bool is_webp, bool is_png) {
+                                          PhotoFormat format) {
   CHECK(size_ptr != nullptr);
 
   tl_object_ptr<telegram_api::fileLocationToBeDeprecated> location;
@@ -322,7 +339,7 @@ Variant<PhotoSize, string> get_photo_size(FileManager *file_manager, PhotoSizeSo
   }
 
   res.file_id = register_photo(file_manager, source, id, access_hash, file_reference, std::move(location),
-                               owner_dialog_id, res.size, dc_id, is_webp, is_png);
+                               owner_dialog_id, res.size, dc_id, format);
 
   if (!content.empty()) {
     file_manager->set_content(res.file_id, std::move(content));
@@ -521,7 +538,7 @@ Photo get_photo(FileManager *file_manager, tl_object_ptr<telegram_api::photo> &&
   for (auto &size_ptr : photo->sizes_) {
     auto photo_size = get_photo_size(file_manager, {FileType::Photo, 0}, photo->id_, photo->access_hash_,
                                      photo->file_reference_.as_slice().str(), DcId::create(photo->dc_id_),
-                                     owner_dialog_id, std::move(size_ptr), false, false);
+                                     owner_dialog_id, std::move(size_ptr), PhotoFormat::Jpeg);
     if (photo_size.get_offset() == 0) {
       PhotoSize &size = photo_size.get<0>();
       if (size.type == 0 || size.type == 't' || size.type == 'i') {

td/telegram/Photo.h

@@ -90,12 +90,14 @@ bool operator!=(const DialogPhoto &lhs, const DialogPhoto &rhs);
 
 StringBuilder &operator<<(StringBuilder &string_builder, const DialogPhoto &dialog_photo);
 
+enum class PhotoFormat : int32 { Jpeg, Png, Webp, Tgs };
+
 PhotoSize get_secret_thumbnail_photo_size(FileManager *file_manager, BufferSlice bytes, DialogId owner_dialog_id,
                                           int32 width, int32 height);
 Variant<PhotoSize, string> get_photo_size(FileManager *file_manager, PhotoSizeSource source, int64 id,
                                           int64 access_hash, string file_reference, DcId dc_id,
                                           DialogId owner_dialog_id, tl_object_ptr<telegram_api::PhotoSize> &&size_ptr,
-                                          bool is_webp, bool is_png);
+                                          PhotoFormat format);
 PhotoSize get_web_document_photo_size(FileManager *file_manager, FileType file_type, DialogId owner_dialog_id,
                                       tl_object_ptr<telegram_api::WebDocument> web_document_ptr);
 td_api::object_ptr<td_api::photoSize> get_photo_size_object(FileManager *file_manager, const PhotoSize *photo_size);

td/telegram/PhotoSizeSource.hpp

@@ -8,6 +8,7 @@
 
 #include "td/telegram/PhotoSizeSource.h"
 
+#include "td/utils/logging.h"
 #include "td/utils/tl_helpers.h"
 
 namespace td {
@@ -69,7 +70,7 @@ void parse(PhotoSizeSource::DialogPhoto &source, ParserT &parser) {
   switch (source.dialog_id.get_type()) {
     case DialogType::SecretChat:
     case DialogType::None:
-      return parser.set_error("Invalid chat identifier");
+      return parser.set_error(PSTRING() << "Invalid chat identifier " << source.dialog_id.get());
     default:
       break;
   }

td/telegram/PollManager.cpp

@@ -1297,7 +1297,8 @@ PollId PollManager::on_get_poll(PollId poll_id, tl_object_ptr<telegram_api::poll
     poll_id = PollId(poll_server->id_);
   }
   if (!poll_id.is_valid() || is_local_poll_id(poll_id)) {
-    LOG(ERROR) << "Receive " << poll_id << " from server";
+    LOG(ERROR) << "Receive " << poll_id << " from server: " << oneline(to_string(poll_server)) << " "
+               << oneline(to_string(poll_results));
     return PollId();
   }
   if (poll_server != nullptr && poll_server->id_ != poll_id.get()) {

td/telegram/SecureValue.cpp

@@ -757,7 +757,7 @@ static Status check_gender(string &gender) {
 
 static Result<string> get_personal_details(td_api::object_ptr<td_api::personalDetails> &&personal_details) {
   if (personal_details == nullptr) {
-    return Status::Error(400, "Personal details must not be empty");
+    return Status::Error(400, "Personal details must be non-empty");
   }
   TRY_STATUS(check_name(personal_details->first_name_));
   TRY_STATUS(check_name(personal_details->middle_name_));
@@ -767,7 +767,7 @@ static Result<string> get_personal_details(td_api::object_ptr<td_api::personalDe
   TRY_STATUS(check_name(personal_details->native_last_name_));
   TRY_RESULT(birthdate, get_date(std::move(personal_details->birthdate_)));
   if (birthdate.empty()) {
-    return Status::Error(400, "Birthdate must not be empty");
+    return Status::Error(400, "Birthdate must be non-empty");
   }
   TRY_STATUS(check_gender(personal_details->gender_));
   TRY_STATUS(check_country_code(personal_details->country_code_));
@@ -808,7 +808,7 @@ static Result<td_api::object_ptr<td_api::personalDetails>> get_personal_details_
   TRY_RESULT(native_last_name, get_json_object_string_field(object, "last_name_native", true));
   TRY_RESULT(birthdate, get_json_object_string_field(object, "birth_date", true));
   if (birthdate.empty()) {
-    return Status::Error(400, "Birthdate must not be empty");
+    return Status::Error(400, "Birthdate must be non-empty");
   }
   TRY_RESULT(gender, get_json_object_string_field(object, "gender", true));
   TRY_RESULT(country_code, get_json_object_string_field(object, "country_code", true));
@@ -836,7 +836,7 @@ static Status check_document_number(string &number) {
     return Status::Error(400, "Document number must be encoded in UTF-8");
   }
   if (number.empty()) {
-    return Status::Error(400, "Document number must not be empty");
+    return Status::Error(400, "Document number must be non-empty");
   }
   if (utf8_length(number) > 24) {
     return Status::Error(400, "Document number is too long");
@@ -867,7 +867,7 @@ static Result<SecureValue> get_identity_document(SecureValueType type, FileManag
                                                  td_api::object_ptr<td_api::inputIdentityDocument> &&identity_document,
                                                  bool need_reverse_side) {
   if (identity_document == nullptr) {
-    return Status::Error(400, "Identity document must not be empty");
+    return Status::Error(400, "Identity document must be non-empty");
   }
   TRY_STATUS(check_document_number(identity_document->number_));
   TRY_RESULT(date, get_date(std::move(identity_document->expiry_date_)));
@@ -950,7 +950,7 @@ static Result<SecureValue> get_personal_document(
     SecureValueType type, FileManager *file_manager,
     td_api::object_ptr<td_api::inputPersonalDocument> &&personal_document) {
   if (personal_document == nullptr) {
-    return Status::Error(400, "Personal document must not be empty");
+    return Status::Error(400, "Personal document must be non-empty");
   }
 
   SecureValue res;
@@ -988,7 +988,7 @@ static Status check_email_address(string &email_address) {
 Result<SecureValue> get_secure_value(FileManager *file_manager,
                                      td_api::object_ptr<td_api::InputPassportElement> &&input_passport_element) {
   if (input_passport_element == nullptr) {
-    return Status::Error(400, "InputPassportElement must not be empty");
+    return Status::Error(400, "InputPassportElement must be non-empty");
   }
 
   SecureValue res;

td/telegram/StickersManager.cpp

@@ -12,6 +12,7 @@
 
 #include "td/telegram/AccessRights.h"
 #include "td/telegram/AuthManager.h"
+#include "td/telegram/ConfigManager.h"
 #include "td/telegram/ConfigShared.h"
 #include "td/telegram/ContactsManager.h"
 #include "td/telegram/DialogId.h"
@@ -1592,9 +1593,10 @@ std::pair<int64, FileId> StickersManager::on_get_sticker_document(
 
   PhotoSize thumbnail;
   for (auto &thumb : document->thumbs_) {
-    auto photo_size = get_photo_size(td_->file_manager_.get(), {FileType::Thumbnail, 0}, document_id,
-                                     document->access_hash_, document->file_reference_.as_slice().str(), dc_id,
-                                     DialogId(), std::move(thumb), has_webp_thumbnail(sticker), false);
+    auto photo_size =
+        get_photo_size(td_->file_manager_.get(), {FileType::Thumbnail, 0}, document_id, document->access_hash_,
+                       document->file_reference_.as_slice().str(), dc_id, DialogId(), std::move(thumb),
+                       has_webp_thumbnail(sticker) ? PhotoFormat::Webp : PhotoFormat::Jpeg);
     if (photo_size.get_offset() == 0) {
       thumbnail = std::move(photo_size.get<0>());
       break;
@@ -1661,9 +1663,11 @@ StickerSetId StickersManager::get_sticker_set_id(const tl_object_ptr<telegram_ap
       return search_sticker_set(static_cast<const telegram_api::inputStickerSetShortName *>(set_ptr.get())->short_name_,
                                 Auto());
     case telegram_api::inputStickerSetAnimatedEmoji::ID:
-    case telegram_api::inputStickerSetDice::ID:
       LOG(ERROR) << "Receive special sticker set " << to_string(set_ptr);
       return add_special_sticker_set(SpecialStickerSetType(set_ptr).type_).id_;
+    case telegram_api::inputStickerSetDice::ID:
+      LOG(ERROR) << "Receive special sticker set " << to_string(set_ptr);
+      return StickerSetId();
     default:
       UNREACHABLE();
       return StickerSetId();
@@ -1687,9 +1691,11 @@ StickerSetId StickersManager::add_sticker_set(tl_object_ptr<telegram_api::InputS
       return search_sticker_set(set->short_name_, Auto());
     }
     case telegram_api::inputStickerSetAnimatedEmoji::ID:
-    case telegram_api::inputStickerSetDice::ID:
       LOG(ERROR) << "Receive special sticker set " << to_string(set_ptr);
       return add_special_sticker_set(SpecialStickerSetType(set_ptr).type_).id_;
+    case telegram_api::inputStickerSetDice::ID:
+      LOG(ERROR) << "Receive special sticker set " << to_string(set_ptr);
+      return StickerSetId();
     default:
       UNREACHABLE();
       return StickerSetId();
@@ -1784,9 +1790,9 @@ bool StickersManager::merge_stickers(FileId new_id, FileId old_id, bool can_dele
     Sticker *new_ = new_it->second.get();
     CHECK(new_ != nullptr);
 
-    if (old_->alt != new_->alt || old_->set_id != new_->set_id ||
-        (!old_->is_animated && !new_->is_animated && old_->dimensions.width != 0 && old_->dimensions.height != 0 &&
-         old_->dimensions != new_->dimensions)) {
+    if (old_->set_id == new_->set_id && (old_->alt != new_->alt || old_->set_id != new_->set_id ||
+                                         (!old_->is_animated && !new_->is_animated && old_->dimensions.width != 0 &&
+                                          old_->dimensions.height != 0 && old_->dimensions != new_->dimensions))) {
       LOG(ERROR) << "Sticker has changed: alt = (" << old_->alt << ", " << new_->alt << "), set_id = (" << old_->set_id
                  << ", " << new_->set_id << "), dimensions = (" << old_->dimensions << ", " << new_->dimensions << ")";
     }
@@ -1878,11 +1884,14 @@ StickerSetId StickersManager::on_get_input_sticker_set(FileId sticker_file_id,
               }
             }));
       }
-      return set_id;
+      // always return empty StickerSetId, because we can't trust the set_id provided by the peer in the secret chat
+      // the real sticker set id will be set in on_get_sticker if and only if the sticker is really from the set
+      return StickerSetId();
     }
     case telegram_api::inputStickerSetAnimatedEmoji::ID:
-    case telegram_api::inputStickerSetDice::ID:
       return add_special_sticker_set(SpecialStickerSetType(set_ptr).type_).id_;
+    case telegram_api::inputStickerSetDice::ID:
+      return StickerSetId();
     default:
       UNREACHABLE();
       return StickerSetId();
@@ -1946,7 +1955,9 @@ void StickersManager::create_sticker(FileId file_id, PhotoSize thumbnail, Dimens
       }
     }
   }
-  s->is_animated = is_animated;
+  if (s->set_id.is_valid()) {
+    s->is_animated = is_animated;
+  }
   on_get_sticker(std::move(s), sticker != nullptr);
 }
 
@@ -2108,7 +2119,8 @@ StickerSetId StickersManager::on_get_sticker_set(tl_object_ptr<telegram_api::sti
   PhotoSize thumbnail;
   if (set->thumb_ != nullptr) {
     auto photo_size = get_photo_size(td_->file_manager_.get(), {set_id.get(), s->access_hash}, 0, 0, "",
-                                     DcId::create(set->thumb_dc_id_), DialogId(), std::move(set->thumb_), true, false);
+                                     DcId::create(set->thumb_dc_id_), DialogId(), std::move(set->thumb_),
+                                     is_animated ? PhotoFormat::Tgs : PhotoFormat::Webp);
     if (photo_size.get_offset() == 0) {
       thumbnail = std::move(photo_size.get<0>());
     } else {
@@ -3082,6 +3094,9 @@ void StickersManager::load_installed_sticker_sets(bool is_masks, Promise<Unit> &
 }
 
 void StickersManager::on_load_installed_sticker_sets_from_database(bool is_masks, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (value.empty()) {
     LOG(INFO) << "Installed " << (is_masks ? "mask " : "") << "sticker sets aren't found in database";
     reload_installed_sticker_sets(is_masks, true);
@@ -3193,7 +3208,7 @@ string StickersManager::get_sticker_set_database_value(const StickerSet *s, bool
 void StickersManager::update_sticker_set(StickerSet *sticker_set) {
   CHECK(sticker_set != nullptr);
   if (sticker_set->is_changed || sticker_set->need_save_to_database) {
-    if (G()->parameters().use_file_db) {
+    if (G()->parameters().use_file_db && !G()->close_flag()) {
       LOG(INFO) << "Save " << sticker_set->id << " to database";
       if (sticker_set->is_inited) {
         G()->td_db()->get_sqlite_pmc()->set(get_sticker_set_database_key(sticker_set->id),
@@ -3288,6 +3303,9 @@ void StickersManager::load_sticker_sets_without_stickers(vector<StickerSetId> &&
 }
 
 void StickersManager::on_load_sticker_set_from_database(StickerSetId sticker_set_id, bool with_stickers, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
   StickerSet *sticker_set = get_sticker_set(sticker_set_id);
   CHECK(sticker_set != nullptr);
   if (sticker_set->was_loaded) {
@@ -3411,12 +3429,22 @@ void StickersManager::on_update_dice_emojis() {
     return;
   }
 
-  auto dice_emojis_str = G()->shared_config().get_option_string("dice_emojis", "🎲\x01🎯");
+  auto dice_emojis_str = G()->shared_config().get_option_string("dice_emojis", "🎲\x01🎯\x01🏀");
   if (dice_emojis_str == dice_emojis_str_) {
     return;
   }
   dice_emojis_str_ = std::move(dice_emojis_str);
-  dice_emojis_ = full_split(dice_emojis_str_, '\x01');
+  auto new_dice_emojis = full_split(dice_emojis_str_, '\x01');
+  for (auto &emoji : new_dice_emojis) {
+    if (!td::contains(dice_emojis_, emoji)) {
+      auto &special_sticker_set = add_special_sticker_set(SpecialStickerSetType::animated_dice(emoji));
+      CHECK(!special_sticker_set.id_.is_valid());
+
+      LOG(INFO) << "Load new dice sticker set for emoji " << emoji;
+      load_special_sticker_set(special_sticker_set);
+    }
+  }
+  dice_emojis_ = std::move(new_dice_emojis);
 
   send_closure(G()->td(), &Td::send_update, get_update_dice_emojis_object());
 }
@@ -3430,7 +3458,7 @@ void StickersManager::on_update_dice_success_values() {
     return;
   }
 
-  auto dice_success_values_str = G()->shared_config().get_option_string("dice_success_values", "0,0");
+  auto dice_success_values_str = G()->shared_config().get_option_string("dice_success_values", "0,0,0");
   if (dice_success_values_str == dice_success_values_str_) {
     return;
   }
@@ -3464,7 +3492,12 @@ void StickersManager::register_dice(const string &emoji, int32 value, FullMessag
   bool is_inserted = dice_messages_[emoji].insert(full_message_id).second;
   LOG_CHECK(is_inserted) << source << " " << emoji << " " << value << " " << full_message_id;
 
-  if (!td::contains(dice_emojis_, emoji) && !full_message_id.get_message_id().is_any_server()) {
+  if (!td::contains(dice_emojis_, emoji)) {
+    if (full_message_id.get_message_id().is_any_server() &&
+        full_message_id.get_dialog_id().get_type() != DialogType::SecretChat) {
+      send_closure(G()->config_manager(), &ConfigManager::get_app_config,
+                   Promise<td_api::object_ptr<td_api::JsonValue>>());
+    }
     return;
   }
 
@@ -3697,6 +3730,10 @@ void StickersManager::on_old_featured_sticker_sets_invalidated() {
 }
 
 void StickersManager::invalidate_old_featured_sticker_sets() {
+  if (G()->close_flag()) {
+    return;
+  }
+
   LOG(INFO) << "Invalidate old featured sticker sets";
   if (G()->parameters().use_file_db) {
     G()->td_db()->get_binlog_pmc()->erase("invalidate_old_featured_sticker_sets");
@@ -3801,7 +3838,7 @@ void StickersManager::on_get_featured_sticker_sets(
 
   if (offset >= 0) {
     if (generation == old_featured_sticker_set_generation_) {
-      if (G()->parameters().use_file_db) {
+      if (G()->parameters().use_file_db && !G()->close_flag()) {
         LOG(INFO) << "Save old trending sticker sets to database with offset " << old_featured_sticker_set_ids_.size();
         CHECK(old_featured_sticker_set_ids_.size() % OLD_FEATURED_STICKER_SET_SLICE_SIZE == 0);
         StickerSetListLogEvent log_event(featured_sticker_set_ids);
@@ -3819,7 +3856,7 @@ void StickersManager::on_get_featured_sticker_sets(
 
   LOG_IF(ERROR, featured_sticker_sets_hash_ != featured_stickers->hash_) << "Trending sticker sets hash mismatch";
 
-  if (!G()->parameters().use_file_db) {
+  if (!G()->parameters().use_file_db || G()->close_flag()) {
     return;
   }
 
@@ -3874,6 +3911,9 @@ void StickersManager::load_featured_sticker_sets(Promise<Unit> &&promise) {
 }
 
 void StickersManager::on_load_featured_sticker_sets_from_database(string value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (value.empty()) {
     LOG(INFO) << "Trending sticker sets aren't found in database";
     reload_featured_sticker_sets(true);
@@ -3950,6 +3990,9 @@ void StickersManager::load_old_featured_sticker_sets(Promise<Unit> &&promise) {
 }
 
 void StickersManager::on_load_old_featured_sticker_sets_from_database(uint32 generation, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (generation != old_featured_sticker_set_generation_) {
     return;
   }
@@ -4155,7 +4198,7 @@ string &StickersManager::get_input_sticker_emojis(td_api::InputSticker *sticker)
 
 Result<std::tuple<FileId, bool, bool, bool>> StickersManager::prepare_input_sticker(td_api::InputSticker *sticker) {
   if (sticker == nullptr) {
-    return Status::Error(3, "Input sticker must not be empty");
+    return Status::Error(3, "Input sticker must be non-empty");
   }
 
   if (!clean_input_string(get_input_sticker_emojis(sticker))) {
@@ -4849,7 +4892,7 @@ void StickersManager::send_update_installed_sticker_sets(bool from_database) {
         installed_sticker_sets_hash_[is_masks] = get_sticker_sets_hash(installed_sticker_set_ids_[is_masks]);
         send_closure(G()->td(), &Td::send_update, get_update_installed_sticker_sets_object(is_masks));
 
-        if (G()->parameters().use_file_db && !from_database) {
+        if (G()->parameters().use_file_db && !from_database && !G()->close_flag()) {
           LOG(INFO) << "Save installed " << (is_masks ? "mask " : "") << "sticker sets to database";
           StickerSetListLogEvent log_event(installed_sticker_set_ids_[is_masks]);
           G()->td_db()->get_sqlite_pmc()->set(is_masks ? "sss1" : "sss0", log_event_store(log_event).as_slice().str(),
@@ -4936,6 +4979,9 @@ void StickersManager::load_recent_stickers(bool is_attached, Promise<Unit> &&pro
 }
 
 void StickersManager::on_load_recent_stickers_from_database(bool is_attached, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (value.empty()) {
     LOG(INFO) << "Recent " << (is_attached ? "attached " : "") << "stickers aren't found in database";
     reload_recent_stickers(is_attached, true);
@@ -5256,7 +5302,7 @@ void StickersManager::send_update_recent_stickers(bool from_database) {
 }
 
 void StickersManager::save_recent_stickers_to_database(bool is_attached) {
-  if (G()->parameters().use_file_db) {
+  if (G()->parameters().use_file_db && !G()->close_flag()) {
     LOG(INFO) << "Save recent " << (is_attached ? "attached " : "") << "stickers to database";
     StickerListLogEvent log_event(recent_sticker_ids_[is_attached]);
     G()->td_db()->get_sqlite_pmc()->set(is_attached ? "ssr1" : "ssr0", log_event_store(log_event).as_slice().str(),
@@ -5356,6 +5402,9 @@ void StickersManager::load_favorite_stickers(Promise<Unit> &&promise) {
 }
 
 void StickersManager::on_load_favorite_stickers_from_database(const string &value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (value.empty()) {
     LOG(INFO) << "Favorite stickers aren't found in database";
     reload_favorite_stickers(true);
@@ -5620,7 +5669,7 @@ void StickersManager::send_update_favorite_stickers(bool from_database) {
 }
 
 void StickersManager::save_favorite_stickers_to_database() {
-  if (G()->parameters().use_file_db) {
+  if (G()->parameters().use_file_db && !G()->close_flag()) {
     LOG(INFO) << "Save favorite stickers to database";
     StickerListLogEvent log_event(favorite_sticker_ids_);
     G()->td_db()->get_sqlite_pmc()->set("ssfav", log_event_store(log_event).as_slice().str(), Auto());
@@ -5790,7 +5839,10 @@ void StickersManager::on_get_language_codes(const string &key, Result<vector<str
   CHECK(it != emoji_language_codes_.end());
   if (it->second != language_codes) {
     LOG(INFO) << "Update emoji language codes for " << key << " to " << language_codes;
-    G()->td_db()->get_sqlite_pmc()->set(key, implode(language_codes, '$'), Auto());
+    if (!G()->close_flag()) {
+      CHECK(G()->parameters().use_file_db);
+      G()->td_db()->get_sqlite_pmc()->set(key, implode(language_codes, '$'), Auto());
+    }
     it->second = std::move(language_codes);
   }
 
@@ -5924,7 +5976,8 @@ void StickersManager::on_get_emoji_keywords(
             is_good = false;
           }
         }
-        if (is_good) {
+        if (is_good && !G()->close_flag()) {
+          CHECK(G()->parameters().use_file_db);
           G()->td_db()->get_sqlite_pmc()->set(get_language_emojis_database_key(language_code, text),
                                               implode(keyword->emoticons_, '$'), mpas.get_promise());
         }
@@ -5937,10 +5990,13 @@ void StickersManager::on_get_emoji_keywords(
         UNREACHABLE();
     }
   }
-  G()->td_db()->get_sqlite_pmc()->set(get_emoji_language_code_version_database_key(language_code), to_string(version),
-                                      mpas.get_promise());
-  G()->td_db()->get_sqlite_pmc()->set(get_emoji_language_code_last_difference_time_database_key(language_code),
-                                      to_string(G()->unix_time()), mpas.get_promise());
+  if (!G()->close_flag()) {
+    CHECK(G()->parameters().use_file_db);
+    G()->td_db()->get_sqlite_pmc()->set(get_emoji_language_code_version_database_key(language_code), to_string(version),
+                                        mpas.get_promise());
+    G()->td_db()->get_sqlite_pmc()->set(get_emoji_language_code_last_difference_time_database_key(language_code),
+                                        to_string(G()->unix_time()), mpas.get_promise());
+  }
   emoji_language_code_versions_[language_code] = version;
   emoji_language_code_last_difference_times_[language_code] = static_cast<int32>(Time::now_cached());
 

td/telegram/Td.cpp

@@ -99,11 +99,11 @@
 
 #include "td/db/binlog/BinlogEvent.h"
 
-#include "td/mtproto/crypto.h"
 #include "td/mtproto/DhHandshake.h"
 #include "td/mtproto/Handshake.h"
 #include "td/mtproto/HandshakeActor.h"
 #include "td/mtproto/RawConnection.h"
+#include "td/mtproto/RSA.h"
 #include "td/mtproto/TransportType.h"
 
 #include "td/utils/buffer.h"
@@ -587,10 +587,10 @@ class TestProxyRequest : public RequestOnceActor {
     }
 
     auto dc_options = ConnectionCreator::get_default_dc_options(false);
-    IPAddress mtproto_ip;
+    IPAddress mtproto_ip_address;
     for (auto &dc_option : dc_options.dc_options) {
       if (dc_option.get_dc_id().get_raw_id() == dc_id_) {
-        mtproto_ip = dc_option.get_ip_address();
+        mtproto_ip_address = dc_option.get_ip_address();
         break;
       }
     }
@@ -601,8 +601,8 @@ class TestProxyRequest : public RequestOnceActor {
         });
 
     child_ =
-        ConnectionCreator::prepare_connection(r_socket_fd.move_as_ok(), proxy_, mtproto_ip, get_transport(), "Test",
-                                              "TestPingDC2", nullptr, {}, false, std::move(connection_promise));
+        ConnectionCreator::prepare_connection(r_socket_fd.move_as_ok(), proxy_, mtproto_ip_address, get_transport(),
+                                              "Test", "TestPingDC2", nullptr, {}, false, std::move(connection_promise));
   }
 
   void on_connection_data(Result<ConnectionCreator::ConnectionData> r_data) {
@@ -4313,6 +4313,10 @@ void Td::init_file_manager() {
       send_closure(G()->storage_manager(), &StorageManager::on_new_file, size, real_size, cnt);
     }
 
+    void destroy_file_source(FileId file_id) final {
+      td_->file_reference_manager_->memory_cleanup(file_id);
+    }
+
     void on_file_updated(FileId file_id) final {
       send_closure(G()->td(), &Td::send_update,
                    make_tl_object<td_api::updateFile>(td_->file_manager_->get_file_object(file_id)));
@@ -5125,6 +5129,8 @@ void Td::on_request(uint64 id, td_api::optimizeStorage &request) {
   documents_manager_->memory_cleanup();
   video_notes_manager_->memory_cleanup();
   videos_manager_->memory_cleanup();
+  audios_manager_->memory_cleanup();
+  animations_manager_->memory_cleanup();
   file_manager_->memory_cleanup();
 
   malloc_trim(0);
@@ -5798,7 +5804,7 @@ void Td::on_request(uint64 id, td_api::createCall &request) {
   });
 
   if (!request.protocol_) {
-    return query_promise.set_error(Status::Error(5, "Call protocol must not be empty"));
+    return query_promise.set_error(Status::Error(5, "Call protocol must be non-empty"));
   }
 
   UserId user_id(request.user_id_);
@@ -5826,7 +5832,7 @@ void Td::on_request(uint64 id, td_api::acceptCall &request) {
   CHECK_IS_USER();
   CREATE_OK_REQUEST_PROMISE();
   if (!request.protocol_) {
-    return promise.set_error(Status::Error(5, "Call protocol must not be empty"));
+    return promise.set_error(Status::Error(5, "Call protocol must be non-empty"));
   }
   send_closure(G()->call_manager(), &CallManager::accept_call, CallId(request.call_id_),
                CallProtocol::from_td_api(*request.protocol_), std::move(promise));
@@ -6232,7 +6238,7 @@ void Td::on_request(uint64 id, const td_api::getBlockedUsers &request) {
 void Td::on_request(uint64 id, td_api::addContact &request) {
   CHECK_IS_USER();
   if (request.contact_ == nullptr) {
-    return send_error_raw(id, 5, "Contact must not be empty");
+    return send_error_raw(id, 5, "Contact must be non-empty");
   }
   CLEAN_INPUT_STRING(request.contact_->phone_number_);
   CLEAN_INPUT_STRING(request.contact_->first_name_);
@@ -6245,7 +6251,7 @@ void Td::on_request(uint64 id, td_api::importContacts &request) {
   CHECK_IS_USER();
   for (auto &contact : request.contacts_) {
     if (contact == nullptr) {
-      return send_error_raw(id, 5, "Contact must not be empty");
+      return send_error_raw(id, 5, "Contact must be non-empty");
     }
     CLEAN_INPUT_STRING(contact->phone_number_);
     CLEAN_INPUT_STRING(contact->first_name_);
@@ -6279,7 +6285,7 @@ void Td::on_request(uint64 id, td_api::changeImportedContacts &request) {
   CHECK_IS_USER();
   for (auto &contact : request.contacts_) {
     if (contact == nullptr) {
-      return send_error_raw(id, 5, "Contact must not be empty");
+      return send_error_raw(id, 5, "Contact must be non-empty");
     }
     CLEAN_INPUT_STRING(contact->phone_number_);
     CLEAN_INPUT_STRING(contact->first_name_);
@@ -6598,7 +6604,7 @@ void Td::on_request(uint64 id, const td_api::getChatNotificationSettingsExceptio
 void Td::on_request(uint64 id, const td_api::getScopeNotificationSettings &request) {
   CHECK_IS_USER();
   if (request.scope_ == nullptr) {
-    return send_error_raw(id, 400, "Scope must not be empty");
+    return send_error_raw(id, 400, "Scope must be non-empty");
   }
   CREATE_REQUEST(GetScopeNotificationSettingsRequest, get_notification_settings_scope(request.scope_));
 }
@@ -6646,7 +6652,7 @@ void Td::on_request(uint64 id, td_api::setChatNotificationSettings &request) {
 void Td::on_request(uint64 id, td_api::setScopeNotificationSettings &request) {
   CHECK_IS_USER();
   if (request.scope_ == nullptr) {
-    return send_error_raw(id, 400, "Scope must not be empty");
+    return send_error_raw(id, 400, "Scope must be non-empty");
   }
   answer_ok_query(id, messages_manager_->set_scope_notification_settings(
                           get_notification_settings_scope(request.scope_), std::move(request.notification_settings_)));
@@ -7201,7 +7207,7 @@ void Td::on_request(uint64 id, td_api::sendPaymentForm &request) {
   CLEAN_INPUT_STRING(request.order_info_id_);
   CLEAN_INPUT_STRING(request.shipping_option_id_);
   if (request.credentials_ == nullptr) {
-    return send_error_raw(id, 400, "Input payments credentials must not be empty");
+    return send_error_raw(id, 400, "Input payments credentials must be non-empty");
   }
   CREATE_REQUEST_PROMISE();
   messages_manager_->send_payment_form({DialogId(request.chat_id_), MessageId(request.message_id_)},
@@ -7250,7 +7256,7 @@ void Td::on_request(uint64 id, td_api::getPassportElement &request) {
   CHECK_IS_USER();
   CLEAN_INPUT_STRING(request.password_);
   if (request.type_ == nullptr) {
-    return send_error_raw(id, 400, "Type must not be empty");
+    return send_error_raw(id, 400, "Type must be non-empty");
   }
   CREATE_REQUEST_PROMISE();
   send_closure(secure_manager_, &SecureManager::get_secure_value, std::move(request.password_),
@@ -7280,7 +7286,7 @@ void Td::on_request(uint64 id, td_api::setPassportElement &request) {
 void Td::on_request(uint64 id, const td_api::deletePassportElement &request) {
   CHECK_IS_USER();
   if (request.type_ == nullptr) {
-    return send_error_raw(id, 400, "Type must not be empty");
+    return send_error_raw(id, 400, "Type must be non-empty");
   }
   CREATE_OK_REQUEST_PROMISE();
   send_closure(secure_manager_, &SecureManager::delete_secure_value, get_secure_value_type_td_api(request.type_),
@@ -7376,7 +7382,7 @@ void Td::on_request(uint64 id, td_api::sendPassportAuthorizationForm &request) {
   CHECK_IS_USER();
   for (auto &type : request.types_) {
     if (type == nullptr) {
-      return send_error_raw(id, 400, "Type must not be empty");
+      return send_error_raw(id, 400, "Type must be non-empty");
     }
   }
 

td/telegram/WebPagesManager.cpp

@@ -429,11 +429,6 @@ WebPageId WebPagesManager::on_get_web_page(tl_object_ptr<telegram_api::WebPage>
       LOG(INFO) << "Got empty " << web_page_id;
       const WebPage *web_page_to_delete = get_web_page(web_page_id);
       if (web_page_to_delete != nullptr) {
-        if (web_page_to_delete->logevent_id != 0) {
-          LOG(INFO) << "Erase " << web_page_id << " from binlog";
-          binlog_erase(G()->td_db()->get_binlog(), web_page_to_delete->logevent_id);
-          web_page_to_delete->logevent_id = 0;
-        }
         if (web_page_to_delete->file_source_id.is_valid()) {
           td_->file_manager_->change_files_source(web_page_to_delete->file_source_id,
                                                   get_web_page_file_ids(web_page_to_delete), vector<FileId>());
@@ -667,9 +662,7 @@ void WebPagesManager::on_get_web_page_instant_view_view_count(WebPageId web_page
   }
 
   auto *instant_view = &web_pages_[web_page_id]->instant_view;
-  if (instant_view->is_empty) {
-    return;
-  }
+  if (instant_view->is_empty) { return; }
   if (instant_view->view_count >= view_count) {
     return;
   }
@@ -707,9 +700,7 @@ void WebPagesManager::register_web_page(WebPageId web_page_id, FullMessageId ful
 
   LOG(INFO) << "Register " << web_page_id << " from " << full_message_id << " from " << source;
   bool is_inserted = web_page_messages_[web_page_id].insert(full_message_id).second;
-  if (!is_inserted) {
-    return;
-  }
+  if (!is_inserted) { return; }
 
   if (!td_->auth_manager_->is_bot() && !have_web_page_force(web_page_id)) {
     LOG(INFO) << "Waiting for " << web_page_id << " needed in " << full_message_id;
@@ -728,6 +719,7 @@ void WebPagesManager::unregister_web_page(WebPageId web_page_id, FullMessageId f
   if (!is_deleted) {
     return;
   }
+  if (!is_deleted) { return; }
 
   if (message_ids.empty()) {
     web_page_messages_.erase(web_page_id);
@@ -773,13 +765,9 @@ void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const st
 
 void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const string &url, WebPageId web_page_id,
                                                       Promise<Unit> &&promise) {
-  if (!(web_page_id == WebPageId() || have_web_page(web_page_id))) {
-    return;
-  }
+  if (!(web_page_id == WebPageId() || have_web_page(web_page_id))) { return; }
 
-  if (!(got_web_page_previews_.find(request_id) == got_web_page_previews_.end())) {
-    return;
-  }
+  if (!(got_web_page_previews_.find(request_id) == got_web_page_previews_.end())) { return; }
   got_web_page_previews_[request_id] = web_page_id;
 
   if (web_page_id.is_valid() && !url.empty()) {
@@ -792,9 +780,7 @@ void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const st
 void WebPagesManager::on_get_web_page_preview_fail(int64 request_id, const string &url, Status error,
                                                    Promise<Unit> &&promise) {
   LOG(INFO) << "Clean up getting of web page preview with url \"" << url << '"';
-  if (!error.is_error()) {
-    return;
-  }
+  if (!error.is_error()) { return; }
   promise.set_error(std::move(error));
 }
 
@@ -927,9 +913,7 @@ void WebPagesManager::load_web_page_instant_view(WebPageId web_page_id, bool for
 void WebPagesManager::reload_web_page_instant_view(WebPageId web_page_id) {
   LOG(INFO) << "Reload " << web_page_id << " instant view";
   const WebPage *web_page = get_web_page(web_page_id);
-  if (!(web_page != nullptr && !web_page->instant_view.is_empty)) {
-    return;
-  }
+  if (!(web_page != nullptr && !web_page->instant_view.is_empty)) { return; }
 
   auto promise = PromiseCreator::lambda([web_page_id](Result<> result) {
     send_closure(G()->web_pages_manager(), &WebPagesManager::update_web_page_instant_view_load_requests, web_page_id,
@@ -945,9 +929,10 @@ void WebPagesManager::reload_web_page_instant_view(WebPageId web_page_id) {
 }
 
 void WebPagesManager::on_load_web_page_instant_view_from_database(WebPageId web_page_id, string value) {
-  if(!(G()->parameters().use_message_db)) {
+  if (G()->close_flag()) {
     return;
   }
+  if (!(G()->parameters().use_message_db)) { return; }
   LOG(INFO) << "Successfully loaded " << web_page_id << " instant view of size " << value.size() << " from database";
   //  G()->td_db()->get_sqlite_pmc()->erase(get_web_page_instant_view_database_key(web_page_id), Auto());
   //  return;
@@ -1100,6 +1085,9 @@ void WebPagesManager::load_web_page_by_url(const string &url, Promise<Unit> &&pr
 
 void WebPagesManager::on_load_web_page_id_by_url_from_database(const string &url, string value,
                                                                Promise<Unit> &&promise) {
+  if (G()->close_flag()) {
+    return;
+  }
   LOG(INFO) << "Successfully loaded url \"" << url << "\" of size " << value.size() << " from database";
   //  G()->td_db()->get_sqlite_pmc()->erase(get_web_page_url_database_key(web_page_id), Auto());
   //  return;
@@ -1138,9 +1126,7 @@ void WebPagesManager::on_load_web_page_id_by_url_from_database(const string &url
 void WebPagesManager::on_load_web_page_by_url_from_database(WebPageId web_page_id, const string &url,
                                                             Promise<Unit> &&promise, Result<> result) {
   if (result.is_error()) {
-    if(!(G()->close_flag())) {
-      return;
-    }
+    CHECK(G()->close_flag());
     promise.set_error(Status::Error(500, "Request aborted"));
     return;
   }
@@ -1218,9 +1204,7 @@ void WebPagesManager::on_web_page_changed(WebPageId web_page_id, bool have_web_p
     for (auto full_message_id : it->second) {
       full_message_ids.push_back(full_message_id);
     }
-    if (full_message_ids.empty()) {
-      return;
-    }
+    if (full_message_ids.empty()) { return; }
     for (auto full_message_id : full_message_ids) {
       if (!have_web_page) {
         td_->messages_manager_->delete_pending_message_web_page(full_message_id);
@@ -1229,13 +1213,9 @@ void WebPagesManager::on_web_page_changed(WebPageId web_page_id, bool have_web_p
       }
     }
     if (have_web_page) {
-      if (!(web_page_messages_[web_page_id].size() == full_message_ids.size())) {
-        return;
-      }
+      if (!(web_page_messages_[web_page_id].size() == full_message_ids.size())) { return; }
     } else {
-      if (!(web_page_messages_.count(web_page_id) == 0)) {
-        return;
-      }
+      if (!(web_page_messages_.count(web_page_id) == 0)) { return; }
     }
   }
   auto get_it = pending_get_web_pages_.find(web_page_id);
@@ -1475,6 +1455,9 @@ string WebPagesManager::get_web_page_database_key(WebPageId web_page_id) {
 }
 
 void WebPagesManager::on_save_web_page_to_database(WebPageId web_page_id, bool success) {
+  if (G()->close_flag()) {
+    return;
+  }
   const WebPage *web_page = get_web_page(web_page_id);
   if (web_page == nullptr) {
     LOG(ERROR) << "Can't find " << (success ? "saved " : "failed to save ") << web_page_id;
@@ -1513,6 +1496,9 @@ void WebPagesManager::load_web_page_from_database(WebPageId web_page_id, Promise
 }
 
 void WebPagesManager::on_load_web_page_from_database(WebPageId web_page_id, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (!loaded_from_database_web_pages_.insert(web_page_id).second) {
     return;
   }
@@ -1521,9 +1507,7 @@ void WebPagesManager::on_load_web_page_from_database(WebPageId web_page_id, stri
   vector<Promise<Unit>> promises;
   if (it != load_web_page_from_database_queries_.end()) {
     promises = std::move(it->second);
-    if (promises.empty()) {
-      return;
-    }
+    if (promises.empty()) { return; }
     load_web_page_from_database_queries_.erase(it);
   }
 

td/telegram/cli.cpp

@@ -911,6 +911,8 @@ class CliClient final : public Actor {
     if (disable_network_) {
       send_request(td_api::make_object<td_api::setNetworkType>(td_api::make_object<td_api::networkTypeNone>()));
     }
+
+    on_cmd("v0");
   }
 #ifndef USE_READLINE
   size_t buffer_pos_ = 0;
@@ -1994,7 +1996,7 @@ class CliClient final : public Actor {
 
       send_request(td_api::make_object<td_api::getChatMessageCount>(
           as_chat_id(chat_id), get_search_messages_filter(filter), as_bool(return_local)));
-    } else if (op == "gup" || op == "GetUserPhotos") {
+    } else if (op == "gup" || op == "gupf") {
       string user_id;
       string offset;
       string limit;

td/telegram/files/FileId.h

@@ -9,6 +9,7 @@
 #include "td/utils/common.h"
 #include "td/utils/StringBuilder.h"
 
+#include <ctime>
 #include <functional>
 #include <type_traits>
 
@@ -17,6 +18,7 @@ namespace td {
 class FileId {
   int32 id = 0;
   int32 remote_id = 0;
+  int64 time_ = INT64_MAX;
 
  public:
   FileId() = default;
@@ -34,10 +36,26 @@ class FileId {
     return id > 0;
   }
 
+  int32 fast_get() const {
+    return id;
+  }
+
   int32 get() const {
     return id;
   }
 
+  void set_time() {
+    time_ = std::time(nullptr);
+  }
+
+  int64 get_time() const {
+    return time_;
+  }
+
+  void reset_time() {
+    time_ = INT64_MAX;
+  }
+
   int32 get_remote() const {
     return remote_id;
   }

td/telegram/files/FileManager.cpp

@@ -48,6 +48,8 @@
 #include <tuple>
 #include <utility>
 
+#define FILE_TTL 20
+
 namespace td {
 namespace {
 constexpr int64 MAX_FILE_SIZE = 1500 * (1 << 20) /* 1500MB */;
@@ -661,7 +663,14 @@ int64 FileView::remote_size() const {
 }
 
 string FileView::path() const {
-  return "";
+  switch (node_->local_.type()) {
+    case LocalFileLocation::Type::Full:
+      return node_->local_.full().path_;
+    case LocalFileLocation::Type::Partial:
+      return node_->local_.partial().path_;
+    default:
+      return "";
+  }
 }
 
 bool FileView::has_url() const {
@@ -974,8 +983,7 @@ bool FileManager::try_fix_partial_local_location(FileNodePtr node) {
 }
 
 FileManager::FileIdInfo *FileManager::get_file_id_info(FileId file_id) {
-  LOG_CHECK(0 <= file_id.get() && file_id.get() < static_cast<int32>(file_id_info_.size()))
-      << file_id << " " << file_id_info_.size();
+  file_id.set_time();
   return &file_id_info_[file_id.get()];
 }
 
@@ -1011,7 +1019,7 @@ void FileManager::try_forget_file_id(FileId file_id) {
   bool is_removed = td::remove(file_node->file_ids_, file_id);
   CHECK(is_removed);
   *info = FileIdInfo();
-  empty_file_ids_.push_back(file_id.get());
+  file_id_info_.erase(file_id.get());
 }
 
 FileId FileManager::register_empty(FileType type) {
@@ -1131,13 +1139,13 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi
   // create FileNode
   auto file_node_id = next_file_node_id();
   auto &node = file_nodes_[file_node_id];
-  node = td::make_unique<FileNode>(std::move(data.local_), NewRemoteFileLocation(data.remote_, file_location_source),
+  node = FileNode(std::move(data.local_), NewRemoteFileLocation(data.remote_, file_location_source),
                                    std::move(data.generate_), data.size_, data.expected_size_,
                                    std::move(data.remote_name_), std::move(data.url_), data.owner_dialog_id_,
                                    std::move(data.encryption_key_), file_id, static_cast<int8>(has_remote));
-  node->pmc_id_ = FileDbId(data.pmc_id_);
+  node.pmc_id_ = FileDbId(data.pmc_id_);
   get_file_id_info(file_id)->node_id_ = file_node_id;
-  node->file_ids_.push_back(file_id);
+  node.file_ids_.push_back(file_id);
 
   FileView file_view(get_file_node(file_id));
 
@@ -1184,7 +1192,7 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi
 
   int new_cnt = new_remote + new_local + new_generate;
   if (data.pmc_id_ == 0 && file_db_ && new_cnt > 0) {
-    node->need_load_from_pmc_ = true;
+    node.need_load_from_pmc_ = true;
   }
   bool no_sync_merge = to_merge.size() == 1 && new_cnt == 0;
   for (auto id : to_merge) {
@@ -1363,7 +1371,260 @@ void FileManager::do_cancel_generate(FileNodePtr node) {
 }
 
 Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) {
-  return Status::Error("stub");
+  LOG(DEBUG) << "Merge new file " << x_file_id << " and old file " << y_file_id;
+
+  if (!x_file_id.is_valid()) {
+    return Status::Error("First file_id is invalid");
+  }
+  FileNodePtr x_node = no_sync ? get_file_node(x_file_id) : get_sync_file_node(x_file_id);
+  if (!x_node) {
+    return Status::Error(PSLICE() << "Can't merge files. First id is invalid: " << x_file_id << " and " << y_file_id);
+  }
+
+  if (!y_file_id.is_valid()) {
+    LOG(DEBUG) << "Old file is invalid";
+    return x_node->main_file_id_;
+  }
+  FileNodePtr y_node = get_file_node(y_file_id);
+  if (!y_node) {
+    return Status::Error(PSLICE() << "Can't merge files. Second id is invalid: " << x_file_id << " and " << y_file_id);
+  }
+
+  if (x_file_id == x_node->upload_pause_) {
+    x_node->set_upload_pause(FileId());
+  }
+  if (x_node.get() == y_node.get()) {
+    LOG(DEBUG) << "Files are already merged";
+    return x_node->main_file_id_;
+  }
+  if (y_file_id == y_node->upload_pause_) {
+    y_node->set_upload_pause(FileId());
+  }
+
+  if (x_node->remote_.full && y_node->remote_.full && !x_node->remote_.full.value().is_web() &&
+      !y_node->remote_.full.value().is_web() && y_node->remote_.is_full_alive &&
+      x_node->remote_.full_source == FileLocationSource::FromServer &&
+      y_node->remote_.full_source == FileLocationSource::FromServer &&
+      x_node->remote_.full.value().get_dc_id() != y_node->remote_.full.value().get_dc_id()) {
+  }
+
+  bool drop_last_successful_force_reupload_time = x_node->last_successful_force_reupload_time_ <= 0 &&
+                                                  x_node->remote_.full &&
+                                                  x_node->remote_.full_source == FileLocationSource::FromServer;
+
+  auto count_local = [](auto &node) {
+    return std::accumulate(node->file_ids_.begin(), node->file_ids_.end(), 0,
+                           [](const auto &x, const auto &y) { return x + (y.get_remote() != 0); });
+  };
+  if (count_local(x_node) + count_local(y_node) > 100) {
+  }
+
+  FileNodePtr nodes[] = {x_node, y_node, x_node};
+  FileNodeId node_ids[] = {get_file_id_info(x_file_id)->node_id_, get_file_id_info(y_file_id)->node_id_};
+  int trusted_by_source = merge_choose_file_source_location(x_node->remote_.full_source, y_node->remote_.full_source);
+
+  int local_i = merge_choose_local_location(x_node->local_, y_node->local_);
+  int remote_i = merge_choose_remote_location(x_node->remote_, y_node->remote_);
+  int generate_i = merge_choose_generate_location(x_node->generate_, y_node->generate_);
+  int size_i = merge_choose_size(x_node->size_, y_node->size_);
+  int expected_size_i = merge_choose_expected_size(x_node->expected_size_, y_node->expected_size_);
+  int remote_name_i = merge_choose_name(x_node->remote_name_, y_node->remote_name_);
+  int url_i = merge_choose_name(x_node->url_, y_node->url_);
+  int owner_i = merge_choose_owner(x_node->owner_dialog_id_, y_node->owner_dialog_id_);
+  int encryption_key_i = merge_choose_encryption_key(x_node->encryption_key_, y_node->encryption_key_);
+  int main_file_id_i = merge_choose_main_file_id(x_node->main_file_id_, x_node->main_file_id_priority_,
+                                                 y_node->main_file_id_, y_node->main_file_id_priority_);
+
+  if (size_i == -1) {
+    return Status::Error(PSLICE() << "Can't merge files. Different size: " << x_node->size_ << " and "
+                                  << y_node->size_);
+  }
+  if (encryption_key_i == -1) {
+    if (nodes[remote_i]->remote_.full && nodes[local_i]->local_.type() != LocalFileLocation::Type::Partial) {
+      LOG(ERROR) << "Different encryption key in files, but lets choose same key as remote location";
+      encryption_key_i = remote_i;
+    } else {
+      return Status::Error("Can't merge files. Different encryption keys");
+    }
+  }
+
+  // prefer more trusted source
+  if (remote_name_i == 2) {
+    remote_name_i = trusted_by_source;
+  }
+  if (url_i == 2) {
+    url_i = trusted_by_source;
+  }
+  if (expected_size_i == 2) {
+    expected_size_i = trusted_by_source;
+  }
+
+  int node_i =
+      std::make_tuple(y_node->pmc_id_.is_valid(), x_node->pmc_id_, y_node->file_ids_.size(), main_file_id_i == 1) >
+      std::make_tuple(x_node->pmc_id_.is_valid(), y_node->pmc_id_, x_node->file_ids_.size(), main_file_id_i == 0);
+
+  auto other_node_i = 1 - node_i;
+  FileNodePtr node = nodes[node_i];
+  FileNodePtr other_node = nodes[other_node_i];
+  auto file_view = FileView(node);
+
+  LOG(DEBUG) << "Have x_node->pmc_id_ = " << x_node->pmc_id_.get() << ", y_node->pmc_id_ = " << y_node->pmc_id_.get()
+             << ", x_node_size = " << x_node->file_ids_.size() << ", y_node_size = " << y_node->file_ids_.size()
+             << ", node_i = " << node_i << ", local_i = " << local_i << ", remote_i = " << remote_i
+             << ", generate_i = " << generate_i << ", size_i = " << size_i << ", remote_name_i = " << remote_name_i
+             << ", url_i = " << url_i << ", owner_i = " << owner_i << ", encryption_key_i = " << encryption_key_i
+             << ", main_file_id_i = " << main_file_id_i << ", trusted_by_source = " << trusted_by_source
+             << ", x_source = " << x_node->remote_.full_source << ", y_source = " << y_node->remote_.full_source;
+  if (local_i == other_node_i) {
+    do_cancel_download(node);
+    node->set_download_offset(other_node->download_offset_);
+    node->set_local_location(other_node->local_, other_node->local_ready_size_, other_node->download_offset_,
+                             other_node->local_ready_prefix_size_);
+    node->download_id_ = other_node->download_id_;
+    node->download_was_update_file_reference_ = other_node->download_was_update_file_reference_;
+    node->is_download_started_ |= other_node->is_download_started_;
+    node->set_download_priority(other_node->download_priority_);
+    other_node->download_id_ = 0;
+    other_node->download_was_update_file_reference_ = false;
+    other_node->is_download_started_ = false;
+    other_node->download_priority_ = 0;
+    other_node->download_offset_ = 0;
+    other_node->local_ready_prefix_size_ = 0;
+
+    //do_cancel_generate(node);
+    //node->set_generate_location(std::move(other_node->generate_));
+    //node->generate_id_ = other_node->generate_id_;
+    //node->set_generate_priority(other_node->generate_download_priority_, other_node->generate_upload_priority_);
+    //other_node->generate_id_ = 0;
+    //other_node->generate_was_update_ = false;
+    //other_node->generate_priority_ = 0;
+    //other_node->generate_download_priority_ = 0;
+    //other_node->generate_upload_priority_ = 0;
+  } else {
+    do_cancel_download(other_node);
+    //do_cancel_generate(other_node);
+  }
+
+  if (remote_i == other_node_i) {
+    do_cancel_upload(node);
+    node->set_new_remote_location(std::move(other_node->remote_));
+    node->upload_id_ = other_node->upload_id_;
+    node->upload_was_update_file_reference_ = other_node->upload_was_update_file_reference_;
+    node->set_upload_priority(other_node->upload_priority_);
+    node->set_upload_pause(other_node->upload_pause_);
+    other_node->upload_id_ = 0;
+    other_node->upload_was_update_file_reference_ = false;
+    other_node->upload_priority_ = 0;
+    other_node->set_upload_pause(FileId());
+  } else {
+    do_cancel_upload(other_node);
+  }
+
+  if (generate_i == other_node_i) {
+    do_cancel_generate(node);
+    node->set_generate_location(std::move(other_node->generate_));
+    node->generate_id_ = other_node->generate_id_;
+    node->set_generate_priority(other_node->generate_download_priority_, other_node->generate_upload_priority_);
+    other_node->generate_id_ = 0;
+    other_node->generate_priority_ = 0;
+    other_node->generate_download_priority_ = 0;
+    other_node->generate_upload_priority_ = 0;
+  } else {
+    do_cancel_generate(other_node);
+  }
+
+  if (size_i == other_node_i) {
+    node->set_size(other_node->size_);
+  }
+
+  if (expected_size_i == other_node_i) {
+    node->set_expected_size(other_node->expected_size_);
+  }
+
+  if (remote_name_i == other_node_i) {
+    node->set_remote_name(other_node->remote_name_);
+  }
+
+  if (url_i == other_node_i) {
+    node->set_url(other_node->url_);
+  }
+
+  if (owner_i == other_node_i) {
+    node->set_owner_dialog_id(other_node->owner_dialog_id_);
+  }
+
+  if (encryption_key_i == other_node_i) {
+    node->set_encryption_key(other_node->encryption_key_);
+    nodes[node_i]->set_encryption_key(nodes[encryption_key_i]->encryption_key_);
+  }
+  node->need_load_from_pmc_ |= other_node->need_load_from_pmc_;
+  node->can_search_locally_ &= other_node->can_search_locally_;
+
+  if (drop_last_successful_force_reupload_time) {
+    node->last_successful_force_reupload_time_ = -1e10;
+  } else if (other_node->last_successful_force_reupload_time_ > node->last_successful_force_reupload_time_) {
+    node->last_successful_force_reupload_time_ = other_node->last_successful_force_reupload_time_;
+  }
+
+  if (main_file_id_i == other_node_i) {
+    context_->on_merge_files(other_node->main_file_id_, node->main_file_id_);
+    node->main_file_id_ = other_node->main_file_id_;
+    node->main_file_id_priority_ = other_node->main_file_id_priority_;
+  } else {
+    context_->on_merge_files(node->main_file_id_, other_node->main_file_id_);
+  }
+
+  bool send_updates_flag = false;
+  auto other_pmc_id = other_node->pmc_id_;
+  node->file_ids_.insert(node->file_ids_.end(), other_node->file_ids_.begin(), other_node->file_ids_.end());
+
+  for (auto file_id : other_node->file_ids_) {
+    auto file_id_info = get_file_id_info(file_id);
+    LOG_CHECK(file_id_info->node_id_ == node_ids[other_node_i])
+        << node_ids[node_i] << " " << node_ids[other_node_i] << " " << file_id << " " << file_id_info->node_id_;
+    file_id_info->node_id_ = node_ids[node_i];
+    send_updates_flag |= file_id_info->send_updates_flag_;
+  }
+  other_node = {this};
+
+  if (send_updates_flag) {
+    // node might not changed, but other_node might changed, so we need to send update anyway
+    VLOG(update_file) << "File " << node->main_file_id_ << " has been merged";
+    node->on_info_changed();
+  }
+
+  // Check is some download/upload queries are ready
+  for (auto file_id : vector<FileId>(node->file_ids_)) {
+    auto *info = get_file_id_info(file_id);
+    if (info->download_priority_ != 0 && file_view.has_local_location()) {
+      info->download_priority_ = 0;
+      if (info->download_callback_) {
+        info->download_callback_->on_download_ok(file_id);
+        info->download_callback_.reset();
+      }
+    }
+    if (info->upload_priority_ != 0 && file_view.has_active_upload_remote_location()) {
+      info->upload_priority_ = 0;
+      if (info->upload_callback_) {
+        info->upload_callback_->on_upload_ok(file_id, nullptr);
+        info->upload_callback_.reset();
+      }
+    }
+  }
+
+  file_nodes_.erase(node_ids[other_node_i]);
+
+  run_generate(node);
+  run_download(node);
+  run_upload(node, {});
+
+  if (other_pmc_id.is_valid()) {
+    // node might not changed, but we need to merge nodes in pmc anyway
+    node->on_pmc_changed();
+  }
+  try_flush_node_full(node, node_i != remote_i, node_i != local_i, node_i != generate_i, other_pmc_id);
+
+  return node->main_file_id_;
 }
 
 void FileManager::add_file_source(FileId file_id, FileSourceId file_source_id) {
@@ -1544,7 +1805,7 @@ void FileManager::flush_to_pmc(FileNodePtr node, bool new_remote, bool new_local
 }
 
 FileNode *FileManager::get_file_node_raw(FileId file_id, FileNodeId *file_node_id) {
-  if (file_id.get() <= 0 || file_id.get() >= static_cast<int32>(file_id_info_.size())) {
+  if (file_id.get() <= 0) {
     return nullptr;
   }
   FileNodeId node_id = file_id_info_[file_id.get()].node_id_;
@@ -1554,7 +1815,7 @@ FileNode *FileManager::get_file_node_raw(FileId file_id, FileNodeId *file_node_i
   if (file_node_id != nullptr) {
     *file_node_id = node_id;
   }
-  return file_nodes_[node_id].get();
+  return &file_nodes_[node_id];
 }
 
 FileNodePtr FileManager::get_sync_file_node(FileId file_id) {
@@ -2931,29 +3192,15 @@ string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::Inp
 }
 
 FileId FileManager::next_file_id() {
-  if (!empty_file_ids_.empty()) {
-    auto res = empty_file_ids_.back();
-    empty_file_ids_.pop_back();
-    file_id_insert_time_[res] = std::time(nullptr);
-    return FileId{res, 0};
-  }
-
-  auto id = (int32) file_id_info_.size();
+  auto id = file_id_seqno++;
   FileId res(static_cast<int32>(id), 0);
-  file_id_insert_time_[id] = std::time(nullptr);
-  file_id_info_.push_back({});
+  file_id_info_[id] = {};
+  res.set_time();
   return res;
 }
 
 FileManager::FileNodeId FileManager::next_file_node_id() {
-  if (!empty_node_ids_.empty()) {
-    auto res = empty_node_ids_.back();
-    empty_node_ids_.pop_back();
-    return static_cast<FileNodeId>(res);
-  }
-
-  auto res = static_cast<FileNodeId>(file_nodes_.size());
-  file_nodes_.emplace_back(nullptr);
+  auto res = static_cast<FileNodeId>(file_node_seqno++);
   return res;
 }
 
@@ -3077,6 +3324,7 @@ void FileManager::on_download_ok(QueryId query_id, const FullLocalFileLocation &
     if (is_new) {
       context_->on_new_file(size, get_file_view(r_new_file_id.ok()).get_allocated_local_size(), 1);
     }
+    LOG_STATUS(merge(r_new_file_id.ok(), file_id));
   }
 }
 
@@ -3461,60 +3709,199 @@ void FileManager::hangup() {
   stop();
 }
 
+void FileManager::destroy_query(int32 file_id) {
+  for (auto &query_id : queries_container_.ids()) {
+    auto query = queries_container_.get(query_id);
+    if (query != nullptr && file_id == query->file_id_.fast_get()) {
+      on_error(query_id, Status::Error(400, "FILE_DOWNLOAD_RESTART"));
+    }
+  }
+}
+
+
 void FileManager::memory_cleanup() {
-  if (file_id_info_.size() > 1000 || empty_file_ids_.size() < 200) {
-    is_closed_ = true;
+  /* DESTROY OLD file_id_info_ */
+  {
+    auto it = file_id_info_.begin();
     auto time = std::time(nullptr);
+    std::vector<int32> file_to_be_deleted = {};
+
+    while (it != file_id_info_.end()) {
+      if (it->second.node_id_ != 0) {
+        auto &node = file_nodes_[it->second.node_id_];
+
+        if (time - node.main_file_id_.get_time() > FILE_TTL) {
+          auto can_reset = node.download_priority_ == 0;
+          can_reset &= node.generate_download_priority_ == 0;
+          can_reset &= node.download_id_ == 0;
+
+          if (can_reset) {
+            auto file_ids_it = node.file_ids_.begin();
+
+            while (file_ids_it != node.file_ids_.end() && can_reset) {
+              auto &file = file_id_info_[file_ids_it->fast_get()];
+              can_reset &= file.download_priority_ == 0;
+              can_reset &= time - file_ids_it->get_time() > FILE_TTL;
+              ++file_ids_it;
+            }
+          }
+
+          if (can_reset) {
+            node.main_file_id_.reset_time();
+
+            for (auto &file_id : node.file_ids_) {
+              file_id.reset_time();
 
-    for (unsigned int i = 1; i < file_id_info_.size(); i++) {
-      auto main_node_id = file_id_info_[i].node_id_;
-      if (main_node_id != 0) {
-        auto &node = file_nodes_[main_node_id];
-        if (node != nullptr && ((int32) i) == node->main_file_id_.get()) {
-          if (time - file_id_insert_time_[i] > 60 * 5 /* MAIN FILE TTL */) {
-            file_id_insert_time_[i] = INT64_MAX;
-            for (auto &file_id : node->file_ids_) {
               /* DESTROY ASSOCIATED QUERIES */
-              for (auto &query_id : queries_container_.ids()) {
-                auto query = queries_container_.get(query_id);
-                if (query != nullptr && file_id.get() == query->file_id_.get()) {
-                  on_error(query_id, Status::Error(400, "FILE_DOWNLOAD_RESTART"));
-                }
-              }
+              destroy_query(file_id.fast_get());
 
-              /* DESTROY ASSOCIATED FILE */
-              file_id_info_[file_id.get()] = FileIdInfo();
-              empty_file_ids_.push_back(file_id.get());
+              /* DESTROY ASSOCIATED LATE */
+              file_to_be_deleted.push_back(file_id.fast_get());
             }
 
             /* DESTROY MAIN QUERY */
-            for (auto &query_id : queries_container_.ids()) {
-              auto query = queries_container_.get(query_id);
-              if (query != nullptr && ((int32)i) == query->file_id_.get()) {
-                on_error(query_id, Status::Error(400, "FILE_DOWNLOAD_RESTART"));
-              }
-            }
-
-            /* DESTROY MAIN FILE */
-            empty_file_ids_.push_back(i);
-            file_id_info_[i] = FileIdInfo();
+            destroy_query(it->first);
 
             /* DESTROY MAIN NODE */
-            node.reset();
-            file_nodes_[main_node_id] = nullptr;
-            empty_node_ids_.push_back(main_node_id);
+            file_nodes_.erase(it->first);
+
+            /* DESTROY MAIN FILE LATE */
+            file_to_be_deleted.push_back(it->first);
+          }
+        }
+      } else {
+        file_to_be_deleted.push_back(it->first);
+      }
+
+      ++it;
+    }
+
+    for (auto file_id : file_to_be_deleted) {
+      context_->destroy_file_source({file_id, 0});
+      file_id_info_.erase(file_id);
+    }
+  }
+
+  /* DESTROY INVALID FILES */
+  {
+    auto it = file_id_info_.begin();
+    while (it != file_id_info_.end()) {
+      if (it->second.node_id_ != 0) {
+        if (file_nodes_[it->second.node_id_].empty) {
+          destroy_query(it->first);
+          context_->destroy_file_source({it->first, 0});
+          file_id_info_.erase(it++);
+          file_nodes_.erase(it->second.node_id_);
+        } else {
+          ++it;
+        }
+      } else {
+        ++it;
+      }
+    }
+  }
+
+  /* DESTROY INVALID file_nodes_ */
+  {
+    auto it = file_nodes_.begin();
+    while (it != file_nodes_.end()) {
+      if (it->second.empty) {
+        file_nodes_.erase(it++);
+      } else {
+        if (it->second.main_file_id_.empty()) {
+          file_nodes_.erase(it++);
+        } else {
+          if (file_id_info_[it->second.main_file_id_.get()].node_id_ == 0) {
+            for (auto &file_id : it->second.file_ids_) {
+              context_->destroy_file_source(file_id);
+              file_id_info_.erase(file_id.get());
+            }
+            file_id_info_.erase(it->second.main_file_id_.get());
+            file_nodes_.erase(it++);
+          } else {
+            ++it;
           }
         }
       }
     }
-
-    file_id_info_.shrink_to_fit();
-    empty_file_ids_.shrink_to_fit();
-    empty_node_ids_.shrink_to_fit();
-
-    LOG(INFO) << empty_file_ids_.size() << " empty ids and " <<  queries_container_.size() << " running queries";
-    is_closed_ = false;
   }
+
+  /* DESTROY INVALID file_hash_to_file_id_ */
+  {
+    auto it = file_hash_to_file_id_.begin();
+    while (it != file_hash_to_file_id_.end()) {
+      auto &file = file_id_info_[it->second.fast_get()];
+      if (file_nodes_[file.node_id_].empty) {
+        file_hash_to_file_id_.erase(it++);
+        file_nodes_.erase(file.node_id_);
+      } else {
+        ++it;
+      }
+    }
+  }
+
+  /* DESTROY INVALID local_location_to_file_id_ */
+  {
+    auto it = local_location_to_file_id_.begin();
+    while (it != local_location_to_file_id_.end()) {
+      auto &file = file_id_info_[it->second.fast_get()];
+      if (file_nodes_[file.node_id_].empty) {
+        it = local_location_to_file_id_.erase(it++);
+        file_nodes_.erase(file.node_id_);
+      } else {
+        ++it;
+      }
+    }
+  }
+
+  /* DESTROY INVALID generate_location_to_file_id_ */
+  {
+    auto it = generate_location_to_file_id_.begin();
+    while (it != generate_location_to_file_id_.end()) {
+      auto &file = file_id_info_[it->second.fast_get()];
+      if (file_nodes_[file.node_id_].empty) {
+        it = generate_location_to_file_id_.erase(it++);
+        file_nodes_.erase(file.node_id_);
+      } else {
+        ++it;
+      }
+    }
+  }
+
+  /* DESTROY INVALID remote_location_info_ */
+  {
+    auto map = remote_location_info_.get_map();
+    auto it = map.begin();
+    while (it != map.end()) {
+      auto &file = file_id_info_[it->first.file_id_.fast_get()];
+      if (file_nodes_[file.node_id_].empty) {
+        remote_location_info_.erase(it->second);
+        map.erase(it++);
+        file_nodes_.erase(file.node_id_);
+      } else {
+        ++it;
+      }
+    }
+  }
+
+  /* DESTROY NULL file_id_info_ */
+  {
+    auto it = file_id_info_.begin();
+    while (it != file_id_info_.end()) {
+      if (file_nodes_[it->second.node_id_].empty) {
+        context_->destroy_file_source({it->first, 0});
+        file_id_info_.erase(it++);
+      } else {
+        ++it;
+      }
+    }
+  }
+
+  file_nodes_.rehash(0);
+  file_hash_to_file_id_.rehash(0);
+  file_id_info_.rehash(0);
+
+  LOG(ERROR) << "registered ids: " << file_id_info_.size() << " registered nodes: " << file_nodes_.size();
 }
 
 void FileManager::tear_down() {

td/telegram/files/FileManager.h

@@ -68,6 +68,7 @@ struct NewRemoteFileLocation {
 
 class FileNode {
  public:
+  FileNode() {empty = true;}
   FileNode(LocalFileLocation local, NewRemoteFileLocation remote, unique_ptr<FullGenerateFileLocation> generate,
            int64 size, int64 expected_size, string remote_name, string url, DialogId owner_dialog_id,
            FileEncryptionKey key, FileId main_file_id, int8 main_file_id_priority)
@@ -119,6 +120,7 @@ class FileNode {
   void on_info_flushed();
 
   string suggested_name() const;
+  bool empty = false;
 
  private:
   friend class FileView;
@@ -146,7 +148,7 @@ class FileNode {
   FileEncryptionKey encryption_key_;
   FileDbId pmc_id_;
   std::vector<FileId> file_ids_;
-  FileId main_file_id_;
+  FileId main_file_id_ = {};
 
   double last_successful_force_reupload_time_ = -1e10;
 
@@ -369,6 +371,8 @@ class FileManager : public FileLoadManager::Callback {
 
     virtual void on_file_updated(FileId size) = 0;
 
+    virtual void destroy_file_source(FileId file_id) = 0;
+
     virtual bool add_file_source(FileId file_id, FileSourceId file_source_id) = 0;
 
     virtual bool remove_file_source(FileId file_id, FileSourceId file_source_id) = 0;
@@ -483,6 +487,8 @@ class FileManager : public FileLoadManager::Callback {
   static constexpr char PERSISTENT_ID_VERSION_MAP = 3;
   static constexpr char PERSISTENT_ID_VERSION = 4;
 
+  void destroy_query(int32 file_id);
+
   Result<FileId> check_input_file_id(FileType type, Result<FileId> result, bool is_encrypted, bool allow_zero,
                                      bool is_secure) TD_WARN_UNUSED_RESULT;
 
@@ -550,17 +556,17 @@ class FileManager : public FileLoadManager::Callback {
   };
   Enumerator<RemoteInfo> remote_location_info_;
 
+  FileNodeId file_node_seqno = 0;
+  int32 file_id_seqno = 0;
+
   std::unordered_map<string, FileId> file_hash_to_file_id_;
 
   std::map<FullLocalFileLocation, FileId> local_location_to_file_id_;
   std::map<FullGenerateFileLocation, FileId> generate_location_to_file_id_;
   std::map<FileDbId, int32> pmc_id_to_file_node_id_;
-  std::unordered_map<int32, int64> file_id_insert_time_;
 
-  std::vector<FileIdInfo> file_id_info_;
-  vector<int32> empty_file_ids_;
-  vector<int32> empty_node_ids_;
-  vector<unique_ptr<FileNode>> file_nodes_;
+  std::unordered_map<int32, FileIdInfo> file_id_info_;
+  std::unordered_map<FileNodeId, FileNode> file_nodes_;
   ActorOwn<FileLoadManager> file_load_manager_;
   ActorOwn<FileGenerateManager> file_generate_manager_;
 

td/telegram/misc.cpp

@@ -345,7 +345,7 @@ Result<string> check_url(Slice url) {
     return PSTRING() << (is_tg ? "tg" : "ton") << "://" << http_url.host_ << query;
   }
 
-  if (http_url.host_.find('.') == string::npos) {
+  if (http_url.host_.find('.') == string::npos && !http_url.is_ipv6_) {
     return Status::Error("Wrong HTTP URL");
   }
   return http_url.get_url();

td/telegram/net/ConnectionCreator.cpp

@@ -379,8 +379,8 @@ void ConnectionCreator::ping_proxy_resolved(int32 proxy_id, IPAddress ip_address
   CHECK(proxy.use_proxy());
   auto token = next_token();
   auto ref =
-      prepare_connection(std::move(socket_fd), proxy, extra.mtproto_ip, extra.transport_type, "Ping", extra.debug_str,
-                         nullptr, create_reference(token), false, std::move(connection_promise));
+      prepare_connection(std::move(socket_fd), proxy, extra.mtproto_ip_address, extra.transport_type, "Ping",
+                         extra.debug_str, nullptr, create_reference(token), false, std::move(connection_promise));
   if (!ref.empty()) {
     children_[token] = {false, std::move(ref)};
   }
@@ -661,9 +661,9 @@ Result<mtproto::TransportType> ConnectionCreator::get_transport_type(const Proxy
     if (!proxy.user().empty() || !proxy.password().empty()) {
       proxy_authorization = "|basic " + base64_encode(PSLICE() << proxy.user() << ':' << proxy.password());
     }
-    return mtproto::TransportType{
-        mtproto::TransportType::Http, 0,
-        mtproto::ProxySecret::from_raw(PSTRING() << info.option->get_ip_address().get_ip_str() << proxy_authorization)};
+    return mtproto::TransportType{mtproto::TransportType::Http, 0,
+                                  mtproto::ProxySecret::from_raw(
+                                      PSTRING() << info.option->get_ip_address().get_ip_host() << proxy_authorization)};
   }
 
   if (info.use_http) {
@@ -675,7 +675,7 @@ Result<mtproto::TransportType> ConnectionCreator::get_transport_type(const Proxy
 
 Result<SocketFd> ConnectionCreator::find_connection(const Proxy &proxy, const IPAddress &proxy_ip_address, DcId dc_id,
                                                     bool allow_media_only, FindConnectionExtra &extra) {
-  extra.debug_str = PSTRING() << "Failed to find valid IP for " << dc_id;
+  extra.debug_str = PSTRING() << "Failed to find valid IP address for " << dc_id;
   bool prefer_ipv6 =
       G()->shared_config().get_option_boolean("prefer_ipv6") || (proxy.use_proxy() && proxy_ip_address.is_ipv6());
   bool only_http = proxy.use_http_caching_proxy();
@@ -697,9 +697,9 @@ Result<SocketFd> ConnectionCreator::find_connection(const Proxy &proxy, const IP
   extra.check_mode |= info.should_check;
 
   if (proxy.use_proxy()) {
-    extra.mtproto_ip = info.option->get_ip_address();
+    extra.mtproto_ip_address = info.option->get_ip_address();
     extra.debug_str = PSTRING() << (proxy.use_socks5_proxy() ? "Socks5" : (only_http ? "HTTP_ONLY" : "HTTP_TCP")) << ' '
-                                << proxy_ip_address << " --> " << extra.mtproto_ip << extra.debug_str;
+                                << proxy_ip_address << " --> " << extra.mtproto_ip_address << extra.debug_str;
     VLOG(connections) << "Create: " << extra.debug_str;
     return SocketFd::open(proxy_ip_address);
   } else {
@@ -709,12 +709,10 @@ Result<SocketFd> ConnectionCreator::find_connection(const Proxy &proxy, const IP
   }
 }
 
-ActorOwn<> ConnectionCreator::prepare_connection(SocketFd socket_fd, const Proxy &proxy, const IPAddress &mtproto_ip,
-                                                 mtproto::TransportType transport_type, Slice actor_name_prefix,
-                                                 Slice debug_str,
-                                                 unique_ptr<mtproto::RawConnection::StatsCallback> stats_callback,
-                                                 ActorShared<> parent, bool use_connection_token,
-                                                 Promise<ConnectionData> promise) {
+ActorOwn<> ConnectionCreator::prepare_connection(
+    SocketFd socket_fd, const Proxy &proxy, const IPAddress &mtproto_ip_address, mtproto::TransportType transport_type,
+    Slice actor_name_prefix, Slice debug_str, unique_ptr<mtproto::RawConnection::StatsCallback> stats_callback,
+    ActorShared<> parent, bool use_connection_token, Promise<ConnectionData> promise) {
   if (proxy.use_socks5_proxy() || proxy.use_http_tcp_proxy() || transport_type.secret.emulate_tls()) {
     VLOG(connections) << "Create new transparent proxy connection " << debug_str;
     class Callback : public TransparentProxy::Callback {
@@ -765,11 +763,11 @@ ActorOwn<> ConnectionCreator::prepare_connection(SocketFd socket_fd, const Proxy
                                           !proxy.use_socks5_proxy());
     if (proxy.use_socks5_proxy()) {
       return ActorOwn<>(create_actor<Socks5>(PSLICE() << actor_name_prefix << "Socks5", std::move(socket_fd),
-                                             mtproto_ip, proxy.user().str(), proxy.password().str(),
+                                             mtproto_ip_address, proxy.user().str(), proxy.password().str(),
                                              std::move(callback), std::move(parent)));
     } else if (proxy.use_http_tcp_proxy()) {
       return ActorOwn<>(create_actor<HttpProxy>(PSLICE() << actor_name_prefix << "HttpProxy", std::move(socket_fd),
-                                                mtproto_ip, proxy.user().str(), proxy.password().str(),
+                                                mtproto_ip_address, proxy.user().str(), proxy.password().str(),
                                                 std::move(callback), std::move(parent)));
     } else if (transport_type.secret.emulate_tls()) {
       return ActorOwn<>(create_actor<mtproto::TlsInit>(
@@ -909,7 +907,7 @@ void ConnectionCreator::client_loop(ClientInfo &client) {
         td::make_unique<detail::StatsCallback>(client.is_media ? media_net_stats_callback_ : common_net_stats_callback_,
                                                actor_id(this), client.hash, extra.stat);
     auto token = next_token();
-    auto ref = prepare_connection(std::move(socket_fd), proxy, extra.mtproto_ip, extra.transport_type, Slice(),
+    auto ref = prepare_connection(std::move(socket_fd), proxy, extra.mtproto_ip_address, extra.transport_type, Slice(),
                                   extra.debug_str, std::move(stats_callback), create_reference(token), true,
                                   std::move(promise));
     if (!ref.empty()) {

td/telegram/net/ConnectionCreator.h

@@ -91,7 +91,7 @@ class ConnectionCreator : public NetQueryCallback {
 
   static DcOptions get_default_dc_options(bool is_test);
 
-  static ActorOwn<> prepare_connection(SocketFd socket_fd, const Proxy &proxy, const IPAddress &mtproto_ip,
+  static ActorOwn<> prepare_connection(SocketFd socket_fd, const Proxy &proxy, const IPAddress &mtproto_ip_address,
                                        mtproto::TransportType transport_type, Slice actor_name_prefix, Slice debug_str,
                                        unique_ptr<mtproto::RawConnection::StatsCallback> stats_callback,
                                        ActorShared<> parent, bool use_connection_token,
@@ -229,7 +229,7 @@ class ConnectionCreator : public NetQueryCallback {
     DcOptionsSet::Stat *stat{nullptr};
     mtproto::TransportType transport_type;
     string debug_str;
-    IPAddress mtproto_ip;
+    IPAddress mtproto_ip_address;
     bool check_mode{false};
   };
 

td/telegram/net/PublicRsaKeyShared.h

@@ -8,7 +8,7 @@
 
 #include "td/telegram/net/DcId.h"
 
-#include "td/mtproto/crypto.h"
+#include "td/mtproto/RSA.h"
 
 #include "td/utils/common.h"
 #include "td/utils/port/RwMutex.h"

td/telegram/net/PublicRsaKeyWatchdog.cpp

@@ -12,7 +12,7 @@
 
 #include "td/telegram/telegram_api.h"
 
-#include "td/mtproto/crypto.h"
+#include "td/mtproto/RSA.h"
 
 #include "td/utils/logging.h"
 #include "td/utils/Time.h"

td/telegram/net/Session.cpp

@@ -18,11 +18,11 @@
 #include "td/telegram/StateManager.h"
 #include "td/telegram/UniqueId.h"
 
-#include "td/mtproto/crypto.h"
 #include "td/mtproto/DhHandshake.h"
 #include "td/mtproto/Handshake.h"
 #include "td/mtproto/HandshakeActor.h"
 #include "td/mtproto/RawConnection.h"
+#include "td/mtproto/RSA.h"
 #include "td/mtproto/SessionConnection.h"
 #include "td/mtproto/TransportType.h"
 

tdnet/td/net/GetHostByNameActor.cpp

@@ -34,12 +34,7 @@ class GoogleDnsResolver : public Actor {
   double begin_time_ = 0;
 
   void start_up() override {
-    auto r_address = IPAddress::get_ipv4_address(host_);
-    if (r_address.is_ok()) {
-      promise_.set_value(r_address.move_as_ok());
-      return stop();
-    }
-    r_address = IPAddress::get_ipv6_address(host_);
+    auto r_address = IPAddress::get_ip_address(host_);
     if (r_address.is_ok()) {
       promise_.set_value(r_address.move_as_ok());
       return stop();

tdnet/td/net/HttpProxy.cpp

@@ -20,7 +20,7 @@ void HttpProxy::send_connect() {
   CHECK(state_ == State::SendConnect);
   state_ = State::WaitConnectResponse;
 
-  string host = PSTRING() << ip_address_.get_ip_str() << ':' << ip_address_.get_port();
+  string host = PSTRING() << ip_address_.get_ip_host() << ':' << ip_address_.get_port();
   string proxy_authorization;
   if (!username_.empty() || !password_.empty()) {
     auto userinfo = PSTRING() << username_ << ':' << password_;
@@ -47,7 +47,7 @@ Status HttpProxy::wait_connect_response() {
     size_t len = min(sizeof(buf), it.size());
     it.advance(len, MutableSlice{buf, sizeof(buf)});
     VLOG(proxy) << "Failed to connect: " << format::escaped(Slice(buf, len));
-    return Status::Error(PSLICE() << "Failed to connect to " << ip_address_.get_ip_str() << ':'
+    return Status::Error(PSLICE() << "Failed to connect to " << ip_address_.get_ip_host() << ':'
                                   << ip_address_.get_port());
   }
 

tdnet/td/net/HttpReader.cpp

@@ -148,6 +148,7 @@ Result<size_t> HttpReader::read_next(HttpQuery *query) {
             end_p--;
           }
 
+          CHECK(p != nullptr);
           Slice boundary(p, static_cast<size_t>(end_p - p));
           if (boundary.empty() || boundary.size() > MAX_BOUNDARY_LENGTH) {
             return Status::Error(400, "Bad Request: boundary too big or empty");

tdnet/td/net/Socks5.cpp

@@ -110,7 +110,7 @@ void Socks5::send_ip_address() {
     request += static_cast<char>((ipv4 >> 24) & 255);
   } else {
     request += '\x04';
-    request += ip_address_.get_ipv6().str();
+    request += ip_address_.get_ipv6();
   }
   auto port = ip_address_.get_port();
   request += static_cast<char>((port >> 8) & 255);

tdnet/td/net/SslStream.cpp

@@ -10,6 +10,7 @@
 #include "td/utils/common.h"
 #include "td/utils/logging.h"
 #include "td/utils/misc.h"
+#include "td/utils/port/IPAddress.h"
 #include "td/utils/port/wstring_convert.h"
 #include "td/utils/StackAllocator.h"
 #include "td/utils/Status.h"
@@ -235,6 +236,9 @@ class SslStreamImpl {
     options |= SSL_OP_NO_SSLv3;
 #endif
     SSL_CTX_set_options(ssl_ctx, options);
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+    SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_VERSION);
+#endif
     SSL_CTX_set_mode(ssl_ctx, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_ENABLE_PARTIAL_WRITE);
 
     if (cert_file.empty()) {
@@ -318,12 +322,18 @@ class SslStreamImpl {
       SSL_free(ssl_handle);
     };
 
+    auto r_ip_address = IPAddress::get_ip_address(host);
+
 #if OPENSSL_VERSION_NUMBER >= 0x10002000L
     X509_VERIFY_PARAM *param = SSL_get0_param(ssl_handle);
-    /* Enable automatic hostname checks */
-    // TODO: X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS
     X509_VERIFY_PARAM_set_hostflags(param, 0);
-    X509_VERIFY_PARAM_set1_host(param, host.c_str(), 0);
+    if (r_ip_address.is_ok()) {
+      LOG(DEBUG) << "Set verification IP address to " << r_ip_address.ok().get_ip_str();
+      X509_VERIFY_PARAM_set1_ip_asc(param, r_ip_address.ok().get_ip_str().c_str());
+    } else {
+      LOG(DEBUG) << "Set verification host to " << host;
+      X509_VERIFY_PARAM_set1_host(param, host.c_str(), 0);
+    }
 #else
 #warning DANGEROUS! HTTPS HOST WILL NOT BE CHECKED. INSTALL OPENSSL >= 1.0.2 OR IMPLEMENT HTTPS HOST CHECK MANUALLY
 #endif
@@ -333,8 +343,11 @@ class SslStreamImpl {
     SSL_set_bio(ssl_handle, bio, bio);
 
 #if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT)
-    auto host_str = host.str();
-    SSL_set_tlsext_host_name(ssl_handle, MutableCSlice(host_str).begin());
+    if (r_ip_address.is_error()) {  // IP address must not be send as SNI
+      LOG(DEBUG) << "Set SNI host name to " << host;
+      auto host_str = host.str();
+      SSL_set_tlsext_host_name(ssl_handle, MutableCSlice(host_str).begin());
+    }
 #endif
     SSL_set_connect_state(ssl_handle);
 
@@ -466,25 +479,26 @@ class SslStreamImpl {
         LOG(ERROR) << "SSL_get_error returned no error";
         return 0;
       case SSL_ERROR_ZERO_RETURN:
-        LOG(DEBUG) << "SSL_ERROR_ZERO_RETURN";
+        LOG(DEBUG) << "SSL_ZERO_RETURN";
         return 0;
       case SSL_ERROR_WANT_READ:
-        LOG(DEBUG) << "SSL_ERROR_WANT_READ";
+        LOG(DEBUG) << "SSL_WANT_READ";
         return 0;
       case SSL_ERROR_WANT_WRITE:
-        LOG(DEBUG) << "SSL_ERROR_WANT_WRITE";
+        LOG(DEBUG) << "SSL_WANT_WRITE";
         return 0;
       case SSL_ERROR_WANT_CONNECT:
       case SSL_ERROR_WANT_ACCEPT:
       case SSL_ERROR_WANT_X509_LOOKUP:
-        LOG(DEBUG) << "SSL_ERROR: CONNECT ACCEPT LOOKUP";
+        LOG(DEBUG) << "SSL: CONNECT ACCEPT LOOKUP";
         return 0;
       case SSL_ERROR_SYSCALL:
-        LOG(DEBUG) << "SSL_ERROR_SYSCALL";
         if (ERR_peek_error() == 0) {
           if (os_error.code() != 0) {
+            LOG(DEBUG) << "SSL_ERROR_SYSCALL";
             return std::move(os_error);
           } else {
+            LOG(DEBUG) << "SSL_SYSCALL";
             return 0;
           }
         }
@@ -501,6 +515,7 @@ int strm_read(BIO *b, char *buf, int len) {
   auto *stream = static_cast<SslStreamImpl *>(BIO_get_data(b));
   CHECK(stream != nullptr);
   BIO_clear_retry_flags(b);
+  CHECK(buf != nullptr);
   int res = narrow_cast<int>(stream->flow_read(MutableSlice(buf, len)));
   if (res == 0) {
     BIO_set_retry_read(b);
@@ -512,6 +527,7 @@ int strm_write(BIO *b, const char *buf, int len) {
   auto *stream = static_cast<SslStreamImpl *>(BIO_get_data(b));
   CHECK(stream != nullptr);
   BIO_clear_retry_flags(b);
+  CHECK(buf != nullptr);
   return narrow_cast<int>(stream->flow_write(Slice(buf, len)));
 }
 }  // namespace

tdutils/td/utils/Enumerator.h

@@ -19,25 +19,46 @@ class Enumerator {
  public:
   using Key = int32;
 
+  std::map<ValueT, int32> get_map() const {
+    return map_;
+  }
+
+  std::pair<Key, bool> next() {
+    if (!empty_id_.empty()) {
+      auto res = empty_id_.back();
+      empty_id_.pop_back();
+      return std::make_pair(res, true);
+    }
+
+    return std::make_pair((Key) (arr_.size() + 1), false);
+  }
+
+  void erase(Key key_y) {
+    empty_id_.push_back(key_y);
+  }
+
   Key add(ValueT v) {
-    CHECK(arr_.size() < static_cast<size_t>(std::numeric_limits<int32>::max() - 1));
-    int32 next_id = static_cast<int32>(arr_.size() + 1);
+    auto next_id = next();
     bool was_inserted;
     decltype(map_.begin()) it;
-    std::tie(it, was_inserted) = map_.emplace(std::move(v), next_id);
-    if (was_inserted) {
+    std::tie(it, was_inserted) = map_.emplace(std::move(v), next_id.first);
+    if (was_inserted && next_id.second) {
+      arr_[next_id.first - 1] = &it->first;
+    } else if (was_inserted) {
       arr_.push_back(&it->first);
+    } else if (next_id.second) {
+      empty_id_.push_back(next_id.first);
     }
     return it->second;
   }
 
   const ValueT &get(Key key) const {
-    auto pos = static_cast<size_t>(key - 1);
-    CHECK(pos < arr_.size());
+    auto pos = static_cast<Key>(key - 1);
     return *arr_[pos];
   }
 
  private:
+  std::vector<Key> empty_id_;
   std::map<ValueT, int32> map_;
   std::vector<const ValueT *> arr_;
 };

tdutils/td/utils/HttpUrl.cpp

@@ -10,6 +10,7 @@
 #include "td/utils/logging.h"
 #include "td/utils/misc.h"
 #include "td/utils/Parser.h"
+#include "td/utils/port/IPAddress.h"
 
 namespace td {
 
@@ -29,13 +30,7 @@ string HttpUrl::get_url() const {
     result += userinfo_;
     result += '@';
   }
-  if (is_ipv6_) {
-    result += '[';
-  }
   result += host_;
-  if (is_ipv6_) {
-    result += ']';
-  }
   if (specified_port_ > 0) {
     result += ':';
     result += to_string(specified_port_);
@@ -88,8 +83,11 @@ Result<HttpUrl> parse_url(Slice url, HttpUrl::Protocol default_protocol) {
 
   bool is_ipv6 = false;
   if (!host.empty() && host[0] == '[' && host.back() == ']') {
-    host.remove_prefix(1);
-    host.remove_suffix(1);
+    IPAddress ip_address;
+    if (ip_address.init_ipv6_port(host.str(), 1).is_error()) {
+      return Status::Error("Wrong IPv6 address specified in the URL");
+    }
+    CHECK(ip_address.is_ipv6());
     is_ipv6 = true;
   }
   if (host.empty()) {

tdutils/td/utils/Parser.h

@@ -45,7 +45,7 @@ class ParserImpl {
     return ptr_ == end_;
   }
   void clear() {
-    ptr_ = nullptr;
+    ptr_ = SliceT().begin();
     end_ = ptr_;
     status_ = Status::OK();
   }

tdutils/td/utils/port/IPAddress.cpp

@@ -267,11 +267,11 @@ uint32 IPAddress::get_ipv4() const {
   return htonl(ipv4_addr_.sin_addr.s_addr);
 }
 
-Slice IPAddress::get_ipv6() const {
+string IPAddress::get_ipv6() const {
   static_assert(sizeof(ipv6_addr_.sin6_addr) == 16, "ipv6 size == 16");
   CHECK(is_valid());
   CHECK(!is_ipv4());
-  return Slice(ipv6_addr_.sin6_addr.s6_addr, 16);
+  return Slice(ipv6_addr_.sin6_addr.s6_addr, 16).str();
 }
 
 IPAddress IPAddress::get_any_addr() const {
@@ -308,7 +308,12 @@ void IPAddress::init_ipv6_any() {
 Status IPAddress::init_ipv6_port(CSlice ipv6, int port) {
   is_valid_ = false;
   if (port <= 0 || port >= (1 << 16)) {
-    return Status::Error(PSLICE() << "Invalid [port=" << port << "]");
+    return Status::Error(PSLICE() << "Invalid [IPv6 address port=" << port << "]");
+  }
+  string ipv6_plain;
+  if (ipv6.size() > 2 && ipv6[0] == '[' && ipv6.back() == ']') {
+    ipv6_plain.assign(ipv6.begin() + 1, ipv6.size() - 2);
+    ipv6 = ipv6_plain;
   }
   std::memset(&ipv6_addr_, 0, sizeof(ipv6_addr_));
   ipv6_addr_.sin6_family = AF_INET6;
@@ -330,7 +335,7 @@ Status IPAddress::init_ipv6_as_ipv4_port(CSlice ipv4, int port) {
 Status IPAddress::init_ipv4_port(CSlice ipv4, int port) {
   is_valid_ = false;
   if (port <= 0 || port >= (1 << 16)) {
-    return Status::Error(PSLICE() << "Invalid [port=" << port << "]");
+    return Status::Error(PSLICE() << "Invalid [IPv4 address port=" << port << "]");
   }
   std::memset(&ipv4_addr_, 0, sizeof(ipv4_addr_));
   ipv4_addr_.sin_family = AF_INET;
@@ -345,6 +350,18 @@ Status IPAddress::init_ipv4_port(CSlice ipv4, int port) {
   return Status::OK();
 }
 
+Result<IPAddress> IPAddress::get_ip_address(CSlice host) {
+  auto r_address = get_ipv4_address(host);
+  if (r_address.is_ok()) {
+    return r_address.move_as_ok();
+  }
+  r_address = get_ipv6_address(host);
+  if (r_address.is_ok()) {
+    return r_address.move_as_ok();
+  }
+  return Status::Error("Not a valid IP address");
+}
+
 Result<IPAddress> IPAddress::get_ipv4_address(CSlice host) {
   // sometimes inet_addr allows much more valid IPv4 hosts than inet_pton,
   // like 0x12.0x34.0x56.0x78, or 0x12345678, or 0x7f.001
@@ -372,6 +389,10 @@ Result<IPAddress> IPAddress::get_ipv6_address(CSlice host) {
 }
 
 Status IPAddress::init_host_port(CSlice host, int port, bool prefer_ipv6) {
+  if (host.size() > 2 && host[0] == '[' && host.back() == ']') {
+    return init_ipv6_port(host, port == 0 ? 1 : port);
+  }
+
   return init_host_port(host, PSLICE() << port, prefer_ipv6);
 }
 
@@ -388,6 +409,11 @@ Status IPAddress::init_host_port(CSlice host, CSlice port, bool prefer_ipv6) {
   TRY_RESULT(ascii_host, idn_to_ascii(host));
   host = ascii_host;  // assign string to CSlice
 
+  if (host[0] == '[' && host.back() == ']') {
+    auto port_int = to_integer<int>(port);
+    return init_ipv6_port(host, port_int == 0 ? 1 : port_int);
+  }
+
   // some getaddrinfo implementations use inet_pton instead of inet_aton and support only decimal-dotted IPv4 form,
   // and so doesn't recognize 0x12.0x34.0x56.0x78, or 0x12345678, or 0x7f.001 as valid IPv4 addresses
   auto ipv4_numeric_addr = inet_addr(host.c_str());
@@ -494,19 +520,19 @@ Status IPAddress::init_peer_address(const SocketFd &socket_fd) {
   return Status::OK();
 }
 
-CSlice IPAddress::ipv4_to_str(uint32 ipv4) {
+string IPAddress::ipv4_to_str(uint32 ipv4) {
   ipv4 = ntohl(ipv4);
-  return ::td::get_ip_str(AF_INET, &ipv4);
+  return ::td::get_ip_str(AF_INET, &ipv4).str();
 }
 
-CSlice IPAddress::ipv6_to_str(Slice ipv6) {
+string IPAddress::ipv6_to_str(Slice ipv6) {
   CHECK(ipv6.size() == 16);
-  return ::td::get_ip_str(AF_INET6, ipv6.ubegin());
+  return ::td::get_ip_str(AF_INET6, ipv6.ubegin()).str();
 }
 
-Slice IPAddress::get_ip_str() const {
+CSlice IPAddress::get_ip_str() const {
   if (!is_valid()) {
-    return Slice("0.0.0.0");
+    return CSlice("0.0.0.0");
   }
 
   switch (get_address_family()) {
@@ -516,7 +542,23 @@ Slice IPAddress::get_ip_str() const {
       return ::td::get_ip_str(AF_INET, &ipv4_addr_.sin_addr);
     default:
       UNREACHABLE();
-      return Slice();
+      return CSlice();
+  }
+}
+
+string IPAddress::get_ip_host() const {
+  if (!is_valid()) {
+    return "0.0.0.0";
+  }
+
+  switch (get_address_family()) {
+    case AF_INET6:
+      return PSTRING() << '[' << ::td::get_ip_str(AF_INET6, &ipv6_addr_.sin6_addr) << ']';
+    case AF_INET:
+      return ::td::get_ip_str(AF_INET, &ipv4_addr_.sin_addr).str();
+    default:
+      UNREACHABLE();
+      return string();
   }
 }
 
@@ -599,12 +641,7 @@ StringBuilder &operator<<(StringBuilder &builder, const IPAddress &address) {
   if (!address.is_valid()) {
     return builder << "[invalid]";
   }
-  if (address.get_address_family() == AF_INET) {
-    return builder << "[" << address.get_ip_str() << ":" << address.get_port() << "]";
-  } else {
-    CHECK(address.get_address_family() == AF_INET6);
-    return builder << "[[" << address.get_ip_str() << "]:" << address.get_port() << "]";
-  }
+  return builder << "[" << address.get_ip_host() << ":" << address.get_port() << "]";
 }
 
 }  // namespace td

tdutils/td/utils/port/IPAddress.h

@@ -39,11 +39,20 @@ class IPAddress {
   void set_port(int port);
 
   uint32 get_ipv4() const;
-  Slice get_ipv6() const;
-  Slice get_ip_str() const;
+  string get_ipv6() const;
+
+  // returns result in a static thread-local buffer, which may be overwritten by any subsequent method call
+  CSlice get_ip_str() const;
+
+  // returns IP address as a host, i.e. IPv4 or [IPv6]
+  string get_ip_host() const;
+
+  static string ipv4_to_str(uint32 ipv4);
+  static string ipv6_to_str(Slice ipv6);
 
   IPAddress get_any_addr() const;
 
+  static Result<IPAddress> get_ip_address(CSlice host);  // host must be any IPv4 or IPv6
   static Result<IPAddress> get_ipv4_address(CSlice host);
   static Result<IPAddress> get_ipv6_address(CSlice host);
 
@@ -63,8 +72,6 @@ class IPAddress {
   const sockaddr *get_sockaddr() const;
   size_t get_sockaddr_len() const;
   int get_address_family() const;
-  static CSlice ipv4_to_str(uint32 ipv4);
-  static CSlice ipv6_to_str(Slice ipv6);
   Status init_sockaddr(sockaddr *addr);
   Status init_sockaddr(sockaddr *addr, socklen_t len) TD_WARN_UNUSED_RESULT;
 

tdutils/test/heap.cpp

@@ -10,31 +10,27 @@
 #include "td/utils/Heap.h"
 #include "td/utils/Random.h"
 
-#include <algorithm>
 #include <cstdio>
-#include <cstdlib>
 #include <set>
 #include <utility>
 
 REGISTER_TESTS(heap)
 
-using namespace td;
-
 TEST(Heap, sort_random_perm) {
   int n = 1000000;
 
-  std::vector<int> v(n);
+  td::vector<int> v(n);
   for (int i = 0; i < n; i++) {
     v[i] = i;
   }
 
   // random shuffle
   for (int i = 1; i < n; i++) {
-    std::swap(v[Random::fast(0, i)], v[i]);
+    std::swap(v[td::Random::fast(0, i)], v[i]);
   }
 
-  std::vector<HeapNode> nodes(n);
-  KHeap<int> kheap;
+  td::vector<td::HeapNode> nodes(n);
+  td::KHeap<int> kheap;
   for (int i = 0; i < n; i++) {
     kheap.insert(v[i], &nodes[i]);
   }
@@ -55,7 +51,7 @@ class CheckedHeap {
       nodes[i].value = i;
     }
   }
-  static void xx(int key, const HeapNode *heap_node) {
+  static void xx(int key, const td::HeapNode *heap_node) {
     const Node *node = static_cast<const Node *>(heap_node);
     std::fprintf(stderr, "(%d;%d)", node->key, node->value);
   }
@@ -70,9 +66,9 @@ class CheckedHeap {
   }
   int random_id() const {
     CHECK(!empty());
-    return ids[Random::fast(0, static_cast<int>(ids.size() - 1))];
+    return ids[td::Random::fast(0, static_cast<int>(ids.size() - 1))];
   }
-  size_t size() const {
+  std::size_t size() const {
     return ids.size();
   }
   bool empty() const {
@@ -144,19 +140,19 @@ class CheckedHeap {
   }
 
  private:
-  struct Node : public HeapNode {
+  struct Node : public td::HeapNode {
     Node() = default;
     Node(int key, int value) : key(key), value(value) {
     }
     int key = 0;
     int value = 0;
   };
-  vector<int> ids;
-  vector<int> rev_ids;
-  vector<int> free_ids;
-  vector<Node> nodes;
+  td::vector<int> ids;
+  td::vector<int> rev_ids;
+  td::vector<int> free_ids;
+  td::vector<Node> nodes;
   std::set<std::pair<int, int>> set_heap;
-  KHeap<int> kheap;
+  td::KHeap<int> kheap;
 };
 
 TEST(Heap, random_events) {
@@ -167,11 +163,11 @@ TEST(Heap, random_events) {
       heap.top_key();
     }
 
-    int x = Random::fast(0, 4);
+    int x = td::Random::fast(0, 4);
     if (heap.empty() || (x < 2 && heap.size() < 1000)) {
-      heap.insert(Random::fast(0, 99));
+      heap.insert(td::Random::fast(0, 99));
     } else if (x < 3) {
-      heap.fix_key(Random::fast(0, 99), heap.random_id());
+      heap.fix_key(td::Random::fast(0, 99), heap.random_id());
     } else if (x < 4) {
       heap.erase(heap.random_id());
     } else if (x < 5) {

test/http.cpp

@@ -36,7 +36,6 @@
 #include "data.h"
 
 #include <algorithm>
-#include <cstdlib>
 #include <limits>
 
 REGISTER_TESTS(http)

test/message_entities.cpp

@@ -201,7 +201,7 @@ TEST(MessageEntities, bank_card_number) {
   check_bank_card_number(" - - - - 1 - -- 234 - - 56- - 7890150000  - - - -", {"1 - -- 234 - - 56- - 7890150000"});
   check_bank_card_number("4916-3385-0608-2832; 5280 9342 8317 1080 ;345936346788903",
                          {"4916-3385-0608-2832", "5280 9342 8317 1080", "345936346788903"});
-  check_bank_card_number("4556728228023269,4916141675244747020,49161416752447470,4556728228023269",
+  check_bank_card_number("4556728228023269, 4916141675244747020, 49161416752447470, 4556728228023269",
                          {"4556728228023269", "4916141675244747020", "4556728228023269"});
   check_bank_card_number("a1234567890128", {});
   check_bank_card_number("1234567890128a", {});
@@ -210,7 +210,7 @@ TEST(MessageEntities, bank_card_number) {
   check_bank_card_number("1234567890128_", {});
   check_bank_card_number("_1234567890128", {});
   check_bank_card_number("1234567890128/", {"1234567890128"});
-  check_bank_card_number(",1234567890128", {"1234567890128"});
+  check_bank_card_number("\"1234567890128", {"1234567890128"});
   check_bank_card_number("+1234567890128", {});
 }
 
@@ -625,26 +625,35 @@ TEST(MessageEntities, fix_formatted_text) {
   str += "a  \r\n  ";
   fixed_str += "a  \n  ";
 
+  for (td::int32 i = 33; i <= 35; i++) {
+    td::vector<td::MessageEntity> entities;
+    entities.emplace_back(td::MessageEntity::Type::Pre, 0, i);
+
+    td::vector<td::MessageEntity> fixed_entities = entities;
+    fixed_entities.back().length = i - 1;
+    check_fix_formatted_text(str, entities, fixed_str, fixed_entities, true, false, false, true);
+
+    td::string expected_str = fixed_str.substr(0, 33);
+    fixed_entities.back().length = i == 33 ? 32 : 33;
+    check_fix_formatted_text(str, entities, expected_str, fixed_entities, false, false, false, false);
+  }
+
   for (td::int32 i = 33; i <= 35; i++) {
     td::vector<td::MessageEntity> entities;
     entities.emplace_back(td::MessageEntity::Type::Bold, 0, i);
 
     td::vector<td::MessageEntity> fixed_entities;
     if (i != 33) {
-      fixed_entities = entities;
-      fixed_entities.back().length = i - 1;
+      fixed_entities.emplace_back(td::MessageEntity::Type::Bold, 32, i - 33);
     }
     check_fix_formatted_text(str, entities, fixed_str, fixed_entities, true, false, false, true);
 
     td::string expected_str;
     if (i != 33) {
-      fixed_entities = entities;
-      fixed_entities.back().length = 33;
-      expected_str = fixed_str.substr(0, 33);
-    } else {
-      fixed_entities.clear();
-      expected_str = "a";
+      fixed_entities.back().offset = 0;
+      fixed_entities.back().length = 1;
     }
+    expected_str = "a";
     check_fix_formatted_text(str, entities, expected_str, fixed_entities, false, false, false, false);
   }
 
@@ -702,6 +711,12 @@ TEST(MessageEntities, fix_formatted_text) {
           while (static_cast<size_t>(fixed_offset + fixed_length) > fixed_str.size()) {
             fixed_length--;
           }
+          if (type == td::MessageEntity::Type::Bold || type == td::MessageEntity::Type::Url) {
+            while (fixed_length > 0 && (fixed_str[fixed_offset] == ' ' || fixed_str[fixed_offset] == '\n')) {
+              fixed_offset++;
+              fixed_length--;
+            }
+          }
 
           td::vector<td::MessageEntity> entities;
           entities.emplace_back(type, offset, length);
@@ -727,22 +742,29 @@ TEST(MessageEntities, fix_formatted_text) {
     for (td::int32 offset = -10; offset <= 10; offset++) {
       td::vector<td::MessageEntity> entities;
       entities.emplace_back(td::MessageEntity::Type::Bold, offset, length);
-      td::vector<td::MessageEntity> fixed_entities;
       if (length < 0 || offset < 0 || (length > 0 && static_cast<size_t>(length + offset) > str.size())) {
         check_fix_formatted_text(str, entities, true, false, false, false);
         check_fix_formatted_text(str, entities, false, false, false, true);
         continue;
       }
 
-      if (length > 0 && (length >= 2 || offset != 3)) {
-        fixed_entities.emplace_back(td::MessageEntity::Type::Bold, offset, length);
+      td::vector<td::MessageEntity> fixed_entities;
+      if (length > 0) {
+        if (offset == 3) {
+          if (length >= 2) {
+            fixed_entities.emplace_back(td::MessageEntity::Type::Bold, offset + 1, length - 1);
+          }
+        } else {
+          fixed_entities.emplace_back(td::MessageEntity::Type::Bold, offset, length);
+        }
       }
+
       check_fix_formatted_text(str, entities, str, fixed_entities, true, false, false, false);
       check_fix_formatted_text(str, entities, str, fixed_entities, false, false, false, true);
     }
   }
 
-  str = "aba caba";
+  str = "abadcaba";
   for (td::int32 length = 1; length <= 7; length++) {
     for (td::int32 offset = 0; offset <= 8 - length; offset++) {
       for (td::int32 length2 = 1; length2 <= 7; length2++) {
@@ -926,7 +948,7 @@ TEST(MessageEntities, fix_formatted_text) {
   check_fix_formatted_text("example.com a", {{td::MessageEntity::Type::Italic, 0, 13}}, "example.com a",
                            {{td::MessageEntity::Type::Url, 0, 11},
                             {td::MessageEntity::Type::Italic, 0, 11},
-                            {td::MessageEntity::Type::Italic, 11, 2}});
+                            {td::MessageEntity::Type::Italic, 12, 1}});
   check_fix_formatted_text("a example.com", {{td::MessageEntity::Type::Italic, 0, 13}}, "a example.com",
                            {{td::MessageEntity::Type::Italic, 0, 2},
                             {td::MessageEntity::Type::Url, 2, 11},

test/mtproto.cpp

@@ -10,7 +10,6 @@
 #include "td/actor/PromiseFuture.h"
 
 #include "td/mtproto/AuthData.h"
-#include "td/mtproto/crypto.h"
 #include "td/mtproto/DhHandshake.h"
 #include "td/mtproto/Handshake.h"
 #include "td/mtproto/HandshakeActor.h"
@@ -18,6 +17,7 @@
 #include "td/mtproto/PingConnection.h"
 #include "td/mtproto/ProxySecret.h"
 #include "td/mtproto/RawConnection.h"
+#include "td/mtproto/RSA.h"
 #include "td/mtproto/TlsInit.h"
 #include "td/mtproto/TransportType.h"
 
@@ -86,10 +86,13 @@ TEST(Mtproto, GetHostByNameActor) {
                                       " ",
                                       "a",
                                       "\x80",
+                                      "[]",
                                       "127.0.0.1.",
                                       "0x12.0x34.0x56.0x78",
                                       "0x7f.001",
-                                      "2001:0db8:85a3:0000:0000:8a2e:0370:7334"};
+                                      "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
+                                      "[2001:0db8:85a3:0000:0000:8a2e:0370:7334]",
+                                      "[[2001:0db8:85a3:0000:0000:8a2e:0370:7334]]"};
     for (auto types : {vector<GetHostByNameActor::ResolverType>{GetHostByNameActor::ResolverType::Native},
                        vector<GetHostByNameActor::ResolverType>{GetHostByNameActor::ResolverType::Google},
                        vector<GetHostByNameActor::ResolverType>{GetHostByNameActor::ResolverType::Google,
@@ -105,7 +108,7 @@ TEST(Mtproto, GetHostByNameActor) {
 
       for (auto host : hosts) {
         for (auto prefer_ipv6 : {false, true}) {
-          bool allow_ok = host.size() > 2;
+          bool allow_ok = host.size() > 2 && host[1] != '[';
           bool allow_both = host == "127.0.0.1." || host == "localhost" || (host == "москва.рф" && prefer_ipv6);
           bool allow_error = !allow_ok || allow_both;
           run(actor_id, host, prefer_ipv6, allow_ok, allow_error);
@@ -434,11 +437,11 @@ class Socks5TestActor : public Actor {
 
     IPAddress socks5_ip;
     socks5_ip.init_ipv4_port("131.191.89.104", 43077).ensure();
-    IPAddress mtproto_ip = get_default_ip_address();
+    IPAddress mtproto_ip_address = get_default_ip_address();
 
     auto r_socket = SocketFd::open(socks5_ip);
-    create_actor<Socks5>("socks5", r_socket.move_as_ok(), mtproto_ip, "", "", make_unique<Callback>(std::move(promise)),
-                         actor_shared())
+    create_actor<Socks5>("socks5", r_socket.move_as_ok(), mtproto_ip_address, "", "",
+                         make_unique<Callback>(std::move(promise)), actor_shared())
         .release();
   }
 

test/secret.cpp

@@ -26,6 +26,7 @@
 #include "td/tl/tl_object_parse.h"
 #include "td/tl/tl_object_store.h"
 
+#include "td/utils/as.h"
 #include "td/utils/base64.h"
 #include "td/utils/buffer.h"
 #include "td/utils/common.h"
@@ -885,20 +886,17 @@ class Master : public Actor {
   }
   void process_net_query(my_api::messages_sendEncryptedService &&message, NetQueryPtr net_query,
                          ActorShared<NetQueryCallback> callback) {
-    process_net_query_send_enrypted(std::move(message.data_), std::move(net_query), std::move(callback));
+    process_net_query_send_encrypted(std::move(message.data_), std::move(net_query), std::move(callback));
   }
   void process_net_query(my_api::messages_sendEncrypted &&message, NetQueryPtr net_query,
                          ActorShared<NetQueryCallback> callback) {
-    process_net_query_send_enrypted(std::move(message.data_), std::move(net_query), std::move(callback));
+    process_net_query_send_encrypted(std::move(message.data_), std::move(net_query), std::move(callback));
   }
-  void process_net_query_send_enrypted(BufferSlice data, NetQueryPtr net_query,
-                                       ActorShared<NetQueryCallback> callback) {
-    my_api::messages_sentEncryptedMessage sent_message;
-    sent_message.date_ = 0;
-    auto storer = TLObjectStorer<my_api::messages_sentEncryptedMessage>(sent_message);
-    BufferSlice answer(storer.size());
-    auto real_size = storer.store(answer.as_slice().ubegin());
-    CHECK(real_size == answer.size());
+  void process_net_query_send_encrypted(BufferSlice data, NetQueryPtr net_query,
+                                        ActorShared<NetQueryCallback> callback) {
+    BufferSlice answer(8);
+    answer.as_slice().fill(0);
+    as<int32>(answer.as_slice().begin()) = my_api::messages_sentEncryptedMessage::ID;
     net_query->set_ok(std::move(answer));
     send_closure(std::move(callback), &NetQueryCallback::on_result, std::move(net_query));