From d985e3cc84a13d8c0add3cdb52596f60cb0198fd Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Aug 2021 03:43:48 +0300 Subject: [PATCH 01/51] Use preincrement for iterators. --- td/telegram/MessageEntity.cpp | 8 ++++---- td/telegram/SecureManager.cpp | 8 ++++---- td/telegram/SuggestedAction.cpp | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index e4dfd459f..23b68eba9 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -1440,7 +1440,7 @@ static void remove_entities_intersecting_blockquote(vector &entit while (blockquote_it != blockquote_entities.end() && (blockquote_it->type != MessageEntity::Type::BlockQuote || blockquote_it->offset + blockquote_it->length <= entities[i].offset)) { - blockquote_it++; + ++blockquote_it; } if (blockquote_it != blockquote_entities.end() && (blockquote_it->offset + blockquote_it->length < entities[i].offset + entities[i].length || @@ -1546,17 +1546,17 @@ static vector merge_entities(vector old_entities, for (auto &old_entity : old_entities) { while (new_it != new_end && new_it->offset + new_it->length <= old_entity.offset) { result.push_back(std::move(*new_it)); - new_it++; + ++new_it; } auto old_entity_end = old_entity.offset + old_entity.length; result.push_back(std::move(old_entity)); while (new_it != new_end && new_it->offset < old_entity_end) { - new_it++; + ++new_it; } } while (new_it != new_end) { result.push_back(std::move(*new_it)); - new_it++; + ++new_it; } return result; diff --git a/td/telegram/SecureManager.cpp b/td/telegram/SecureManager.cpp index 41db3ed1b..ccecc5784 100644 --- a/td/telegram/SecureManager.cpp +++ b/td/telegram/SecureManager.cpp @@ -441,8 +441,8 @@ void SetSecureValue::start_up() { for (auto it = secure_value_.files.begin(); it != secure_value_.files.end();) { auto file_id = file_manager->get_file_view(it->file_id).file_id(); bool is_duplicate = false; - for (auto pit = secure_value_.files.begin(); pit != it; pit++) { - if (file_id == file_manager->get_file_view(pit->file_id).file_id()) { + for (auto other_it = secure_value_.files.begin(); other_it != it; ++other_it) { + if (file_id == file_manager->get_file_view(other_it->file_id).file_id()) { is_duplicate = true; break; } @@ -458,8 +458,8 @@ void SetSecureValue::start_up() { for (auto it = secure_value_.translations.begin(); it != secure_value_.translations.end();) { auto file_id = file_manager->get_file_view(it->file_id).file_id(); bool is_duplicate = file_id == front_side_file_id || file_id == reverse_side_file_id || file_id == selfie_file_id; - for (auto pit = secure_value_.translations.begin(); pit != it; pit++) { - if (file_id == file_manager->get_file_view(pit->file_id).file_id()) { + for (auto other_it = secure_value_.translations.begin(); other_it != it; ++other_it) { + if (file_id == file_manager->get_file_view(other_it->file_id).file_id()) { is_duplicate = true; break; } diff --git a/td/telegram/SuggestedAction.cpp b/td/telegram/SuggestedAction.cpp index a89e27098..2aa3026fd 100644 --- a/td/telegram/SuggestedAction.cpp +++ b/td/telegram/SuggestedAction.cpp @@ -136,8 +136,8 @@ void update_suggested_actions(vector &suggested_actions, } else if (old_it == suggested_actions.end() || *new_it < *old_it) { added_actions.push_back(*new_it++); } else { - old_it++; - new_it++; + ++old_it; + ++new_it; } } CHECK(!added_actions.empty() || !removed_actions.empty()); From 126fe070dc72b9c79a968be4a92f1db3c68cfcd4 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Aug 2021 05:09:24 +0300 Subject: [PATCH 02/51] Remove unused secure_storage::EncryptedFile class. --- td/telegram/SecureStorage.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/td/telegram/SecureStorage.h b/td/telegram/SecureStorage.h index 0a4d45979..768f47ed9 100644 --- a/td/telegram/SecureStorage.h +++ b/td/telegram/SecureStorage.h @@ -182,10 +182,6 @@ struct EncryptedValue { BufferSlice data; ValueHash hash; }; -struct EncryptedFile { - std::string path; - ValueHash hash; -}; Result encrypt_value(const Secret &secret, Slice data); Result encrypt_file(const Secret &secret, std::string src, std::string dest); From 5b4a657228da74f24dfb9777afa997b9fa305dbf Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 5 Aug 2021 01:09:39 +0300 Subject: [PATCH 03/51] Minor include fixes. --- td/telegram/SecretChatActor.cpp | 1 - td/telegram/files/FileLoaderUtils.cpp | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/td/telegram/SecretChatActor.cpp b/td/telegram/SecretChatActor.cpp index 955412e25..6c105b558 100644 --- a/td/telegram/SecretChatActor.cpp +++ b/td/telegram/SecretChatActor.cpp @@ -30,7 +30,6 @@ #include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/misc.h" -#include "td/utils/overloaded.h" #include "td/utils/Random.h" #include "td/utils/ScopeGuard.h" #include "td/utils/SliceBuilder.h" diff --git a/td/telegram/files/FileLoaderUtils.cpp b/td/telegram/files/FileLoaderUtils.cpp index c30c728b5..23e5ac973 100644 --- a/td/telegram/files/FileLoaderUtils.cpp +++ b/td/telegram/files/FileLoaderUtils.cpp @@ -19,6 +19,7 @@ #include "td/utils/port/Clocks.h" #include "td/utils/port/FileFd.h" #include "td/utils/port/path.h" +#include "td/utils/port/Stat.h" #include "td/utils/Random.h" #include "td/utils/SliceBuilder.h" #include "td/utils/StringBuilder.h" From 86c6115d4931c0ad7f68c06a7021e3343eaf147a Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 5 Aug 2021 21:29:40 +0300 Subject: [PATCH 04/51] Faster switching back from getDifference to updates. --- td/telegram/UpdatesManager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/td/telegram/UpdatesManager.cpp b/td/telegram/UpdatesManager.cpp index d644048de..51461cb3c 100644 --- a/td/telegram/UpdatesManager.cpp +++ b/td/telegram/UpdatesManager.cpp @@ -1356,9 +1356,9 @@ void UpdatesManager::on_get_difference(tl_object_ptrintermediate_state_); if (get_pts() != std::numeric_limits::max() && state->date_ == get_date() && (state->pts_ == get_pts() || - (min_postponed_update_pts_ != 0 && state->pts_ - 1000 >= min_postponed_update_pts_)) && + (min_postponed_update_pts_ != 0 && state->pts_ - 500 >= min_postponed_update_pts_)) && (state->qts_ == get_qts() || - (min_postponed_update_qts_ != 0 && state->qts_ - 1000 >= min_postponed_update_qts_))) { + (min_postponed_update_qts_ != 0 && state->qts_ - 500 >= min_postponed_update_qts_))) { on_get_updates_state(std::move(state), "get difference final slice"); VLOG(get_difference) << "Trying to switch back from getDifference to update processing"; break; From 2f5020ad0219e4a3698763d6edd73816cb5149b8 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 6 Aug 2021 06:54:44 +0300 Subject: [PATCH 05/51] Improve logging. --- td/telegram/MessagesManager.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index ac56e6edc..080ec6d54 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -28255,6 +28255,7 @@ void MessagesManager::send_update_unread_message_count(DialogList &list, DialogI } if (!from_database) { + LOG(INFO) << "Save unread message count in " << dialog_list_id; G()->td_db()->get_binlog_pmc()->set( PSTRING() << "unread_message_count" << dialog_list_id.get(), PSTRING() << list.unread_message_total_count_ << ' ' << list.unread_message_muted_count_); @@ -32968,7 +32969,7 @@ void MessagesManager::delete_message_from_database(Dialog *d, MessageId message_ if (old_message_id.is_valid()) { bool have_old_message = get_message(d, old_message_id) != nullptr; LOG(WARNING) << "Sent " << FullMessageId{d->dialog_id, message_id} - << " was deleted before it was received. Have old message = " << have_old_message; + << " was deleted before it was received. Have old " << old_message_id << " = " << have_old_message; send_closure_later(actor_id(this), &MessagesManager::delete_messages, d->dialog_id, vector{old_message_id}, false, Promise()); delete_update_message_id(d->dialog_id, message_id); From 5a8f8fbe8289dbbb08ec098759d8546d771fa3f7 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 6 Aug 2021 07:07:03 +0300 Subject: [PATCH 06/51] Support awaited updates with promise in process_get_channel_difference_updates. --- td/telegram/MessagesManager.cpp | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 080ec6d54..8b7bf6a08 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -35563,7 +35563,11 @@ void MessagesManager::process_get_channel_difference_updates( // extract awaited sent messages, which were edited or deleted after that auto postponed_updates_it = postponed_channel_updates_.find(dialog_id); - std::map> awaited_messages; + struct AwaitedMessage { + tl_object_ptr message; + Promise promise; + }; + std::map awaited_messages; if (postponed_updates_it != postponed_channel_updates_.end()) { auto &updates = postponed_updates_it->second; while (!updates.empty()) { @@ -35577,14 +35581,17 @@ void MessagesManager::process_get_channel_difference_updates( auto promise = std::move(it->second.promise); updates.erase(it); - if (!promise && update->get_id() == telegram_api::updateNewChannelMessage::ID) { + if (update->get_id() == telegram_api::updateNewChannelMessage::ID) { auto update_new_channel_message = static_cast(update.get()); auto message_id = get_message_id(update_new_channel_message->message_, false); FullMessageId full_message_id(dialog_id, message_id); if (update_message_ids_.find(full_message_id) != update_message_ids_.end() && changed_message_ids.find(message_id) != changed_message_ids.end()) { changed_message_ids.erase(message_id); - awaited_messages.emplace(message_id, std::move(update_new_channel_message->message_)); + AwaitedMessage awaited_message; + awaited_message.message = std::move(update_new_channel_message->message_); + awaited_message.promise = std::move(promise); + awaited_messages.emplace(message_id, std::move(awaited_message)); continue; } } @@ -35605,21 +35612,28 @@ void MessagesManager::process_get_channel_difference_updates( for (auto &message : new_messages) { auto message_id = get_message_id(message, false); while (it != awaited_messages.end() && it->first < message_id) { - on_get_message(std::move(it->second), true, true, false, true, true, "postponed channel update"); + on_get_message(std::move(it->second.message), true, true, false, true, true, "postponed channel update"); + it->second.promise.set_value(Unit()); ++it; } + Promise promise; if (it != awaited_messages.end() && it->first == message_id) { if (is_edited_message(message)) { // the new message is edited, apply postponed one and move this to updateEditChannelMessage other_updates.push_back(make_tl_object(std::move(message), new_pts, 0)); - message = std::move(it->second); + message = std::move(it->second.message); + promise = std::move(it->second.promise); + } else { + it->second.promise.set_value(Unit()); } ++it; } on_get_message(std::move(message), true, true, false, true, true, "get channel difference"); + promise.set_value(Unit()); } while (it != awaited_messages.end()) { - on_get_message(std::move(it->second), true, true, false, true, true, "postponed channel update 2"); + on_get_message(std::move(it->second.message), true, true, false, true, true, "postponed channel update 2"); + it->second.promise.set_value(Unit()); ++it; } From 6c3294684ce4a24e81ce79412fbe380205beb169 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 7 Aug 2021 08:46:56 +0300 Subject: [PATCH 07/51] Fix parsing of PhotoSizeSource::FullLegacy. --- td/telegram/PhotoSizeSource.hpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/td/telegram/PhotoSizeSource.hpp b/td/telegram/PhotoSizeSource.hpp index fb3c952a1..838a70439 100644 --- a/td/telegram/PhotoSizeSource.hpp +++ b/td/telegram/PhotoSizeSource.hpp @@ -109,9 +109,7 @@ void parse(PhotoSizeSource::FullLegacy &source, ParserT &parser) { parse(source.volume_id, parser); parse(source.secret, parser); parse(source.local_id, parser); - if (source.local_id < 0) { - parser.set_error("Wrong local_id"); - } + // source.local_id can be negative in secret chat thumbnails } template From 3c7cf8421600beead16e87a51de9885ef890a848 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 7 Aug 2021 09:10:28 +0300 Subject: [PATCH 08/51] Don't call ExportChannelMessageLinkQuery for bots. --- td/telegram/MessagesManager.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 8b7bf6a08..17659dbd3 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -17345,8 +17345,10 @@ Result> MessagesManager::get_message_link(FullMessageId } } - td_->create_handler(Promise()) - ->send(dialog_id.get_channel_id(), m->message_id, for_group, true); + if (!td_->auth_manager_->is_bot()) { + td_->create_handler(Promise()) + ->send(dialog_id.get_channel_id(), m->message_id, for_group, true); + } SliceBuilder sb; sb << G()->shared_config().get_option_string("t_me_url", "https://t.me/"); From 6c0a5823b5c1f13d9f1b677f6cad096266e6c989 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 7 Aug 2021 09:59:49 +0300 Subject: [PATCH 09/51] Remove mentions of deprecated public interfaces from README. --- README.md | 8 +++----- example/python/README.md | 4 ++-- example/swift/README.md | 4 ++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 3de02bd61..f833b6a77 100644 --- a/README.md +++ b/README.md @@ -131,11 +131,9 @@ For C++ projects that use CMake, the best approach is to build `TDLib` as part o There are several libraries that you could use in your CMake project: * Td::TdJson, Td::TdJsonStatic — dynamic and static version of a JSON interface. This has a simple C interface, so it can be easily used with any programming language that is able to execute C functions. - See [td_json_client](https://core.telegram.org/tdlib/docs/td__json__client_8h.html) and [td_log](https://core.telegram.org/tdlib/docs/td__log_8h.html) documentation for more information. + See [td_json_client](https://core.telegram.org/tdlib/docs/td__json__client_8h.html) documentation for more information. * Td::TdStatic — static library with C++ interface for general usage. - See [Client](https://core.telegram.org/tdlib/docs/classtd_1_1_client.html) and [Log](https://core.telegram.org/tdlib/docs/classtd_1_1_log.html) documentation for more information. -* Td::TdCoreStatic — static library with low-level C++ interface intended mostly for internal usage. - See [ClientActor](https://core.telegram.org/tdlib/docs/classtd_1_1_client_actor.html) and [Log](https://core.telegram.org/tdlib/docs/classtd_1_1_log.html) documentation for more information. + See [ClientManager](https://core.telegram.org/tdlib/docs/classtd_1_1_client_manager.html) and [Client](https://core.telegram.org/tdlib/docs/classtd_1_1_client.html) documentation for more information. For example, part of your CMakeLists.txt may look like this: ``` @@ -173,7 +171,7 @@ git checkout td/telegram/Client.h td/telegram/Log.h td/tl/TlObject.h ## Using from other programming languages `TDLib` provides efficient native C++, Java, and .NET interfaces. But for most use cases we suggest to use the JSON interface, which can be easily used with any programming language that is able to execute C functions. -See [td_json_client](https://core.telegram.org/tdlib/docs/td__json__client_8h.html) and [td_log](https://core.telegram.org/tdlib/docs/td__log_8h.html) documentation for detailed JSON interface description, +See [td_json_client](https://core.telegram.org/tdlib/docs/td__json__client_8h.html) documentation for detailed JSON interface description, the [td_api.tl](https://github.com/tdlib/td/blob/master/td/generate/scheme/td_api.tl) scheme or the automatically generated [HTML documentation](https://core.telegram.org/tdlib/docs/td__api_8h.html) for a list of all available `TDLib` [methods](https://core.telegram.org/tdlib/docs/classtd_1_1td__api_1_1_function.html) and [classes](https://core.telegram.org/tdlib/docs/classtd_1_1td__api_1_1_object.html). diff --git a/example/python/README.md b/example/python/README.md index c4a4ad376..c3c171d90 100644 --- a/example/python/README.md +++ b/example/python/README.md @@ -7,5 +7,5 @@ Then you can run the example: python tdjson_example.py ``` -Description of all available classes and methods can be found at [td_json_client](https://core.telegram.org/tdlib/docs/td__json__client_8h.html), -[td_log](https://core.telegram.org/tdlib/docs/td__log_8h.html) and [td_api](https://core.telegram.org/tdlib/docs/td__api_8h.html) documentation. +Description of all available classes and methods can be found at [td_json_client](https://core.telegram.org/tdlib/docs/td__json__client_8h.html) +and [td_api](https://core.telegram.org/tdlib/docs/td__api_8h.html) documentation. diff --git a/example/swift/README.md b/example/swift/README.md index 113a606bb..7c57dcf64 100644 --- a/example/swift/README.md +++ b/example/swift/README.md @@ -11,5 +11,5 @@ cmake --build . --target install Then you can open and build the example with the latest Xcode. -Description of all available classes and methods can be found at [td_json_client](https://core.telegram.org/tdlib/docs/td__json__client_8h.html), -[td_log](https://core.telegram.org/tdlib/docs/td__log_8h.html) and [td_api](https://core.telegram.org/tdlib/docs/td__api_8h.html) documentation. +Description of all available classes and methods can be found at [td_json_client](https://core.telegram.org/tdlib/docs/td__json__client_8h.html) +and [td_api](https://core.telegram.org/tdlib/docs/td__api_8h.html) documentation. From a4b35648aeca1de42b1bdc22ec68dc64551a34f6 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 8 Aug 2021 11:14:26 +0300 Subject: [PATCH 10/51] Remove stickers_emoji_cache_time key from appConfig. --- td/telegram/ConfigManager.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/td/telegram/ConfigManager.cpp b/td/telegram/ConfigManager.cpp index 4d767ae38..a620ad5dd 100644 --- a/td/telegram/ConfigManager.cpp +++ b/td/telegram/ConfigManager.cpp @@ -1485,7 +1485,8 @@ void ConfigManager::process_app_config(tl_object_ptr &c for (auto &key_value : static_cast(config.get())->value_) { Slice key = key_value->key_; telegram_api::JSONValue *value = key_value->value_.get(); - if (key == "test" || key == "wallet_enabled" || key == "wallet_blockchain_name" || key == "wallet_config") { + if (key == "test" || key == "wallet_enabled" || key == "wallet_blockchain_name" || key == "wallet_config" || + key == "stickers_emoji_cache_time") { continue; } if (key == "ignore_restriction_reasons") { From 5c0fb8874236c7d3e68160d2cc1b515d88839e41 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 8 Aug 2021 11:26:51 +0300 Subject: [PATCH 11/51] Move GetDeepLinkInfoQuery to LinkManager. --- td/telegram/LinkManager.cpp | 66 +++++++++++++++++++++++++++++++++++++ td/telegram/LinkManager.h | 2 ++ td/telegram/Td.cpp | 63 +---------------------------------- 3 files changed, 69 insertions(+), 62 deletions(-) diff --git a/td/telegram/LinkManager.cpp b/td/telegram/LinkManager.cpp index e68063f3d..55f252d8b 100644 --- a/td/telegram/LinkManager.cpp +++ b/td/telegram/LinkManager.cpp @@ -16,6 +16,7 @@ #include "td/telegram/MessageEntity.h" #include "td/telegram/MessageId.h" #include "td/telegram/MessagesManager.h" +#include "td/telegram/misc.h" #include "td/telegram/ServerMessageId.h" #include "td/telegram/Td.h" #include "td/telegram/TdDb.h" @@ -329,6 +330,55 @@ class LinkManager::InternalLinkVoiceChat final : public InternalLink { } }; +class GetDeepLinkInfoQuery final : public Td::ResultHandler { + Promise> promise_; + + public: + explicit GetDeepLinkInfoQuery(Promise> &&promise) + : promise_(std::move(promise)) { + } + + void send(Slice link) { + send_query(G()->net_query_creator().create_unauth(telegram_api::help_getDeepLinkInfo(link.str()))); + } + + void on_result(uint64 id, BufferSlice packet) final { + auto result_ptr = fetch_result(packet); + if (result_ptr.is_error()) { + return on_error(id, result_ptr.move_as_error()); + } + + auto result = result_ptr.move_as_ok(); + switch (result->get_id()) { + case telegram_api::help_deepLinkInfoEmpty::ID: + return promise_.set_value(nullptr); + case telegram_api::help_deepLinkInfo::ID: { + auto info = telegram_api::move_object_as(result); + bool need_update = (info->flags_ & telegram_api::help_deepLinkInfo::UPDATE_APP_MASK) != 0; + + auto entities = get_message_entities(nullptr, std::move(info->entities_), "GetDeepLinkInfoQuery"); + auto status = fix_formatted_text(info->message_, entities, true, true, true, true); + if (status.is_error()) { + LOG(ERROR) << "Receive error " << status << " while parsing deep link info " << info->message_; + if (!clean_input_string(info->message_)) { + info->message_.clear(); + } + entities = find_entities(info->message_, true); + } + FormattedText text{std::move(info->message_), std::move(entities)}; + return promise_.set_value( + td_api::make_object(get_formatted_text_object(text, true), need_update)); + } + default: + UNREACHABLE(); + } + } + + void on_error(uint64 id, Status status) final { + promise_.set_error(std::move(status)); + } +}; + class RequestUrlAuthQuery final : public Td::ResultHandler { Promise> promise_; string url_; @@ -1008,6 +1058,22 @@ void LinkManager::update_autologin_domains(string autologin_token, vector> &&promise) { + Slice link_scheme("tg:"); + if (begins_with(link, link_scheme)) { + link.remove_prefix(link_scheme.size()); + if (begins_with(link, "//")) { + link.remove_prefix(2); + } + } + size_t pos = 0; + while (pos < link.size() && link[pos] != '/' && link[pos] != '?' && link[pos] != '#') { + pos++; + } + link.truncate(pos); + td_->create_handler(std::move(promise))->send(link); +} + void LinkManager::get_external_link_info(string &&link, Promise> &&promise) { auto default_result = td_api::make_object(link, false); if (G()->close_flag()) { diff --git a/td/telegram/LinkManager.h b/td/telegram/LinkManager.h index 8ae0cccc5..6a58bacf2 100644 --- a/td/telegram/LinkManager.h +++ b/td/telegram/LinkManager.h @@ -54,6 +54,8 @@ class LinkManager final : public Actor { void update_autologin_domains(string autologin_token, vector autologin_domains, vector url_auth_domains); + void get_deep_link_info(Slice link, Promise> &&promise); + void get_external_link_info(string &&link, Promise> &&promise); void get_login_url_info(FullMessageId full_message_id, int32 button_id, diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index 49b39095d..33f7509c8 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -436,67 +436,6 @@ class GetInviteTextQuery final : public Td::ResultHandler { } }; -class GetDeepLinkInfoQuery final : public Td::ResultHandler { - Promise> promise_; - - public: - explicit GetDeepLinkInfoQuery(Promise> &&promise) - : promise_(std::move(promise)) { - } - - void send(Slice link) { - Slice link_scheme("tg:"); - if (begins_with(link, link_scheme)) { - link.remove_prefix(link_scheme.size()); - if (begins_with(link, "//")) { - link.remove_prefix(2); - } - } - size_t pos = 0; - while (pos < link.size() && link[pos] != '/' && link[pos] != '?' && link[pos] != '#') { - pos++; - } - link.truncate(pos); - send_query(G()->net_query_creator().create_unauth(telegram_api::help_getDeepLinkInfo(link.str()))); - } - - void on_result(uint64 id, BufferSlice packet) final { - auto result_ptr = fetch_result(packet); - if (result_ptr.is_error()) { - return on_error(id, result_ptr.move_as_error()); - } - - auto result = result_ptr.move_as_ok(); - switch (result->get_id()) { - case telegram_api::help_deepLinkInfoEmpty::ID: - return promise_.set_value(nullptr); - case telegram_api::help_deepLinkInfo::ID: { - auto info = telegram_api::move_object_as(result); - bool need_update = (info->flags_ & telegram_api::help_deepLinkInfo::UPDATE_APP_MASK) != 0; - - auto entities = get_message_entities(nullptr, std::move(info->entities_), "GetDeepLinkInfoQuery"); - auto status = fix_formatted_text(info->message_, entities, true, true, true, true); - if (status.is_error()) { - LOG(ERROR) << "Receive error " << status << " while parsing deep link info " << info->message_; - if (!clean_input_string(info->message_)) { - info->message_.clear(); - } - entities = find_entities(info->message_, true); - } - FormattedText text{std::move(info->message_), std::move(entities)}; - return promise_.set_value( - td_api::make_object(get_formatted_text_object(text, true), need_update)); - } - default: - UNREACHABLE(); - } - } - - void on_error(uint64 id, Status status) final { - promise_.set_error(std::move(status)); - } -}; - class SaveAppLogQuery final : public Td::ResultHandler { Promise promise_; @@ -8217,7 +8156,7 @@ void Td::on_request(uint64 id, const td_api::getApplicationDownloadLink &request void Td::on_request(uint64 id, td_api::getDeepLinkInfo &request) { CLEAN_INPUT_STRING(request.link_); CREATE_REQUEST_PROMISE(); - create_handler(std::move(promise))->send(request.link_); + link_manager_->get_deep_link_info(request.link_, std::move(promise)); } void Td::on_request(uint64 id, const td_api::getApplicationConfig &request) { From 6766eb81c3a8bda7dd14407dfbb8a631fba7ab89 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 9 Aug 2021 16:18:59 +0300 Subject: [PATCH 12/51] Unify limit documentation. --- td/generate/scheme/td_api.tl | 40 ++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index 1dec08a74..f1cd485e6 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -4017,10 +4017,10 @@ getFile file_id:int32 = File; getRemoteFile remote_file_id:string file_type:FileType = File; //@description Returns an ordered list of chats in a chat list. Chats are sorted by the pair (chat.position.order, chat.id) in descending order. (For example, to get a list of chats from the beginning, the offset_order should be equal to a biggest signed 64-bit number 9223372036854775807 == 2^63 - 1). -//-For optimal performance the number of returned chats is chosen by the library +//-For optimal performance, the number of returned chats is chosen by TDLib //@chat_list The chat list in which to return chats //@offset_order Chat order to return chats from @offset_chat_id Chat identifier to return chats from -//@limit The maximum number of chats to be returned. It is possible that fewer chats than the limit are returned even if the end of the list is not reached +//@limit The maximum number of chats to be returned. For optimal performance, the number of returned chats is chosen by TDLib and can be smaller than the specified limit, even if the end of the list is not reached getChats chat_list:ChatList offset_order:int64 offset_chat_id:int53 limit:int32 = Chats; //@description Searches a public chat by its username. Currently only private chats, supergroups and channels can be public. Returns the chat if found; otherwise an error is returned @username Username to be resolved @@ -4074,21 +4074,21 @@ getGroupsInCommon user_id:int32 offset_chat_id:int53 limit:int32 = Chats; //@description Returns messages in a chat. The messages are returned in a reverse chronological order (i.e., in order of decreasing message_id). -//-For optimal performance the number of returned messages is chosen by the library. This is an offline request if only_local is true +//-For optimal performance, the number of returned messages is chosen by TDLib. This is an offline request if only_local is true //@chat_id Chat identifier //@from_message_id Identifier of the message starting from which history must be fetched; use 0 to get results from the last message //@offset Specify 0 to get results from exactly the from_message_id or a negative offset up to 99 to get additionally some newer messages -//@limit The maximum number of messages to be returned; must be positive and can't be greater than 100. If the offset is negative, the limit must be greater than or equal to -offset. Fewer messages may be returned than specified by the limit, even if the end of the message history has not been reached +//@limit The maximum number of messages to be returned; must be positive and can't be greater than 100. If the offset is negative, the limit must be greater than or equal to -offset. For optimal performance, the number of returned messages is chosen by TDLib and can be smaller than the specified limit //@only_local If true, returns only messages that are available locally without sending network requests getChatHistory chat_id:int53 from_message_id:int53 offset:int32 limit:int32 only_local:Bool = Messages; //@description Returns messages in a message thread of a message. Can be used only if message.can_get_message_thread == true. Message thread of a channel message is in the channel's linked supergroup. -//-The messages are returned in a reverse chronological order (i.e., in order of decreasing message_id). For optimal performance the number of returned messages is chosen by the library +//-The messages are returned in a reverse chronological order (i.e., in order of decreasing message_id). For optimal performance, the number of returned messages is chosen by TDLib //@chat_id Chat identifier //@message_id Message identifier, which thread history needs to be returned //@from_message_id Identifier of the message starting from which history must be fetched; use 0 to get results from the last message //@offset Specify 0 to get results from exactly the from_message_id or a negative offset up to 99 to get additionally some newer messages -//@limit The maximum number of messages to be returned; must be positive and can't be greater than 100. If the offset is negative, the limit must be greater than or equal to -offset. Fewer messages may be returned than specified by the limit, even if the end of the message thread history has not been reached +//@limit The maximum number of messages to be returned; must be positive and can't be greater than 100. If the offset is negative, the limit must be greater than or equal to -offset. For optimal performance, the number of returned messages is chosen by TDLib and can be smaller than the specified limit getMessageThreadHistory chat_id:int53 message_id:int53 from_message_id:int53 offset:int32 limit:int32 = Messages; //@description Deletes all messages in the chat. Use Chat.can_be_deleted_only_for_self and Chat.can_be_deleted_for_all_users fields to find whether and how the method can be applied to the chat @@ -4099,41 +4099,41 @@ deleteChatHistory chat_id:int53 remove_from_chat_list:Bool revoke:Bool = Ok; deleteChat chat_id:int53 = Ok; //@description Searches for messages with given words in the chat. Returns the results in reverse chronological order, i.e. in order of decreasing message_id. Cannot be used in secret chats with a non-empty query -//-(searchSecretMessages should be used instead), or without an enabled message database. For optimal performance the number of returned messages is chosen by the library +//-(searchSecretMessages should be used instead), or without an enabled message database. For optimal performance, the number of returned messages is chosen by TDLib and can be smaller than the specified limit //@chat_id Identifier of the chat in which to search messages //@query Query to search for //@sender If not null, only messages sent by the specified sender will be returned. Not supported in secret chats //@from_message_id Identifier of the message starting from which history must be fetched; use 0 to get results from the last message //@offset Specify 0 to get results from exactly the from_message_id or a negative offset to get the specified message and some newer messages -//@limit The maximum number of messages to be returned; must be positive and can't be greater than 100. If the offset is negative, the limit must be greater than -offset. Fewer messages may be returned than specified by the limit, even if the end of the message history has not been reached +//@limit The maximum number of messages to be returned; must be positive and can't be greater than 100. If the offset is negative, the limit must be greater than -offset. For optimal performance, the number of returned messages is chosen by TDLib and can be smaller than the specified limit //@filter Filter for message content in the search results //@message_thread_id If not 0, only messages in the specified thread will be returned; supergroups only searchChatMessages chat_id:int53 query:string sender:MessageSender from_message_id:int53 offset:int32 limit:int32 filter:SearchMessagesFilter message_thread_id:int53 = Messages; //@description Searches for messages in all chats except secret chats. Returns the results in reverse chronological order (i.e., in order of decreasing (date, chat_id, message_id)). -//-For optimal performance the number of returned messages is chosen by the library +//-For optimal performance, the number of returned messages is chosen by TDLib and can be smaller than the specified limit //@chat_list Chat list in which to search messages; pass null to search in all chats regardless of their chat list. Only Main and Archive chat lists are supported //@query Query to search for //@offset_date The date of the message starting from which the results should be fetched. Use 0 or any date in the future to get results from the last message //@offset_chat_id The chat identifier of the last found message, or 0 for the first request //@offset_message_id The message identifier of the last found message, or 0 for the first request -//@limit The maximum number of messages to be returned; up to 100. Fewer messages may be returned than specified by the limit, even if the end of the message history has not been reached +//@limit The maximum number of messages to be returned; up to 100. For optimal performance, the number of returned messages is chosen by TDLib and can be smaller than the specified limit //@filter Filter for message content in the search results; searchMessagesFilterCall, searchMessagesFilterMissedCall, searchMessagesFilterMention, searchMessagesFilterUnreadMention, searchMessagesFilterFailedToSend and searchMessagesFilterPinned are unsupported in this function //@min_date If not 0, the minimum date of the messages to return //@max_date If not 0, the maximum date of the messages to return searchMessages chat_list:ChatList query:string offset_date:int32 offset_chat_id:int53 offset_message_id:int53 limit:int32 filter:SearchMessagesFilter min_date:int32 max_date:int32 = Messages; -//@description Searches for messages in secret chats. Returns the results in reverse chronological order. For optimal performance the number of returned messages is chosen by the library +//@description Searches for messages in secret chats. Returns the results in reverse chronological order. For optimal performance, the number of returned messages is chosen by TDLib //@chat_id Identifier of the chat in which to search. Specify 0 to search in all secret chats //@query Query to search for. If empty, searchChatMessages should be used instead //@offset Offset of the first entry to return as received from the previous request; use empty string to get first chunk of results -//@limit The maximum number of messages to be returned; up to 100. Fewer messages may be returned than specified by the limit, even if the end of the message history has not been reached +//@limit The maximum number of messages to be returned; up to 100. For optimal performance, the number of returned messages is chosen by TDLib and can be smaller than the specified limit //@filter A filter for message content in the search results searchSecretMessages chat_id:int53 query:string offset:string limit:int32 filter:SearchMessagesFilter = FoundMessages; -//@description Searches for call messages. Returns the results in reverse chronological order (i. e., in order of decreasing message_id). For optimal performance the number of returned messages is chosen by the library +//@description Searches for call messages. Returns the results in reverse chronological order (i. e., in order of decreasing message_id). For optimal performance, the number of returned messages is chosen by TDLib //@from_message_id Identifier of the message from which to search; use 0 to get results from the last message -//@limit The maximum number of messages to be returned; up to 100. Fewer messages may be returned than specified by the limit, even if the end of the message history has not been reached @only_missed If true, returns only messages with missed calls +//@limit The maximum number of messages to be returned; up to 100. For optimal performance, the number of returned messages is chosen by TDLib and can be smaller than the specified limit @only_missed If true, returns only messages with missed calls searchCallMessages from_message_id:int53 limit:int32 only_missed:Bool = Messages; //@description Deletes all call messages @revoke Pass true to delete the messages for all users @@ -4154,11 +4154,11 @@ getChatMessageCount chat_id:int53 filter:SearchMessagesFilter return_local:Bool //@description Returns all scheduled messages in a chat. The messages are returned in a reverse chronological order (i.e., in order of decreasing message_id) @chat_id Chat identifier getChatScheduledMessages chat_id:int53 = Messages; -//@description Returns forwarded copies of a channel message to different public channels. For optimal performance the number of returned messages is chosen by the library +//@description Returns forwarded copies of a channel message to different public channels. For optimal performance, the number of returned messages is chosen by TDLib //@chat_id Chat identifier of the message //@message_id Message identifier //@offset Offset of the first entry to return as received from the previous request; use empty string to get first chunk of results -//@limit The maximum number of messages to be returned; must be positive and can't be greater than 100. Fewer messages may be returned than specified by the limit, even if the end of the list has not been reached +//@limit The maximum number of messages to be returned; must be positive and can't be greater than 100. For optimal performance, the number of returned messages is chosen by TDLib and can be smaller than the specified limit getMessagePublicForwards chat_id:int53 message_id:int53 offset:string limit:int32 = FoundMessages; @@ -4331,11 +4331,11 @@ getJsonString json_value:JsonValue = Text; //@option_ids 0-based identifiers of answer options, chosen by the user. User can choose more than 1 answer option only is the poll allows multiple answers setPollAnswer chat_id:int53 message_id:int53 option_ids:vector = Ok; -//@description Returns users voted for the specified option in a non-anonymous polls. For the optimal performance the number of returned users is chosen by the library +//@description Returns users voted for the specified option in a non-anonymous polls. For optimal performance, the number of returned users is chosen by TDLib //@chat_id Identifier of the chat to which the poll belongs @message_id Identifier of the message containing the poll //@option_id 0-based identifier of the answer option //@offset Number of users to skip in the result; must be non-negative -//@limit The maximum number of users to be returned; must be positive and can't be greater than 50. Fewer users may be returned than specified by the limit, even if the end of the voter list has not been reached +//@limit The maximum number of users to be returned; must be positive and can't be greater than 50. For optimal performance, the number of returned users is chosen by TDLib and can be smaller than the specified limit, even if the end of the voter list has not been reached getPollVoters chat_id:int53 message_id:int53 option_id:int32 offset:int32 limit:int32 = Users; //@description Stops a poll. A poll in a message can be stopped when the message has can_be_edited flag set @@ -4935,9 +4935,9 @@ getInstalledStickerSets is_masks:Bool = StickerSets; //@description Returns a list of archived sticker sets @is_masks Pass true to return mask stickers sets; pass false to return ordinary sticker sets @offset_sticker_set_id Identifier of the sticker set from which to return the result @limit The maximum number of sticker sets to return getArchivedStickerSets is_masks:Bool offset_sticker_set_id:int64 limit:int32 = StickerSets; -//@description Returns a list of trending sticker sets. For the optimal performance the number of returned sticker sets is chosen by the library +//@description Returns a list of trending sticker sets. For optimal performance, the number of returned sticker sets is chosen by TDLib //@offset The offset from which to return the sticker sets; must be non-negative -//@limit The maximum number of sticker sets to be returned; must be non-negative. Fewer sticker sets may be returned than specified by the limit, even if the end of the list has not been reached +//@limit The maximum number of sticker sets to be returned; must be non-negative. For optimal performance, the number of returned sticker sets is chosen by TDLib and can be smaller than the specified limit, even if the end of the list has not been reached getTrendingStickerSets offset:int32 limit:int32 = StickerSets; //@description Returns a list of sticker sets attached to a file. Currently only photos and videos can have attached sticker sets @file_id File identifier From ab71d6a200a0c6c068c5a2564e6e8fa6f7cc190d Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 26 Jul 2021 07:53:36 +0300 Subject: [PATCH 13/51] Add textEntityTypeMediaTimestamp. --- td/generate/scheme/td_api.tl | 9 ++++++--- td/telegram/MessageEntity.cpp | 22 ++++++++++++++++++++-- td/telegram/MessageEntity.h | 1 + 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index f1cd485e6..7678b79a6 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -1852,6 +1852,9 @@ textEntityTypeTextUrl url:string = TextEntityType; //@description A text shows instead of a raw mention of the user (e.g., when the user has no username) @user_id Identifier of the mentioned user textEntityTypeMentionName user_id:int32 = TextEntityType; +//@description A media timestamp @media_timestamp Timestamp from which a video/audio/video note/voice note playing should start, in seconds. The media can be in the content or the web page preview of the current message, or in the same places in the replied message +textEntityTypeMediaTimestamp media_timestamp:int32 = TextEntityType; + //@description A thumbnail to be sent along with a file; must be in JPEG or WEBP format for stickers, and less than 200 KB in size @thumbnail Thumbnail file to send. Sending thumbnails by file_id is currently not supported //@width Thumbnail width, usually shouldn't exceed 320. Use 0 if unknown @height Thumbnail height, usually shouldn't exceed 320. Use 0 if unknown @@ -3119,7 +3122,7 @@ messageLink link:string is_public:Bool = MessageLink; //@is_public True, if the link is a public link for a message in a chat //@chat_id If found, identifier of the chat to which the message belongs, 0 otherwise //@message If found, the linked message; may be null -//@media_timestamp Timestamp from which the video/audio/video note/voice note playing should start, in seconds; 0 if not specified. The media can be in the message content or in its link preview +//@media_timestamp Timestamp from which the video/audio/video note/voice note playing should start, in seconds; 0 if not specified. The media can be in the message content or in its web page preview //@for_album True, if the whole media album to which the message belongs is linked //@for_comment True, if the message is linked as a channel post comment or from a message thread messageLinkInfo is_public:Bool chat_id:int53 message:message media_timestamp:int32 for_album:Bool for_comment:Bool = MessageLinkInfo; @@ -4172,7 +4175,7 @@ removeNotificationGroup notification_group_id:int32 max_notification_id:int32 = //@description Returns an HTTPS link to a message in a chat. Available only for already sent messages in supergroups and channels. This is an offline request //@chat_id Identifier of the chat to which the message belongs //@message_id Identifier of the message -//@media_timestamp If not 0, timestamp from which the video/audio/video note/voice note playing should start, in seconds. The media can be in the message content or in its link preview +//@media_timestamp If not 0, timestamp from which the video/audio/video note/voice note playing should start, in seconds. The media can be in the message content or in its web page preview //@for_album Pass true to create a link for the whole media album //@for_comment Pass true to create a link to the message as a channel post comment, or from a message thread getMessageLink chat_id:int53 message_id:int53 media_timestamp:int32 for_album:Bool for_comment:Bool = MessageLink; @@ -4425,7 +4428,7 @@ openMessageContent chat_id:int53 message_id:int53 = Ok; //@description Returns information about the type of an internal link. Returns a 404 error if the link is not internal. Can be called before authorization @link The link getInternalLinkType link:string = InternalLinkType; -//@description Returns information about an action to be done when the current user clicks an external link. Don't use this method for links from secret chats if link preview is disabled in secret chats @link The link +//@description Returns information about an action to be done when the current user clicks an external link. Don't use this method for links from secret chats if web page preview is disabled in secret chats @link The link getExternalLinkInfo link:string = LoginUrlInfo; //@description Returns an HTTP URL which can be used to automatically authorize the current user on a website after clicking an HTTP link. Use the method getExternalLinkInfo to find whether a prior user confirmation is needed diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index 23b68eba9..f1281276f 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -29,7 +29,7 @@ namespace td { int MessageEntity::get_type_priority(Type type) { - static const int types[] = {50, 50, 50, 50, 50, 90, 91, 20, 11, 10, 49, 49, 50, 50, 92, 93, 0, 50}; + static const int types[] = {50, 50, 50, 50, 50, 90, 91, 20, 11, 10, 49, 49, 50, 50, 92, 93, 0, 50, 50}; static_assert(sizeof(types) / sizeof(types[0]) == static_cast(MessageEntity::Type::Size), ""); return types[static_cast(type)]; } @@ -72,6 +72,8 @@ StringBuilder &operator<<(StringBuilder &string_builder, const MessageEntity::Ty return string_builder << "PhoneNumber"; case MessageEntity::Type::BankCardNumber: return string_builder << "BankCardNumber"; + case MessageEntity::Type::MediaTimestamp: + return string_builder << "MediaTimestamp"; default: UNREACHABLE(); return string_builder << "Impossible"; @@ -130,6 +132,8 @@ tl_object_ptr MessageEntity::get_text_entity_type_object return make_tl_object(); case MessageEntity::Type::BankCardNumber: return make_tl_object(); + case MessageEntity::Type::MediaTimestamp: + return make_tl_object(to_integer(argument)); default: UNREACHABLE(); return nullptr; @@ -1291,7 +1295,8 @@ static constexpr int32 get_continuous_entities_mask() { get_entity_type_mask(MessageEntity::Type::EmailAddress) | get_entity_type_mask(MessageEntity::Type::TextUrl) | get_entity_type_mask(MessageEntity::Type::MentionName) | get_entity_type_mask(MessageEntity::Type::Cashtag) | get_entity_type_mask(MessageEntity::Type::PhoneNumber) | - get_entity_type_mask(MessageEntity::Type::BankCardNumber); + get_entity_type_mask(MessageEntity::Type::BankCardNumber) | + get_entity_type_mask(MessageEntity::Type::MediaTimestamp); } static constexpr int32 get_pre_entities_mask() { @@ -1616,6 +1621,8 @@ string get_first_url(Slice text, const vector &entities) { break; case MessageEntity::Type::BankCardNumber: break; + case MessageEntity::Type::MediaTimestamp: + break; default: UNREACHABLE(); } @@ -3030,6 +3037,8 @@ vector> get_input_secret_message_entiti break; case MessageEntity::Type::MentionName: break; + case MessageEntity::Type::MediaTimestamp: + break; default: UNREACHABLE(); } @@ -3121,6 +3130,15 @@ Result> get_message_entities(const ContactsManager *contac entities.emplace_back(entity->offset_, entity->length_, user_id); break; } + case td_api::textEntityTypeMediaTimestamp::ID: { + auto entity_media_timestamp = static_cast(entity->type_.get()); + if (entity_media_timestamp->media_timestamp_ <= 0) { + return Status::Error(400, "Invalid media timestamp specified"); + } + entities.emplace_back(MessageEntity::Type::MediaTimestamp, entity->offset_, entity->length_, + to_string(entity_media_timestamp->media_timestamp_)); + break; + } default: UNREACHABLE(); } diff --git a/td/telegram/MessageEntity.h b/td/telegram/MessageEntity.h index aa8b70bdc..537582cf9 100644 --- a/td/telegram/MessageEntity.h +++ b/td/telegram/MessageEntity.h @@ -48,6 +48,7 @@ class MessageEntity { Strikethrough, BlockQuote, BankCardNumber, + MediaTimestamp, Size }; Type type; From 389e92f1f4d83788cb5d0daf048f753d62ac9505 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 26 Jul 2021 22:48:33 +0300 Subject: [PATCH 14/51] Add skip_media_timestamps parameter to get_formatted_text. --- td/telegram/DraftMessage.cpp | 2 +- td/telegram/InputMessageText.cpp | 9 +++++---- td/telegram/LinkManager.cpp | 4 ++-- td/telegram/MessageContent.cpp | 6 +++--- td/telegram/MessageEntity.cpp | 9 +++++---- td/telegram/MessageEntity.h | 5 +++-- td/telegram/MessagesManager.cpp | 7 ++++--- td/telegram/PollManager.cpp | 2 +- td/telegram/Td.cpp | 6 +++--- td/telegram/TermsOfService.cpp | 2 +- td/telegram/WebPagesManager.cpp | 2 +- test/message_entities.cpp | 16 ++++++++-------- 12 files changed, 37 insertions(+), 33 deletions(-) diff --git a/td/telegram/DraftMessage.cpp b/td/telegram/DraftMessage.cpp index 85dffce23..aa73d3a58 100644 --- a/td/telegram/DraftMessage.cpp +++ b/td/telegram/DraftMessage.cpp @@ -45,7 +45,7 @@ unique_ptr get_draft_message(ContactsManager *contacts_manager, } auto entities = get_message_entities(contacts_manager, std::move(draft->entities_), "draftMessage"); - auto status = fix_formatted_text(draft->message_, entities, true, true, true, true); + auto status = fix_formatted_text(draft->message_, entities, true, true, true, true, true); if (status.is_error()) { LOG(ERROR) << "Receive error " << status << " while parsing draft " << draft->message_; if (!clean_input_string(draft->message_)) { diff --git a/td/telegram/InputMessageText.cpp b/td/telegram/InputMessageText.cpp index 352a49dd4..d8262a66b 100644 --- a/td/telegram/InputMessageText.cpp +++ b/td/telegram/InputMessageText.cpp @@ -39,15 +39,16 @@ Result process_input_message_text(const ContactsManager *conta } TRY_RESULT(entities, get_message_entities(contacts_manager, std::move(input_message_text->text_->entities_))); - auto need_skip_commands = need_always_skip_bot_commands(contacts_manager, dialog_id, is_bot); + auto need_skip_bot_commands = need_always_skip_bot_commands(contacts_manager, dialog_id, is_bot); bool parse_markdown = G()->shared_config().get_option_boolean("always_parse_markdown"); TRY_STATUS(fix_formatted_text(input_message_text->text_->text_, entities, for_draft, parse_markdown, - need_skip_commands, for_draft)); + need_skip_bot_commands, is_bot || for_draft || parse_markdown, for_draft)); InputMessageText result{FormattedText{std::move(input_message_text->text_->text_), std::move(entities)}, input_message_text->disable_web_page_preview_, input_message_text->clear_draft_}; - if (G()->shared_config().get_option_boolean("always_parse_markdown")) { + if (parse_markdown) { result.text = parse_markdown_v3(std::move(result.text)); - fix_formatted_text(result.text.text, result.text.entities, for_draft, false, need_skip_commands, for_draft) + fix_formatted_text(result.text.text, result.text.entities, for_draft, false, need_skip_bot_commands, + is_bot || for_draft, for_draft) .ensure(); } return std::move(result); diff --git a/td/telegram/LinkManager.cpp b/td/telegram/LinkManager.cpp index 55f252d8b..ad44404f2 100644 --- a/td/telegram/LinkManager.cpp +++ b/td/telegram/LinkManager.cpp @@ -357,7 +357,7 @@ class GetDeepLinkInfoQuery final : public Td::ResultHandler { bool need_update = (info->flags_ & telegram_api::help_deepLinkInfo::UPDATE_APP_MASK) != 0; auto entities = get_message_entities(nullptr, std::move(info->entities_), "GetDeepLinkInfoQuery"); - auto status = fix_formatted_text(info->message_, entities, true, true, true, true); + auto status = fix_formatted_text(info->message_, entities, true, true, true, true, true); if (status.is_error()) { LOG(ERROR) << "Receive error " << status << " while parsing deep link info " << info->message_; if (!clean_input_string(info->message_)) { @@ -1005,7 +1005,7 @@ unique_ptr LinkManager::get_internal_link_message_dra } else { full_text.text = url.str(); } - if (fix_formatted_text(full_text.text, full_text.entities, false, false, false, true).is_error()) { + if (fix_formatted_text(full_text.text, full_text.entities, false, false, false, true, true).is_error()) { return nullptr; } if (full_text.text[0] == '@') { diff --git a/td/telegram/MessageContent.cpp b/td/telegram/MessageContent.cpp index 3599fb4f1..501fd9c61 100644 --- a/td/telegram/MessageContent.cpp +++ b/td/telegram/MessageContent.cpp @@ -1461,7 +1461,7 @@ InlineMessageContent create_inline_message_content(Td *td, FileId file_id, auto inline_message = move_tl_object_as(bot_inline_message); auto entities = get_message_entities(td->contacts_manager_.get(), std::move(inline_message->entities_), "botInlineMessageText"); - auto status = fix_formatted_text(inline_message->message_, entities, false, true, true, false); + auto status = fix_formatted_text(inline_message->message_, entities, false, true, true, false, false); if (status.is_error()) { LOG(ERROR) << "Receive error " << status << " while parsing botInlineMessageText " << inline_message->message_; break; @@ -1525,7 +1525,7 @@ InlineMessageContent create_inline_message_content(Td *td, FileId file_id, auto inline_message = move_tl_object_as(bot_inline_message); auto caption = get_message_text(td->contacts_manager_.get(), inline_message->message_, std::move(inline_message->entities_), - true, 0, false, "create_inline_message_content"); + true, false, 0, false, "create_inline_message_content"); if (allowed_media_content_id == td_api::inputMessageAnimation::ID) { result.message_content = make_unique(file_id, std::move(caption)); } else if (allowed_media_content_id == td_api::inputMessageAudio::ID) { @@ -3823,7 +3823,7 @@ unique_ptr get_secret_message_content( } auto entities = get_message_entities(std::move(secret_entities)); - auto status = fix_formatted_text(message_text, entities, true, false, true, false); + auto status = fix_formatted_text(message_text, entities, true, false, true, true, false); if (status.is_error()) { LOG(WARNING) << "Receive error " << status << " while parsing secret message \"" << message_text << "\" with entities " << format::as_array(entities); diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index f1281276f..5a6fd691d 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -3777,7 +3777,7 @@ static void merge_new_entities(vector &entities, vector &entities, bool allow_empty, bool skip_new_entities, - bool skip_bot_commands, bool for_draft) { + bool skip_bot_commands, bool skip_media_timestamps, bool for_draft) { string result; if (entities.empty()) { // fast path @@ -3898,11 +3898,12 @@ Status fix_formatted_text(string &text, vector &entities, bool al FormattedText get_message_text(const ContactsManager *contacts_manager, string message_text, vector> &&server_entities, - bool skip_new_entities, int32 send_date, bool from_album, const char *source) { + bool skip_new_entities, bool skip_media_timestamps, int32 send_date, bool from_album, + const char *source) { auto entities = get_message_entities(contacts_manager, std::move(server_entities), source); auto debug_message_text = message_text; auto debug_entities = entities; - auto status = fix_formatted_text(message_text, entities, true, skip_new_entities, true, false); + auto status = fix_formatted_text(message_text, entities, true, skip_new_entities, true, skip_media_timestamps, false); if (status.is_error()) { // message entities in media albums can be wrong because of a long time ago fixed server-side bug if (!from_album && (send_date == 0 || send_date > 1600340000)) { // approximate fix date @@ -3957,7 +3958,7 @@ Result process_input_caption(const ContactsManager *contacts_mana } TRY_RESULT(entities, get_message_entities(contacts_manager, std::move(caption->entities_))); TRY_STATUS(fix_formatted_text(caption->text_, entities, true, false, - need_always_skip_bot_commands(contacts_manager, dialog_id, is_bot), false)); + need_always_skip_bot_commands(contacts_manager, dialog_id, is_bot), is_bot, false)); return FormattedText{std::move(caption->text_), std::move(entities)}; } diff --git a/td/telegram/MessageEntity.h b/td/telegram/MessageEntity.h index 537582cf9..3374935fb 100644 --- a/td/telegram/MessageEntity.h +++ b/td/telegram/MessageEntity.h @@ -179,11 +179,12 @@ vector get_message_entities(vector &entities, bool allow_empty, bool skip_new_entities, - bool skip_bot_commands, bool for_draft) TD_WARN_UNUSED_RESULT; + bool skip_bot_commands, bool skip_media_timestamps, bool for_draft) TD_WARN_UNUSED_RESULT; FormattedText get_message_text(const ContactsManager *contacts_manager, string message_text, vector> &&server_entities, - bool skip_new_entities, int32 send_date, bool from_album, const char *source); + bool skip_new_entities, bool skip_media_timestamps, int32 send_date, bool from_album, + const char *source); td_api::object_ptr extract_input_caption( tl_object_ptr &input_message_content); diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 17659dbd3..d409b921a 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -6523,7 +6523,7 @@ void MessagesManager::on_update_service_notification(tl_object_ptrauth_manager_->is_bot(); auto contacts_manager = is_authorized ? td_->contacts_manager_.get() : nullptr; auto message_text = get_message_text(contacts_manager, std::move(update->message_), std::move(update->entities_), - skip_new_entities, date, false, "on_update_service_notification"); + skip_new_entities, !is_user, date, false, "on_update_service_notification"); DialogId owner_dialog_id = is_user ? get_service_notifications_dialog()->dialog_id : DialogId(); auto content = get_message_content(td_, std::move(message_text), std::move(update->media_), owner_dialog_id, false, UserId(), &ttl); @@ -13213,7 +13213,8 @@ MessagesManager::MessageInfo MessagesManager::parse_telegram_api_message( message_info.content = get_message_content( td_, get_message_text(td_->contacts_manager_.get(), std::move(message->message_), std::move(message->entities_), - true, message_info.forward_header ? message_info.forward_header->date_ : message_info.date, + true, td_->auth_manager_->is_bot(), + message_info.forward_header ? message_info.forward_header->date_ : message_info.date, message_info.media_album_id != 0, new_source.c_str()), std::move(message->media_), message_info.dialog_id, is_content_read, message_info.via_bot_user_id, &message_info.ttl); @@ -14207,7 +14208,7 @@ void MessagesManager::on_update_sent_text_message(int64 random_id, const FormattedText *old_message_text = get_message_content_text(m->content.get()); CHECK(old_message_text != nullptr); FormattedText new_message_text = get_message_text( - td_->contacts_manager_.get(), old_message_text->text, std::move(entities), true, + td_->contacts_manager_.get(), old_message_text->text, std::move(entities), true, td_->auth_manager_->is_bot(), m->forward_info ? m->forward_info->date : m->date, m->media_album_id != 0, "on_update_sent_text_message"); auto new_content = get_message_content(td_, std::move(new_message_text), std::move(message_media), dialog_id, true /*likely ignored*/, UserId() /*likely ignored*/, nullptr /*ignored*/); diff --git a/td/telegram/PollManager.cpp b/td/telegram/PollManager.cpp index 472e8855c..323b058f6 100644 --- a/td/telegram/PollManager.cpp +++ b/td/telegram/PollManager.cpp @@ -1544,7 +1544,7 @@ PollId PollManager::on_get_poll(PollId poll_id, tl_object_ptrcontacts_manager_.get(), std::move(poll_results->solution_entities_), "on_get_poll"); - auto status = fix_formatted_text(poll_results->solution_, entities, true, true, true, false); + auto status = fix_formatted_text(poll_results->solution_, entities, true, true, true, true, false); if (status.is_error()) { if (!clean_input_string(poll_results->solution_)) { poll_results->solution_.clear(); diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index 33f7509c8..bc2f6f104 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -8368,13 +8368,13 @@ td_api::object_ptr Td::do_static_request(td_api::parseMarkdown & return make_error(400, r_entities.error().message()); } auto entities = r_entities.move_as_ok(); - auto status = fix_formatted_text(request.text_->text_, entities, true, true, true, true); + auto status = fix_formatted_text(request.text_->text_, entities, true, true, true, true, true); if (status.is_error()) { return make_error(400, status.error().message()); } auto parsed_text = parse_markdown_v3({std::move(request.text_->text_), std::move(entities)}); - fix_formatted_text(parsed_text.text, parsed_text.entities, true, true, true, true).ensure(); + fix_formatted_text(parsed_text.text, parsed_text.entities, true, true, true, true, true).ensure(); return get_formatted_text_object(parsed_text, true); } @@ -8388,7 +8388,7 @@ td_api::object_ptr Td::do_static_request(td_api::getMarkdownText return make_error(400, r_entities.error().message()); } auto entities = r_entities.move_as_ok(); - auto status = fix_formatted_text(request.text_->text_, entities, true, true, true, true); + auto status = fix_formatted_text(request.text_->text_, entities, true, true, true, true, true); if (status.is_error()) { return make_error(400, status.error().message()); } diff --git a/td/telegram/TermsOfService.cpp b/td/telegram/TermsOfService.cpp index 654be4697..c2954a014 100644 --- a/td/telegram/TermsOfService.cpp +++ b/td/telegram/TermsOfService.cpp @@ -95,7 +95,7 @@ TermsOfService::TermsOfService(telegram_api::object_ptrid_->data_); auto entities = get_message_entities(nullptr, std::move(terms->entities_), "TermsOfService"); - auto status = fix_formatted_text(terms->text_, entities, true, true, true, false); + auto status = fix_formatted_text(terms->text_, entities, true, true, true, true, false); if (status.is_error()) { if (!clean_input_string(terms->text_)) { terms->text_.clear(); diff --git a/td/telegram/WebPagesManager.cpp b/td/telegram/WebPagesManager.cpp index b5fe736f0..a738383bc 100644 --- a/td/telegram/WebPagesManager.cpp +++ b/td/telegram/WebPagesManager.cpp @@ -792,7 +792,7 @@ int64 WebPagesManager::get_web_page_preview(td_api::object_ptrtext_, entities, true, false, true, false); + auto result = fix_formatted_text(text->text_, entities, true, false, true, true, false); if (result.is_error() || text->text_.empty()) { promise.set_value(Unit()); return 0; diff --git a/test/message_entities.cpp b/test/message_entities.cpp index 6583b9129..964e47725 100644 --- a/test/message_entities.cpp +++ b/test/message_entities.cpp @@ -655,16 +655,16 @@ static void check_fix_formatted_text(td::string str, td::vector &expected_entities, bool allow_empty = true, bool skip_new_entities = false, bool skip_bot_commands = false, bool for_draft = true) { - ASSERT_TRUE( - td::fix_formatted_text(str, entities, allow_empty, skip_new_entities, skip_bot_commands, for_draft).is_ok()); + ASSERT_TRUE(td::fix_formatted_text(str, entities, allow_empty, skip_new_entities, skip_bot_commands, true, for_draft) + .is_ok()); ASSERT_STREQ(expected_str, str); ASSERT_EQ(expected_entities, entities); } static void check_fix_formatted_text(td::string str, td::vector entities, bool allow_empty, bool skip_new_entities, bool skip_bot_commands, bool for_draft) { - ASSERT_TRUE( - fix_formatted_text(str, entities, allow_empty, skip_new_entities, skip_bot_commands, for_draft).is_error()); + ASSERT_TRUE(td::fix_formatted_text(str, entities, allow_empty, skip_new_entities, skip_bot_commands, true, for_draft) + .is_error()); } TEST(MessageEntities, fix_formatted_text) { @@ -1064,7 +1064,7 @@ TEST(MessageEntities, fix_formatted_text) { return result; }; auto old_type_mask = get_type_mask(str.size(), entities); - ASSERT_TRUE(td::fix_formatted_text(str, entities, false, false, true, false).is_ok()); + ASSERT_TRUE(td::fix_formatted_text(str, entities, false, false, true, true, false).is_ok()); auto new_type_mask = get_type_mask(str.size(), entities); auto splittable_mask = (1 << 5) | (1 << 6) | (1 << 14) | (1 << 15); auto pre_mask = (1 << 7) | (1 << 8) | (1 << 9); @@ -1384,7 +1384,7 @@ static void check_parse_markdown_v3(td::string text, td::vector Date: Wed, 28 Jul 2021 06:48:52 +0300 Subject: [PATCH 15/51] Move fix_entity_offsets to a separate function. --- td/telegram/MessageEntity.cpp | 87 ++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 41 deletions(-) diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index 5a6fd691d..490c398bc 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -1461,6 +1461,51 @@ static void remove_entities_intersecting_blockquote(vector &entit entities.erase(entities.begin() + left_entities, entities.end()); } +// keeps only non-intersecting entities +// fixes entity offsets from UTF-8 to UTF-16 offsets +static void fix_entity_offsets(Slice text, vector &entities) { + if (entities.empty()) { + return; + } + + sort_entities(entities); + + remove_intersecting_entities(entities); + + const unsigned char *begin = text.ubegin(); + const unsigned char *ptr = begin; + const unsigned char *end = text.uend(); + + int32 utf16_pos = 0; + for (auto &entity : entities) { + int cnt = 2; + auto entity_begin = entity.offset; + auto entity_end = entity.offset + entity.length; + + int32 pos = static_cast(ptr - begin); + if (entity_begin == pos) { + cnt--; + entity.offset = utf16_pos; + } + + while (ptr != end && cnt > 0) { + unsigned char c = ptr[0]; + utf16_pos += 1 + (c >= 0xf0); + ptr = next_utf8_unsafe(ptr, nullptr, "fix_entity_offsets"); + + pos = static_cast(ptr - begin); + if (entity_begin == pos) { + cnt--; + entity.offset = utf16_pos; + } else if (entity_end == pos) { + cnt--; + entity.length = utf16_pos - entity.offset; + } + } + CHECK(cnt == 0); + } +} + vector find_entities(Slice text, bool skip_bot_commands) { vector entities; @@ -1490,47 +1535,7 @@ vector find_entities(Slice text, bool skip_bot_commands) { entities.emplace_back(type, offset, length); } - if (entities.empty()) { - return entities; - } - - sort_entities(entities); - - remove_intersecting_entities(entities); - - // fix offsets to UTF-16 offsets - const unsigned char *begin = text.ubegin(); - const unsigned char *ptr = begin; - const unsigned char *end = text.uend(); - - int32 utf16_pos = 0; - for (auto &entity : entities) { - int cnt = 2; - auto entity_begin = entity.offset; - auto entity_end = entity.offset + entity.length; - - int32 pos = static_cast(ptr - begin); - if (entity_begin == pos) { - cnt--; - entity.offset = utf16_pos; - } - - while (ptr != end && cnt > 0) { - unsigned char c = ptr[0]; - utf16_pos += 1 + (c >= 0xf0); - ptr = next_utf8_unsafe(ptr, nullptr, "find_entities"); - - pos = static_cast(ptr - begin); - if (entity_begin == pos) { - cnt--; - entity.offset = utf16_pos; - } else if (entity_end == pos) { - cnt--; - entity.length = utf16_pos - entity.offset; - } - } - CHECK(cnt == 0); - } + fix_entity_offsets(text, entities); return entities; } From 14cf908017782ac428e6933b649da722a493ef17 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 28 Jul 2021 08:30:22 +0300 Subject: [PATCH 16/51] Implement find_media_timestamps. --- td/telegram/MessageEntity.cpp | 105 ++++++++++++++++++++++++++++++++++ td/telegram/MessageEntity.h | 3 +- test/message_entities.cpp | 41 +++++++++++++ 3 files changed, 148 insertions(+), 1 deletion(-) diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index 490c398bc..27bb5f460 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -434,6 +434,57 @@ static vector match_cashtags(Slice str) { return result; } +static vector match_media_timestamps(Slice str) { + vector result; + const unsigned char *begin = str.ubegin(); + const unsigned char *end = str.uend(); + const unsigned char *ptr = begin; + + while (true) { + ptr = static_cast(std::memchr(ptr, ':', narrow_cast(end - ptr))); + if (ptr == nullptr) { + break; + } + + auto media_timestamp_begin = ptr; + while (media_timestamp_begin != begin && + (media_timestamp_begin[-1] == ':' || is_digit(media_timestamp_begin[-1]))) { + media_timestamp_begin--; + } + auto media_timestamp_end = ptr; + while (media_timestamp_end + 1 != end && (media_timestamp_end[1] == ':' || is_digit(media_timestamp_end[1]))) { + media_timestamp_end++; + } + media_timestamp_end++; + + if (media_timestamp_begin != ptr && media_timestamp_end != ptr + 1 && is_digit(ptr[1])) { + ptr = media_timestamp_end; + + if (media_timestamp_begin != begin) { + uint32 prev; + next_utf8_unsafe(prev_utf8_unsafe(media_timestamp_begin), &prev, "match_media_timestamps 1"); + + if (is_word_character(prev)) { + continue; + } + } + if (media_timestamp_end != end) { + uint32 next; + next_utf8_unsafe(media_timestamp_end, &next, "match_media_timestamps 2"); + + if (is_word_character(next)) { + continue; + } + } + + result.emplace_back(media_timestamp_begin, media_timestamp_end); + } else { + ptr = media_timestamp_end; + } + } + return result; +} + static vector match_bank_card_numbers(Slice str) { vector result; const unsigned char *begin = str.ubegin(); @@ -1251,6 +1302,42 @@ vector> find_urls(Slice str) { return result; } +vector> find_media_timestamps(Slice str) { + vector> result; + for (auto media_timestamp : match_media_timestamps(str)) { + vector parts = full_split(media_timestamp, ':'); + CHECK(parts.size() >= 2); + if (parts.size() > 3 || parts.back().size() != 2) { + continue; + } + auto seconds = to_integer(parts.back()); + if (seconds >= 60) { + continue; + } + if (parts.size() == 2) { + if (parts[0].size() > 4 || parts[0].empty()) { + continue; + } + + auto minutes = to_integer(parts[0]); + result.emplace_back(media_timestamp, minutes * 60 + seconds); + continue; + } else { + if (parts[0].size() > 2 || parts[1].size() > 2 || parts[0].empty() || parts[1].empty()) { + continue; + } + + auto minutes = to_integer(parts[1]); + if (minutes >= 60) { + continue; + } + auto hours = to_integer(parts[0]); + result.emplace_back(media_timestamp, hours * 3600 + minutes * 60 + seconds); + } + } + return result; +} + static int32 text_length(Slice text) { return narrow_cast(utf8_utf16_length(text)); } @@ -1540,6 +1627,21 @@ vector find_entities(Slice text, bool skip_bot_commands) { return entities; } +static vector find_media_timestamp_entities(Slice text) { + vector entities; + + auto new_entities = find_media_timestamps(text); + for (auto &entity : new_entities) { + auto offset = narrow_cast(entity.first.begin() - text.begin()); + auto length = narrow_cast(entity.first.size()); + entities.emplace_back(MessageEntity::Type::MediaTimestamp, offset, length, to_string(entity.second)); + } + + fix_entity_offsets(text, entities); + + return entities; +} + static vector merge_entities(vector old_entities, vector new_entities) { if (new_entities.empty()) { return old_entities; @@ -3892,6 +3994,9 @@ Status fix_formatted_text(string &text, vector &entities, bool al if (!skip_new_entities) { merge_new_entities(entities, find_entities(text, skip_bot_commands)); } + if (!skip_media_timestamps) { + merge_new_entities(entities, find_media_timestamp_entities(text)); + } // new whitespace-only entities could be added after splitting of entities remove_invalid_entities(text, entities); diff --git a/td/telegram/MessageEntity.h b/td/telegram/MessageEntity.h index 3374935fb..9dd5257ba 100644 --- a/td/telegram/MessageEntity.h +++ b/td/telegram/MessageEntity.h @@ -146,7 +146,8 @@ vector find_cashtags(Slice str); vector find_bank_card_numbers(Slice str); vector find_tg_urls(Slice str); bool is_email_address(Slice str); -vector> find_urls(Slice str); // slice + is_email_address +vector> find_urls(Slice str); // slice + is_email_address +vector> find_media_timestamps(Slice str); // slice + media_timestamp string get_first_url(Slice text, const vector &entities); diff --git a/test/message_entities.cpp b/test/message_entities.cpp index 964e47725..4e8935aca 100644 --- a/test/message_entities.cpp +++ b/test/message_entities.cpp @@ -172,6 +172,47 @@ TEST(MessageEntities, cashtag) { check_cashtag(u8"\u2122$ABC\u2122", {"$ABC"}); } +static void check_media_timestamp(const td::string &str, const td::vector> &expected) { + auto result = td::find_media_timestamps(str); + if (result != expected) { + LOG(FATAL) << td::tag("text", str) << td::tag("got", td::format::as_array(result)) + << td::tag("expected", td::format::as_array(expected)); + } +} + +TEST(MessageEntities, media_timestamp) { + check_media_timestamp("", {}); + check_media_timestamp(":", {}); + check_media_timestamp(":1", {}); + check_media_timestamp("a:1", {}); + check_media_timestamp("01", {}); + check_media_timestamp("01:", {}); + check_media_timestamp("01::", {}); + check_media_timestamp("01::", {}); + check_media_timestamp("a1:1a", {}); + check_media_timestamp("a1::01a", {}); + check_media_timestamp("2001:db8::8a2e:f70:13a4", {}); + check_media_timestamp("0:00", {{"0:00", 0}}); + check_media_timestamp("+0:00", {{"0:00", 0}}); + check_media_timestamp("0:00+", {{"0:00", 0}}); + check_media_timestamp("a0:00", {}); + check_media_timestamp("0:00a", {}); + check_media_timestamp("б0:00", {}); + check_media_timestamp("0:00б", {}); + check_media_timestamp("_0:00", {}); + check_media_timestamp("0:00_", {}); + check_media_timestamp("00:00:00:00", {}); + check_media_timestamp("1:1:01 1:1:1", {{"1:1:01", 3661}}); + check_media_timestamp("0:0:00 00:00 000:00 0000:00 00000:00 00:00:00 000:00:00 00:000:00 00:00:000", + {{"0:0:00", 0}, {"00:00", 0}, {"000:00", 0}, {"0000:00", 0}, {"00:00:00", 0}}); + check_media_timestamp("00:0:00 0:00:00 00::00 :00:00 00:00: 00:00:0 00:00:", {{"00:0:00", 0}, {"0:00:00", 0}}); + check_media_timestamp("1:1:59 1:1:-1 1:1:60", {{"1:1:59", 3719}}); + check_media_timestamp("1:59:00 1:-1:00 1:60:00", {{"1:59:00", 7140}, {"1:00", 60}}); + check_media_timestamp("59:59 60:00", {{"59:59", 3599}, {"60:00", 3600}}); + check_media_timestamp("9999:59 99:59:59 99:60:59", {{"9999:59", 599999}, {"99:59:59", 360000 - 1}}); + check_media_timestamp("2001:db8::8a2e:f70:13a4", {}); +} + static void check_bank_card_number(const td::string &str, const td::vector &expected) { auto result_slice = td::find_bank_card_numbers(str); td::vector result; From 45bf232417e9252ee6768dbe7be3bbc2f162eab4 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Aug 2021 04:26:41 +0300 Subject: [PATCH 17/51] Fimd media timestamp entities in find_entities. --- td/telegram/DraftMessage.cpp | 2 +- td/telegram/LinkManager.cpp | 2 +- td/telegram/MessageContent.cpp | 6 +++--- td/telegram/MessageEntity.cpp | 24 ++++++++++++++++-------- td/telegram/MessageEntity.h | 2 +- td/telegram/PollManager.cpp | 2 +- td/telegram/Td.cpp | 2 +- td/telegram/TermsOfService.cpp | 2 +- td/telegram/WebPagesManager.cpp | 2 +- 9 files changed, 26 insertions(+), 18 deletions(-) diff --git a/td/telegram/DraftMessage.cpp b/td/telegram/DraftMessage.cpp index aa73d3a58..2f55263ae 100644 --- a/td/telegram/DraftMessage.cpp +++ b/td/telegram/DraftMessage.cpp @@ -51,7 +51,7 @@ unique_ptr get_draft_message(ContactsManager *contacts_manager, if (!clean_input_string(draft->message_)) { draft->message_.clear(); } - entities = find_entities(draft->message_, false); + entities = find_entities(draft->message_, false, true); } result->input_message_text.text = FormattedText{std::move(draft->message_), std::move(entities)}; result->input_message_text.disable_web_page_preview = (flags & telegram_api::draftMessage::NO_WEBPAGE_MASK) != 0; diff --git a/td/telegram/LinkManager.cpp b/td/telegram/LinkManager.cpp index ad44404f2..8b40734ff 100644 --- a/td/telegram/LinkManager.cpp +++ b/td/telegram/LinkManager.cpp @@ -363,7 +363,7 @@ class GetDeepLinkInfoQuery final : public Td::ResultHandler { if (!clean_input_string(info->message_)) { info->message_.clear(); } - entities = find_entities(info->message_, true); + entities = find_entities(info->message_, true, true); } FormattedText text{std::move(info->message_), std::move(entities)}; return promise_.set_value( diff --git a/td/telegram/MessageContent.cpp b/td/telegram/MessageContent.cpp index 501fd9c61..bc3159ba2 100644 --- a/td/telegram/MessageContent.cpp +++ b/td/telegram/MessageContent.cpp @@ -1030,7 +1030,7 @@ static void parse_caption(FormattedText &caption, ParserT &parser) { if (!check_utf8(caption.text)) { caption.text.clear(); } - caption.entities = find_entities(caption.text, false); + caption.entities = find_entities(caption.text, false, true); } } @@ -3823,14 +3823,14 @@ unique_ptr get_secret_message_content( } auto entities = get_message_entities(std::move(secret_entities)); - auto status = fix_formatted_text(message_text, entities, true, false, true, true, false); + auto status = fix_formatted_text(message_text, entities, true, false, true, td->auth_manager_->is_bot(), false); if (status.is_error()) { LOG(WARNING) << "Receive error " << status << " while parsing secret message \"" << message_text << "\" with entities " << format::as_array(entities); if (!clean_input_string(message_text)) { message_text.clear(); } - entities = find_entities(message_text, true); + entities = find_entities(message_text, true, td->auth_manager_->is_bot()); } // support of old layer and old constructions diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index 27bb5f460..f86e6f525 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -1593,7 +1593,7 @@ static void fix_entity_offsets(Slice text, vector &entities) { } } -vector find_entities(Slice text, bool skip_bot_commands) { +vector find_entities(Slice text, bool skip_bot_commands, bool skip_media_timestamps) { vector entities; auto add_entities = [&entities, &text](MessageEntity::Type type, vector (*find_entities_f)(Slice)) mutable { @@ -1622,6 +1622,15 @@ vector find_entities(Slice text, bool skip_bot_commands) { entities.emplace_back(type, offset, length); } + if (!skip_media_timestamps) { + auto media_timestamps = find_media_timestamps(text); + for (auto &entity : media_timestamps) { + auto offset = narrow_cast(entity.first.begin() - text.begin()); + auto length = narrow_cast(entity.first.size()); + entities.emplace_back(MessageEntity::Type::MediaTimestamp, offset, length, to_string(entity.second)); + } + } + fix_entity_offsets(text, entities); return entities; @@ -1630,8 +1639,8 @@ vector find_entities(Slice text, bool skip_bot_commands) { static vector find_media_timestamp_entities(Slice text) { vector entities; - auto new_entities = find_media_timestamps(text); - for (auto &entity : new_entities) { + auto media_timestamps = find_media_timestamps(text); + for (auto &entity : media_timestamps) { auto offset = narrow_cast(entity.first.begin() - text.begin()); auto length = narrow_cast(entity.first.size()); entities.emplace_back(MessageEntity::Type::MediaTimestamp, offset, length, to_string(entity.second)); @@ -2353,7 +2362,7 @@ static vector find_splittable_entities_v3(Slice text, const vecto } } - auto found_entities = find_entities(text, false); + auto found_entities = find_entities(text, false, true); td::remove_if(found_entities, [](const auto &entity) { return entity.type == MessageEntity::Type::EmailAddress || entity.type == MessageEntity::Type::Url; }); @@ -3992,9 +4001,8 @@ Status fix_formatted_text(string &text, vector &entities, bool al } if (!skip_new_entities) { - merge_new_entities(entities, find_entities(text, skip_bot_commands)); - } - if (!skip_media_timestamps) { + merge_new_entities(entities, find_entities(text, skip_bot_commands, skip_media_timestamps)); + } else if (!skip_media_timestamps) { merge_new_entities(entities, find_media_timestamp_entities(text)); } @@ -4024,7 +4032,7 @@ FormattedText get_message_text(const ContactsManager *contacts_manager, string m if (!clean_input_string(message_text)) { message_text.clear(); } - entities = find_entities(message_text, false); + entities = find_entities(message_text, false, skip_media_timestamps); } return FormattedText{std::move(message_text), std::move(entities)}; } diff --git a/td/telegram/MessageEntity.h b/td/telegram/MessageEntity.h index 9dd5257ba..f9e682fb1 100644 --- a/td/telegram/MessageEntity.h +++ b/td/telegram/MessageEntity.h @@ -137,7 +137,7 @@ vector> get_text_entities_object(const vector< td_api::object_ptr get_formatted_text_object(const FormattedText &text, bool skip_bot_commands); -vector find_entities(Slice text, bool skip_bot_commands); +vector find_entities(Slice text, bool skip_bot_commands, bool skip_media_timestamps); vector find_mentions(Slice str); vector find_bot_commands(Slice str); diff --git a/td/telegram/PollManager.cpp b/td/telegram/PollManager.cpp index 323b058f6..266a6fe4e 100644 --- a/td/telegram/PollManager.cpp +++ b/td/telegram/PollManager.cpp @@ -1549,7 +1549,7 @@ PollId PollManager::on_get_poll(PollId poll_id, tl_object_ptrsolution_)) { poll_results->solution_.clear(); } - entities = find_entities(poll_results->solution_, true); + entities = find_entities(poll_results->solution_, true, true); } FormattedText explanation{std::move(poll_results->solution_), std::move(entities)}; diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index bc2f6f104..78ff4e1e6 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -8319,7 +8319,7 @@ td_api::object_ptr Td::do_static_request(const td_api::getTextEn if (!check_utf8(request.text_)) { return make_error(400, "Text must be encoded in UTF-8"); } - auto text_entities = find_entities(request.text_, false); + auto text_entities = find_entities(request.text_, false, false); return make_tl_object(get_text_entities_object(text_entities, false)); } diff --git a/td/telegram/TermsOfService.cpp b/td/telegram/TermsOfService.cpp index c2954a014..c4d792c67 100644 --- a/td/telegram/TermsOfService.cpp +++ b/td/telegram/TermsOfService.cpp @@ -100,7 +100,7 @@ TermsOfService::TermsOfService(telegram_api::object_ptrtext_)) { terms->text_.clear(); } - entities = find_entities(terms->text_, true); + entities = find_entities(terms->text_, true, true); } if (terms->text_.empty()) { id_.clear(); diff --git a/td/telegram/WebPagesManager.cpp b/td/telegram/WebPagesManager.cpp index a738383bc..f4c6933d5 100644 --- a/td/telegram/WebPagesManager.cpp +++ b/td/telegram/WebPagesManager.cpp @@ -1185,7 +1185,7 @@ tl_object_ptr WebPagesManager::get_web_page_object(WebPageId we FormattedText description; description.text = web_page->description; - description.entities = find_entities(web_page->description, true); + description.entities = find_entities(web_page->description, true, true); auto r_url = parse_url(web_page->display_url); if (r_url.is_ok()) { From 304280df690a169660ec31c4319cc14af14b90bb Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Aug 2021 07:57:27 +0300 Subject: [PATCH 18/51] Add message.can_get_media_timestamp_links. --- td/generate/scheme/td_api.tl | 5 +- td/telegram/MessagesManager.cpp | 83 ++++++++++++++++++++++----------- td/telegram/MessagesManager.h | 2 + td/telegram/WebPagesManager.cpp | 2 +- 4 files changed, 62 insertions(+), 30 deletions(-) diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index 7678b79a6..22f39d57c 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -759,6 +759,7 @@ messageSendingStateFailed error_code:int32 error_message:string can_retry:Bool r //@can_be_deleted_for_all_users True, if the message can be deleted for all users //@can_get_statistics True, if the message statistics are available //@can_get_message_thread True, if the message thread info is available +//@can_get_media_timestamp_links True, if media timestamp links can be generated for media timestamp entities in the message text, caption or web page description //@is_channel_post True, if the message is a channel post. All messages to channels are channel posts, all other messages are not channel posts //@contains_unread_mention True, if the message contains an unread mention for the current user //@date Point in time (Unix timestamp) when the message was sent @@ -776,7 +777,7 @@ messageSendingStateFailed error_code:int32 error_message:string can_retry:Bool r //@restriction_reason If non-empty, contains a human-readable description of the reason why access to this message must be restricted //@content Content of the message //@reply_markup Reply markup for the message; may be null -message id:int53 sender:MessageSender chat_id:int53 sending_state:MessageSendingState scheduling_state:MessageSchedulingState is_outgoing:Bool is_pinned:Bool can_be_edited:Bool can_be_forwarded:Bool can_be_deleted_only_for_self:Bool can_be_deleted_for_all_users:Bool can_get_statistics:Bool can_get_message_thread:Bool is_channel_post:Bool contains_unread_mention:Bool date:int32 edit_date:int32 forward_info:messageForwardInfo interaction_info:messageInteractionInfo reply_in_chat_id:int53 reply_to_message_id:int53 message_thread_id:int53 ttl:int32 ttl_expires_in:double via_bot_user_id:int32 author_signature:string media_album_id:int64 restriction_reason:string content:MessageContent reply_markup:ReplyMarkup = Message; +message id:int53 sender:MessageSender chat_id:int53 sending_state:MessageSendingState scheduling_state:MessageSchedulingState is_outgoing:Bool is_pinned:Bool can_be_edited:Bool can_be_forwarded:Bool can_be_deleted_only_for_self:Bool can_be_deleted_for_all_users:Bool can_get_statistics:Bool can_get_message_thread:Bool can_get_media_timestamp_links:Bool is_channel_post:Bool contains_unread_mention:Bool date:int32 edit_date:int32 forward_info:messageForwardInfo interaction_info:messageInteractionInfo reply_in_chat_id:int53 reply_to_message_id:int53 message_thread_id:int53 ttl:int32 ttl_expires_in:double via_bot_user_id:int32 author_signature:string media_album_id:int64 restriction_reason:string content:MessageContent reply_markup:ReplyMarkup = Message; //@description Contains a list of messages @total_count Approximate total count of messages found @messages List of messages; messages may be null messages total_count:int32 messages:vector = Messages; @@ -4172,7 +4173,7 @@ removeNotification notification_group_id:int32 notification_id:int32 = Ok; removeNotificationGroup notification_group_id:int32 max_notification_id:int32 = Ok; -//@description Returns an HTTPS link to a message in a chat. Available only for already sent messages in supergroups and channels. This is an offline request +//@description Returns an HTTPS link to a message in a chat. Available only for already sent messages in supergroups and channels, or if message.can_get_media_timestamp_links and a media timestamp link is generated. This is an offline request //@chat_id Identifier of the chat to which the message belongs //@message_id Identifier of the message //@media_timestamp If not 0, timestamp from which the video/audio/video note/voice note playing should start, in seconds. The media can be in the message content or in its web page preview diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index d409b921a..438be4c8f 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -696,7 +696,9 @@ class ExportChannelMessageLinkQuery final : public Td::ResultHandler { for_group_ = for_group; ignore_result_ = ignore_result; auto input_channel = td->contacts_manager_->get_input_channel(channel_id); - CHECK(input_channel != nullptr); + if (input_channel == nullptr) { + return on_error(0, Status::Error(400, "Can't access the chat")); + } int32 flags = 0; if (for_group) { flags |= telegram_api::channels_exportMessageLink::GROUPED_MASK; @@ -17294,6 +17296,35 @@ bool MessagesManager::is_message_edited_recently(FullMessageId full_message_id, return m->edit_date >= G()->unix_time() - seconds; } +Status MessagesManager::can_get_media_timestamp_link(DialogId dialog_id, const Message *m) { + if (m == nullptr) { + return Status::Error(400, "Message not found"); + } + + if (dialog_id.get_type() != DialogType::Channel) { + auto forward_info = m->forward_info.get(); + if (!can_message_content_have_media_timestamp(m->content.get()) || forward_info == nullptr || + forward_info->is_imported || is_forward_info_sender_hidden(forward_info) || + !forward_info->message_id.is_valid() || !m->forward_info->message_id.is_server() || + !forward_info->sender_dialog_id.is_valid() || + forward_info->sender_dialog_id.get_type() != DialogType::Channel) { + return Status::Error(400, "Message links are available only for messages in supergroups and channel chats"); + } + return Status::OK(); + } + + if (m->message_id.is_yet_unsent()) { + return Status::Error(400, "Message is not sent yet"); + } + if (m->message_id.is_scheduled()) { + return Status::Error(400, "Message is scheduled"); + } + if (!m->message_id.is_server()) { + return Status::Error(400, "Message is local"); + } + return Status::OK(); +} + Result> MessagesManager::get_message_link(FullMessageId full_message_id, int32 media_timestamp, bool for_group, bool for_comment) { auto dialog_id = full_message_id.get_dialog_id(); @@ -17304,26 +17335,23 @@ Result> MessagesManager::get_message_link(FullMessageId if (!have_input_peer(dialog_id, AccessRights::Read)) { return Status::Error(400, "Can't access the chat"); } - if (dialog_id.get_type() != DialogType::Channel) { - return Status::Error(400, "Public message links are available only for messages in supergroups and channel chats"); - } auto *m = get_message_force(d, full_message_id.get_message_id(), "get_message_link"); - if (m == nullptr) { - return Status::Error(400, "Message not found"); - } - if (m->message_id.is_yet_unsent()) { - return Status::Error(400, "Message is not sent yet"); - } - if (m->message_id.is_scheduled()) { - return Status::Error(400, "Message is scheduled"); - } - if (!m->message_id.is_server()) { - return Status::Error(400, "Message is local"); - } + TRY_STATUS(can_get_media_timestamp_link(dialog_id, m)); + auto message_id = m->message_id; + if (dialog_id.get_type() != DialogType::Channel) { + CHECK(m != nullptr); + CHECK(m->forward_info != nullptr); + CHECK(m->forward_info->sender_dialog_id.get_type() == DialogType::Channel); - if (m->media_album_id == 0) { - for_group = true; // default is true + dialog_id = m->forward_info->sender_dialog_id; + message_id = m->forward_info->message_id; + for_group = false; + for_comment = false; + } else { + if (m->media_album_id == 0) { + for_group = true; // default is true + } } if (!m->top_thread_message_id.is_valid() || !m->top_thread_message_id.is_server()) { @@ -17340,6 +17368,7 @@ Result> MessagesManager::get_message_link(FullMessageId media_timestamp = 0; } if (media_timestamp != 0) { + for_group = false; auto duration = get_message_content_duration(m->content.get(), td_); if (duration != 0 && media_timestamp > duration) { media_timestamp = 0; @@ -17348,7 +17377,7 @@ Result> MessagesManager::get_message_link(FullMessageId if (!td_->auth_manager_->is_bot()) { td_->create_handler(Promise()) - ->send(dialog_id.get_channel_id(), m->message_id, for_group, true); + ->send(dialog_id.get_channel_id(), message_id, for_group, true); } SliceBuilder sb; @@ -17368,7 +17397,7 @@ Result> MessagesManager::get_message_link(FullMessageId linked_message_id.is_server() && have_input_peer(linked_dialog_id, AccessRights::Read) && !channel_username.empty()) { sb << channel_username << '/' << linked_message_id.get_server_message_id().get() - << "?comment=" << m->message_id.get_server_message_id().get(); + << "?comment=" << message_id.get_server_message_id().get(); if (!for_group) { sb << "&single"; } @@ -17384,8 +17413,7 @@ Result> MessagesManager::get_message_link(FullMessageId bool is_public = !dialog_username.empty(); if (m->content->get_type() == MessageContentType::VideoNote && is_broadcast_channel(dialog_id) && is_public) { return std::make_pair( - PSTRING() << "https://telesco.pe/" << dialog_username << '/' << m->message_id.get_server_message_id().get(), - true); + PSTRING() << "https://telesco.pe/" << dialog_username << '/' << message_id.get_server_message_id().get(), true); } if (is_public) { @@ -17393,7 +17421,7 @@ Result> MessagesManager::get_message_link(FullMessageId } else { sb << "c/" << dialog_id.get_channel_id().get(); } - sb << '/' << m->message_id.get_server_message_id().get(); + sb << '/' << message_id.get_server_message_id().get(); char separator = '?'; if (for_comment) { @@ -22800,6 +22828,7 @@ tl_object_ptr MessagesManager::get_message_object(DialogId dial bool can_be_forwarded = for_event_log ? false : can_forward_message(dialog_id, m); bool can_get_statistics = for_event_log ? false : can_get_message_statistics(dialog_id, m); bool can_get_message_thread = for_event_log ? false : get_top_thread_full_message_id(dialog_id, m).is_ok(); + bool can_get_media_timestamp_links = for_event_log ? false : can_get_media_timestamp_link(dialog_id, m).is_ok(); auto via_bot_user_id = td_->contacts_manager_->get_user_id_object(m->via_bot_user_id, "via_bot_user_id"); auto media_album_id = for_event_log ? static_cast(0) : m->media_album_id; auto reply_to_message_id = for_event_log ? static_cast(0) : m->reply_to_message_id.get(); @@ -22817,10 +22846,10 @@ tl_object_ptr MessagesManager::get_message_object(DialogId dial m->message_id.get(), get_message_sender_object_const(m->sender_user_id, m->sender_dialog_id, source.c_str()), dialog_id.get(), std::move(sending_state), std::move(scheduling_state), is_outgoing, is_pinned, can_be_edited, can_be_forwarded, can_delete_for_self, can_delete_for_all_users, can_get_statistics, can_get_message_thread, - m->is_channel_post, contains_unread_mention, date, edit_date, get_message_forward_info_object(m->forward_info), - get_message_interaction_info_object(dialog_id, m), reply_in_dialog_id.get(), reply_to_message_id, - top_thread_message_id, ttl, ttl_expires_in, via_bot_user_id, m->author_signature, media_album_id, - get_restriction_reason_description(m->restriction_reasons), + can_get_media_timestamp_links, m->is_channel_post, contains_unread_mention, date, edit_date, + get_message_forward_info_object(m->forward_info), get_message_interaction_info_object(dialog_id, m), + reply_in_dialog_id.get(), reply_to_message_id, top_thread_message_id, ttl, ttl_expires_in, via_bot_user_id, + m->author_signature, media_album_id, get_restriction_reason_description(m->restriction_reasons), get_message_content_object(m->content.get(), td_, dialog_id, live_location_date, m->is_content_secret, skip_bot_commands), get_reply_markup_object(m->reply_markup)); diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index 4ede96d8c..106d30ef5 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -1776,6 +1776,8 @@ class MessagesManager final : public Actor { Status can_pin_messages(DialogId dialog_id) const; + static Status can_get_media_timestamp_link(DialogId dialog_id, const Message *m); + void cancel_edit_message_media(DialogId dialog_id, Message *m, Slice error_message); void on_message_media_edited(DialogId dialog_id, MessageId message_id, FileId file_id, FileId thumbnail_file_id, diff --git a/td/telegram/WebPagesManager.cpp b/td/telegram/WebPagesManager.cpp index f4c6933d5..4bd43d72b 100644 --- a/td/telegram/WebPagesManager.cpp +++ b/td/telegram/WebPagesManager.cpp @@ -1185,7 +1185,7 @@ tl_object_ptr WebPagesManager::get_web_page_object(WebPageId we FormattedText description; description.text = web_page->description; - description.entities = find_entities(web_page->description, true, true); + description.entities = find_entities(web_page->description, true, false); auto r_url = parse_url(web_page->display_url); if (r_url.is_ok()) { From da3b8f93a0a3fbbd607277d02d0234541dc1c7c3 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Aug 2021 09:28:53 +0300 Subject: [PATCH 19/51] Hide too big media timestamps. --- td/telegram/Game.cpp | 2 +- td/telegram/InputMessageText.cpp | 3 ++- td/telegram/LinkManager.cpp | 4 +-- td/telegram/MessageContent.cpp | 43 +++++++++++++++++--------------- td/telegram/MessageContent.h | 3 ++- td/telegram/MessageEntity.cpp | 13 +++++++--- td/telegram/MessageEntity.h | 5 ++-- td/telegram/MessagesManager.cpp | 21 +++++++++------- td/telegram/PollManager.cpp | 2 +- td/telegram/Td.cpp | 10 +++++--- td/telegram/TermsOfService.h | 2 +- td/telegram/WebPagesManager.cpp | 11 +++++--- 12 files changed, 70 insertions(+), 49 deletions(-) diff --git a/td/telegram/Game.cpp b/td/telegram/Game.cpp index 6704dbc10..5a954f5fa 100644 --- a/td/telegram/Game.cpp +++ b/td/telegram/Game.cpp @@ -93,7 +93,7 @@ const FormattedText &Game::get_text() const { tl_object_ptr Game::get_game_object(Td *td, bool skip_bot_commands) const { return make_tl_object( - id_, short_name_, title_, get_formatted_text_object(text_, skip_bot_commands), description_, + id_, short_name_, title_, get_formatted_text_object(text_, skip_bot_commands, -1), description_, get_photo_object(td->file_manager_.get(), photo_), td->animations_manager_->get_animation_object(animation_file_id_, "get_game_object")); } diff --git a/td/telegram/InputMessageText.cpp b/td/telegram/InputMessageText.cpp index d8262a66b..8d596fecd 100644 --- a/td/telegram/InputMessageText.cpp +++ b/td/telegram/InputMessageText.cpp @@ -54,8 +54,9 @@ Result process_input_message_text(const ContactsManager *conta return std::move(result); } +// used only for draft td_api::object_ptr get_input_message_text_object(const InputMessageText &input_message_text) { - return td_api::make_object(get_formatted_text_object(input_message_text.text, false), + return td_api::make_object(get_formatted_text_object(input_message_text.text, false, -1), input_message_text.disable_web_page_preview, input_message_text.clear_draft); } diff --git a/td/telegram/LinkManager.cpp b/td/telegram/LinkManager.cpp index 8b40734ff..26dd976d5 100644 --- a/td/telegram/LinkManager.cpp +++ b/td/telegram/LinkManager.cpp @@ -192,7 +192,7 @@ class LinkManager::InternalLinkMessageDraft final : public InternalLink { bool contains_link_ = false; td_api::object_ptr get_internal_link_type_object() const final { - return td_api::make_object(get_formatted_text_object(text_, true), + return td_api::make_object(get_formatted_text_object(text_, true, -1), contains_link_); } @@ -367,7 +367,7 @@ class GetDeepLinkInfoQuery final : public Td::ResultHandler { } FormattedText text{std::move(info->message_), std::move(entities)}; return promise_.set_value( - td_api::make_object(get_formatted_text_object(text, true), need_update)); + td_api::make_object(get_formatted_text_object(text, true, -1), need_update)); } default: UNREACHABLE(); diff --git a/td/telegram/MessageContent.cpp b/td/telegram/MessageContent.cpp index bc3159ba2..48b0e0ffb 100644 --- a/td/telegram/MessageContent.cpp +++ b/td/telegram/MessageContent.cpp @@ -2914,9 +2914,9 @@ void merge_message_contents(Td *td, const MessageContent *old_content, MessageCo if (old_->text.text != new_->text.text) { if (need_message_changed_warning && need_message_text_changed_warning(old_, new_)) { LOG(ERROR) << "Message text has changed from " - << to_string(get_message_content_object(old_content, td, dialog_id, -1, false, false)) + << to_string(get_message_content_object(old_content, td, dialog_id, -1, false, false, -1)) << ". New content is " - << to_string(get_message_content_object(new_content, td, dialog_id, -1, false, false)); + << to_string(get_message_content_object(new_content, td, dialog_id, -1, false, false, -1)); } need_update = true; } @@ -2926,9 +2926,9 @@ void merge_message_contents(Td *td, const MessageContent *old_content, MessageCo old_->text.entities.size() <= MAX_CUSTOM_ENTITIES_COUNT && need_message_entities_changed_warning(old_->text.entities, new_->text.entities)) { LOG(WARNING) << "Entities has changed from " - << to_string(get_message_content_object(old_content, td, dialog_id, -1, false, false)) + << to_string(get_message_content_object(old_content, td, dialog_id, -1, false, false, -1)) << ". New content is " - << to_string(get_message_content_object(new_content, td, dialog_id, -1, false, false)); + << to_string(get_message_content_object(new_content, td, dialog_id, -1, false, false, -1)); } need_update = true; } @@ -4647,19 +4647,21 @@ unique_ptr get_action_message_content(Td *td, tl_object_ptr get_message_content_object(const MessageContent *content, Td *td, DialogId dialog_id, int32 message_date, - bool is_content_secret, bool skip_bot_commands) { + bool is_content_secret, bool skip_bot_commands, + int32 max_media_timestamp) { CHECK(content != nullptr); switch (content->get_type()) { case MessageContentType::Animation: { const MessageAnimation *m = static_cast(content); return make_tl_object( td->animations_manager_->get_animation_object(m->file_id, "get_message_content_object"), - get_formatted_text_object(m->caption, skip_bot_commands), is_content_secret); + get_formatted_text_object(m->caption, skip_bot_commands, max_media_timestamp), is_content_secret); } case MessageContentType::Audio: { const MessageAudio *m = static_cast(content); - return make_tl_object(td->audios_manager_->get_audio_object(m->file_id), - get_formatted_text_object(m->caption, skip_bot_commands)); + return make_tl_object( + td->audios_manager_->get_audio_object(m->file_id), + get_formatted_text_object(m->caption, skip_bot_commands, max_media_timestamp)); } case MessageContentType::Contact: { const MessageContact *m = static_cast(content); @@ -4669,7 +4671,7 @@ tl_object_ptr get_message_content_object(const MessageCo const MessageDocument *m = static_cast(content); return make_tl_object( td->documents_manager_->get_document_object(m->file_id, PhotoFormat::Jpeg), - get_formatted_text_object(m->caption, skip_bot_commands)); + get_formatted_text_object(m->caption, skip_bot_commands, max_media_timestamp)); } case MessageContentType::Game: { const MessageGame *m = static_cast(content); @@ -4694,9 +4696,9 @@ tl_object_ptr get_message_content_object(const MessageCo } case MessageContentType::Photo: { const MessagePhoto *m = static_cast(content); - return make_tl_object(get_photo_object(td->file_manager_.get(), m->photo), - get_formatted_text_object(m->caption, skip_bot_commands), - is_content_secret); + return make_tl_object( + get_photo_object(td->file_manager_.get(), m->photo), + get_formatted_text_object(m->caption, skip_bot_commands, max_media_timestamp), is_content_secret); } case MessageContentType::Sticker: { const MessageSticker *m = static_cast(content); @@ -4704,8 +4706,9 @@ tl_object_ptr get_message_content_object(const MessageCo } case MessageContentType::Text: { const MessageText *m = static_cast(content); - return make_tl_object(get_formatted_text_object(m->text, skip_bot_commands), - td->web_pages_manager_->get_web_page_object(m->web_page_id)); + return make_tl_object( + get_formatted_text_object(m->text, skip_bot_commands, max_media_timestamp), + td->web_pages_manager_->get_web_page_object(m->web_page_id)); } case MessageContentType::Unsupported: return make_tl_object(); @@ -4715,9 +4718,9 @@ tl_object_ptr get_message_content_object(const MessageCo } case MessageContentType::Video: { const MessageVideo *m = static_cast(content); - return make_tl_object(td->videos_manager_->get_video_object(m->file_id), - get_formatted_text_object(m->caption, skip_bot_commands), - is_content_secret); + return make_tl_object( + td->videos_manager_->get_video_object(m->file_id), + get_formatted_text_object(m->caption, skip_bot_commands, max_media_timestamp), is_content_secret); } case MessageContentType::VideoNote: { const MessageVideoNote *m = static_cast(content); @@ -4726,9 +4729,9 @@ tl_object_ptr get_message_content_object(const MessageCo } case MessageContentType::VoiceNote: { const MessageVoiceNote *m = static_cast(content); - return make_tl_object(td->voice_notes_manager_->get_voice_note_object(m->file_id), - get_formatted_text_object(m->caption, skip_bot_commands), - m->is_listened); + return make_tl_object( + td->voice_notes_manager_->get_voice_note_object(m->file_id), + get_formatted_text_object(m->caption, skip_bot_commands, max_media_timestamp), m->is_listened); } case MessageContentType::ChatCreate: { const MessageChatCreate *m = static_cast(content); diff --git a/td/telegram/MessageContent.h b/td/telegram/MessageContent.h index d127710d6..f41ac3346 100644 --- a/td/telegram/MessageContent.h +++ b/td/telegram/MessageContent.h @@ -198,7 +198,8 @@ unique_ptr get_action_message_content(Td *td, tl_object_ptr get_message_content_object(const MessageContent *content, Td *td, DialogId dialog_id, int32 message_date, - bool is_content_secret, bool skip_bot_commands); + bool is_content_secret, bool skip_bot_commands, + int32 max_media_timestamp); const FormattedText *get_message_content_text(const MessageContent *content); diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index f86e6f525..3c0b2da28 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -145,7 +145,7 @@ tl_object_ptr MessageEntity::get_text_entity_object() const } vector> get_text_entities_object(const vector &entities, - bool skip_bot_commands) { + bool skip_bot_commands, int32 max_media_timestamp) { vector> result; result.reserve(entities.size()); @@ -153,6 +153,10 @@ vector> get_text_entities_object(const vector< if (skip_bot_commands && entity.type == MessageEntity::Type::BotCommand) { continue; } + if (entity.type == MessageEntity::Type::MediaTimestamp && + (max_media_timestamp < 0 || max_media_timestamp < to_integer(entity.argument))) { + continue; + } auto entity_object = entity.get_text_entity_object(); if (entity_object->type_ != nullptr) { result.push_back(std::move(entity_object)); @@ -166,9 +170,10 @@ StringBuilder &operator<<(StringBuilder &string_builder, const FormattedText &te return string_builder << '"' << text.text << "\" with entities " << text.entities; } -td_api::object_ptr get_formatted_text_object(const FormattedText &text, bool skip_bot_commands) { - return td_api::make_object(text.text, - get_text_entities_object(text.entities, skip_bot_commands)); +td_api::object_ptr get_formatted_text_object(const FormattedText &text, bool skip_bot_commands, + int32 max_media_timestamp) { + return td_api::make_object( + text.text, get_text_entities_object(text.entities, skip_bot_commands, max_media_timestamp)); } static bool is_word_character(uint32 code) { diff --git a/td/telegram/MessageEntity.h b/td/telegram/MessageEntity.h index f9e682fb1..470f560f3 100644 --- a/td/telegram/MessageEntity.h +++ b/td/telegram/MessageEntity.h @@ -133,9 +133,10 @@ Result> get_message_entities(const ContactsManager *contac bool allow_all = false); vector> get_text_entities_object(const vector &entities, - bool skip_bot_commands); + bool skip_bot_commands, int32 max_media_timestamp); -td_api::object_ptr get_formatted_text_object(const FormattedText &text, bool skip_bot_commands); +td_api::object_ptr get_formatted_text_object(const FormattedText &text, bool skip_bot_commands, + int32 max_media_timestamp); vector find_entities(Slice text, bool skip_bot_commands, bool skip_media_timestamps); diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 438be4c8f..6c11041ee 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -6534,8 +6534,8 @@ void MessagesManager::on_update_service_notification(tl_object_ptrflags_ & telegram_api::updateServiceNotification::POPUP_MASK) != 0) { send_closure(G()->td(), &Td::send_update, td_api::make_object( - update->type_, - get_message_content_object(content.get(), td_, owner_dialog_id, date, is_content_secret, true))); + update->type_, get_message_content_object(content.get(), td_, owner_dialog_id, date, + is_content_secret, true, -1))); } if (has_date && is_user) { Dialog *d = get_service_notifications_dialog(); @@ -13512,7 +13512,7 @@ std::pair> MessagesManager::creat LOG(ERROR) << "Receive media group identifier " << message_info.media_album_id << " in " << message_id << " from " << dialog_id << " with content " << oneline(to_string(get_message_content_object(message->content.get(), td_, dialog_id, message->date, - is_content_secret, false))); + is_content_secret, false, -1))); } else { message->media_album_id = message_info.media_album_id; } @@ -22840,7 +22840,8 @@ tl_object_ptr MessagesManager::get_message_object(DialogId dial auto date = is_scheduled ? 0 : m->date; auto edit_date = m->hide_edit_date ? 0 : m->edit_date; auto is_pinned = for_event_log || is_scheduled ? false : m->is_pinned; - bool skip_bot_commands = for_event_log || need_skip_bot_commands(dialog_id, m); + bool skip_bot_commands = for_event_log ? true : need_skip_bot_commands(dialog_id, m); + int32 max_media_timestamp = for_event_log ? -1 : get_message_content_duration(m->content.get(), td_); string source = PSTRING() << dialog_id << ' ' << m->message_id; return make_tl_object( m->message_id.get(), get_message_sender_object_const(m->sender_user_id, m->sender_dialog_id, source.c_str()), @@ -22851,7 +22852,7 @@ tl_object_ptr MessagesManager::get_message_object(DialogId dial reply_in_dialog_id.get(), reply_to_message_id, top_thread_message_id, ttl, ttl_expires_in, via_bot_user_id, m->author_signature, media_album_id, get_restriction_reason_description(m->restriction_reasons), get_message_content_object(m->content.get(), td_, dialog_id, live_location_date, m->is_content_secret, - skip_bot_commands), + skip_bot_commands, max_media_timestamp), get_reply_markup_object(m->reply_markup)); } @@ -24042,7 +24043,7 @@ void MessagesManager::do_send_message_group(int64 media_album_id) { << file_view.has_active_download_remote_location() << " " << file_view.is_encrypted() << " " << is_web << " " << file_view.has_url() << " " << to_string(get_message_content_object(m->content.get(), td_, dialog_id, m->date, - m->is_content_secret, false)); + m->is_content_secret, false, -1)); } auto entities = get_input_message_entities(td_->contacts_manager_.get(), caption, "do_send_message_group"); int32 input_single_media_flags = 0; @@ -28144,8 +28145,10 @@ void MessagesManager::send_update_message_content(DialogId dialog_id, const Mess void MessagesManager::send_update_message_content_impl(DialogId dialog_id, const Message *m, const char *source) const { CHECK(m != nullptr); LOG(INFO) << "Send updateMessageContent for " << m->message_id << " in " << dialog_id << " from " << source; - auto content_object = get_message_content_object(m->content.get(), td_, dialog_id, m->is_failed_to_send ? 0 : m->date, - m->is_content_secret, need_skip_bot_commands(dialog_id, m)); + int32 max_media_timestamp = get_message_content_duration(m->content.get(), td_); + auto content_object = + get_message_content_object(m->content.get(), td_, dialog_id, m->is_failed_to_send ? 0 : m->date, + m->is_content_secret, need_skip_bot_commands(dialog_id, m), max_media_timestamp); send_closure(G()->td(), &Td::send_update, td_api::make_object(dialog_id.get(), m->message_id.get(), std::move(content_object))); @@ -30486,7 +30489,7 @@ void MessagesManager::on_send_dialog_action_timeout(DialogId dialog_id) { if (!file_id.is_valid()) { LOG(ERROR) << "Have no file in " << to_string(get_message_content_object(m->content.get(), td_, dialog_id, m->date, m->is_content_secret, - false)); + false, -1)); return; } auto file_view = td_->file_manager_->get_file_view(file_id); diff --git a/td/telegram/PollManager.cpp b/td/telegram/PollManager.cpp index 266a6fe4e..c7e62846c 100644 --- a/td/telegram/PollManager.cpp +++ b/td/telegram/PollManager.cpp @@ -558,7 +558,7 @@ td_api::object_ptr PollManager::get_poll_object(PollId poll_id, co auto correct_option_id = is_local_poll_id(poll_id) ? -1 : poll->correct_option_id; poll_type = td_api::make_object( correct_option_id, - get_formatted_text_object(is_local_poll_id(poll_id) ? FormattedText() : poll->explanation, true)); + get_formatted_text_object(is_local_poll_id(poll_id) ? FormattedText() : poll->explanation, true, -1)); } else { poll_type = td_api::make_object(poll->allow_multiple_answers); } diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index 78ff4e1e6..961ad8baf 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -8320,7 +8320,8 @@ td_api::object_ptr Td::do_static_request(const td_api::getTextEn return make_error(400, "Text must be encoded in UTF-8"); } auto text_entities = find_entities(request.text_, false, false); - return make_tl_object(get_text_entities_object(text_entities, false)); + return make_tl_object( + get_text_entities_object(text_entities, false, std::numeric_limits::max())); } td_api::object_ptr Td::do_static_request(td_api::parseTextEntities &request) { @@ -8355,7 +8356,7 @@ td_api::object_ptr Td::do_static_request(td_api::parseTextEntiti } return make_tl_object(std::move(request.text_), - get_text_entities_object(r_entities.ok(), false)); + get_text_entities_object(r_entities.ok(), false, -1)); } td_api::object_ptr Td::do_static_request(td_api::parseMarkdown &request) { @@ -8375,7 +8376,7 @@ td_api::object_ptr Td::do_static_request(td_api::parseMarkdown & auto parsed_text = parse_markdown_v3({std::move(request.text_->text_), std::move(entities)}); fix_formatted_text(parsed_text.text, parsed_text.entities, true, true, true, true, true).ensure(); - return get_formatted_text_object(parsed_text, true); + return get_formatted_text_object(parsed_text, false, std::numeric_limits::max()); } td_api::object_ptr Td::do_static_request(td_api::getMarkdownText &request) { @@ -8393,7 +8394,8 @@ td_api::object_ptr Td::do_static_request(td_api::getMarkdownText return make_error(400, status.error().message()); } - return get_formatted_text_object(get_markdown_v3({std::move(request.text_->text_), std::move(entities)}), true); + return get_formatted_text_object(get_markdown_v3({std::move(request.text_->text_), std::move(entities)}), false, + std::numeric_limits::max()); } td_api::object_ptr Td::do_static_request(const td_api::getFileMimeType &request) { diff --git a/td/telegram/TermsOfService.h b/td/telegram/TermsOfService.h index 4c3cc4fcf..344c57e0a 100644 --- a/td/telegram/TermsOfService.h +++ b/td/telegram/TermsOfService.h @@ -42,7 +42,7 @@ class TermsOfService { return nullptr; } - return td_api::make_object(get_formatted_text_object(text_, true), min_user_age_, + return td_api::make_object(get_formatted_text_object(text_, true, -1), min_user_age_, show_popup_); } diff --git a/td/telegram/WebPagesManager.cpp b/td/telegram/WebPagesManager.cpp index 4bd43d72b..0ff25cd3b 100644 --- a/td/telegram/WebPagesManager.cpp +++ b/td/telegram/WebPagesManager.cpp @@ -1254,11 +1254,16 @@ tl_object_ptr WebPagesManager::get_web_page_object(WebPageId we } } + int32 max_media_timestamp = -1; + if (web_page->document.type == Document::Type::Audio || web_page->document.type == Document::Type::Video || + web_page->document.type == Document::Type::VideoNote || web_page->document.type == Document::Type::VoiceNote) { + max_media_timestamp = web_page->duration == 0 ? std::numeric_limits::max() : web_page->duration; + } return make_tl_object( web_page->url, web_page->display_url, web_page->type, web_page->site_name, web_page->title, - get_formatted_text_object(description, true), get_photo_object(td_->file_manager_.get(), web_page->photo), - web_page->embed_url, web_page->embed_type, web_page->embed_dimensions.width, web_page->embed_dimensions.height, - web_page->duration, web_page->author, + get_formatted_text_object(description, true, max_media_timestamp), + get_photo_object(td_->file_manager_.get(), web_page->photo), web_page->embed_url, web_page->embed_type, + web_page->embed_dimensions.width, web_page->embed_dimensions.height, web_page->duration, web_page->author, web_page->document.type == Document::Type::Animation ? td_->animations_manager_->get_animation_object(web_page->document.file_id, "get_web_page_object") : nullptr, From c667f6c9bbc22a0d380f1a38659d010adca46517 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 5 Aug 2021 01:23:16 +0300 Subject: [PATCH 20/51] Improve test. --- test/message_entities.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/message_entities.cpp b/test/message_entities.cpp index 4e8935aca..49a635df6 100644 --- a/test/message_entities.cpp +++ b/test/message_entities.cpp @@ -6,6 +6,7 @@ // #include "td/telegram/MessageEntity.h" +#include "td/utils/algorithm.h" #include "td/utils/common.h" #include "td/utils/format.h" #include "td/utils/logging.h" @@ -17,6 +18,7 @@ #include "td/utils/utf8.h" #include +#include static void check_mention(const td::string &str, const td::vector &expected) { auto result_slice = td::find_mentions(str); @@ -172,8 +174,9 @@ TEST(MessageEntities, cashtag) { check_cashtag(u8"\u2122$ABC\u2122", {"$ABC"}); } -static void check_media_timestamp(const td::string &str, const td::vector> &expected) { - auto result = td::find_media_timestamps(str); +static void check_media_timestamp(const td::string &str, const td::vector> &expected) { + auto result = td::transform(td::find_media_timestamps(str), + [](auto &&entity) { return std::make_pair(entity.first.str(), entity.second); }); if (result != expected) { LOG(FATAL) << td::tag("text", str) << td::tag("got", td::format::as_array(result)) << td::tag("expected", td::format::as_array(expected)); From 9653cc9e3ee24e57052e62d25560703d0ef90696 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 5 Aug 2021 02:09:04 +0300 Subject: [PATCH 21/51] Add separate get_message_content_media_duration. --- td/telegram/MessageContent.cpp | 32 ++++++++++++++++++++++++++++---- td/telegram/MessageContent.h | 2 ++ td/telegram/MessagesManager.cpp | 8 ++++---- td/telegram/WebPagesManager.cpp | 23 ++++++++++++++--------- td/telegram/WebPagesManager.h | 4 +++- 5 files changed, 51 insertions(+), 18 deletions(-) diff --git a/td/telegram/MessageContent.cpp b/td/telegram/MessageContent.cpp index 48b0e0ffb..08b0a7e30 100644 --- a/td/telegram/MessageContent.cpp +++ b/td/telegram/MessageContent.cpp @@ -4921,10 +4921,6 @@ int32 get_message_content_duration(const MessageContent *content, const Td *td) auto audio_file_id = static_cast(content)->file_id; return td->audios_manager_->get_audio_duration(audio_file_id); } - case MessageContentType::Text: { - auto web_page_id = static_cast(content)->web_page_id; - return td->web_pages_manager_->get_web_page_duration(web_page_id); - } case MessageContentType::Video: { auto video_file_id = static_cast(content)->file_id; return td->videos_manager_->get_video_duration(video_file_id); @@ -4942,6 +4938,34 @@ int32 get_message_content_duration(const MessageContent *content, const Td *td) } } +int32 get_message_content_media_duration(const MessageContent *content, const Td *td) { + CHECK(content != nullptr); + switch (content->get_type()) { + case MessageContentType::Audio: { + auto audio_file_id = static_cast(content)->file_id; + return td->audios_manager_->get_audio_duration(audio_file_id); + } + case MessageContentType::Text: { + auto web_page_id = static_cast(content)->web_page_id; + return td->web_pages_manager_->get_web_page_media_duration(web_page_id); + } + case MessageContentType::Video: { + auto video_file_id = static_cast(content)->file_id; + return td->videos_manager_->get_video_duration(video_file_id); + } + case MessageContentType::VideoNote: { + auto video_note_file_id = static_cast(content)->file_id; + return td->video_notes_manager_->get_video_note_duration(video_note_file_id); + } + case MessageContentType::VoiceNote: { + auto voice_file_id = static_cast(content)->file_id; + return td->voice_notes_manager_->get_voice_note_duration(voice_file_id); + } + default: + return -1; + } +} + FileId get_message_content_upload_file_id(const MessageContent *content) { switch (content->get_type()) { case MessageContentType::Animation: diff --git a/td/telegram/MessageContent.h b/td/telegram/MessageContent.h index f41ac3346..6f66670a5 100644 --- a/td/telegram/MessageContent.h +++ b/td/telegram/MessageContent.h @@ -207,6 +207,8 @@ const FormattedText *get_message_content_caption(const MessageContent *content); int32 get_message_content_duration(const MessageContent *content, const Td *td); +int32 get_message_content_media_duration(const MessageContent *content, const Td *td); + FileId get_message_content_upload_file_id(const MessageContent *content); FileId get_message_content_any_file_id(const MessageContent *content); diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 6c11041ee..7ce98a167 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -17369,7 +17369,7 @@ Result> MessagesManager::get_message_link(FullMessageId } if (media_timestamp != 0) { for_group = false; - auto duration = get_message_content_duration(m->content.get(), td_); + auto duration = get_message_content_media_duration(m->content.get(), td_); if (duration != 0 && media_timestamp > duration) { media_timestamp = 0; } @@ -17630,7 +17630,7 @@ td_api::object_ptr MessagesManager::get_message_link_in for_album = !info.is_single && m->media_album_id != 0; for_comment = (info.comment_dialog_id.is_valid() || info.for_comment) && m->top_thread_message_id.is_valid(); if (can_message_content_have_media_timestamp(m->content.get())) { - auto duration = get_message_content_duration(m->content.get(), td_); + auto duration = get_message_content_media_duration(m->content.get(), td_); if (duration == 0 || info.media_timestamp <= duration) { media_timestamp = info.media_timestamp; } @@ -22841,7 +22841,7 @@ tl_object_ptr MessagesManager::get_message_object(DialogId dial auto edit_date = m->hide_edit_date ? 0 : m->edit_date; auto is_pinned = for_event_log || is_scheduled ? false : m->is_pinned; bool skip_bot_commands = for_event_log ? true : need_skip_bot_commands(dialog_id, m); - int32 max_media_timestamp = for_event_log ? -1 : get_message_content_duration(m->content.get(), td_); + int32 max_media_timestamp = for_event_log ? -1 : get_message_content_media_duration(m->content.get(), td_); string source = PSTRING() << dialog_id << ' ' << m->message_id; return make_tl_object( m->message_id.get(), get_message_sender_object_const(m->sender_user_id, m->sender_dialog_id, source.c_str()), @@ -28145,7 +28145,7 @@ void MessagesManager::send_update_message_content(DialogId dialog_id, const Mess void MessagesManager::send_update_message_content_impl(DialogId dialog_id, const Message *m, const char *source) const { CHECK(m != nullptr); LOG(INFO) << "Send updateMessageContent for " << m->message_id << " in " << dialog_id << " from " << source; - int32 max_media_timestamp = get_message_content_duration(m->content.get(), td_); + int32 max_media_timestamp = get_message_content_media_duration(m->content.get(), td_); auto content_object = get_message_content_object(m->content.get(), td_, dialog_id, m->is_failed_to_send ? 0 : m->date, m->is_content_secret, need_skip_bot_commands(dialog_id, m), max_media_timestamp); diff --git a/td/telegram/WebPagesManager.cpp b/td/telegram/WebPagesManager.cpp index 0ff25cd3b..097634f02 100644 --- a/td/telegram/WebPagesManager.cpp +++ b/td/telegram/WebPagesManager.cpp @@ -1254,14 +1254,10 @@ tl_object_ptr WebPagesManager::get_web_page_object(WebPageId we } } - int32 max_media_timestamp = -1; - if (web_page->document.type == Document::Type::Audio || web_page->document.type == Document::Type::Video || - web_page->document.type == Document::Type::VideoNote || web_page->document.type == Document::Type::VoiceNote) { - max_media_timestamp = web_page->duration == 0 ? std::numeric_limits::max() : web_page->duration; - } + auto duration = get_web_page_media_duration(web_page); return make_tl_object( web_page->url, web_page->display_url, web_page->type, web_page->site_name, web_page->title, - get_formatted_text_object(description, true, max_media_timestamp), + get_formatted_text_object(description, true, duration == 0 ? std::numeric_limits::max() : duration), get_photo_object(td_->file_manager_.get(), web_page->photo), web_page->embed_url, web_page->embed_type, web_page->embed_dimensions.width, web_page->embed_dimensions.height, web_page->duration, web_page->author, web_page->document.type == Document::Type::Animation @@ -1695,12 +1691,21 @@ string WebPagesManager::get_web_page_search_text(WebPageId web_page_id) const { return PSTRING() << web_page->title + " " + web_page->description; } -int32 WebPagesManager::get_web_page_duration(WebPageId web_page_id) const { +int32 WebPagesManager::get_web_page_media_duration(WebPageId web_page_id) const { const WebPage *web_page = get_web_page(web_page_id); if (web_page == nullptr) { - return 0; + return -1; } - return web_page->duration; + return get_web_page_media_duration(web_page); +} + +int32 WebPagesManager::get_web_page_media_duration(const WebPage *web_page) { + if (web_page->document.type == Document::Type::Audio || web_page->document.type == Document::Type::Video || + web_page->document.type == Document::Type::VideoNote || web_page->document.type == Document::Type::VoiceNote || web_page->embed_type == "iframe") { + return web_page->duration; + } + + return -1; } vector WebPagesManager::get_web_page_file_ids(const WebPage *web_page) const { diff --git a/td/telegram/WebPagesManager.h b/td/telegram/WebPagesManager.h index 4d5e582ef..c7beff080 100644 --- a/td/telegram/WebPagesManager.h +++ b/td/telegram/WebPagesManager.h @@ -87,7 +87,7 @@ class WebPagesManager final : public Actor { string get_web_page_search_text(WebPageId web_page_id) const; - int32 get_web_page_duration(WebPageId web_page_id) const; + int32 get_web_page_media_duration(WebPageId web_page_id) const; private: static constexpr int32 WEBPAGE_FLAG_HAS_TYPE = 1 << 0; @@ -170,6 +170,8 @@ class WebPagesManager final : public Actor { void tear_down() final; + static int32 get_web_page_media_duration(const WebPage *web_page); + FileSourceId get_web_page_file_source_id(WebPage *web_page); vector get_web_page_file_ids(const WebPage *web_page) const; From 2605cd374b95399fbc3a69e6e5657800e4bbf5a7 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 5 Aug 2021 05:41:24 +0300 Subject: [PATCH 22/51] Add has_media_timestamps. --- td/telegram/MessageEntity.cpp | 15 +++++++++++++++ td/telegram/MessageEntity.h | 2 ++ 2 files changed, 17 insertions(+) diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index 3c0b2da28..b9f213542 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -4096,6 +4096,21 @@ void add_formatted_text_dependencies(Dependencies &dependencies, const Formatted } } +bool has_media_timestamps(const FormattedText *text, int32 min_timestamp, int32 max_timestamp) { + if (text == nullptr) { + return false; + } + for (auto &entity : text->entities) { + if (entity.type == MessageEntity::Type::MediaTimestamp) { + int32 timestamp = to_integer(entity.argument); + if (min_timestamp <= timestamp && timestamp <= max_timestamp) { + return true; + } + } + } + return false; +} + bool has_bot_commands(const FormattedText *text) { if (text == nullptr) { return false; diff --git a/td/telegram/MessageEntity.h b/td/telegram/MessageEntity.h index 470f560f3..78023238f 100644 --- a/td/telegram/MessageEntity.h +++ b/td/telegram/MessageEntity.h @@ -196,6 +196,8 @@ Result process_input_caption(const ContactsManager *contacts_mana void add_formatted_text_dependencies(Dependencies &dependencies, const FormattedText *text); +bool has_media_timestamps(const FormattedText *text, int32 min_timestamp, int32 max_timestamp); + bool has_bot_commands(const FormattedText *text); bool need_always_skip_bot_commands(const ContactsManager *contacts_manager, DialogId dialog_id, bool is_bot); From 4ae305d8c41de001321a43e0b000d8e56101b691 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 5 Aug 2021 09:59:43 +0300 Subject: [PATCH 23/51] Update max_media_timestamp when message content is changed. --- td/telegram/MessagesManager.cpp | 123 +++++++++++++++++++++++++++----- td/telegram/MessagesManager.h | 15 +++- 2 files changed, 121 insertions(+), 17 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 7ce98a167..a653781cc 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -4858,6 +4858,7 @@ void MessagesManager::Message::store(StorerT &storer) const { bool has_send_emoji = !send_emoji.empty(); bool is_imported = is_forwarded && forward_info->is_imported; bool has_ttl_period = ttl_period != 0; + bool has_max_reply_media_timestamp = max_reply_media_timestamp >= 0; BEGIN_STORE_FLAGS(); STORE_FLAG(is_channel_post); STORE_FLAG(is_outgoing); @@ -4919,6 +4920,7 @@ void MessagesManager::Message::store(StorerT &storer) const { STORE_FLAG(has_send_emoji); STORE_FLAG(is_imported); STORE_FLAG(has_ttl_period); // 25 + STORE_FLAG(has_max_reply_media_timestamp); END_STORE_FLAGS(); } @@ -5034,6 +5036,9 @@ void MessagesManager::Message::store(StorerT &storer) const { if (has_ttl_period) { store(ttl_period, storer); } + if (has_max_reply_media_timestamp) { + store(max_reply_media_timestamp, storer); + } } // do not forget to resolve message dependencies @@ -5075,6 +5080,7 @@ void MessagesManager::Message::parse(ParserT &parser) { bool has_send_emoji = false; bool is_imported = false; bool has_ttl_period = false; + bool has_max_reply_media_timestamp = false; BEGIN_PARSE_FLAGS(); PARSE_FLAG(is_channel_post); PARSE_FLAG(is_outgoing); @@ -5136,6 +5142,7 @@ void MessagesManager::Message::parse(ParserT &parser) { PARSE_FLAG(has_send_emoji); PARSE_FLAG(is_imported); PARSE_FLAG(has_ttl_period); + PARSE_FLAG(has_max_reply_media_timestamp); END_PARSE_FLAGS(); } @@ -5258,6 +5265,9 @@ void MessagesManager::Message::parse(ParserT &parser) { if (has_ttl_period) { parse(ttl_period, parser); } + if (has_max_reply_media_timestamp) { + parse(max_reply_media_timestamp, parser); + } is_content_secret |= is_secret_message_content(ttl, content->get_type()); // repair is_content_secret for old messages @@ -6950,11 +6960,11 @@ bool MessagesManager::need_skip_bot_commands(DialogId dialog_id, const Message * } void MessagesManager::on_external_update_message_content(FullMessageId full_message_id) { - const Dialog *d = get_dialog(full_message_id.get_dialog_id()); + Dialog *d = get_dialog(full_message_id.get_dialog_id()); CHECK(d != nullptr); - const Message *m = get_message(d, full_message_id.get_message_id()); + Message *m = get_message(d, full_message_id.get_message_id()); CHECK(m != nullptr); - send_update_message_content(d->dialog_id, m, "on_external_update_message_content"); + send_update_message_content(d, m, "on_external_update_message_content"); if (m->message_id == d->last_message_id) { send_update_chat_last_message_impl(d, "on_external_update_message_content"); } @@ -12136,7 +12146,7 @@ void MessagesManager::on_message_ttl_expired(Dialog *d, Message *m) { remove_message_file_sources(d->dialog_id, m); on_message_ttl_expired_impl(d, m); register_message_content(td_, m->content.get(), {d->dialog_id, m->message_id}, "on_message_ttl_expired"); - send_update_message_content(d->dialog_id, m, "on_message_ttl_expired"); + send_update_message_content(d, m, "on_message_ttl_expired"); } void MessagesManager::on_message_ttl_expired_impl(Dialog *d, Message *m) { @@ -12164,6 +12174,7 @@ void MessagesManager::on_message_ttl_expired_impl(Dialog *d, Message *m) { update_message_contains_unread_mention(d, m, false, "on_message_ttl_expired_impl"); m->contains_mention = false; m->reply_to_message_id = MessageId(); + m->max_reply_media_timestamp = -1; m->reply_in_dialog_id = DialogId(); m->top_thread_message_id = MessageId(); m->linked_top_thread_message_id = MessageId(); @@ -14233,6 +14244,7 @@ void MessagesManager::on_update_sent_text_message(int64 random_id, if (need_update) { send_update_message_content(dialog_id, m, "on_update_sent_text_message"); } + // there is no need to call on_message_changed, because only content was changed } void MessagesManager::delete_pending_message_web_page(FullMessageId full_message_id) { @@ -22841,7 +22853,8 @@ tl_object_ptr MessagesManager::get_message_object(DialogId dial auto edit_date = m->hide_edit_date ? 0 : m->edit_date; auto is_pinned = for_event_log || is_scheduled ? false : m->is_pinned; bool skip_bot_commands = for_event_log ? true : need_skip_bot_commands(dialog_id, m); - int32 max_media_timestamp = for_event_log ? -1 : get_message_content_media_duration(m->content.get(), td_); + int32 max_media_timestamp = + for_event_log ? get_message_own_max_media_timestamp(m) : get_message_max_media_timestamp(m); string source = PSTRING() << dialog_id << ' ' << m->message_id; return make_tl_object( m->message_id.get(), get_message_sender_object_const(m->sender_user_id, m->sender_dialog_id, source.c_str()), @@ -23860,6 +23873,7 @@ void MessagesManager::on_upload_message_media_success(DialogId dialog_id, Messag false, UserId(), nullptr); update_message_content(dialog_id, m, std::move(content), true, true, true); + // there is no need to call on_message_changed, because only content was changed auto input_media = get_input_media(m->content.get(), td_, m->ttl, m->send_emoji, true); Status result; @@ -25424,6 +25438,74 @@ bool MessagesManager::has_message_sender_user_id(DialogId dialog_id, const Messa return true; } +int32 MessagesManager::get_message_own_max_media_timestamp(const Message *m) const { + auto duration = get_message_content_media_duration(m->content.get(), td_); + return duration == 0 ? std::numeric_limits::max() : duration; +} + +int32 MessagesManager::get_message_max_media_timestamp(const Message *m) { + return m->max_own_media_timestamp != -1 ? m->max_own_media_timestamp : m->max_reply_media_timestamp; +} + +void MessagesManager::update_message_max_reply_media_timestamp(const Dialog *d, Message *m, + bool need_send_update_message_content) { + if (td_->auth_manager_->is_bot()) { + return; + } + + auto new_max_reply_media_timestamp = m->max_reply_media_timestamp; + if (!m->reply_to_message_id.is_valid()) { + new_max_reply_media_timestamp = -1; + } else { + auto replied_m = get_message(d, m->reply_to_message_id); + if (replied_m != nullptr) { + new_max_reply_media_timestamp = get_message_own_max_media_timestamp(replied_m); + } else if (d->deleted_message_ids.count(m->reply_to_message_id) || + m->reply_to_message_id <= d->last_clear_history_message_id || + m->reply_to_message_id <= d->max_unavailable_message_id) { + new_max_reply_media_timestamp = -1; + } else { + // replied message isn't deleted and isn't loaded yet + return; + } + } + + if (m->max_reply_media_timestamp == new_max_reply_media_timestamp) { + return; + } + + LOG(INFO) << "Set max_reply_media_timestamp in " << m->message_id << " in " << d->dialog_id << " to " + << new_max_reply_media_timestamp; + auto old_max_media_timestamp = get_message_max_media_timestamp(m); + m->max_reply_media_timestamp = new_max_reply_media_timestamp; + auto new_max_media_timestamp = get_message_max_media_timestamp(m); + if (need_send_update_message_content && old_max_media_timestamp != new_max_media_timestamp) { + if (old_max_media_timestamp > new_max_media_timestamp) { + std::swap(old_max_media_timestamp, new_max_media_timestamp); + } + + if (has_media_timestamps(get_message_content_text(m->content.get()), old_max_media_timestamp + 1, + new_max_media_timestamp)) { + send_update_message_content_impl(d->dialog_id, m, "update_message_max_reply_media_timestamp"); + } + } +} + +void MessagesManager::update_message_max_own_media_timestamp(const Dialog *d, Message *m) { + if (td_->auth_manager_->is_bot()) { + return; + } + auto new_max_own_media_timestamp = get_message_own_max_media_timestamp(m); + if (m->max_own_media_timestamp == new_max_own_media_timestamp) { + return; + } + + LOG(INFO) << "Set max_own_media_timestamp in " << m->message_id << " in " << d->dialog_id << " to " + << new_max_own_media_timestamp; + m->max_own_media_timestamp = new_max_own_media_timestamp; + // TODO update replied messages +} + bool MessagesManager::get_message_disable_web_page_preview(const Message *m) { // m->disable_web_page_preview is known only for sent from this client messages if (m->disable_web_page_preview) { @@ -28133,22 +28215,28 @@ void MessagesManager::send_update_message_send_succeeded(Dialog *d, MessageId ol make_tl_object(get_message_object(d->dialog_id, m), old_message_id.get())); } -void MessagesManager::send_update_message_content(DialogId dialog_id, const Message *m, const char *source) { +void MessagesManager::send_update_message_content(DialogId dialog_id, Message *m, const char *source) { + Dialog *d = get_dialog(dialog_id); + LOG_CHECK(d != nullptr) << "Send updateMessageContent in unknown " << dialog_id << " from " << source + << " with load count " << loaded_dialogs_.count(dialog_id); + send_update_message_content(d, m, source); +} + +void MessagesManager::send_update_message_content(const Dialog *d, Message *m, const char *source) { + CHECK(d != nullptr); CHECK(m != nullptr); - LOG_CHECK(have_dialog(dialog_id)) << "Send updateMessageContent in unknown " << dialog_id << " from " << source - << " with load count " << loaded_dialogs_.count(dialog_id); - delete_bot_command_message_id(dialog_id, m->message_id); - try_add_bot_command_message_id(dialog_id, m); - send_update_message_content_impl(dialog_id, m, source); + delete_bot_command_message_id(d->dialog_id, m->message_id); + try_add_bot_command_message_id(d->dialog_id, m); + update_message_max_own_media_timestamp(d, m); + send_update_message_content_impl(d->dialog_id, m, source); } void MessagesManager::send_update_message_content_impl(DialogId dialog_id, const Message *m, const char *source) const { CHECK(m != nullptr); LOG(INFO) << "Send updateMessageContent for " << m->message_id << " in " << dialog_id << " from " << source; - int32 max_media_timestamp = get_message_content_media_duration(m->content.get(), td_); - auto content_object = - get_message_content_object(m->content.get(), td_, dialog_id, m->is_failed_to_send ? 0 : m->date, - m->is_content_secret, need_skip_bot_commands(dialog_id, m), max_media_timestamp); + auto content_object = get_message_content_object(m->content.get(), td_, dialog_id, m->is_failed_to_send ? 0 : m->date, + m->is_content_secret, need_skip_bot_commands(dialog_id, m), + get_message_max_media_timestamp(m)); send_closure(G()->td(), &Td::send_update, td_api::make_object(dialog_id.get(), m->message_id.get(), std::move(content_object))); @@ -28664,7 +28752,7 @@ FullMessageId MessagesManager::on_send_message_success(int64 random_id, MessageI // } if (merge_message_content_file_id(td_, sent_message->content.get(), new_file_id)) { - send_update_message_content(dialog_id, sent_message.get(), source); + send_update_message_content(d, sent_message.get(), source); } if (old_message_id.is_valid() && new_message_id < old_message_id && !can_overflow_message_id(dialog_id)) { @@ -32436,6 +32524,8 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, uniq } } } + update_message_max_reply_media_timestamp(d, message.get(), false); + update_message_max_own_media_timestamp(d, message.get()); const Message *m = message.get(); if (m->message_id.is_yet_unsent() && m->reply_to_message_id.is_valid() && !m->reply_to_message_id.is_yet_unsent()) { @@ -33305,6 +33395,7 @@ bool MessagesManager::update_message(Dialog *d, Message *old_message, unique_ptr if (new_message->reply_to_message_id == MessageId() || replace_legacy) { LOG(DEBUG) << "Drop message reply_to_message_id"; old_message->reply_to_message_id = MessageId(); + update_message_max_reply_media_timestamp(d, old_message, true); need_send_update = true; } else if (is_new_available) { LOG(ERROR) << message_id << " in " << dialog_id << " has changed message it is reply to from " diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index 106d30ef5..f2f301693 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -1059,6 +1059,9 @@ class MessagesManager final : public Actor { NotificationId notification_id; NotificationId removed_notification_id; + int32 max_reply_media_timestamp = -1; + int32 max_own_media_timestamp = -1; + int32 view_count = 0; int32 forward_count = 0; MessageReplyInfo reply_info; @@ -2143,6 +2146,10 @@ class MessagesManager final : public Actor { bool update_message_content(DialogId dialog_id, Message *old_message, unique_ptr new_content, bool need_send_update_message_content, bool need_merge_files, bool is_message_in_dialog); + void update_message_max_reply_media_timestamp(const Dialog *d, Message *m, bool need_send_update_message_content); + + void update_message_max_own_media_timestamp(const Dialog *d, Message *m); + void send_update_new_message(const Dialog *d, const Message *m); static bool is_from_mention_notification_group(const Dialog *d, const Message *m); @@ -2199,7 +2206,9 @@ class MessagesManager final : public Actor { void send_update_message_send_succeeded(Dialog *d, MessageId old_message_id, const Message *m) const; - void send_update_message_content(DialogId dialog_id, const Message *m, const char *source); + void send_update_message_content(DialogId dialog_id, Message *m, const char *source); + + void send_update_message_content(const Dialog *d, Message *m, const char *source); void send_update_message_content_impl(DialogId dialog_id, const Message *m, const char *source) const; @@ -2602,6 +2611,10 @@ class MessagesManager final : public Actor { bool has_message_sender_user_id(DialogId dialog_id, const Message *m) const; + int32 get_message_own_max_media_timestamp(const Message *m) const; + + static int32 get_message_max_media_timestamp(const Message *m); + static bool get_message_disable_web_page_preview(const Message *m); static int32 get_message_flags(const Message *m); From 0e6584db8711777381d657645c25eefd23edb446 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 5 Aug 2021 10:13:06 +0300 Subject: [PATCH 24/51] Send updateChatLastMessage if content of the last yyet unsent message changes. --- td/telegram/MessagesManager.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index a653781cc..f73b28756 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -12147,6 +12147,7 @@ void MessagesManager::on_message_ttl_expired(Dialog *d, Message *m) { on_message_ttl_expired_impl(d, m); register_message_content(td_, m->content.get(), {d->dialog_id, m->message_id}, "on_message_ttl_expired"); send_update_message_content(d, m, "on_message_ttl_expired"); + // the caller must call on_message_changed } void MessagesManager::on_message_ttl_expired_impl(Dialog *d, Message *m) { @@ -14205,12 +14206,14 @@ void MessagesManager::on_update_sent_text_message(int64 random_id, } auto full_message_id = it->second; - auto m = get_message_force(full_message_id, "on_update_sent_text_message"); + auto dialog_id = full_message_id.get_dialog_id(); + Dialog *d = get_dialog(dialog_id); + auto m = get_message_force(d, full_message_id.get_message_id(), "on_update_sent_text_message"); if (m == nullptr) { // message has already been deleted return; } - auto dialog_id = full_message_id.get_dialog_id(); + CHECK(m->message_id.is_yet_unsent()); full_message_id = FullMessageId(dialog_id, m->message_id); if (m->content->get_type() != MessageContentType::Text) { @@ -14243,8 +14246,10 @@ void MessagesManager::on_update_sent_text_message(int64 random_id, } if (need_update) { send_update_message_content(dialog_id, m, "on_update_sent_text_message"); + if (m->message_id == d->last_message_id) { + send_update_chat_last_message_impl(d, "on_update_sent_text_message"); + } } - // there is no need to call on_message_changed, because only content was changed } void MessagesManager::delete_pending_message_web_page(FullMessageId full_message_id) { @@ -23872,8 +23877,10 @@ void MessagesManager::on_upload_message_media_success(DialogId dialog_id, Messag auto content = get_message_content(td_, caption == nullptr ? FormattedText() : *caption, std::move(media), dialog_id, false, UserId(), nullptr); - update_message_content(dialog_id, m, std::move(content), true, true, true); - // there is no need to call on_message_changed, because only content was changed + if (update_message_content(dialog_id, m, std::move(content), true, true, true) && + m->message_id == d->last_message_id) { + send_update_chat_last_message_impl(d, "on_upload_message_media_success"); + } auto input_media = get_input_media(m->content.get(), td_, m->ttl, m->send_emoji, true); Status result; @@ -33727,7 +33734,7 @@ bool MessagesManager::update_message_content(DialogId dialog_id, Message *old_me if (need_update && need_send_update_message_content) { send_update_message_content(dialog_id, old_message, "update_message_content"); } - return is_content_changed || need_update; + return need_update; } MessagesManager::Dialog *MessagesManager::get_dialog_by_message_id(MessageId message_id) { From 40d953ba2039a9013ad53de02fe780dca1221ac6 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 5 Aug 2021 10:41:13 +0300 Subject: [PATCH 25/51] Init max media timestamps for scheduled messages. --- td/telegram/MessagesManager.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index f73b28756..cc119d3d3 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -31950,6 +31950,7 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(DialogId dialog return add_message_to_dialog(d, std::move(message), from_update, need_update, need_update_dialog_pos, source); } +// keep synced with add_scheduled_message_to_dialog MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, unique_ptr message, bool from_update, bool *need_update, bool *need_update_dialog_pos, const char *source) { @@ -32797,6 +32798,9 @@ MessagesManager::Message *MessagesManager::add_scheduled_message_to_dialog(Dialo LOG(INFO) << "Adding not found " << message_id << " to " << dialog_id << " from " << source; + update_message_max_reply_media_timestamp(d, message.get(), false); + update_message_max_own_media_timestamp(d, message.get()); + const Message *m = message.get(); if (m->message_id.is_yet_unsent() && m->reply_to_message_id.is_valid() && !m->reply_to_message_id.is_yet_unsent()) { replied_by_yet_unsent_messages_[FullMessageId{dialog_id, m->reply_to_message_id}]++; From eee773901c61a903b3243b3eee7903e0ce9f4c8d Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 6 Aug 2021 05:55:43 +0300 Subject: [PATCH 26/51] Update max_reply_media_timestamp in replies when needed. --- td/telegram/MessagesManager.cpp | 106 +++++++++++++++++++++++++++++--- td/telegram/MessagesManager.h | 14 ++++- 2 files changed, 110 insertions(+), 10 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index cc119d3d3..b99a8605f 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -12173,6 +12173,7 @@ void MessagesManager::on_message_ttl_expired_impl(Dialog *d, Message *m) { } remove_message_notification_id(d, m, true, true); update_message_contains_unread_mention(d, m, false, "on_message_ttl_expired_impl"); + unregister_message_reply(d, m); m->contains_mention = false; m->reply_to_message_id = MessageId(); m->max_reply_media_timestamp = -1; @@ -15040,8 +15041,6 @@ unique_ptr MessagesManager::do_delete_message(Dialog * m->have_previous, m->have_next, source); } - delete_bot_command_message_id(d->dialog_id, message_id); - bool need_get_history = false; if (!only_from_memory) { LOG(INFO) << "Deleting " << full_message_id << " with have_previous = " << m->have_previous @@ -15251,7 +15250,9 @@ void MessagesManager::on_message_deleted(Dialog *d, Message *m, bool is_permanen } ttl_unregister_message(d->dialog_id, m, source); ttl_period_unregister_message(d->dialog_id, m); + delete_bot_command_message_id(d->dialog_id, m->message_id); unregister_message_content(td_, m->content.get(), {d->dialog_id, m->message_id}, "on_message_deleted"); + unregister_message_reply(d, m); if (m->notification_id.is_valid()) { delete_notification_id_to_message_id_correspondence(d, m->notification_id, m->message_id); } @@ -15298,6 +15299,7 @@ unique_ptr MessagesManager::do_delete_scheduled_messag cancel_send_deleted_message(d->dialog_id, result.get(), is_permanently_deleted); unregister_message_content(td_, result->content.get(), {d->dialog_id, message_id}, "do_delete_scheduled_message"); + unregister_message_reply(d, m); return result; } @@ -25451,7 +25453,7 @@ int32 MessagesManager::get_message_own_max_media_timestamp(const Message *m) con } int32 MessagesManager::get_message_max_media_timestamp(const Message *m) { - return m->max_own_media_timestamp != -1 ? m->max_own_media_timestamp : m->max_reply_media_timestamp; + return m->max_own_media_timestamp >= 0 ? m->max_own_media_timestamp : m->max_reply_media_timestamp; } void MessagesManager::update_message_max_reply_media_timestamp(const Dialog *d, Message *m, @@ -25502,6 +25504,7 @@ void MessagesManager::update_message_max_own_media_timestamp(const Dialog *d, Me if (td_->auth_manager_->is_bot()) { return; } + auto new_max_own_media_timestamp = get_message_own_max_media_timestamp(m); if (m->max_own_media_timestamp == new_max_own_media_timestamp) { return; @@ -25510,7 +25513,79 @@ void MessagesManager::update_message_max_own_media_timestamp(const Dialog *d, Me LOG(INFO) << "Set max_own_media_timestamp in " << m->message_id << " in " << d->dialog_id << " to " << new_max_own_media_timestamp; m->max_own_media_timestamp = new_max_own_media_timestamp; - // TODO update replied messages + + update_message_max_reply_media_timestamp_in_replied_messages(d->dialog_id, m->message_id); +} + +void MessagesManager::update_message_max_reply_media_timestamp_in_replied_messages(DialogId dialog_id, + MessageId reply_to_message_id) { + if (reply_to_message_id.is_scheduled()) { + return; + } + CHECK(reply_to_message_id.is_valid()); + + FullMessageId full_message_id{dialog_id, reply_to_message_id}; + auto it = replied_by_media_timestamp_messages_.find(full_message_id); + if (it == replied_by_media_timestamp_messages_.end()) { + return; + } + + LOG(INFO) << "Update max_reply_media_timestamp for replies of " << reply_to_message_id << " in " << dialog_id; + + Dialog *d = get_dialog(dialog_id); + CHECK(d != nullptr); + for (auto message_id : it->second) { + auto m = get_message(d, message_id); + CHECK(m != nullptr); + CHECK(m->reply_to_message_id == reply_to_message_id); + update_message_max_reply_media_timestamp(d, m, true); + } +} + +void MessagesManager::register_message_reply(const Dialog *d, const Message *m) { + if (!m->reply_to_message_id.is_valid() || td_->auth_manager_->is_bot()) { + return; + } + + if (has_media_timestamps(get_message_content_text(m->content.get()), 0, std::numeric_limits::max())) { + LOG(INFO) << "Register " << m->message_id << " in " << d->dialog_id << " as reply to " << m->reply_to_message_id; + FullMessageId full_message_id{d->dialog_id, m->reply_to_message_id}; + bool is_inserted = replied_by_media_timestamp_messages_[full_message_id].insert(m->message_id).second; + CHECK(is_inserted); + } +} + +void MessagesManager::reregister_message_reply(const Dialog *d, const Message *m) { + if (!m->reply_to_message_id.is_valid() || td_->auth_manager_->is_bot()) { + return; + } + + auto it = replied_by_media_timestamp_messages_.find({d->dialog_id, m->reply_to_message_id}); + bool was_registered = it != replied_by_media_timestamp_messages_.end() && it->second.count(m->message_id) > 0; + bool need_register = + has_media_timestamps(get_message_content_text(m->content.get()), 0, std::numeric_limits::max()); + if (was_registered == need_register) { + return; + } + if (was_registered) { + unregister_message_reply(d, m); + } else { + register_message_reply(d, m); + } +} + +void MessagesManager::unregister_message_reply(const Dialog *d, const Message *m) { + auto it = replied_by_media_timestamp_messages_.find({d->dialog_id, m->reply_to_message_id}); + if (it == replied_by_media_timestamp_messages_.end()) { + return; + } + + LOG(INFO) << "Unregister " << m->message_id << " in " << d->dialog_id << " as reply to " << m->reply_to_message_id; + auto is_deleted = it->second.erase(m->message_id) > 0; + CHECK(is_deleted); + if (it->second.empty()) { + replied_by_media_timestamp_messages_.erase(it); + } } bool MessagesManager::get_message_disable_web_page_preview(const Message *m) { @@ -28234,6 +28309,8 @@ void MessagesManager::send_update_message_content(const Dialog *d, Message *m, c CHECK(m != nullptr); delete_bot_command_message_id(d->dialog_id, m->message_id); try_add_bot_command_message_id(d->dialog_id, m); + reregister_message_reply(d, m); + update_message_max_reply_media_timestamp(d, m, false); // because the message reply can be just registered update_message_max_own_media_timestamp(d, m); send_update_message_content_impl(d->dialog_id, m, source); } @@ -32532,8 +32609,6 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, uniq } } } - update_message_max_reply_media_timestamp(d, message.get(), false); - update_message_max_own_media_timestamp(d, message.get()); const Message *m = message.get(); if (m->message_id.is_yet_unsent() && m->reply_to_message_id.is_valid() && !m->reply_to_message_id.is_yet_unsent()) { @@ -32600,6 +32675,8 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, uniq register_message_content(td_, m->content.get(), {dialog_id, m->message_id}, "add_message_to_dialog"); + register_message_reply(d, m); + if (*need_update && m->message_id.is_server() && message_content_type == MessageContentType::PinMessage) { auto pinned_message_id = get_message_content_pinned_message_id(m->content.get()); if (d->is_last_pinned_message_id_inited && pinned_message_id > d->last_pinned_message_id) { @@ -32688,6 +32765,10 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, uniq try_add_bot_command_message_id(dialog_id, m); + // must be called after the message is added to correctly update replies + update_message_max_reply_media_timestamp(d, result_message, false); + update_message_max_own_media_timestamp(d, result_message); + result_message->debug_source = source; d->being_added_message_id = MessageId(); @@ -32798,9 +32879,6 @@ MessagesManager::Message *MessagesManager::add_scheduled_message_to_dialog(Dialo LOG(INFO) << "Adding not found " << message_id << " to " << dialog_id << " from " << source; - update_message_max_reply_media_timestamp(d, message.get(), false); - update_message_max_own_media_timestamp(d, message.get()); - const Message *m = message.get(); if (m->message_id.is_yet_unsent() && m->reply_to_message_id.is_valid() && !m->reply_to_message_id.is_yet_unsent()) { replied_by_yet_unsent_messages_[FullMessageId{dialog_id, m->reply_to_message_id}]++; @@ -32816,6 +32894,12 @@ MessagesManager::Message *MessagesManager::add_scheduled_message_to_dialog(Dialo register_message_content(td_, m->content.get(), {dialog_id, m->message_id}, "add_scheduled_message_to_dialog"); + // must be called after register_message_content, which loads web page + update_message_max_reply_media_timestamp(d, message.get(), false); + update_message_max_own_media_timestamp(d, message.get()); + + register_message_reply(d, m); + if (from_update) { update_sent_message_contents(dialog_id, m); update_used_hashtags(dialog_id, m); @@ -33097,6 +33181,9 @@ void MessagesManager::delete_message_from_database(Dialog *d, MessageId message_ // don't store failed to send message identifiers for bots to reduce memory usage if (m == nullptr || !td_->auth_manager_->is_bot() || !m->is_failed_to_send) { d->deleted_message_ids.insert(message_id); + send_closure_later(actor_id(this), + &MessagesManager::update_message_max_reply_media_timestamp_in_replied_messages, d->dialog_id, + message_id); } } @@ -33405,6 +33492,7 @@ bool MessagesManager::update_message(Dialog *d, Message *old_message, unique_ptr // can change message tree and invalidate reference to old_message if (new_message->reply_to_message_id == MessageId() || replace_legacy) { LOG(DEBUG) << "Drop message reply_to_message_id"; + unregister_message_reply(d, old_message); old_message->reply_to_message_id = MessageId(); update_message_max_reply_media_timestamp(d, old_message, true); need_send_update = true; diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index f2f301693..5e33881e0 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -1060,7 +1060,7 @@ class MessagesManager final : public Actor { NotificationId removed_notification_id; int32 max_reply_media_timestamp = -1; - int32 max_own_media_timestamp = -1; + int32 max_own_media_timestamp = -2; // to update replied messages on the first load int32 view_count = 0; int32 forward_count = 0; @@ -2150,6 +2150,14 @@ class MessagesManager final : public Actor { void update_message_max_own_media_timestamp(const Dialog *d, Message *m); + void update_message_max_reply_media_timestamp_in_replied_messages(DialogId dialog_id, MessageId reply_to_message_id); + + void register_message_reply(const Dialog *d, const Message *m); + + void reregister_message_reply(const Dialog *d, const Message *m); + + void unregister_message_reply(const Dialog *d, const Message *m); + void send_update_new_message(const Dialog *d, const Message *m); static bool is_from_mention_notification_group(const Dialog *d, const Message *m); @@ -3219,6 +3227,10 @@ class MessagesManager final : public Actor { std::unordered_map replied_by_yet_unsent_messages_; + // full_message_id -> replies with media timestamps + std::unordered_map, FullMessageIdHash> + replied_by_media_timestamp_messages_; + struct ActiveDialogAction { MessageId top_thread_message_id; UserId user_id; From 2098f10441a0e98d4e46839b817bd2e19360d41b Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 6 Aug 2021 09:14:52 +0300 Subject: [PATCH 27/51] Store media_timestamp in a separate field. --- td/telegram/MessageEntity.cpp | 23 ++++++++++------------- td/telegram/MessageEntity.h | 17 +++++++++++------ td/telegram/MessageEntity.hpp | 6 ++++++ 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index b9f213542..6ac590870 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -133,7 +133,7 @@ tl_object_ptr MessageEntity::get_text_entity_type_object case MessageEntity::Type::BankCardNumber: return make_tl_object(); case MessageEntity::Type::MediaTimestamp: - return make_tl_object(to_integer(argument)); + return make_tl_object(media_timestamp); default: UNREACHABLE(); return nullptr; @@ -153,8 +153,7 @@ vector> get_text_entities_object(const vector< if (skip_bot_commands && entity.type == MessageEntity::Type::BotCommand) { continue; } - if (entity.type == MessageEntity::Type::MediaTimestamp && - (max_media_timestamp < 0 || max_media_timestamp < to_integer(entity.argument))) { + if (entity.type == MessageEntity::Type::MediaTimestamp && max_media_timestamp < entity.media_timestamp) { continue; } auto entity_object = entity.get_text_entity_object(); @@ -1632,7 +1631,7 @@ vector find_entities(Slice text, bool skip_bot_commands, bool ski for (auto &entity : media_timestamps) { auto offset = narrow_cast(entity.first.begin() - text.begin()); auto length = narrow_cast(entity.first.size()); - entities.emplace_back(MessageEntity::Type::MediaTimestamp, offset, length, to_string(entity.second)); + entities.emplace_back(MessageEntity::Type::MediaTimestamp, offset, length, entity.second); } } @@ -1648,7 +1647,7 @@ static vector find_media_timestamp_entities(Slice text) { for (auto &entity : media_timestamps) { auto offset = narrow_cast(entity.first.begin() - text.begin()); auto length = narrow_cast(entity.first.size()); - entities.emplace_back(MessageEntity::Type::MediaTimestamp, offset, length, to_string(entity.second)); + entities.emplace_back(MessageEntity::Type::MediaTimestamp, offset, length, entity.second); } fix_entity_offsets(text, entities); @@ -3253,11 +3252,11 @@ Result> get_message_entities(const ContactsManager *contac } case td_api::textEntityTypeMediaTimestamp::ID: { auto entity_media_timestamp = static_cast(entity->type_.get()); - if (entity_media_timestamp->media_timestamp_ <= 0) { + if (entity_media_timestamp->media_timestamp_ < 0) { return Status::Error(400, "Invalid media timestamp specified"); } entities.emplace_back(MessageEntity::Type::MediaTimestamp, entity->offset_, entity->length_, - to_string(entity_media_timestamp->media_timestamp_)); + entity_media_timestamp->media_timestamp_); break; } default: @@ -4096,16 +4095,14 @@ void add_formatted_text_dependencies(Dependencies &dependencies, const Formatted } } -bool has_media_timestamps(const FormattedText *text, int32 min_timestamp, int32 max_timestamp) { +bool has_media_timestamps(const FormattedText *text, int32 min_media_timestamp, int32 max_media_timestamp) { if (text == nullptr) { return false; } for (auto &entity : text->entities) { - if (entity.type == MessageEntity::Type::MediaTimestamp) { - int32 timestamp = to_integer(entity.argument); - if (min_timestamp <= timestamp && timestamp <= max_timestamp) { - return true; - } + if (entity.type == MessageEntity::Type::MediaTimestamp && min_media_timestamp <= entity.media_timestamp && + entity.media_timestamp <= max_media_timestamp) { + return true; } } return false; diff --git a/td/telegram/MessageEntity.h b/td/telegram/MessageEntity.h index 78023238f..ff3d54bc3 100644 --- a/td/telegram/MessageEntity.h +++ b/td/telegram/MessageEntity.h @@ -51,19 +51,24 @@ class MessageEntity { MediaTimestamp, Size }; - Type type; - int32 offset; - int32 length; + Type type = Type::Size; + int32 offset = -1; + int32 length = -1; + int32 media_timestamp = -1; string argument; UserId user_id; MessageEntity() = default; MessageEntity(Type type, int32 offset, int32 length, string argument = "") - : type(type), offset(offset), length(length), argument(std::move(argument)), user_id() { + : type(type), offset(offset), length(length), media_timestamp(-1), argument(std::move(argument)), user_id() { } MessageEntity(int32 offset, int32 length, UserId user_id) - : type(Type::MentionName), offset(offset), length(length), argument(), user_id(user_id) { + : type(Type::MentionName), offset(offset), length(length), media_timestamp(-1), argument(), user_id(user_id) { + } + MessageEntity(Type type, int32 offset, int32 length, int32 media_timestamp) + : type(type), offset(offset), length(length), media_timestamp(media_timestamp), argument(), user_id(user_id) { + CHECK(type == Type::MediaTimestamp); } tl_object_ptr get_text_entity_object() const; @@ -196,7 +201,7 @@ Result process_input_caption(const ContactsManager *contacts_mana void add_formatted_text_dependencies(Dependencies &dependencies, const FormattedText *text); -bool has_media_timestamps(const FormattedText *text, int32 min_timestamp, int32 max_timestamp); +bool has_media_timestamps(const FormattedText *text, int32 min_media_timestamp, int32 max_media_timestamp); bool has_bot_commands(const FormattedText *text); diff --git a/td/telegram/MessageEntity.hpp b/td/telegram/MessageEntity.hpp index 23aa58313..e81aa4c89 100644 --- a/td/telegram/MessageEntity.hpp +++ b/td/telegram/MessageEntity.hpp @@ -24,6 +24,9 @@ void MessageEntity::store(StorerT &storer) const { if (type == Type::MentionName) { store(user_id, storer); } + if (type == Type::MediaTimestamp) { + store(media_timestamp, storer); + } } template @@ -38,6 +41,9 @@ void MessageEntity::parse(ParserT &parser) { if (type == Type::MentionName) { parse(user_id, parser); } + if (type == Type::MediaTimestamp) { + parse(media_timestamp, parser); + } } template From 0c3a9aebd835c63c8a798dc6df511b1f8b07ef0c Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 9 Aug 2021 17:48:57 +0300 Subject: [PATCH 28/51] Fix CHECK. --- td/telegram/MessagesManager.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index b99a8605f..13d8e7395 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -25580,11 +25580,12 @@ void MessagesManager::unregister_message_reply(const Dialog *d, const Message *m return; } - LOG(INFO) << "Unregister " << m->message_id << " in " << d->dialog_id << " as reply to " << m->reply_to_message_id; auto is_deleted = it->second.erase(m->message_id) > 0; - CHECK(is_deleted); - if (it->second.empty()) { - replied_by_media_timestamp_messages_.erase(it); + if (is_deleted) { + LOG(INFO) << "Unregister " << m->message_id << " in " << d->dialog_id << " as reply to " << m->reply_to_message_id; + if (it->second.empty()) { + replied_by_media_timestamp_messages_.erase(it); + } } } From 4b631b16bf1d5dad1916479ebd4d6d3375fd5427 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 9 Aug 2021 19:37:47 +0300 Subject: [PATCH 29/51] Find MediaTimestamp entities in old messages. --- td/telegram/MessageContent.cpp | 4 ++++ td/telegram/MessageContent.h | 2 ++ td/telegram/MessageEntity.cpp | 3 +++ td/telegram/MessageEntity.h | 4 ++-- td/telegram/MessagesManager.cpp | 23 ++++++++++++++++++++++- td/telegram/MessagesManager.h | 1 + 6 files changed, 34 insertions(+), 3 deletions(-) diff --git a/td/telegram/MessageContent.cpp b/td/telegram/MessageContent.cpp index 08b0a7e30..332e72622 100644 --- a/td/telegram/MessageContent.cpp +++ b/td/telegram/MessageContent.cpp @@ -4880,6 +4880,10 @@ tl_object_ptr get_message_content_object(const MessageCo return nullptr; } +FormattedText *get_message_content_text_mutable(MessageContent *content) { + return const_cast(get_message_content_text(content)); +} + const FormattedText *get_message_content_text(const MessageContent *content) { switch (content->get_type()) { case MessageContentType::Text: diff --git a/td/telegram/MessageContent.h b/td/telegram/MessageContent.h index 6f66670a5..d65994c13 100644 --- a/td/telegram/MessageContent.h +++ b/td/telegram/MessageContent.h @@ -201,6 +201,8 @@ tl_object_ptr get_message_content_object(const MessageCo bool is_content_secret, bool skip_bot_commands, int32 max_media_timestamp); +FormattedText *get_message_content_text_mutable(MessageContent *content); + const FormattedText *get_message_content_text(const MessageContent *content); const FormattedText *get_message_content_caption(const MessageContent *content); diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index 6ac590870..1d5c0355c 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -83,6 +83,9 @@ StringBuilder &operator<<(StringBuilder &string_builder, const MessageEntity::Ty StringBuilder &operator<<(StringBuilder &string_builder, const MessageEntity &message_entity) { string_builder << '[' << message_entity.type << ", offset = " << message_entity.offset << ", length = " << message_entity.length; + if (message_entity.media_timestamp >= 0) { + string_builder << ", media_timestamp = \"" << message_entity.media_timestamp << "\""; + } if (!message_entity.argument.empty()) { string_builder << ", argument = \"" << message_entity.argument << "\""; } diff --git a/td/telegram/MessageEntity.h b/td/telegram/MessageEntity.h index ff3d54bc3..28857bd90 100644 --- a/td/telegram/MessageEntity.h +++ b/td/telegram/MessageEntity.h @@ -74,8 +74,8 @@ class MessageEntity { tl_object_ptr get_text_entity_object() const; bool operator==(const MessageEntity &other) const { - return offset == other.offset && length == other.length && type == other.type && argument == other.argument && - user_id == other.user_id; + return offset == other.offset && length == other.length && type == other.type && + media_timestamp == other.media_timestamp && argument == other.argument && user_id == other.user_id; } bool operator<(const MessageEntity &other) const { diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 13d8e7395..4ace18ebb 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -4859,6 +4859,8 @@ void MessagesManager::Message::store(StorerT &storer) const { bool is_imported = is_forwarded && forward_info->is_imported; bool has_ttl_period = ttl_period != 0; bool has_max_reply_media_timestamp = max_reply_media_timestamp >= 0; + bool are_message_media_timestamp_entities_found = true; + bool has_flags3 = false; BEGIN_STORE_FLAGS(); STORE_FLAG(is_channel_post); STORE_FLAG(is_outgoing); @@ -4919,8 +4921,10 @@ void MessagesManager::Message::store(StorerT &storer) const { STORE_FLAG(has_interaction_info_update_date); STORE_FLAG(has_send_emoji); STORE_FLAG(is_imported); - STORE_FLAG(has_ttl_period); // 25 + STORE_FLAG(has_ttl_period); STORE_FLAG(has_max_reply_media_timestamp); + STORE_FLAG(are_message_media_timestamp_entities_found); + STORE_FLAG(has_flags3); END_STORE_FLAGS(); } @@ -5081,6 +5085,7 @@ void MessagesManager::Message::parse(ParserT &parser) { bool is_imported = false; bool has_ttl_period = false; bool has_max_reply_media_timestamp = false; + bool has_flags3 = false; BEGIN_PARSE_FLAGS(); PARSE_FLAG(is_channel_post); PARSE_FLAG(is_outgoing); @@ -5143,6 +5148,12 @@ void MessagesManager::Message::parse(ParserT &parser) { PARSE_FLAG(is_imported); PARSE_FLAG(has_ttl_period); PARSE_FLAG(has_max_reply_media_timestamp); + PARSE_FLAG(are_media_timestamp_entities_found); + PARSE_FLAG(has_flags3); + END_PARSE_FLAGS(); + } + if (has_flags3) { + BEGIN_PARSE_FLAGS(); END_PARSE_FLAGS(); } @@ -32426,6 +32437,16 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, uniq } } + if (message->from_database && !message->are_media_timestamp_entities_found) { + auto text = get_message_content_text_mutable(message->content.get()); + if (text != nullptr) { + fix_formatted_text(text->text, text->entities, true, true, true, false, false).ensure(); + // always call to save are_media_timestamp_entities_found flag + on_message_changed(d, message.get(), false, "save media timestamp entities"); + } + } + message->are_media_timestamp_entities_found = true; + LOG(INFO) << "Adding not found " << message_id << " to " << dialog_id << " from " << source; if (d->is_empty) { d->is_empty = false; diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index 5e33881e0..82a292f08 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -1038,6 +1038,7 @@ class MessagesManager final : public Actor { bool is_mention_notification_disabled = false; bool is_from_scheduled = false; bool is_pinned = false; + bool are_media_timestamp_entities_found = false; bool is_copy = false; // for send_message bool from_background = false; // for send_message From db9db0eff19d8c97539c4da55a67e00ded3000a0 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 9 Aug 2021 20:47:03 +0300 Subject: [PATCH 30/51] Improve for_group flag in media timestamp links. --- td/telegram/MessagesManager.cpp | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 4ace18ebb..b490af7e4 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -17368,9 +17368,20 @@ Result> MessagesManager::get_message_link(FullMessageId auto *m = get_message_force(d, full_message_id.get_message_id(), "get_message_link"); TRY_STATUS(can_get_media_timestamp_link(dialog_id, m)); + + if (media_timestamp <= 0 || !can_message_content_have_media_timestamp(m->content.get())) { + media_timestamp = 0; + } + if (media_timestamp != 0) { + for_group = false; + auto duration = get_message_content_media_duration(m->content.get(), td_); + if (duration != 0 && media_timestamp > duration) { + media_timestamp = 0; + } + } + auto message_id = m->message_id; if (dialog_id.get_type() != DialogType::Channel) { - CHECK(m != nullptr); CHECK(m->forward_info != nullptr); CHECK(m->forward_info->sender_dialog_id.get_type() == DialogType::Channel); @@ -17378,6 +17389,10 @@ Result> MessagesManager::get_message_link(FullMessageId message_id = m->forward_info->message_id; for_group = false; for_comment = false; + auto channel_message = get_message({dialog_id, message_id}); + if (channel_message != nullptr && channel_message->media_album_id == 0) { + for_group = true; // default is true + } } else { if (m->media_album_id == 0) { for_group = true; // default is true @@ -17394,17 +17409,6 @@ Result> MessagesManager::get_message_link(FullMessageId for_comment = false; } - if (media_timestamp <= 0 || !can_message_content_have_media_timestamp(m->content.get())) { - media_timestamp = 0; - } - if (media_timestamp != 0) { - for_group = false; - auto duration = get_message_content_media_duration(m->content.get(), td_); - if (duration != 0 && media_timestamp > duration) { - media_timestamp = 0; - } - } - if (!td_->auth_manager_->is_bot()) { td_->create_handler(Promise()) ->send(dialog_id.get_channel_id(), message_id, for_group, true); From bed3448fe1b500c4d24bcb3fbae6c9dcd7d7a2f3 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 10 Aug 2021 15:58:39 +0300 Subject: [PATCH 31/51] Don't store server-generated message links. --- td/telegram/MessagesManager.cpp | 13 +++++++------ td/telegram/MessagesManager.h | 6 +++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index b490af7e4..731f72dd2 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -17515,7 +17515,7 @@ string MessagesManager::get_message_embedding_code(FullMessageId full_message_id for_group = true; // default is true } - auto &links = public_message_links_[for_group][dialog_id].links_; + auto &links = message_embedding_codes_[for_group][dialog_id].embedding_codes_; auto it = links.find(m->message_id); if (it == links.end()) { td_->create_handler(std::move(promise)) @@ -17524,14 +17524,15 @@ string MessagesManager::get_message_embedding_code(FullMessageId full_message_id } promise.set_value(Unit()); - return it->second.second; + return it->second; } void MessagesManager::on_get_public_message_link(FullMessageId full_message_id, bool for_group, string url, string html) { LOG_IF(ERROR, url.empty() && html.empty()) << "Receive empty public link for " << full_message_id; - public_message_links_[for_group][full_message_id.get_dialog_id()].links_[full_message_id.get_message_id()] = { - std::move(url), std::move(html)}; + auto dialog_id = full_message_id.get_dialog_id(); + auto message_id = full_message_id.get_message_id(); + message_embedding_codes_[for_group][dialog_id].embedding_codes_[message_id] = std::move(html); } void MessagesManager::get_message_link_info(Slice url, Promise &&promise) { @@ -30435,8 +30436,8 @@ void MessagesManager::on_dialog_username_updated(DialogId dialog_id, const strin update_dialogs_hints(d); } if (old_username != new_username) { - public_message_links_[0].erase(dialog_id); - public_message_links_[1].erase(dialog_id); + message_embedding_codes_[0].erase(dialog_id); + message_embedding_codes_[1].erase(dialog_id); } if (!old_username.empty() && old_username != new_username) { resolved_usernames_.erase(clean_username(old_username)); diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index 82a292f08..e1484ea84 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -3212,10 +3212,10 @@ class MessagesManager final : public Actor { std::unordered_map found_fts_messages_; // random_id -> FoundMessages std::unordered_map found_message_public_forwards_; // random_id -> FoundMessages - struct PublicMessageLinks { - std::unordered_map, MessageIdHash> links_; + struct MessageEmbeddingCodes { + std::unordered_map embedding_codes_; }; - std::unordered_map public_message_links_[2]; + std::unordered_map message_embedding_codes_[2]; std::unordered_map> chat_events_; // random_id -> chat events From 394e3450debccb1caf4e78b4985e42c6ba90ee89 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 10 Aug 2021 16:18:08 +0300 Subject: [PATCH 32/51] Send updateMessageContent only for message known to the app. --- td/telegram/MessagesManager.cpp | 11 ++++++++++- td/telegram/MessagesManager.h | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 731f72dd2..311afbfac 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -22797,6 +22797,8 @@ tl_object_ptr MessagesManager::get_message_object(DialogId dial return nullptr; } + m->is_update_sent = true; + auto sending_state = get_message_sending_state_object(m); if (for_event_log) { @@ -28334,6 +28336,10 @@ void MessagesManager::send_update_message_content(const Dialog *d, Message *m, c void MessagesManager::send_update_message_content_impl(DialogId dialog_id, const Message *m, const char *source) const { CHECK(m != nullptr); + if (!m->is_update_sent) { + LOG(INFO) << "Skip updateMessageContent for " << m->message_id << " in " << dialog_id << " from " << source; + return; + } LOG(INFO) << "Send updateMessageContent for " << m->message_id << " in " << dialog_id << " from " << source; auto content_object = get_message_content_object(m->content.get(), td_, dialog_id, m->is_failed_to_send ? 0 : m->date, m->is_content_secret, need_skip_bot_commands(dialog_id, m), @@ -28354,7 +28360,7 @@ void MessagesManager::send_update_message_edited(DialogId dialog_id, const Messa void MessagesManager::send_update_message_interaction_info(DialogId dialog_id, const Message *m) const { CHECK(m != nullptr); - if (td_->auth_manager_->is_bot()) { + if (td_->auth_manager_->is_bot() || !m->is_update_sent) { return; } @@ -33709,6 +33715,9 @@ bool MessagesManager::update_message(Dialog *d, Message *old_message, unique_ptr if (old_message->last_access_date < new_message->last_access_date) { old_message->last_access_date = new_message->last_access_date; } + if (new_message->is_update_sent) { + old_message->is_update_sent = true; + } if (!is_scheduled) { CHECK(!new_message->have_previous || !new_message->have_next); diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index e1484ea84..0b67c294d 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -1099,6 +1099,7 @@ class MessagesManager final : public Actor { unique_ptr right; mutable int32 last_access_date = 0; + mutable bool is_update_sent = false; // whether the message is known to the app mutable uint64 send_message_log_event_id = 0; From 45790b144cf5c6fc95937c692f7711b57ec30054 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 11 Aug 2021 16:52:34 +0300 Subject: [PATCH 33/51] Fix ChannelFull dependencies. --- td/telegram/ContactsManager.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/td/telegram/ContactsManager.cpp b/td/telegram/ContactsManager.cpp index 7fb90cd96..cab0b7bf2 100644 --- a/td/telegram/ContactsManager.cpp +++ b/td/telegram/ContactsManager.cpp @@ -9535,7 +9535,9 @@ void ContactsManager::on_load_channel_full_from_database(ChannelId channel_id, s Dependencies dependencies; dependencies.channel_ids.insert(channel_id); - add_dialog_and_dependencies(dependencies, DialogId(channel_full->linked_channel_id)); + // must not depend on the linked_dialog_id itself, because message database can be disabled + // the Dialog will be forcely created in update_channel_full + add_dialog_dependencies(dependencies, DialogId(channel_full->linked_channel_id)); dependencies.chat_ids.insert(channel_full->migrated_from_chat_id); dependencies.user_ids.insert(channel_full->bot_user_ids.begin(), channel_full->bot_user_ids.end()); dependencies.user_ids.insert(channel_full->invite_link.get_creator_user_id()); From 01d83bf85c896451b606bd6afb6b5870ac37fdbe Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 12 Aug 2021 20:42:37 +0300 Subject: [PATCH 34/51] Update CentOS 8 repo name. --- build.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.html b/build.html index 123c5f4e6..fecb4a296 100644 --- a/build.html +++ b/build.html @@ -626,7 +626,7 @@ function onOptionsChanged() { cmake = 'cmake3'; packages += ' gperf'; } else { - commands.push(sudo + 'dnf --enablerepo=PowerTools install gperf'); + commands.push(sudo + 'dnf --enablerepo=powertools install gperf'); } packages += ' ' + cmake; if (target === 'JNI') { From 05c993407475eb269cdd210e81bc653cd40ff1dc Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 12 Aug 2021 22:00:27 +0300 Subject: [PATCH 35/51] Add MessageThreadInfo.h. --- CMakeLists.txt | 1 + td/telegram/MessageThreadInfo.h | 21 +++++++++++++++++++++ td/telegram/MessagesManager.h | 5 +---- td/telegram/Td.cpp | 9 +++++---- 4 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 td/telegram/MessageThreadInfo.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 1bf0f9fb6..5c3e36fb3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -539,6 +539,7 @@ set(TDLIB_SOURCE td/telegram/MessageId.h td/telegram/MessageLinkInfo.h td/telegram/MessageReplyInfo.h + td/telegram/MessageThreadInfo.h td/telegram/MessagesDb.h td/telegram/MessageSearchFilter.h td/telegram/MessagesManager.h diff --git a/td/telegram/MessageThreadInfo.h b/td/telegram/MessageThreadInfo.h new file mode 100644 index 000000000..e0514b385 --- /dev/null +++ b/td/telegram/MessageThreadInfo.h @@ -0,0 +1,21 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2021 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/telegram/DialogId.h" +#include "td/telegram/MessageId.h" + +#include "td/utils/common.h" + +namespace td { + +struct MessageThreadInfo { + DialogId dialog_id; + vector message_ids; +}; + +} // namespace td \ No newline at end of file diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index 0b67c294d..419d52d56 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -31,6 +31,7 @@ #include "td/telegram/MessageId.h" #include "td/telegram/MessageLinkInfo.h" #include "td/telegram/MessageReplyInfo.h" +#include "td/telegram/MessageThreadInfo.h" #include "td/telegram/MessagesDb.h" #include "td/telegram/MessageSearchFilter.h" #include "td/telegram/MessageTtlSetting.h" @@ -567,10 +568,6 @@ class MessagesManager final : public Actor { void get_messages_from_server(vector &&message_ids, Promise &&promise, const char *source, tl_object_ptr input_message = nullptr); - struct MessageThreadInfo { - DialogId dialog_id; - vector message_ids; - }; void get_message_thread(DialogId dialog_id, MessageId message_id, Promise &&promise); td_api::object_ptr get_message_thread_info_object(const MessageThreadInfo &info); diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index 961ad8baf..343bfdf59 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -59,6 +59,7 @@ #include "td/telegram/MessageLinkInfo.h" #include "td/telegram/MessageSearchFilter.h" #include "td/telegram/MessagesManager.h" +#include "td/telegram/MessageThreadInfo.h" #include "td/telegram/misc.h" #include "td/telegram/net/ConnectionCreator.h" #include "td/telegram/net/DcId.h" @@ -1002,13 +1003,13 @@ class GetRepliedMessageRequest final : public RequestOnceActor { } }; -class GetMessageThreadRequest final : public RequestActor { +class GetMessageThreadRequest final : public RequestActor { DialogId dialog_id_; MessageId message_id_; - MessagesManager::MessageThreadInfo message_thread_info_; + MessageThreadInfo message_thread_info_; - void do_run(Promise &&promise) final { + void do_run(Promise &&promise) final { if (get_tries() < 2) { promise.set_value(std::move(message_thread_info_)); return; @@ -1016,7 +1017,7 @@ class GetMessageThreadRequest final : public RequestActormessages_manager_->get_message_thread(dialog_id_, message_id_, std::move(promise)); } - void do_set_result(MessagesManager::MessageThreadInfo &&result) final { + void do_set_result(MessageThreadInfo &&result) final { message_thread_info_ = std::move(result); } From bff8f7550fbe02332635dbd3822a3f92e64d2ee0 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 13 Aug 2021 12:26:47 +0300 Subject: [PATCH 36/51] Return MessageThreadInfo from GetDiscussionMessageQuery. --- td/telegram/MessagesManager.cpp | 74 +++++++++++++++------------------ td/telegram/MessagesManager.h | 6 +-- 2 files changed, 36 insertions(+), 44 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 311afbfac..04e0a3cd4 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -395,14 +395,14 @@ class GetDialogUnreadMarksQuery final : public Td::ResultHandler { }; class GetDiscussionMessageQuery final : public Td::ResultHandler { - Promise> promise_; + Promise promise_; DialogId dialog_id_; MessageId message_id_; DialogId expected_dialog_id_; MessageId expected_message_id_; public: - explicit GetDiscussionMessageQuery(Promise> &&promise) : promise_(std::move(promise)) { + explicit GetDiscussionMessageQuery(Promise &&promise) : promise_(std::move(promise)) { } void send(DialogId dialog_id, MessageId message_id, DialogId expected_dialog_id, MessageId expected_message_id) { @@ -16902,15 +16902,14 @@ void MessagesManager::get_message_thread(DialogId dialog_id, MessageId message_i TRY_RESULT_PROMISE(promise, top_thread_full_message_id, get_top_thread_full_message_id(dialog_id, m)); - auto query_promise = - PromiseCreator::lambda([actor_id = actor_id(this), dialog_id, message_id, - promise = std::move(promise)](Result> result) mutable { - if (result.is_error()) { - return promise.set_error(result.move_as_error()); - } - send_closure(actor_id, &MessagesManager::on_get_discussion_message, dialog_id, message_id, result.move_as_ok(), - std::move(promise)); - }); + auto query_promise = PromiseCreator::lambda([actor_id = actor_id(this), dialog_id, message_id, + promise = std::move(promise)](Result result) mutable { + if (result.is_error()) { + return promise.set_error(result.move_as_error()); + } + send_closure(actor_id, &MessagesManager::on_get_discussion_message, dialog_id, message_id, result.move_as_ok(), + std::move(promise)); + }); td_->create_handler(std::move(query_promise)) ->send(dialog_id, message_id, top_thread_full_message_id.get_dialog_id(), @@ -16920,7 +16919,7 @@ void MessagesManager::get_message_thread(DialogId dialog_id, MessageId message_i void MessagesManager::process_discussion_message( telegram_api::object_ptr &&result, DialogId dialog_id, MessageId message_id, DialogId expected_dialog_id, MessageId expected_message_id, - Promise> promise) { + Promise promise) { LOG(INFO) << "Receive discussion message for " << message_id << " in " << dialog_id << ": " << to_string(result); td_->contacts_manager_->on_get_users(std::move(result->users_), "process_discussion_message"); td_->contacts_manager_->on_get_chats(std::move(result->chats_), "process_discussion_message"); @@ -16951,7 +16950,7 @@ void MessagesManager::process_discussion_message( void MessagesManager::process_discussion_message_impl( telegram_api::object_ptr &&result, DialogId dialog_id, MessageId message_id, DialogId expected_dialog_id, MessageId expected_message_id, - Promise> promise) { + Promise promise) { if (G()->close_flag()) { return promise.set_error(Status::Error(500, "Request aborted")); } @@ -16969,20 +16968,22 @@ void MessagesManager::process_discussion_message_impl( last_read_outbox_message_id = MessageId(ServerMessageId(result->read_outbox_max_id_)); } - vector full_message_ids; + MessageThreadInfo message_thread_info; + message_thread_info.dialog_id = expected_dialog_id; MessageId top_message_id; for (auto &message : result->messages_) { auto full_message_id = on_get_message(std::move(message), false, true, false, false, false, "process_discussion_message"); if (full_message_id.get_message_id().is_valid()) { - full_message_ids.push_back(full_message_id); + CHECK(full_message_id.get_dialog_id() == expected_dialog_id); + message_thread_info.message_ids.push_back(full_message_id.get_message_id()); if (full_message_id.get_message_id() == expected_message_id) { top_message_id = expected_message_id; } } } - if (!full_message_ids.empty() && !top_message_id.is_valid()) { - top_message_id = full_message_ids.back().get_message_id(); + if (!message_thread_info.message_ids.empty() && !top_message_id.is_valid()) { + top_message_id = message_thread_info.message_ids.back(); } if (top_message_id.is_valid()) { on_update_read_message_comments(expected_dialog_id, top_message_id, max_message_id, last_read_inbox_message_id, @@ -16992,11 +16993,11 @@ void MessagesManager::process_discussion_message_impl( on_update_read_message_comments(dialog_id, message_id, max_message_id, last_read_inbox_message_id, last_read_outbox_message_id); } - promise.set_value(std::move(full_message_ids)); + promise.set_value(std::move(message_thread_info)); } void MessagesManager::on_get_discussion_message(DialogId dialog_id, MessageId message_id, - vector full_message_ids, + MessageThreadInfo &&message_thread_info, Promise &&promise) { if (G()->close_flag()) { return promise.set_error(Status::Error(500, "Request aborted")); @@ -17008,7 +17009,7 @@ void MessagesManager::on_get_discussion_message(DialogId dialog_id, MessageId me if (m == nullptr) { return promise.set_error(Status::Error(400, "Message not found")); } - if (full_message_ids.empty()) { + if (message_thread_info.message_ids.empty()) { return promise.set_error(Status::Error(400, "Message has no thread")); } @@ -17025,18 +17026,11 @@ void MessagesManager::on_get_discussion_message(DialogId dialog_id, MessageId me expected_dialog_id = dialog_id; } - MessageThreadInfo result; - for (auto full_message_id : full_message_ids) { - if (full_message_id.get_dialog_id() != expected_dialog_id) { - return promise.set_error(Status::Error(500, "Expected messages in a different chat")); - } - result.message_ids.push_back(full_message_id.get_message_id()); - } - if (expected_dialog_id != dialog_id && m->reply_info.is_comment && !result.message_ids.empty() && - m->linked_top_thread_message_id != result.message_ids.back()) { + if (expected_dialog_id != dialog_id && m->reply_info.is_comment && + m->linked_top_thread_message_id != message_thread_info.message_ids.back()) { auto linked_d = get_dialog_force(expected_dialog_id, "on_get_discussion_message 2"); CHECK(linked_d != nullptr); - auto linked_message_id = result.message_ids.back(); + auto linked_message_id = message_thread_info.message_ids.back(); Message *linked_m = get_message_force(linked_d, linked_message_id, "on_get_discussion_message 3"); CHECK(linked_m != nullptr && linked_m->message_id.is_server()); if (linked_m->top_thread_message_id == linked_m->message_id && @@ -17049,8 +17043,7 @@ void MessagesManager::on_get_discussion_message(DialogId dialog_id, MessageId me on_dialog_updated(dialog_id, "on_get_discussion_message"); } } - result.dialog_id = expected_dialog_id; - promise.set_value(std::move(result)); + promise.set_value(std::move(message_thread_info)); } td_api::object_ptr MessagesManager::get_message_thread_info_object( @@ -17610,15 +17603,14 @@ void MessagesManager::on_get_message_link_message(MessageLinkInfo &&info, Dialog return; } - auto query_promise = - PromiseCreator::lambda([actor_id = actor_id(this), info = std::move(info), - promise = std::move(promise)](Result> result) mutable { - if (result.is_error() || result.ok().empty()) { - return promise.set_value(std::move(info)); - } - send_closure(actor_id, &MessagesManager::on_get_message_link_discussion_message, std::move(info), - result.ok()[0].get_dialog_id(), std::move(promise)); - }); + auto query_promise = PromiseCreator::lambda([actor_id = actor_id(this), info = std::move(info), + promise = std::move(promise)](Result result) mutable { + if (result.is_error() || result.ok().message_ids.empty()) { + return promise.set_value(std::move(info)); + } + send_closure(actor_id, &MessagesManager::on_get_message_link_discussion_message, std::move(info), + result.ok().dialog_id, std::move(promise)); + }); td_->create_handler(std::move(query_promise)) ->send(dialog_id, info.message_id, DialogId(m->reply_info.channel_id), MessageId()); diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index 419d52d56..03c7c351e 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -574,7 +574,7 @@ class MessagesManager final : public Actor { void process_discussion_message(telegram_api::object_ptr &&result, DialogId dialog_id, MessageId message_id, DialogId expected_dialog_id, - MessageId expected_message_id, Promise> promise); + MessageId expected_message_id, Promise promise); bool is_message_edited_recently(FullMessageId full_message_id, int32 seconds); @@ -2670,9 +2670,9 @@ class MessagesManager final : public Actor { void process_discussion_message_impl(telegram_api::object_ptr &&result, DialogId dialog_id, MessageId message_id, DialogId expected_dialog_id, - MessageId expected_message_id, Promise> promise); + MessageId expected_message_id, Promise promise); - void on_get_discussion_message(DialogId dialog_id, MessageId message_id, vector full_message_ids, + void on_get_discussion_message(DialogId dialog_id, MessageId message_id, MessageThreadInfo &&message_thread_info, Promise &&promise); static MessageId get_first_database_message_id_by_index(const Dialog *d, MessageSearchFilter filter); From ec55ca1d5c7e7e4e5f2389ba8b6fc9c2ae2f678a Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 13 Aug 2021 13:03:55 +0300 Subject: [PATCH 37/51] Improve logging. --- td/telegram/MessageContent.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/td/telegram/MessageContent.cpp b/td/telegram/MessageContent.cpp index 332e72622..ef625dfb0 100644 --- a/td/telegram/MessageContent.cpp +++ b/td/telegram/MessageContent.cpp @@ -2911,12 +2911,14 @@ void merge_message_contents(Td *td, const MessageContent *old_content, MessageCo case MessageContentType::Text: { auto old_ = static_cast(old_content); auto new_ = static_cast(new_content); + auto get_content_object = [td, dialog_id](const MessageContent *content) { + return to_string( + get_message_content_object(content, td, dialog_id, -1, false, false, std::numeric_limits::max())); + }; if (old_->text.text != new_->text.text) { if (need_message_changed_warning && need_message_text_changed_warning(old_, new_)) { - LOG(ERROR) << "Message text has changed from " - << to_string(get_message_content_object(old_content, td, dialog_id, -1, false, false, -1)) - << ". New content is " - << to_string(get_message_content_object(new_content, td, dialog_id, -1, false, false, -1)); + LOG(ERROR) << "Message text has changed in " << get_content_object(old_content) << ". New content is " + << get_content_object(new_content); } need_update = true; } @@ -2925,10 +2927,8 @@ void merge_message_contents(Td *td, const MessageContent *old_content, MessageCo if (need_message_changed_warning && need_message_text_changed_warning(old_, new_) && old_->text.entities.size() <= MAX_CUSTOM_ENTITIES_COUNT && need_message_entities_changed_warning(old_->text.entities, new_->text.entities)) { - LOG(WARNING) << "Entities has changed from " - << to_string(get_message_content_object(old_content, td, dialog_id, -1, false, false, -1)) - << ". New content is " - << to_string(get_message_content_object(new_content, td, dialog_id, -1, false, false, -1)); + LOG(WARNING) << "Entities has changed in " << get_content_object(old_content) << ". New content is " + << get_content_object(new_content); } need_update = true; } From a59a916f21c5fdf90b93bbf05951d665a8c9f94b Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 13 Aug 2021 13:10:54 +0300 Subject: [PATCH 38/51] Fix MessageEntity constructors. --- td/telegram/MessageEntity.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/td/telegram/MessageEntity.h b/td/telegram/MessageEntity.h index 28857bd90..2a8da6f6e 100644 --- a/td/telegram/MessageEntity.h +++ b/td/telegram/MessageEntity.h @@ -61,13 +61,13 @@ class MessageEntity { MessageEntity() = default; MessageEntity(Type type, int32 offset, int32 length, string argument = "") - : type(type), offset(offset), length(length), media_timestamp(-1), argument(std::move(argument)), user_id() { + : type(type), offset(offset), length(length), argument(std::move(argument)) { } MessageEntity(int32 offset, int32 length, UserId user_id) - : type(Type::MentionName), offset(offset), length(length), media_timestamp(-1), argument(), user_id(user_id) { + : type(Type::MentionName), offset(offset), length(length), user_id(user_id) { } MessageEntity(Type type, int32 offset, int32 length, int32 media_timestamp) - : type(type), offset(offset), length(length), media_timestamp(media_timestamp), argument(), user_id(user_id) { + : type(type), offset(offset), length(length), media_timestamp(media_timestamp) { CHECK(type == Type::MediaTimestamp); } From 4b06cb9f08b47bfe0815106bd2244faffbaf35b6 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 13 Aug 2021 13:19:58 +0300 Subject: [PATCH 39/51] Avoid reply and bot commands registration for messages not in Dialog. --- td/telegram/MessagesManager.cpp | 30 +++++++++++++++++------------- td/telegram/MessagesManager.h | 4 ++-- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 04e0a3cd4..402a04544 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -6975,7 +6975,7 @@ void MessagesManager::on_external_update_message_content(FullMessageId full_mess CHECK(d != nullptr); Message *m = get_message(d, full_message_id.get_message_id()); CHECK(m != nullptr); - send_update_message_content(d, m, "on_external_update_message_content"); + send_update_message_content(d, m, true, "on_external_update_message_content"); if (m->message_id == d->last_message_id) { send_update_chat_last_message_impl(d, "on_external_update_message_content"); } @@ -12157,7 +12157,7 @@ void MessagesManager::on_message_ttl_expired(Dialog *d, Message *m) { remove_message_file_sources(d->dialog_id, m); on_message_ttl_expired_impl(d, m); register_message_content(td_, m->content.get(), {d->dialog_id, m->message_id}, "on_message_ttl_expired"); - send_update_message_content(d, m, "on_message_ttl_expired"); + send_update_message_content(d, m, true, "on_message_ttl_expired"); // the caller must call on_message_changed } @@ -14257,7 +14257,7 @@ void MessagesManager::on_update_sent_text_message(int64 random_id, m->is_content_secret = is_secret_message_content(m->ttl, MessageContentType::Text); } if (need_update) { - send_update_message_content(dialog_id, m, "on_update_sent_text_message"); + send_update_message_content(dialog_id, m, true, "on_update_sent_text_message"); if (m->message_id == d->last_message_id) { send_update_chat_last_message_impl(d, "on_update_sent_text_message"); } @@ -28308,21 +28308,25 @@ void MessagesManager::send_update_message_send_succeeded(Dialog *d, MessageId ol make_tl_object(get_message_object(d->dialog_id, m), old_message_id.get())); } -void MessagesManager::send_update_message_content(DialogId dialog_id, Message *m, const char *source) { +void MessagesManager::send_update_message_content(DialogId dialog_id, Message *m, bool is_message_in_dialog, + const char *source) { Dialog *d = get_dialog(dialog_id); LOG_CHECK(d != nullptr) << "Send updateMessageContent in unknown " << dialog_id << " from " << source << " with load count " << loaded_dialogs_.count(dialog_id); - send_update_message_content(d, m, source); + send_update_message_content(d, m, is_message_in_dialog, source); } -void MessagesManager::send_update_message_content(const Dialog *d, Message *m, const char *source) { +void MessagesManager::send_update_message_content(const Dialog *d, Message *m, bool is_message_in_dialog, + const char *source) { CHECK(d != nullptr); CHECK(m != nullptr); - delete_bot_command_message_id(d->dialog_id, m->message_id); - try_add_bot_command_message_id(d->dialog_id, m); - reregister_message_reply(d, m); - update_message_max_reply_media_timestamp(d, m, false); // because the message reply can be just registered - update_message_max_own_media_timestamp(d, m); + if (is_message_in_dialog) { + delete_bot_command_message_id(d->dialog_id, m->message_id); + try_add_bot_command_message_id(d->dialog_id, m); + reregister_message_reply(d, m); + update_message_max_reply_media_timestamp(d, m, false); // because the message reply can be just registered + update_message_max_own_media_timestamp(d, m); + } send_update_message_content_impl(d->dialog_id, m, source); } @@ -28851,7 +28855,7 @@ FullMessageId MessagesManager::on_send_message_success(int64 random_id, MessageI // } if (merge_message_content_file_id(td_, sent_message->content.get(), new_file_id)) { - send_update_message_content(d, sent_message.get(), source); + send_update_message_content(d, sent_message.get(), false, source); } if (old_message_id.is_valid() && new_message_id < old_message_id && !can_overflow_message_id(dialog_id)) { @@ -33852,7 +33856,7 @@ bool MessagesManager::update_message_content(DialogId dialog_id, Message *old_me } if (need_update && need_send_update_message_content) { - send_update_message_content(dialog_id, old_message, "update_message_content"); + send_update_message_content(dialog_id, old_message, is_message_in_dialog, "update_message_content"); } return need_update; } diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index 03c7c351e..2b7130de3 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -2213,9 +2213,9 @@ class MessagesManager final : public Actor { void send_update_message_send_succeeded(Dialog *d, MessageId old_message_id, const Message *m) const; - void send_update_message_content(DialogId dialog_id, Message *m, const char *source); + void send_update_message_content(DialogId dialog_id, Message *m, bool is_message_in_dialog, const char *source); - void send_update_message_content(const Dialog *d, Message *m, const char *source); + void send_update_message_content(const Dialog *d, Message *m, bool is_message_in_dialog, const char *source); void send_update_message_content_impl(DialogId dialog_id, const Message *m, const char *source) const; From 1885d232a970374871028deaead100d868d168bb Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 13 Aug 2021 13:36:07 +0300 Subject: [PATCH 40/51] Add parameter is_message_in_dialog to update_message. --- td/telegram/MessagesManager.cpp | 13 ++++++------- td/telegram/MessagesManager.h | 3 ++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 402a04544..bd9a6b4be 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -13670,7 +13670,7 @@ FullMessageId MessagesManager::on_get_message(MessageInfo &&message_info, bool f new_message->from_database = false; new_message->have_previous = false; new_message->have_next = false; - update_message(d, old_message.get(), std::move(new_message), &need_update_dialog_pos); + update_message(d, old_message.get(), std::move(new_message), &need_update_dialog_pos, false); new_message = std::move(old_message); set_message_id(new_message, message_id); @@ -28835,7 +28835,7 @@ FullMessageId MessagesManager::on_send_message_success(int64 random_id, MessageI // dump_debug_message_op(d, 5); } - // imitation of update_message(d, sent_message.get(), std::move(new_message), &need_update_dialog_pos); + // imitation of update_message(d, sent_message.get(), std::move(new_message), &need_update_dialog_pos, false); if (date <= 0) { LOG(ERROR) << "Receive " << new_message_id << " in " << dialog_id << " with wrong date " << date << " from " << source; @@ -32281,7 +32281,7 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, uniq auto old_index_mask = get_message_index_mask(dialog_id, m) & INDEX_MASK_MASK; bool was_deleted = delete_active_live_location(dialog_id, m); auto old_file_ids = get_message_content_file_ids(m->content.get(), td_); - bool need_send_update = update_message(d, m, std::move(message), need_update_dialog_pos); + bool need_send_update = update_message(d, m, std::move(message), need_update_dialog_pos, true); if (!need_send_update) { LOG(INFO) << message_id << " in " << dialog_id << " is not changed"; } @@ -32888,7 +32888,7 @@ MessagesManager::Message *MessagesManager::add_scheduled_message_to_dialog(Dialo if (!message->from_database) { auto old_file_ids = get_message_content_file_ids(m->content.get(), td_); bool need_update_dialog_pos = false; - update_message(d, m, std::move(message), &need_update_dialog_pos); + update_message(d, m, std::move(message), &need_update_dialog_pos, true); CHECK(need_update_dialog_pos == false); change_message_files(dialog_id, m, old_file_ids); } @@ -33347,7 +33347,7 @@ void MessagesManager::attach_message_to_next(Dialog *d, MessageId message_id, co } bool MessagesManager::update_message(Dialog *d, Message *old_message, unique_ptr new_message, - bool *need_update_dialog_pos) { + bool *need_update_dialog_pos, bool is_message_in_dialog) { CHECK(d != nullptr); CHECK(old_message != nullptr); CHECK(new_message != nullptr); @@ -33727,8 +33727,7 @@ bool MessagesManager::update_message(Dialog *d, Message *old_message, unique_ptr } if (update_message_content(dialog_id, old_message, std::move(new_message->content), true, - message_id.is_yet_unsent() && new_message->edit_date == 0, - get_message(d, message_id) != nullptr)) { + message_id.is_yet_unsent() && new_message->edit_date == 0, is_message_in_dialog)) { need_send_update = true; } diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index 2b7130de3..99bb506ef 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -2138,7 +2138,8 @@ class MessagesManager final : public Actor { void attach_message_to_next(Dialog *d, MessageId message_id, const char *source); - bool update_message(Dialog *d, Message *old_message, unique_ptr new_message, bool *need_update_dialog_pos); + bool update_message(Dialog *d, Message *old_message, unique_ptr new_message, bool *need_update_dialog_pos, + bool is_message_in_dialog); static bool need_message_changed_warning(const Message *old_message); From e62655efdf44bb198208fa85cc4cfb6de97231ac Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 13 Aug 2021 14:49:24 +0300 Subject: [PATCH 41/51] Add message.has_timestamped_media. --- td/generate/scheme/td_api.tl | 3 ++- td/telegram/MessagesManager.cpp | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index 22f39d57c..3db857179 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -760,6 +760,7 @@ messageSendingStateFailed error_code:int32 error_message:string can_retry:Bool r //@can_get_statistics True, if the message statistics are available //@can_get_message_thread True, if the message thread info is available //@can_get_media_timestamp_links True, if media timestamp links can be generated for media timestamp entities in the message text, caption or web page description +//@has_timestamped_media True, if media timestamp entities refers to a media in this message as opposed to a media in the replied message //@is_channel_post True, if the message is a channel post. All messages to channels are channel posts, all other messages are not channel posts //@contains_unread_mention True, if the message contains an unread mention for the current user //@date Point in time (Unix timestamp) when the message was sent @@ -777,7 +778,7 @@ messageSendingStateFailed error_code:int32 error_message:string can_retry:Bool r //@restriction_reason If non-empty, contains a human-readable description of the reason why access to this message must be restricted //@content Content of the message //@reply_markup Reply markup for the message; may be null -message id:int53 sender:MessageSender chat_id:int53 sending_state:MessageSendingState scheduling_state:MessageSchedulingState is_outgoing:Bool is_pinned:Bool can_be_edited:Bool can_be_forwarded:Bool can_be_deleted_only_for_self:Bool can_be_deleted_for_all_users:Bool can_get_statistics:Bool can_get_message_thread:Bool can_get_media_timestamp_links:Bool is_channel_post:Bool contains_unread_mention:Bool date:int32 edit_date:int32 forward_info:messageForwardInfo interaction_info:messageInteractionInfo reply_in_chat_id:int53 reply_to_message_id:int53 message_thread_id:int53 ttl:int32 ttl_expires_in:double via_bot_user_id:int32 author_signature:string media_album_id:int64 restriction_reason:string content:MessageContent reply_markup:ReplyMarkup = Message; +message id:int53 sender:MessageSender chat_id:int53 sending_state:MessageSendingState scheduling_state:MessageSchedulingState is_outgoing:Bool is_pinned:Bool can_be_edited:Bool can_be_forwarded:Bool can_be_deleted_only_for_self:Bool can_be_deleted_for_all_users:Bool can_get_statistics:Bool can_get_message_thread:Bool can_get_media_timestamp_links:Bool has_timestamped_media:Bool is_channel_post:Bool contains_unread_mention:Bool date:int32 edit_date:int32 forward_info:messageForwardInfo interaction_info:messageInteractionInfo reply_in_chat_id:int53 reply_to_message_id:int53 message_thread_id:int53 ttl:int32 ttl_expires_in:double via_bot_user_id:int32 author_signature:string media_album_id:int64 restriction_reason:string content:MessageContent reply_markup:ReplyMarkup = Message; //@description Contains a list of messages @total_count Approximate total count of messages found @messages List of messages; messages may be null messages total_count:int32 messages:vector = Messages; diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index bd9a6b4be..8a1cf02ef 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -22872,13 +22872,14 @@ tl_object_ptr MessagesManager::get_message_object(DialogId dial bool skip_bot_commands = for_event_log ? true : need_skip_bot_commands(dialog_id, m); int32 max_media_timestamp = for_event_log ? get_message_own_max_media_timestamp(m) : get_message_max_media_timestamp(m); + bool has_timestamped_media = for_event_log || reply_to_message_id == 0 || m->max_own_media_timestamp >= 0; string source = PSTRING() << dialog_id << ' ' << m->message_id; return make_tl_object( m->message_id.get(), get_message_sender_object_const(m->sender_user_id, m->sender_dialog_id, source.c_str()), dialog_id.get(), std::move(sending_state), std::move(scheduling_state), is_outgoing, is_pinned, can_be_edited, can_be_forwarded, can_delete_for_self, can_delete_for_all_users, can_get_statistics, can_get_message_thread, - can_get_media_timestamp_links, m->is_channel_post, contains_unread_mention, date, edit_date, - get_message_forward_info_object(m->forward_info), get_message_interaction_info_object(dialog_id, m), + can_get_media_timestamp_links, has_timestamped_media, m->is_channel_post, contains_unread_mention, date, + edit_date, get_message_forward_info_object(m->forward_info), get_message_interaction_info_object(dialog_id, m), reply_in_dialog_id.get(), reply_to_message_id, top_thread_message_id, ttl, ttl_expires_in, via_bot_user_id, m->author_signature, media_album_id, get_restriction_reason_description(m->restriction_reasons), get_message_content_object(m->content.get(), td_, dialog_id, live_location_date, m->is_content_secret, From 957150a59c7a2d316cbb97986f4e44771eec19e0 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 13 Aug 2021 21:06:54 +0300 Subject: [PATCH 42/51] Add space before supeflous phone number digits. --- td/telegram/CountryInfoManager.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/td/telegram/CountryInfoManager.cpp b/td/telegram/CountryInfoManager.cpp index 6ae7e9302..4c019db7c 100644 --- a/td/telegram/CountryInfoManager.cpp +++ b/td/telegram/CountryInfoManager.cpp @@ -240,8 +240,9 @@ void CountryInfoManager::do_get_phone_number_info(string phone_number_prefix, st result += pattern[current_pattern_pos++]; } if (current_pattern_pos == pattern.size()) { - result += c; - } else if (pattern[current_pattern_pos] == 'X') { + result += ' '; + } + if (current_pattern_pos >= pattern.size() || pattern[current_pattern_pos] == 'X') { result += c; current_pattern_pos++; } else { From 7cb54fddb9a5f6ebe67ff9d71295aafbd70ca0f5 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 14 Aug 2021 22:25:53 +0300 Subject: [PATCH 43/51] Improve some internal link documentation. --- td/generate/scheme/td_api.tl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index 3db857179..2393f061f 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -3063,7 +3063,7 @@ internalLinkTypeChatInvite = InternalLinkType; //@description The link is a link to the filter settings section of the app internalLinkTypeFilterSettings = InternalLinkType; -//@description The link is a link to a game. Call searchPublicChat with the given bot username, check that the user is a bot, ask the current user to select a group to send the game, and then call sendMessage with inputMessageGame +//@description The link is a link to a game. Call searchPublicChat with the given bot username, check that the user is a bot, ask the current user to select a chat to send the game, and then call sendMessage with inputMessageGame //@bot_username Username of the bot that owns the game @game_short_name Short name of the game internalLinkTypeGame bot_username:string game_short_name:string = InternalLinkType; @@ -4188,7 +4188,7 @@ getMessageLink chat_id:int53 message_id:int53 media_timestamp:int32 for_album:Bo //@for_album Pass true to return an HTML code for embedding of the whole media album getMessageEmbeddingCode chat_id:int53 message_id:int53 for_album:Bool = Text; -//@description Returns information about a public or private message link @url The message link +//@description Returns information about a public or private message link. Can be called for any internal link of the type internalLinkTypeMessage @url The message link getMessageLinkInfo url:string = MessageLinkInfo; From 65094f66bd8d8f68ed02116f4c933541272490fd Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 15 Aug 2021 10:46:41 +0300 Subject: [PATCH 44/51] Shrink long KHeap arrays. --- tdutils/td/utils/Heap.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tdutils/td/utils/Heap.h b/tdutils/td/utils/Heap.h index 146cd1086..554550825 100644 --- a/tdutils/td/utils/Heap.h +++ b/tdutils/td/utils/Heap.h @@ -156,6 +156,9 @@ class KHeap { fix_down(pos); fix_up(pos); } + if (array_.capacity() > 50 && array_.size() < array_.capacity() / 4) { + array_.shrink_to_fit(); + } } }; From 99375ff157be8fcc0871ea3a1c3d563079a9b152 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 15 Aug 2021 10:59:12 +0300 Subject: [PATCH 45/51] Improve pos checks in KHeap. --- tdutils/td/utils/Heap.h | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tdutils/td/utils/Heap.h b/tdutils/td/utils/Heap.h index 554550825..697b8b5f8 100644 --- a/tdutils/td/utils/Heap.h +++ b/tdutils/td/utils/Heap.h @@ -20,7 +20,7 @@ struct HeapNode { void remove() { pos_ = -1; } - int pos_ = -1; + int32 pos_ = -1; }; template @@ -45,19 +45,19 @@ class KHeap { CHECK(!empty()); HeapNode *result = array_[0].node_; result->remove(); - erase(0); + erase(static_cast(0)); return result; } void insert(KeyT key, HeapNode *node) { CHECK(!node->in_heap()); array_.push_back({key, node}); - fix_up(static_cast(array_.size()) - 1); + fix_up(array_.size() - 1); } void fix(KeyT key, HeapNode *node) { - CHECK(node->in_heap()); - int pos = node->pos_; + size_t pos = static_cast(node->pos_); + CHECK(pos < array_.size()); KeyT old_key = array_[pos].key_; array_[pos].key_ = key; if (key < old_key) { @@ -68,9 +68,9 @@ class KHeap { } void erase(HeapNode *node) { - CHECK(node->in_heap()); - int pos = node->pos_; + size_t pos = static_cast(node->pos_); node->remove(); + CHECK(pos < array_.size()); erase(pos); } @@ -103,34 +103,34 @@ class KHeap { }; vector array_; - void fix_up(int pos) { + void fix_up(size_t pos) { auto item = array_[pos]; while (pos) { - int parent_pos = (pos - 1) / K; + auto parent_pos = (pos - 1) / K; auto parent_item = array_[parent_pos]; if (parent_item.key_ < item.key_) { break; } - parent_item.node_->pos_ = pos; + parent_item.node_->pos_ = static_cast(pos); array_[pos] = parent_item; pos = parent_pos; } - item.node_->pos_ = pos; + item.node_->pos_ = static_cast(pos); array_[pos] = item; } - void fix_down(int pos) { + void fix_down(size_t pos) { auto item = array_[pos]; while (true) { - int left_pos = pos * K + 1; - int right_pos = min(left_pos + K, static_cast(array_.size())); - int next_pos = pos; + auto left_pos = pos * K + 1; + auto right_pos = min(left_pos + K, array_.size()); + auto next_pos = pos; KeyT next_key = item.key_; - for (int i = left_pos; i < right_pos; i++) { + for (auto i = left_pos; i < right_pos; i++) { KeyT i_key = array_[i].key_; if (i_key < next_key) { next_key = i_key; @@ -141,18 +141,18 @@ class KHeap { break; } array_[pos] = array_[next_pos]; - array_[pos].node_->pos_ = pos; + array_[pos].node_->pos_ = static_cast(pos); pos = next_pos; } - item.node_->pos_ = pos; + item.node_->pos_ = static_cast(pos); array_[pos] = item; } - void erase(int pos) { + void erase(size_t pos) { array_[pos] = array_.back(); array_.pop_back(); - if (pos < static_cast(array_.size())) { + if (pos < array_.size()) { fix_down(pos); fix_up(pos); } From 5349e63c2b01aaaa3f5e08ac6d72d05b8c8dfac6 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 15 Aug 2021 11:15:14 +0300 Subject: [PATCH 46/51] Add Timeout::get_timeout. --- tdactor/td/actor/Timeout.h | 3 +++ tdactor/td/actor/impl/Actor-decl.h | 1 + tdactor/td/actor/impl/Actor.h | 3 +++ tdactor/td/actor/impl/Scheduler-decl.h | 2 ++ tdactor/td/actor/impl/Scheduler.cpp | 6 ++++++ tdactor/td/actor/impl/Scheduler.h | 3 +++ tdutils/td/utils/Heap.h | 6 ++++++ 7 files changed, 24 insertions(+) diff --git a/tdactor/td/actor/Timeout.h b/tdactor/td/actor/Timeout.h index de4216546..63f6cfdf2 100644 --- a/tdactor/td/actor/Timeout.h +++ b/tdactor/td/actor/Timeout.h @@ -35,6 +35,9 @@ class Timeout final : public Actor { bool has_timeout() const { return Actor::has_timeout(); } + double get_timeout() const { + return Actor::get_timeout(); + } void set_timeout_in(double timeout) { Actor::set_timeout_in(timeout); } diff --git a/tdactor/td/actor/impl/Actor-decl.h b/tdactor/td/actor/impl/Actor-decl.h index d6eabd2be..8a107375f 100644 --- a/tdactor/td/actor/impl/Actor-decl.h +++ b/tdactor/td/actor/impl/Actor-decl.h @@ -67,6 +67,7 @@ class Actor : public ObserverBase { void stop(); void do_stop(); bool has_timeout() const; + double get_timeout() const; void set_timeout_in(double timeout_in); void set_timeout_at(double timeout_at); void cancel_timeout(); diff --git a/tdactor/td/actor/impl/Actor.h b/tdactor/td/actor/impl/Actor.h index 94fd9c065..3577093ce 100644 --- a/tdactor/td/actor/impl/Actor.h +++ b/tdactor/td/actor/impl/Actor.h @@ -54,6 +54,9 @@ inline void Actor::do_stop() { inline bool Actor::has_timeout() const { return Scheduler::instance()->has_actor_timeout(this); } +inline double Actor::get_timeout() const { + return Scheduler::instance()->get_actor_timeout(this); +} inline void Actor::set_timeout_in(double timeout_in) { Scheduler::instance()->set_actor_timeout_in(this, timeout_in); } diff --git a/tdactor/td/actor/impl/Scheduler-decl.h b/tdactor/td/actor/impl/Scheduler-decl.h index add1d702d..2b17d90ed 100644 --- a/tdactor/td/actor/impl/Scheduler-decl.h +++ b/tdactor/td/actor/impl/Scheduler-decl.h @@ -126,6 +126,7 @@ class Scheduler { void finish_migrate_actor(Actor *actor); bool has_actor_timeout(const Actor *actor) const; + double get_actor_timeout(const Actor *actor) const; void set_actor_timeout_in(Actor *actor, double timeout); void set_actor_timeout_at(Actor *actor, double timeout_at); void cancel_actor_timeout(Actor *actor); @@ -176,6 +177,7 @@ class Scheduler { void start_migrate_actor(ActorInfo *actor_info, int32 dest_sched_id); bool has_actor_timeout(const ActorInfo *actor_info) const; + double get_actor_timeout(const ActorInfo *actor_info) const; void set_actor_timeout_in(ActorInfo *actor_info, double timeout); void set_actor_timeout_at(ActorInfo *actor_info, double timeout_at); void cancel_actor_timeout(ActorInfo *actor_info); diff --git a/tdactor/td/actor/impl/Scheduler.cpp b/tdactor/td/actor/impl/Scheduler.cpp index c5c5c88a5..3ae5a2468 100644 --- a/tdactor/td/actor/impl/Scheduler.cpp +++ b/tdactor/td/actor/impl/Scheduler.cpp @@ -392,6 +392,7 @@ void Scheduler::do_migrate_actor(ActorInfo *actor_info, int32 dest_sched_id) { void Scheduler::start_migrate_actor(Actor *actor, int32 dest_sched_id) { start_migrate_actor(actor->get_info(), dest_sched_id); } + void Scheduler::start_migrate_actor(ActorInfo *actor_info, int32 dest_sched_id) { VLOG(actor) << "Start migrate actor: " << tag("name", actor_info) << tag("ptr", actor_info) << tag("actor_count", actor_count_); @@ -406,6 +407,11 @@ void Scheduler::start_migrate_actor(ActorInfo *actor_info, int32 dest_sched_id) cancel_actor_timeout(actor_info); } +double Scheduler::get_actor_timeout(const ActorInfo *actor_info) const { + const HeapNode *heap_node = actor_info->get_heap_node(); + return heap_node->in_heap() ? timeout_queue_.get_key(heap_node) - Time::now() : 0.0; +} + void Scheduler::set_actor_timeout_in(ActorInfo *actor_info, double timeout) { if (timeout > 1e10) { timeout = 1e10; diff --git a/tdactor/td/actor/impl/Scheduler.h b/tdactor/td/actor/impl/Scheduler.h index 0d432ca2f..1212509c0 100644 --- a/tdactor/td/actor/impl/Scheduler.h +++ b/tdactor/td/actor/impl/Scheduler.h @@ -302,6 +302,9 @@ inline void Scheduler::finish_migrate_actor(Actor *actor) { inline bool Scheduler::has_actor_timeout(const Actor *actor) const { return has_actor_timeout(actor->get_info()); } +inline double Scheduler::get_actor_timeout(const Actor *actor) const { + return get_actor_timeout(actor->get_info()); +} inline void Scheduler::set_actor_timeout_in(Actor *actor, double timeout) { set_actor_timeout_in(actor->get_info(), timeout); } diff --git a/tdutils/td/utils/Heap.h b/tdutils/td/utils/Heap.h index 697b8b5f8..c9bbbd901 100644 --- a/tdutils/td/utils/Heap.h +++ b/tdutils/td/utils/Heap.h @@ -37,6 +37,12 @@ class KHeap { return array_[0].key_; } + KeyT get_key(const HeapNode *node) const { + size_t pos = static_cast(node->pos_); + CHECK(pos < array_.size()); + return array_[pos].key_; + } + const HeapNode *top() const { return array_[0].node_; } From 9b801645f0740356bf069ee358f91a6ad7f7de4c Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 15 Aug 2021 11:43:00 +0300 Subject: [PATCH 47/51] Set gap timeout if it has decreased. --- td/telegram/UpdatesManager.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/td/telegram/UpdatesManager.cpp b/td/telegram/UpdatesManager.cpp index 51461cb3c..ea4cae858 100644 --- a/td/telegram/UpdatesManager.cpp +++ b/td/telegram/UpdatesManager.cpp @@ -2132,7 +2132,7 @@ void UpdatesManager::process_pending_qts_updates() { } void UpdatesManager::set_pts_gap_timeout(double timeout) { - if (!pts_gap_timeout_.has_timeout()) { + if (!pts_gap_timeout_.has_timeout() || timeout < pts_gap_timeout_.get_timeout()) { pts_gap_timeout_.set_callback(std::move(fill_pts_gap)); pts_gap_timeout_.set_callback_data(static_cast(td_)); pts_gap_timeout_.set_timeout_in(timeout); @@ -2140,7 +2140,7 @@ void UpdatesManager::set_pts_gap_timeout(double timeout) { } void UpdatesManager::set_seq_gap_timeout(double timeout) { - if (!seq_gap_timeout_.has_timeout()) { + if (!seq_gap_timeout_.has_timeout() || timeout < seq_gap_timeout_.get_timeout()) { seq_gap_timeout_.set_callback(std::move(fill_seq_gap)); seq_gap_timeout_.set_callback_data(static_cast(td_)); seq_gap_timeout_.set_timeout_in(timeout); @@ -2148,7 +2148,7 @@ void UpdatesManager::set_seq_gap_timeout(double timeout) { } void UpdatesManager::set_qts_gap_timeout(double timeout) { - if (!qts_gap_timeout_.has_timeout()) { + if (!qts_gap_timeout_.has_timeout() || timeout < qts_gap_timeout_.get_timeout()) { qts_gap_timeout_.set_callback(std::move(fill_qts_gap)); qts_gap_timeout_.set_callback_data(static_cast(td_)); qts_gap_timeout_.set_timeout_in(timeout); From 9a5872fe27a3b8ca386bb50165f46e83b456753c Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 15 Aug 2021 12:46:38 +0300 Subject: [PATCH 48/51] Use update receive time to calculate proper gap time. --- td/telegram/MessagesManager.cpp | 16 ++--- td/telegram/UpdatesManager.cpp | 101 ++++++++++++++++++++------------ td/telegram/UpdatesManager.h | 29 ++++++--- 3 files changed, 94 insertions(+), 52 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 8a1cf02ef..7c1301acb 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -660,7 +660,7 @@ class UnpinAllMessagesQuery final : public Td::ResultHandler { std::move(promise), "unpin all messages"); } else { td->updates_manager_->add_pending_pts_update(make_tl_object(), affected_history->pts_, - affected_history->pts_count_, std::move(promise), + affected_history->pts_count_, Time::now(), std::move(promise), "unpin all messages"); } } else if (affected_history->offset_ <= 0) { @@ -1812,7 +1812,7 @@ class ReadMessagesContentsQuery final : public Td::ResultHandler { if (affected_messages->pts_count_ > 0) { td->updates_manager_->add_pending_pts_update(make_tl_object(), affected_messages->pts_, - affected_messages->pts_count_, Promise(), + affected_messages->pts_count_, Time::now(), Promise(), "read messages content query"); } @@ -2034,7 +2034,7 @@ class ReadHistoryQuery final : public Td::ResultHandler { if (affected_messages->pts_count_ > 0) { td->updates_manager_->add_pending_pts_update(make_tl_object(), affected_messages->pts_, - affected_messages->pts_count_, Promise(), + affected_messages->pts_count_, Time::now(), Promise(), "read history query"); } @@ -2504,7 +2504,7 @@ class DeleteHistoryQuery final : public Td::ResultHandler { if (affected_history->pts_count_ > 0) { td->updates_manager_->add_pending_pts_update(make_tl_object(), affected_history->pts_, - affected_history->pts_count_, Promise(), + affected_history->pts_count_, Time::now(), Promise(), "delete history query"); } @@ -2602,7 +2602,7 @@ class DeletePhoneCallHistoryQuery final : public Td::ResultHandler { auto pts_count = affected_messages->pts_count_; auto update = make_tl_object(std::move(affected_messages->messages_), pts, pts_count); - td->updates_manager_->add_pending_pts_update(std::move(update), pts, pts_count, std::move(promise), + td->updates_manager_->add_pending_pts_update(std::move(update), pts, pts_count, Time::now(), std::move(promise), "delete phone call history query"); } else if (affected_messages->offset_ <= 0) { promise_.set_value(Unit()); @@ -2759,7 +2759,7 @@ class ReadMentionsQuery final : public Td::ResultHandler { td->updates_manager_->get_difference("Wrong messages_readMentions result"); } else { td->updates_manager_->add_pending_pts_update(make_tl_object(), affected_history->pts_, - affected_history->pts_count_, Promise(), + affected_history->pts_count_, Time::now(), Promise(), "read all mentions query"); } } @@ -2899,7 +2899,7 @@ class SendMessageActor final : public NetActorOnce { } td->updates_manager_->add_pending_pts_update(std::move(update), sent_message->pts_, sent_message->pts_count_, - Promise(), "send message actor"); + Time::now(), Promise(), "send message actor"); } void on_error(uint64 id, Status status) final { @@ -3914,7 +3914,7 @@ class DeleteMessagesQuery final : public Td::ResultHandler { if (affected_messages->pts_count_ > 0) { td->updates_manager_->add_pending_pts_update(make_tl_object(), affected_messages->pts_, - affected_messages->pts_count_, Promise(), + affected_messages->pts_count_, Time::now(), Promise(), "delete messages query"); } if (--query_count_ == 0) { diff --git a/td/telegram/UpdatesManager.cpp b/td/telegram/UpdatesManager.cpp index ea4cae858..25ef5c974 100644 --- a/td/telegram/UpdatesManager.cpp +++ b/td/telegram/UpdatesManager.cpp @@ -854,7 +854,7 @@ void UpdatesManager::on_get_updates(tl_object_ptr &&updat auto updates = move_tl_object_as(updates_ptr); td_->contacts_manager_->on_get_users(std::move(updates->users_), "updatesCombined"); td_->contacts_manager_->on_get_chats(std::move(updates->chats_), "updatesCombined"); - on_pending_updates(std::move(updates->updates_), updates->seq_start_, updates->seq_, updates->date_, + on_pending_updates(std::move(updates->updates_), updates->seq_start_, updates->seq_, updates->date_, Time::now(), std::move(promise), "telegram_api::updatesCombined"); break; } @@ -862,8 +862,8 @@ void UpdatesManager::on_get_updates(tl_object_ptr &&updat auto updates = move_tl_object_as(updates_ptr); td_->contacts_manager_->on_get_users(std::move(updates->users_), "updates"); td_->contacts_manager_->on_get_chats(std::move(updates->chats_), "updates"); - on_pending_updates(std::move(updates->updates_), updates->seq_, updates->seq_, updates->date_, std::move(promise), - "telegram_api::updates"); + on_pending_updates(std::move(updates->updates_), updates->seq_, updates->seq_, updates->date_, Time::now(), + std::move(promise), "telegram_api::updates"); break; } case telegram_api::updateShortSentMessage::ID: @@ -1409,11 +1409,12 @@ void UpdatesManager::after_get_difference() { auto updates = std::move(it->second.updates); auto updates_seq_begin = it->second.seq_begin; auto updates_seq_end = it->second.seq_end; + auto receive_time = it->second.receive_time; auto promise = std::move(it->second.promise); // ignore it->second.date, because it may be too old postponed_updates_.erase(it); auto update_count = updates.size(); - on_pending_updates(std::move(updates), updates_seq_begin, updates_seq_end, 0, std::move(promise), + on_pending_updates(std::move(updates), updates_seq_begin, updates_seq_end, 0, receive_time, std::move(promise), "postponed updates"); if (running_get_difference_) { VLOG(get_difference) << "Finish to apply postponed updates with " << postponed_updates_.size() @@ -1433,8 +1434,8 @@ void UpdatesManager::after_get_difference() { LOG(INFO) << "Begin to apply " << postponed_updates.size() << " postponed pts updates"; for (auto &postponed_update : postponed_updates) { auto &update = postponed_update.second; - add_pending_pts_update(std::move(update.update), update.pts, update.pts_count, std::move(update.promise), - "after get difference"); + add_pending_pts_update(std::move(update.update), update.pts, update.pts_count, update.receive_time, + std::move(update.promise), "after get difference"); CHECK(!running_get_difference_); } LOG(INFO) << "Finish to apply postponed pts updates, have " << postponed_pts_updates_.size() @@ -1451,7 +1452,8 @@ void UpdatesManager::after_get_difference() { } void UpdatesManager::on_pending_updates(vector> &&updates, int32 seq_begin, - int32 seq_end, int32 date, Promise &&promise, const char *source) { + int32 seq_end, int32 date, double receive_time, Promise &&promise, + const char *source) { if (get_pts() == -1) { init_state(); } @@ -1485,8 +1487,8 @@ void UpdatesManager::on_pending_updates(vector auto &pending_update = pending_qts_updates_[qts]; if (pending_update.update != nullptr) { LOG(WARNING) << "Receive duplicate update with qts = " << qts; + } else { + pending_update.receive_time = Time::now(); } pending_update.update = std::move(update); pending_update.promises.push_back(std::move(promise)); @@ -1849,7 +1853,8 @@ void UpdatesManager::process_pts_update(tl_object_ptr &&up } void UpdatesManager::add_pending_pts_update(tl_object_ptr &&update, int32 new_pts, - int32 pts_count, Promise &&promise, const char *source) { + int32 pts_count, double receive_time, Promise &&promise, + const char *source) { // do not try to run getDifference from this function CHECK(update != nullptr); CHECK(source != nullptr); @@ -1905,7 +1910,7 @@ void UpdatesManager::add_pending_pts_update(tl_object_ptr if (running_get_difference_) { CHECK(update->get_id() == dummyUpdate::ID || update->get_id() == updateSentMessage::ID); } - postpone_pts_update(std::move(update), new_pts, pts_count, std::move(promise)); + postpone_pts_update(std::move(update), new_pts, pts_count, receive_time, std::move(promise)); return; } @@ -1913,7 +1918,7 @@ void UpdatesManager::add_pending_pts_update(tl_object_ptr LOG(WARNING) << "Have old_pts (= " << old_pts << ") + pts_count (= " << pts_count << ") > new_pts (= " << new_pts << "). Logged in " << G()->shared_config().get_option_integer("authorization_date") << ". Update from " << source << " = " << oneline(to_string(update)); - postpone_pts_update(std::move(update), new_pts, pts_count, std::move(promise)); + postpone_pts_update(std::move(update), new_pts, pts_count, receive_time, std::move(promise)); set_pts_gap_timeout(0.001); return; } @@ -1929,7 +1934,7 @@ void UpdatesManager::add_pending_pts_update(tl_object_ptr << ", pts_count = " << pts_count << ". Logged in " << G()->shared_config().get_option_integer("authorization_date") << ". Update from " << source << " = " << oneline(to_string(update)); - postpone_pts_update(std::move(update), new_pts, pts_count, std::move(promise)); + postpone_pts_update(std::move(update), new_pts, pts_count, receive_time, std::move(promise)); set_pts_gap_timeout(0.001); return; } @@ -1950,10 +1955,11 @@ void UpdatesManager::add_pending_pts_update(tl_object_ptr return; } - pending_pts_updates_.emplace(new_pts, PendingPtsUpdate(std::move(update), new_pts, pts_count, std::move(promise))); + pending_pts_updates_.emplace( + new_pts, PendingPtsUpdate(std::move(update), new_pts, pts_count, receive_time, std::move(promise))); if (old_pts < accumulated_pts_ - accumulated_pts_count_) { - set_pts_gap_timeout(MAX_UNFILLED_GAP_TIME); + set_pts_gap_timeout(receive_time + MAX_UNFILLED_GAP_TIME - Time::now()); last_pts_gap_time_ = Time::now(); return; } @@ -1963,8 +1969,9 @@ void UpdatesManager::add_pending_pts_update(tl_object_ptr } void UpdatesManager::postpone_pts_update(tl_object_ptr &&update, int32 pts, int32 pts_count, - Promise &&promise) { - postponed_pts_updates_.emplace(pts, PendingPtsUpdate(std::move(update), pts, pts_count, std::move(promise))); + double receive_time, Promise &&promise) { + postponed_pts_updates_.emplace(pts, + PendingPtsUpdate(std::move(update), pts, pts_count, receive_time, std::move(promise))); } void UpdatesManager::process_seq_updates(int32 seq_end, int32 date, @@ -2095,7 +2102,15 @@ void UpdatesManager::process_pending_seq_updates() { seq_gap_timeout_.cancel_timeout(); } else { // if after getDifference still have a gap - set_seq_gap_timeout(MAX_UNFILLED_GAP_TIME); + auto update_it = pending_seq_updates_.begin(); + double receive_time = update_it->second.receive_time; + for (size_t i = 0; i < 10; i++) { + if (++update_it == pending_seq_updates_.end()) { + break; + } + receive_time = min(receive_time, update_it->second.receive_time); + } + set_seq_gap_timeout(receive_time + MAX_UNFILLED_GAP_TIME - Time::now()); } } @@ -2127,7 +2142,15 @@ void UpdatesManager::process_pending_qts_updates() { qts_gap_timeout_.cancel_timeout(); } else { // if after getDifference still have a gap - set_qts_gap_timeout(MAX_UNFILLED_GAP_TIME); + auto update_it = pending_qts_updates_.begin(); + double receive_time = update_it->second.receive_time; + for (size_t i = 0; i < 10; i++) { + if (++update_it == pending_qts_updates_.end()) { + break; + } + receive_time = min(receive_time, update_it->second.receive_time); + } + set_qts_gap_timeout(receive_time + MAX_UNFILLED_GAP_TIME - Time::now()); } } @@ -2159,13 +2182,13 @@ void UpdatesManager::on_pending_update(tl_object_ptr updat const char *source) { vector> updates; updates.push_back(std::move(update)); - on_pending_updates(std::move(updates), seq, seq, 0, std::move(promise), source); + on_pending_updates(std::move(updates), seq, seq, 0, Time::now(), std::move(promise), source); } void UpdatesManager::on_update(tl_object_ptr update, Promise &&promise) { int new_pts = update->pts_; int pts_count = update->pts_count_; - add_pending_pts_update(std::move(update), new_pts, pts_count, std::move(promise), "updateNewMessage"); + add_pending_pts_update(std::move(update), new_pts, pts_count, Time::now(), std::move(promise), "updateNewMessage"); } void UpdatesManager::on_update(tl_object_ptr update, Promise &&promise) { @@ -2184,36 +2207,41 @@ void UpdatesManager::on_update(tl_object_ptr &&promise) { int new_pts = update->pts_; int pts_count = update->pts_count_; - add_pending_pts_update(std::move(update), new_pts, pts_count, std::move(promise), "updateReadMessagesContents"); + add_pending_pts_update(std::move(update), new_pts, pts_count, Time::now(), std::move(promise), + "updateReadMessagesContents"); } void UpdatesManager::on_update(tl_object_ptr update, Promise &&promise) { int new_pts = update->pts_; int pts_count = update->pts_count_; - add_pending_pts_update(std::move(update), new_pts, pts_count, std::move(promise), "updateEditMessage"); + add_pending_pts_update(std::move(update), new_pts, pts_count, Time::now(), std::move(promise), "updateEditMessage"); } void UpdatesManager::on_update(tl_object_ptr update, Promise &&promise) { int new_pts = update->pts_; int pts_count = update->pts_count_; if (update->messages_.empty()) { - add_pending_pts_update(make_tl_object(), new_pts, pts_count, Promise(), "updateDeleteMessages"); + add_pending_pts_update(make_tl_object(), new_pts, pts_count, Time::now(), Promise(), + "updateDeleteMessages"); promise.set_value(Unit()); } else { - add_pending_pts_update(std::move(update), new_pts, pts_count, std::move(promise), "updateDeleteMessages"); + add_pending_pts_update(std::move(update), new_pts, pts_count, Time::now(), std::move(promise), + "updateDeleteMessages"); } } void UpdatesManager::on_update(tl_object_ptr update, Promise &&promise) { int new_pts = update->pts_; int pts_count = update->pts_count_; - add_pending_pts_update(std::move(update), new_pts, pts_count, std::move(promise), "updateReadHistoryInbox"); + add_pending_pts_update(std::move(update), new_pts, pts_count, Time::now(), std::move(promise), + "updateReadHistoryInbox"); } void UpdatesManager::on_update(tl_object_ptr update, Promise &&promise) { int new_pts = update->pts_; int pts_count = update->pts_count_; - add_pending_pts_update(std::move(update), new_pts, pts_count, std::move(promise), "updateReadHistoryOutbox"); + add_pending_pts_update(std::move(update), new_pts, pts_count, Time::now(), std::move(promise), + "updateReadHistoryOutbox"); } void UpdatesManager::on_update(tl_object_ptr update, Promise &&promise) { @@ -2325,7 +2353,8 @@ void UpdatesManager::on_update(tl_object_ptr update, Promise &&promise) { int new_pts = update->pts_; int pts_count = update->pts_count_; - add_pending_pts_update(std::move(update), new_pts, pts_count, std::move(promise), "updatePinnedMessages"); + add_pending_pts_update(std::move(update), new_pts, pts_count, Time::now(), std::move(promise), + "updatePinnedMessages"); } void UpdatesManager::on_update(tl_object_ptr update, @@ -2388,7 +2417,7 @@ void UpdatesManager::on_update(tl_object_ptr up void UpdatesManager::on_update(tl_object_ptr update, Promise &&promise) { td_->web_pages_manager_->on_get_web_page(std::move(update->webpage_), DialogId()); - add_pending_pts_update(make_tl_object(), update->pts_, update->pts_count_, Promise(), + add_pending_pts_update(make_tl_object(), update->pts_, update->pts_count_, Time::now(), Promise(), "updateWebPage"); promise.set_value(Unit()); } @@ -2409,8 +2438,8 @@ void UpdatesManager::on_update(tl_object_ptr up } if (update->pts_ > 0) { - add_pending_pts_update(make_tl_object(), update->pts_, update->pts_count_, Promise(), - "updateFolderPeers"); + add_pending_pts_update(make_tl_object(), update->pts_, update->pts_count_, Time::now(), + Promise(), "updateFolderPeers"); } promise.set_value(Unit()); } diff --git a/td/telegram/UpdatesManager.h b/td/telegram/UpdatesManager.h index 4013dd882..66b6a3c77 100644 --- a/td/telegram/UpdatesManager.h +++ b/td/telegram/UpdatesManager.h @@ -95,7 +95,7 @@ class UpdatesManager final : public Actor { void on_get_updates(tl_object_ptr &&updates_ptr, Promise &&promise); void add_pending_pts_update(tl_object_ptr &&update, int32 new_pts, int32 pts_count, - Promise &&promise, const char *source); + double receive_time, Promise &&promise, const char *source); static std::unordered_set get_sent_messages_random_ids(const telegram_api::Updates *updates_ptr); @@ -134,10 +134,16 @@ class UpdatesManager final : public Actor { tl_object_ptr update; int32 pts; int32 pts_count; + double receive_time; Promise promise; - PendingPtsUpdate(tl_object_ptr &&update, int32 pts, int32 pts_count, Promise &&promise) - : update(std::move(update)), pts(pts), pts_count(pts_count), promise(std::move(promise)) { + PendingPtsUpdate(tl_object_ptr &&update, int32 pts, int32 pts_count, double receive_time, + Promise &&promise) + : update(std::move(update)) + , pts(pts) + , pts_count(pts_count) + , receive_time(receive_time) + , promise(std::move(promise)) { } }; @@ -146,17 +152,24 @@ class UpdatesManager final : public Actor { int32 seq_begin; int32 seq_end; int32 date; + double receive_time; vector> updates; Promise promise; - PendingSeqUpdates(int32 seq_begin, int32 seq_end, int32 date, vector> &&updates, - Promise &&promise) - : seq_begin(seq_begin), seq_end(seq_end), date(date), updates(std::move(updates)), promise(std::move(promise)) { + PendingSeqUpdates(int32 seq_begin, int32 seq_end, int32 date, double receive_time, + vector> &&updates, Promise &&promise) + : seq_begin(seq_begin) + , seq_end(seq_end) + , date(date) + , receive_time(receive_time) + , updates(std::move(updates)) + , promise(std::move(promise)) { } }; class PendingQtsUpdate { public: + double receive_time; tl_object_ptr update; vector> promises; }; @@ -243,13 +256,13 @@ class UpdatesManager final : public Actor { void add_pending_qts_update(tl_object_ptr &&update, int32 qts, Promise &&promise); void on_pending_updates(vector> &&updates, int32 seq_begin, int32 seq_end, - int32 date, Promise &&promise, const char *source); + int32 date, double receive_time, Promise &&promise, const char *source); void process_updates(vector> &&updates, bool force_apply, Promise &&promise); void postpone_pts_update(tl_object_ptr &&update, int32 pts, int32 pts_count, - Promise &&promise); + double receive_time, Promise &&promise); void process_pts_update(tl_object_ptr &&update); From 8ac0b02a6def3158f12a79fe3164f7066e3b0f93 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 15 Aug 2021 12:51:57 +0300 Subject: [PATCH 49/51] Improve warnings on long gap fill. --- td/telegram/UpdatesManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/td/telegram/UpdatesManager.cpp b/td/telegram/UpdatesManager.cpp index 25ef5c974..ab2123498 100644 --- a/td/telegram/UpdatesManager.cpp +++ b/td/telegram/UpdatesManager.cpp @@ -1960,7 +1960,6 @@ void UpdatesManager::add_pending_pts_update(tl_object_ptr if (old_pts < accumulated_pts_ - accumulated_pts_count_) { set_pts_gap_timeout(receive_time + MAX_UNFILLED_GAP_TIME - Time::now()); - last_pts_gap_time_ = Time::now(); return; } @@ -2159,6 +2158,7 @@ void UpdatesManager::set_pts_gap_timeout(double timeout) { pts_gap_timeout_.set_callback(std::move(fill_pts_gap)); pts_gap_timeout_.set_callback_data(static_cast(td_)); pts_gap_timeout_.set_timeout_in(timeout); + last_pts_gap_time_ = Time::now(); } } From 6194d9ec149e1fe706c1149358fa13f14c0d3697 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 15 Aug 2021 13:52:00 +0300 Subject: [PATCH 50/51] Drop pending seq/qts updates received before seq/qts overflow. --- td/telegram/UpdatesManager.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/td/telegram/UpdatesManager.cpp b/td/telegram/UpdatesManager.cpp index ab2123498..837101d02 100644 --- a/td/telegram/UpdatesManager.cpp +++ b/td/telegram/UpdatesManager.cpp @@ -1401,7 +1401,7 @@ void UpdatesManager::after_get_difference() { return; } - if (postponed_updates_.size()) { + if (!postponed_updates_.empty()) { VLOG(get_difference) << "Begin to apply " << postponed_updates_.size() << " postponed update chunks"; size_t total_update_count = 0; while (!postponed_updates_.empty()) { @@ -1427,7 +1427,7 @@ void UpdatesManager::after_get_difference() { VLOG(get_difference) << "Finish to apply " << total_update_count << " postponed updates"; } - if (postponed_pts_updates_.size()) { // must be before td_->messages_manager_->after_get_difference() + if (!postponed_pts_updates_.empty()) { // must be before td_->messages_manager_->after_get_difference() auto postponed_updates = std::move(postponed_pts_updates_); postponed_pts_updates_.clear(); @@ -2081,16 +2081,17 @@ void UpdatesManager::process_pending_seq_updates() { while (!pending_seq_updates_.empty() && !running_get_difference_) { auto update_it = pending_seq_updates_.begin(); auto seq_begin = update_it->second.seq_begin; - if (seq_begin > seq_ + 1) { + if (seq_begin - 1 > seq_ && seq_begin - 500000000 <= seq_) { + // the updates will be applied later break; } - if (seq_begin == seq_ + 1) { + if (seq_begin - 1 == seq_) { process_seq_updates(update_it->second.seq_end, update_it->second.date, std::move(update_it->second.updates), std::move(update_it->second.promise)); } else { // old update CHECK(seq_begin != 0); - LOG_IF(ERROR, update_it->second.seq_end > seq_) + LOG_IF(ERROR, update_it->second.seq_end > seq_ && seq_begin - 1 < seq_) << "Strange updates coming with seq_begin = " << seq_begin << ", seq_end = " << update_it->second.seq_end << ", but seq = " << seq_; update_it->second.promise.set_value(Unit()); @@ -2117,12 +2118,15 @@ void UpdatesManager::process_pending_qts_updates() { if (pending_qts_updates_.empty()) { return; } + LOG(DEBUG) << "Process " << pending_qts_updates_.size() << " pending qts updates"; while (!pending_qts_updates_.empty()) { CHECK(!running_get_difference_); auto update_it = pending_qts_updates_.begin(); auto qts = update_it->first; - if (qts > get_qts() + 1) { + auto old_qts = get_qts(); + if (qts - 1 > old_qts && qts - 500000000 <= old_qts) { + // the update will be applied later break; } auto promise = PromiseCreator::lambda([promises = std::move(update_it->second.promises)](Unit) mutable { @@ -2130,13 +2134,14 @@ void UpdatesManager::process_pending_qts_updates() { promise.set_value(Unit()); } }); - if (qts == get_qts() + 1) { + if (qts == old_qts + 1) { process_qts_update(std::move(update_it->second.update), qts, std::move(promise)); } else { promise.set_value(Unit()); } pending_qts_updates_.erase(update_it); } + if (pending_qts_updates_.empty()) { qts_gap_timeout_.cancel_timeout(); } else { From d161323858a782bc500d188b9ae916982526c262 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 15 Aug 2021 14:46:19 +0300 Subject: [PATCH 51/51] Add delay before reading history on server in non-joined channels with wrong server_unread_count. --- td/telegram/MessagesManager.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 7c1301acb..65a482fa6 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -20493,7 +20493,9 @@ void MessagesManager::read_history_on_server(Dialog *d, MessageId max_message_id d->updated_read_history_message_ids.insert(MessageId()); - bool need_delay = d->is_opened && !is_secret && d->server_unread_count > 0; + bool need_delay = d->is_opened && !is_secret && + (d->server_unread_count > 0 || (!need_unread_counter(d->order) && d->last_message_id.is_valid() && + max_message_id < d->last_message_id)); pending_read_history_timeout_.set_timeout_in(dialog_id.get(), need_delay ? MIN_READ_HISTORY_DELAY : 0); } @@ -20523,7 +20525,7 @@ void MessagesManager::read_message_thread_history_on_server(Dialog *d, MessageId d->updated_read_history_message_ids.insert(top_thread_message_id); - bool need_delay = d->is_opened && last_message_id.is_valid() && max_message_id != last_message_id; + bool need_delay = d->is_opened && last_message_id.is_valid() && max_message_id < last_message_id; pending_read_history_timeout_.set_timeout_in(dialog_id.get(), need_delay ? MIN_READ_HISTORY_DELAY : 0); }