From d5a19f0a82db2f6ec7784399999feedf3c661660 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 29 Jul 2021 02:42:53 +0300 Subject: [PATCH 1/6] Don't add last new message in on_get_dialog if it is supposed to be added through get_channel_difference. --- td/telegram/MessagesManager.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index d0a7306a8..e78ddf139 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -14458,7 +14458,7 @@ void MessagesManager::on_get_dialogs(FolderId folder_id, vectorpts == 0 || dialog->pts_ <= d->pts) { auto added_full_message_id = on_get_message(std::move(last_message), false, has_pts, false, false, false, "get chats"); CHECK(d->last_new_message_id == MessageId()); @@ -14468,14 +14468,13 @@ void MessagesManager::on_get_dialogs(FolderId folder_id, vectorlast_new_message_id, "on_get_dialogs"); send_update_chat_last_message(d, "on_get_dialogs"); } + } else { + get_channel_difference(dialog_id, d->pts, true, "on_get_dialogs"); } } if (has_pts && !running_get_channel_difference(dialog_id)) { - int32 channel_pts = dialog->pts_; - LOG_IF(ERROR, channel_pts < d->pts) << "In new " << dialog_id << " pts = " << d->pts - << ", but pts = " << channel_pts << " received in GetChats"; - set_channel_pts(d, channel_pts, "get channel"); + set_channel_pts(d, dialog->pts_, "get channel"); } } bool is_marked_as_unread = (dialog->flags_ & telegram_api::dialog::UNREAD_MARK_MASK) != 0; From be53ea2e1c2aa704df0f1634e6e7dce6154df8df Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 29 Jul 2021 02:52:59 +0300 Subject: [PATCH 2/6] Move retry_get_difference_timeout out of Dialog to allow retries for unknown dialogs. --- td/telegram/MessagesManager.cpp | 18 +++++++++++------- td/telegram/MessagesManager.h | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index e78ddf139..1c8f9455d 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -35614,11 +35614,15 @@ void MessagesManager::on_get_channel_difference( if (difference_ptr == nullptr) { bool have_access = have_input_peer(dialog_id, AccessRights::Read); - if (have_access && d != nullptr) { - channel_get_difference_retry_timeout_.add_timeout_in(dialog_id.get(), d->retry_get_difference_timeout); - d->retry_get_difference_timeout *= 2; - if (d->retry_get_difference_timeout > 60) { - d->retry_get_difference_timeout = Random::fast(60, 80); + if (have_access) { + auto &delay = channel_get_difference_retry_timeouts_[dialog_id]; + if (delay == 0) { + delay = 1; + } + channel_get_difference_retry_timeout_.add_timeout_in(dialog_id.get(), delay); + delay *= 2; + if (delay > 60) { + delay = Random::fast(60, 80); } } else { after_get_channel_difference(dialog_id, false); @@ -35626,6 +35630,8 @@ void MessagesManager::on_get_channel_difference( return; } + channel_get_difference_retry_timeouts_.erase(dialog_id); + LOG(INFO) << "Receive result of getChannelDifference for " << dialog_id << " with pts = " << request_pts << " and limit = " << request_limit << ": " << to_string(difference_ptr); @@ -35671,8 +35677,6 @@ void MessagesManager::on_get_channel_difference( LOG_IF(ERROR, cur_pts != request_pts) << "Channel pts has changed from " << request_pts << " to " << d->pts << " in " << dialog_id << " during getChannelDifference"; - d->retry_get_difference_timeout = 1; - bool is_final = true; int32 timeout = 0; switch (difference_ptr->get_id()) { diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index e55736511..d5db1b10f 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -1240,7 +1240,6 @@ class MessagesManager final : public Actor { int32 pts = 0; // for channels only std::multimap postponed_channel_updates; // for channels only - int32 retry_get_difference_timeout = 1; // for channels only int32 pending_read_channel_inbox_pts = 0; // for channels only MessageId pending_read_channel_inbox_max_message_id; // for channels only int32 pending_read_channel_inbox_server_unread_count = 0; // for channels only @@ -3237,6 +3236,7 @@ class MessagesManager final : public Actor { std::unordered_map active_get_channel_differencies_; std::unordered_map get_channel_difference_to_log_event_id_; + std::unordered_map channel_get_difference_retry_timeouts_; MultiTimeout channel_get_difference_timeout_{"ChannelGetDifferenceTimeout"}; MultiTimeout channel_get_difference_retry_timeout_{"ChannelGetDifferenceRetryTimeout"}; From c07562ed52cde90a4ec35b73481b1fdf2d1d3214 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 29 Jul 2021 03:32:08 +0300 Subject: [PATCH 3/6] Repair last message after getChannelDifference if it is still unknown. --- td/telegram/MessagesManager.cpp | 9 ++++++++- td/telegram/MessagesManager.h | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 1c8f9455d..4a4b19bc0 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -13890,6 +13890,7 @@ void MessagesManager::set_dialog_unread_mention_count(Dialog *d, int32 unread_me void MessagesManager::set_dialog_is_empty(Dialog *d, const char *source) { LOG(INFO) << "Set " << d->dialog_id << " is_empty to true from " << source; + CHECK(d->have_full_history); d->is_empty = true; if (d->server_unread_count + d->local_unread_count > 0) { @@ -14458,7 +14459,7 @@ void MessagesManager::on_get_dialogs(FolderId folder_id, vectorpts == 0 || dialog->pts_ <= d->pts) { + } else if (!has_pts || d->pts == 0 || dialog->pts_ <= d->pts || d->is_channel_difference_finished) { auto added_full_message_id = on_get_message(std::move(last_message), false, has_pts, false, false, false, "get chats"); CHECK(d->last_new_message_id == MessageId()); @@ -35825,6 +35826,7 @@ void MessagesManager::after_get_channel_difference(DialogId dialog_id, bool succ auto d = get_dialog(dialog_id); if (d != nullptr) { + d->is_channel_difference_finished = true; bool have_access = have_input_peer(dialog_id, AccessRights::Read); if (!d->postponed_channel_updates.empty()) { LOG(INFO) << "Begin to apply postponed channel updates"; @@ -35877,6 +35879,11 @@ void MessagesManager::after_get_channel_difference(DialogId dialog_id, bool succ send_closure_later(G()->notification_manager(), &NotificationManager::after_get_chat_difference, d->mention_notification_group.group_id); } + + if (!td_->auth_manager_->is_bot() && have_access && !d->last_message_id.is_valid() && !d->is_empty && + (d->order != DEFAULT_ORDER || is_dialog_sponsored(d))) { + get_history_from_the_end_impl(d, true, false, Auto()); + } } if (postponed_chat_read_inbox_updates_.erase(dialog_id) > 0) { diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index d5db1b10f..7581b78a3 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -1237,6 +1237,7 @@ class MessagesManager final : public Actor { bool is_update_new_chat_sent = false; bool has_unload_timeout = false; + bool is_channel_difference_finished = false; int32 pts = 0; // for channels only std::multimap postponed_channel_updates; // for channels only From c1d69fcc3261be395cad8b15d31941974d75148c Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 29 Jul 2021 03:39:05 +0300 Subject: [PATCH 4/6] Ensure that if Dialog is_empty, than we have_full_history. --- td/telegram/MessagesManager.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 4a4b19bc0..59035a7f7 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -10905,7 +10905,7 @@ void MessagesManager::unload_dialog(DialogId dialog_id) { } if (!unloaded_message_ids.empty()) { - if (!G()->parameters().use_message_db) { + if (!G()->parameters().use_message_db && !d->is_empty) { d->have_full_history = false; } @@ -11013,6 +11013,7 @@ void MessagesManager::on_dialog_deleted(DialogId dialog_id, Promise &&prom delete_all_dialog_messages(d, true, false); if (dialog_id.get_type() != DialogType::SecretChat) { d->have_full_history = false; + d->is_empty = false; d->need_restore_reply_markup = true; } if (remove_recently_found_dialog_internal(dialog_id)) { @@ -13781,7 +13782,7 @@ void MessagesManager::remove_dialog_newer_messages(Dialog *d, MessageId from_mes delete_all_dialog_messages_from_database(d, MessageId::max(), "remove_dialog_newer_messages"); set_dialog_first_database_message_id(d, MessageId(), "remove_dialog_newer_messages"); set_dialog_last_database_message_id(d, MessageId(), source); - if (d->dialog_id.get_type() != DialogType::SecretChat) { + if (d->dialog_id.get_type() != DialogType::SecretChat && !d->is_empty) { d->have_full_history = false; } invalidate_message_indexes(d); @@ -22119,6 +22120,7 @@ void MessagesManager::on_get_history_from_database(DialogId dialog_id, MessageId if (message == nullptr) { if (d->have_full_history) { d->have_full_history = false; + d->is_empty = false; // just in case on_dialog_updated(dialog_id, "drop have_full_history in on_get_history_from_database"); } break; @@ -33889,6 +33891,11 @@ void MessagesManager::fix_new_dialog(Dialog *d, unique_ptr &&last_datab } } + if (d->is_empty && !d->have_full_history) { + LOG(ERROR) << "Drop invalid flag is_empty"; + d->is_empty = false; + } + if (being_added_dialog_id_ != dialog_id && !td_->auth_manager_->is_bot() && !is_dialog_inited(d) && dialog_type != DialogType::SecretChat && have_input_peer(dialog_id, AccessRights::Read)) { // asynchronously get dialog from the server @@ -35556,6 +35563,7 @@ void MessagesManager::on_get_channel_dialog(DialogId dialog_id, MessageId last_m set_dialog_first_database_message_id(d, MessageId(), "on_get_channel_dialog 6"); set_dialog_last_database_message_id(d, MessageId(), "on_get_channel_dialog 7"); d->have_full_history = false; + d->is_empty = false; } invalidate_message_indexes(d); From bc8544299b598080d270c2093d96fe8c4ee19c50 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 29 Jul 2021 06:07:11 +0300 Subject: [PATCH 5/6] Repeat getHistory request if some new server messages aren't received. --- td/telegram/MessagesManager.cpp | 68 +++++++++++++++++++++++---------- td/telegram/MessagesManager.h | 5 ++- 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 59035a7f7..d936bac7c 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -1928,6 +1928,7 @@ class GetHistoryQuery final : public Td::ResultHandler { Promise promise_; DialogId dialog_id_; MessageId from_message_id_; + MessageId old_last_new_message_id_; int32 offset_; int32 limit_; bool from_the_end_; @@ -1936,7 +1937,8 @@ class GetHistoryQuery final : public Td::ResultHandler { explicit GetHistoryQuery(Promise &&promise) : promise_(std::move(promise)) { } - void send(DialogId dialog_id, MessageId from_message_id, int32 offset, int32 limit) { + void send(DialogId dialog_id, MessageId from_message_id, MessageId old_last_new_message_id, int32 offset, + int32 limit) { auto input_peer = td->messages_manager_->get_input_peer(dialog_id, AccessRights::Read); if (input_peer == nullptr) { LOG(ERROR) << "Can't get chat history in " << dialog_id << " because doesn't have info about the chat"; @@ -1946,6 +1948,7 @@ class GetHistoryQuery final : public Td::ResultHandler { dialog_id_ = dialog_id; from_message_id_ = from_message_id; + old_last_new_message_id_ = old_last_new_message_id; offset_ = offset; limit_ = limit; from_the_end_ = false; @@ -1953,7 +1956,7 @@ class GetHistoryQuery final : public Td::ResultHandler { std::move(input_peer), from_message_id.get_server_message_id().get(), 0, offset, limit, 0, 0, 0))); } - void send_get_from_the_end(DialogId dialog_id, int32 limit) { + void send_get_from_the_end(DialogId dialog_id, MessageId old_last_new_message_id, int32 limit) { auto input_peer = td->messages_manager_->get_input_peer(dialog_id, AccessRights::Read); if (input_peer == nullptr) { LOG(ERROR) << "Can't get chat history because doesn't have info about the chat"; @@ -1961,6 +1964,7 @@ class GetHistoryQuery final : public Td::ResultHandler { } dialog_id_ = dialog_id; + old_last_new_message_id_ = old_last_new_message_id; offset_ = 0; limit_ = limit; from_the_end_ = true; @@ -1977,17 +1981,17 @@ class GetHistoryQuery final : public Td::ResultHandler { auto info = td->messages_manager_->on_get_messages(result_ptr.move_as_ok(), "GetHistoryQuery"); td->messages_manager_->get_channel_difference_if_needed( dialog_id_, std::move(info), - PromiseCreator::lambda([td = td, dialog_id = dialog_id_, from_message_id = from_message_id_, offset = offset_, - limit = limit_, from_the_end = from_the_end_, + PromiseCreator::lambda([td = td, dialog_id = dialog_id_, from_message_id = from_message_id_, + old_last_new_message_id = old_last_new_message_id_, offset = offset_, limit = limit_, + from_the_end = from_the_end_, promise = std::move(promise_)](Result &&result) mutable { if (result.is_error()) { promise.set_error(result.move_as_error()); } else { auto info = result.move_as_ok(); // TODO use info.total_count, info.pts - td->messages_manager_->on_get_history(dialog_id, from_message_id, offset, limit, from_the_end, - std::move(info.messages)); - promise.set_value(Unit()); + td->messages_manager_->on_get_history(dialog_id, from_message_id, old_last_new_message_id, offset, limit, + from_the_end, std::move(info.messages), std::move(promise)); } })); } @@ -9249,18 +9253,34 @@ void MessagesManager::on_get_messages(vector> &&messages) { +void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_id, MessageId old_last_new_message_id, + int32 offset, int32 limit, bool from_the_end, + vector> &&messages, Promise &&promise) { LOG(INFO) << "Receive " << messages.size() << " history messages " << (from_the_end ? "from the end " : "") << "in " << dialog_id << " from " << from_message_id << " with offset " << offset << " and limit " << limit; CHECK(-limit < offset && offset <= 0); CHECK(offset < 0 || from_the_end); CHECK(!from_message_id.is_scheduled()); + Dialog *d = get_dialog(dialog_id); + + MessageId last_received_message_id = messages.empty() ? MessageId() : get_message_id(messages[0], false); + if (d != nullptr && old_last_new_message_id < d->last_new_message_id && + (from_the_end || old_last_new_message_id < from_message_id) && + last_received_message_id < d->last_new_message_id) { + // new server messages were added to the dialog since the request was sent, but weren't received + // they should have been received, so we must repeat the request to get them + if (from_the_end) { + get_history_from_the_end_impl(d, false, false, std::move(promise)); + } else { + get_history_impl(d, from_message_id, offset, limit, false, false, std::move(promise)); + } + return; + } + // the server can return less messages than requested if some of messages are deleted during request // but if it happens, it is likely that there are no more messages on the server bool have_full_history = from_the_end && narrow_cast(messages.size()) < limit && messages.size() <= 1; - Dialog *d = get_dialog(dialog_id); if (messages.empty()) { if (d != nullptr) { @@ -9281,6 +9301,7 @@ void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_ // be aware that in some cases an empty answer may be returned, because of the race of getHistory and deleteMessages // and not because there are no more messages + promise.set_value(Unit()); return; } @@ -9298,6 +9319,7 @@ void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_ } // TODO move to ERROR LOG(FATAL) << error; + promise.set_value(Unit()); return; } cur_message_id = message_id; @@ -9310,7 +9332,6 @@ void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_ // be aware that any subset of the returned messages may be already deleted and returned as MessageEmpty bool is_channel_message = dialog_id.get_type() == DialogType::Channel; MessageId first_added_message_id; - MessageId last_received_message_id = get_message_id(messages[0], false); MessageId last_added_message_id; bool have_next = false; @@ -9337,15 +9358,15 @@ void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_ } if (from_the_end && d != nullptr) { - auto last_server_message_id = get_message_id(messages[0], false); - // delete all server messages with ID > last_server_message_id + // delete all server messages with ID > last_received_message_id + // there were no new messages received after the getHistory request was sent, so they are already deleted message vector message_ids; - find_newer_messages(d->messages.get(), last_server_message_id, message_ids); + find_newer_messages(d->messages.get(), last_received_message_id, message_ids); if (!message_ids.empty()) { bool need_update_dialog_pos = false; vector deleted_message_ids; for (auto message_id : message_ids) { - CHECK(message_id > last_server_message_id); + CHECK(message_id > last_received_message_id); if (message_id.is_server()) { auto message = delete_message(d, message_id, true, &need_update_dialog_pos, "on_get_gistory 1"); CHECK(message != nullptr); @@ -9360,10 +9381,10 @@ void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_ send_update_delete_messages(dialog_id, std::move(deleted_message_ids), true, false); message_ids.clear(); - find_newer_messages(d->messages.get(), last_server_message_id, message_ids); + find_newer_messages(d->messages.get(), last_received_message_id, message_ids); } - // connect all messages with ID > last_server_message_id + // connect all messages with ID > last_received_message_id for (size_t i = 0; i + 1 < message_ids.size(); i++) { auto m = get_message(d, message_ids[i]); CHECK(m != nullptr); @@ -9372,6 +9393,8 @@ void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_ attach_message_to_next(d, message_ids[i], "on_get_history 3"); } } + + have_next = true; } } @@ -9420,6 +9443,7 @@ void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_ } if (d == nullptr) { + promise.set_value(Unit()); return; } @@ -9546,6 +9570,7 @@ void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_ } } } + promise.set_value(Unit()); } vector MessagesManager::get_peers_dialog_ids(vector> &&peers) { @@ -20309,6 +20334,7 @@ tl_object_ptr MessagesManager::get_dialog_history(DialogId dia while (*p != nullptr && messages.size() < static_cast(limit)) { messages.push_back(get_message_object(dialog_id, *p)); from_message_id = (*p)->message_id; + from_the_end = false; --p; } } @@ -22327,13 +22353,15 @@ void MessagesManager::get_history_from_the_end_impl(const Dialog *d, bool from_d std::move(promise)); })); } else { - if (only_local || dialog_id.get_type() == DialogType::SecretChat) { + if (only_local || dialog_id.get_type() == DialogType::SecretChat || d->last_message_id.is_valid()) { + // if last message is known, there are no reasons to get message history from server from the end promise.set_value(Unit()); return; } LOG(INFO) << "Get history from the end of " << dialog_id << " from server"; - td_->create_handler(std::move(promise))->send_get_from_the_end(dialog_id, limit); + td_->create_handler(std::move(promise)) + ->send_get_from_the_end(dialog_id, d->last_new_message_id, limit); } } @@ -22385,7 +22413,7 @@ void MessagesManager::get_history_impl(const Dialog *d, MessageId from_message_i LOG(INFO) << "Get history in " << dialog_id << " from " << from_message_id << " with offset " << offset << " and limit " << limit << " from server"; td_->create_handler(std::move(promise)) - ->send(dialog_id, from_message_id.get_next_server_message_id(), offset, limit); + ->send(dialog_id, from_message_id.get_next_server_message_id(), d->last_new_message_id, offset, limit); } } diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index 7581b78a3..56e7ff8d2 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -189,8 +189,9 @@ class MessagesManager final : public Actor { void on_get_messages(vector> &&messages, bool is_channel_message, bool is_scheduled, const char *source); - void on_get_history(DialogId dialog_id, MessageId from_message_id, int32 offset, int32 limit, bool from_the_end, - vector> &&messages); + void on_get_history(DialogId dialog_id, MessageId from_message_id, MessageId old_last_new_message_id, int32 offset, + int32 limit, bool from_the_end, vector> &&messages, + Promise &&promise); void on_get_public_dialogs_search_result(const string &query, vector> &&my_peers, vector> &&peers); From d4dc4f2a50f39b1c05efd955a6e9de0db2b197bc Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 29 Jul 2021 13:02:38 +0300 Subject: [PATCH 6/6] Fix value check. --- td/telegram/ConfigManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/td/telegram/ConfigManager.cpp b/td/telegram/ConfigManager.cpp index f7d9912a2..028033b76 100644 --- a/td/telegram/ConfigManager.cpp +++ b/td/telegram/ConfigManager.cpp @@ -1690,7 +1690,7 @@ void ConfigManager::process_app_config(tl_object_ptr &c if (video_note_setting->value_->get_id() == telegram_api::jsonNumber::ID) { auto setting_value = static_cast( static_cast(video_note_setting->value_.get())->value_); - if (value > 0) { + if (setting_value > 0) { if (video_note_setting->key_ == "diameter") { G()->shared_config().set_option_integer("suggested_video_note_length", setting_value); }