From 4cf1e02479f5eb0d2f30cbc39c5397a165c37bce Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 22 Jul 2022 21:21:30 +0300 Subject: [PATCH 01/23] Add multiarg version of Scheduler::destroy_on_scheduler. --- td/telegram/ContactsManager.cpp | 47 ++++++-------------------- td/telegram/MessagesManager.cpp | 38 ++++++--------------- td/telegram/PollManager.cpp | 7 ++-- td/telegram/StickersManager.cpp | 16 +++------ td/telegram/VoiceNotesManager.cpp | 5 ++- td/telegram/WebPagesManager.cpp | 7 ++-- td/telegram/files/FileManager.cpp | 11 ++---- tdactor/td/actor/impl/Scheduler-decl.h | 3 ++ tdactor/td/actor/impl/Scheduler.h | 7 ++++ 9 files changed, 43 insertions(+), 98 deletions(-) diff --git a/td/telegram/ContactsManager.cpp b/td/telegram/ContactsManager.cpp index 12b14ae82..83fea9a95 100644 --- a/td/telegram/ContactsManager.cpp +++ b/td/telegram/ContactsManager.cpp @@ -3398,43 +3398,16 @@ ContactsManager::ContactsManager(Td *td, ActorShared<> parent) : td_(td), parent } ContactsManager::~ContactsManager() { - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), users_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), users_full_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), user_photos_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), unknown_users_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), pending_user_photos_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), user_profile_photo_file_source_ids_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), my_photo_file_id_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), chats_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), chats_full_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), unknown_chats_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), chat_full_file_source_ids_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), min_channels_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), channels_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), channels_full_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), unknown_channels_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), invalidated_channels_full_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), channel_full_file_source_ids_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), secret_chats_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), unknown_secret_chats_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), secret_chats_with_user_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), invite_link_infos_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), dialog_access_by_invite_link_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), loaded_from_database_users_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), unavailable_user_fulls_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), loaded_from_database_chats_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), unavailable_chat_fulls_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), loaded_from_database_channels_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), unavailable_channel_fulls_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), loaded_from_database_secret_chats_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), dialog_administrators_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), cached_channel_participants_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), resolved_phone_numbers_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), channel_participants_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), all_imported_contacts_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), linked_channel_ids_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), restricted_user_ids_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), restricted_channel_ids_); + Scheduler::instance()->destroy_on_scheduler( + G()->get_gc_scheduler_id(), users_, users_full_, user_photos_, unknown_users_, pending_user_photos_, + user_profile_photo_file_source_ids_, my_photo_file_id_, chats_, chats_full_, unknown_chats_, + chat_full_file_source_ids_, min_channels_, channels_, channels_full_, unknown_channels_, + invalidated_channels_full_, channel_full_file_source_ids_, secret_chats_, unknown_secret_chats_, + secret_chats_with_user_, invite_link_infos_, dialog_access_by_invite_link_, loaded_from_database_users_, + unavailable_user_fulls_, loaded_from_database_chats_, unavailable_chat_fulls_, loaded_from_database_channels_, + unavailable_channel_fulls_, loaded_from_database_secret_chats_, dialog_administrators_, + cached_channel_participants_, resolved_phone_numbers_, channel_participants_, all_imported_contacts_, + linked_channel_ids_, restricted_user_ids_, restricted_channel_ids_); } void ContactsManager::tear_down() { diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 883aa5b8f..410c1895c 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -5809,34 +5809,16 @@ MessagesManager::MessagesManager(Td *td, ActorShared<> parent) } MessagesManager::~MessagesManager() { - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), ttl_nodes_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), ttl_heap_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), being_sent_messages_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), update_message_ids_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), update_scheduled_message_ids_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), message_id_to_dialog_id_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), last_clear_history_message_id_to_dialog_id_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), dialogs_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), postponed_chat_read_inbox_updates_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), found_public_dialogs_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), found_on_server_dialogs_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), found_common_dialogs_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), message_embedding_codes_[0]); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), message_embedding_codes_[1]); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), replied_by_media_timestamp_messages_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), notification_group_id_to_dialog_id_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), active_get_channel_differencies_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), get_channel_difference_to_log_event_id_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), channel_get_difference_retry_timeouts_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), is_channel_difference_finished_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), resolved_usernames_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), inaccessible_resolved_usernames_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), dialog_bot_command_message_ids_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), full_message_id_to_file_source_id_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), last_outgoing_forwarded_message_date_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), dialog_viewed_messages_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), dialog_online_member_counts_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), previous_repaired_read_inbox_max_message_id_); + Scheduler::instance()->destroy_on_scheduler( + G()->get_gc_scheduler_id(), ttl_nodes_, ttl_heap_, being_sent_messages_, update_message_ids_, + update_scheduled_message_ids_, message_id_to_dialog_id_, last_clear_history_message_id_to_dialog_id_, dialogs_, + postponed_chat_read_inbox_updates_, found_public_dialogs_, found_on_server_dialogs_, found_common_dialogs_, + message_embedding_codes_[0], message_embedding_codes_[1], replied_by_media_timestamp_messages_, + notification_group_id_to_dialog_id_, active_get_channel_differencies_, get_channel_difference_to_log_event_id_, + channel_get_difference_retry_timeouts_, is_channel_difference_finished_, resolved_usernames_, + inaccessible_resolved_usernames_, dialog_bot_command_message_ids_, full_message_id_to_file_source_id_, + last_outgoing_forwarded_message_date_, dialog_viewed_messages_, dialog_online_member_counts_, + previous_repaired_read_inbox_max_message_id_); } void MessagesManager::on_channel_get_difference_timeout_callback(void *messages_manager_ptr, int64 dialog_id_int) { diff --git a/td/telegram/PollManager.cpp b/td/telegram/PollManager.cpp index 341d8c613..16875f7b0 100644 --- a/td/telegram/PollManager.cpp +++ b/td/telegram/PollManager.cpp @@ -266,11 +266,8 @@ void PollManager::tear_down() { } PollManager::~PollManager() { - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), polls_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), server_poll_messages_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), other_poll_messages_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), poll_voters_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), loaded_from_database_polls_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), polls_, server_poll_messages_, + other_poll_messages_, poll_voters_, loaded_from_database_polls_); } void PollManager::on_update_poll_timeout_callback(void *poll_manager_ptr, int64 poll_id_int) { diff --git a/td/telegram/StickersManager.cpp b/td/telegram/StickersManager.cpp index 820e15a74..02b422d0b 100644 --- a/td/telegram/StickersManager.cpp +++ b/td/telegram/StickersManager.cpp @@ -1305,18 +1305,10 @@ StickersManager::StickersManager(Td *td, ActorShared<> parent) : td_(td), parent } StickersManager::~StickersManager() { - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), stickers_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), sticker_sets_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), short_name_to_sticker_set_id_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), attached_sticker_sets_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), found_stickers_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), found_sticker_sets_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), emoji_language_codes_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), emoji_language_code_versions_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), emoji_language_code_last_difference_times_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), reloaded_emoji_keywords_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), dice_messages_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), emoji_messages_); + Scheduler::instance()->destroy_on_scheduler( + G()->get_gc_scheduler_id(), stickers_, sticker_sets_, short_name_to_sticker_set_id_, attached_sticker_sets_, + found_stickers_, found_sticker_sets_, emoji_language_codes_, emoji_language_code_versions_, + emoji_language_code_last_difference_times_, reloaded_emoji_keywords_, dice_messages_, emoji_messages_); } void StickersManager::start_up() { diff --git a/td/telegram/VoiceNotesManager.cpp b/td/telegram/VoiceNotesManager.cpp index 593e0b3e0..fddd8dd00 100644 --- a/td/telegram/VoiceNotesManager.cpp +++ b/td/telegram/VoiceNotesManager.cpp @@ -101,9 +101,8 @@ VoiceNotesManager::VoiceNotesManager(Td *td, ActorShared<> parent) : td_(td), pa } VoiceNotesManager::~VoiceNotesManager() { - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), voice_notes_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), voice_note_messages_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), message_voice_notes_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), voice_notes_, voice_note_messages_, + message_voice_notes_); } void VoiceNotesManager::tear_down() { diff --git a/td/telegram/WebPagesManager.cpp b/td/telegram/WebPagesManager.cpp index 28dda3bce..ccaa0885b 100644 --- a/td/telegram/WebPagesManager.cpp +++ b/td/telegram/WebPagesManager.cpp @@ -418,11 +418,8 @@ void WebPagesManager::tear_down() { } WebPagesManager::~WebPagesManager() { - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), web_pages_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), web_page_messages_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), got_web_page_previews_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), url_to_web_page_id_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), url_to_file_source_id_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), web_pages_, web_page_messages_, + got_web_page_previews_, url_to_web_page_id_, url_to_file_source_id_); } WebPageId WebPagesManager::on_get_web_page(tl_object_ptr &&web_page_ptr, diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index b80146d56..6dcc7cd37 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -840,14 +840,9 @@ void FileManager::init_actor() { } FileManager::~FileManager() { - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), remote_location_info_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), file_hash_to_file_id_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), local_location_to_file_id_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), generate_location_to_file_id_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), pmc_id_to_file_node_id_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), file_id_info_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), empty_file_ids_); - Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), file_nodes_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), remote_location_info_, file_hash_to_file_id_, + local_location_to_file_id_, generate_location_to_file_id_, + pmc_id_to_file_node_id_, file_id_info_, empty_file_ids_, file_nodes_); } string FileManager::fix_file_extension(Slice file_name, Slice file_type, Slice file_extension) { diff --git a/tdactor/td/actor/impl/Scheduler-decl.h b/tdactor/td/actor/impl/Scheduler-decl.h index 57f49dbb4..a8ec2aaa9 100644 --- a/tdactor/td/actor/impl/Scheduler-decl.h +++ b/tdactor/td/actor/impl/Scheduler-decl.h @@ -105,6 +105,9 @@ class Scheduler { template void destroy_on_scheduler(int32 sched_id, T &value); + template + void destroy_on_scheduler(int32 sched_id, ArgsT &...values); + template void send_lambda(ActorRef actor_ref, EventT &&lambda); diff --git a/tdactor/td/actor/impl/Scheduler.h b/tdactor/td/actor/impl/Scheduler.h index 182066145..4319dc2c2 100644 --- a/tdactor/td/actor/impl/Scheduler.h +++ b/tdactor/td/actor/impl/Scheduler.h @@ -179,6 +179,13 @@ inline void Scheduler::destroy_on_scheduler(int32 sched_id, T &value) { } } +template +inline void Scheduler::destroy_on_scheduler(int32 sched_id, ArgsT &...values) { + destroy_on_scheduler_impl(sched_id, PromiseCreator::lambda([values = std::make_tuple(std::move(values)...)](Unit) { + // destroy values + })); +} + inline void Scheduler::before_tail_send(const ActorId<> &actor_id) { // TODO } From bc83832e035ce54e0de92c2271979b7306e4f19c Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 23 Jul 2022 11:00:46 +0300 Subject: [PATCH 02/23] Avoid copying when adding users/chats to binlog. --- td/telegram/ContactsManager.cpp | 60 ++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/td/telegram/ContactsManager.cpp b/td/telegram/ContactsManager.cpp index 83fea9a95..4dc5e2134 100644 --- a/td/telegram/ContactsManager.cpp +++ b/td/telegram/ContactsManager.cpp @@ -8767,23 +8767,24 @@ void ContactsManager::on_get_user(tl_object_ptr &&user_ptr, class ContactsManager::UserLogEvent { public: UserId user_id; - User u; + const User *u_in = nullptr; + unique_ptr u_out; UserLogEvent() = default; - UserLogEvent(UserId user_id, const User &u) : user_id(user_id), u(u) { + UserLogEvent(UserId user_id, const User *u) : user_id(user_id), u_in(u) { } template void store(StorerT &storer) const { td::store(user_id, storer); - td::store(u, storer); + td::store(*u_in, storer); } template void parse(ParserT &parser) { td::parse(user_id, parser); - td::parse(u, parser); + td::parse(u_out, parser); } }; @@ -8794,7 +8795,7 @@ void ContactsManager::save_user(User *u, UserId user_id, bool from_binlog) { CHECK(u != nullptr); if (!u->is_saved || !u->is_status_saved) { // TODO more effective handling of !u->is_status_saved if (!from_binlog) { - auto log_event = UserLogEvent(user_id, *u); + auto log_event = UserLogEvent(user_id, u); auto storer = get_log_event_storer(log_event); if (u->log_event_id == 0) { u->log_event_id = binlog_add(G()->td_db()->get_binlog(), LogEvent::HandlerType::Users, storer); @@ -8824,8 +8825,8 @@ void ContactsManager::on_binlog_user_event(BinlogEvent &&event) { } LOG(INFO) << "Add " << user_id << " from binlog"; - User *u = add_user(user_id, "on_binlog_user_event"); - *u = std::move(log_event.u); // users come from binlog before all other events, so just add them + User *u = users_.emplace(user_id, std::move(log_event.u_out)).first->second.get(); + CHECK(u != nullptr); u->log_event_id = event.id_; @@ -9073,23 +9074,24 @@ ContactsManager::User *ContactsManager::get_user_force_impl(UserId user_id) { class ContactsManager::ChatLogEvent { public: ChatId chat_id; - Chat c; + const Chat *c_in = nullptr; + unique_ptr c_out; ChatLogEvent() = default; - ChatLogEvent(ChatId chat_id, const Chat &c) : chat_id(chat_id), c(c) { + ChatLogEvent(ChatId chat_id, const Chat *c) : chat_id(chat_id), c_in(c) { } template void store(StorerT &storer) const { td::store(chat_id, storer); - td::store(c, storer); + td::store(*c_in, storer); } template void parse(ParserT &parser) { td::parse(chat_id, parser); - td::parse(c, parser); + td::parse(c_out, parser); } }; @@ -9100,7 +9102,7 @@ void ContactsManager::save_chat(Chat *c, ChatId chat_id, bool from_binlog) { CHECK(c != nullptr); if (!c->is_saved) { if (!from_binlog) { - auto log_event = ChatLogEvent(chat_id, *c); + auto log_event = ChatLogEvent(chat_id, c); auto storer = get_log_event_storer(log_event); if (c->log_event_id == 0) { c->log_event_id = binlog_add(G()->td_db()->get_binlog(), LogEvent::HandlerType::Chats, storer); @@ -9131,8 +9133,8 @@ void ContactsManager::on_binlog_chat_event(BinlogEvent &&event) { } LOG(INFO) << "Add " << chat_id << " from binlog"; - Chat *c = add_chat(chat_id); - *c = std::move(log_event.c); // chats come from binlog before all other events, so just add them + Chat *c = chats_.emplace(chat_id, std::move(log_event.c_out)).first->second.get(); + CHECK(c != nullptr); c->log_event_id = event.id_; @@ -9310,23 +9312,24 @@ ContactsManager::Chat *ContactsManager::get_chat_force(ChatId chat_id) { class ContactsManager::ChannelLogEvent { public: ChannelId channel_id; - Channel c; + const Channel *c_in = nullptr; + unique_ptr c_out; ChannelLogEvent() = default; - ChannelLogEvent(ChannelId channel_id, const Channel &c) : channel_id(channel_id), c(c) { + ChannelLogEvent(ChannelId channel_id, const Channel *c) : channel_id(channel_id), c_in(c) { } template void store(StorerT &storer) const { td::store(channel_id, storer); - td::store(c, storer); + td::store(*c_in, storer); } template void parse(ParserT &parser) { td::parse(channel_id, parser); - td::parse(c, parser); + td::parse(c_out, parser); } }; @@ -9337,7 +9340,7 @@ void ContactsManager::save_channel(Channel *c, ChannelId channel_id, bool from_b CHECK(c != nullptr); if (!c->is_saved) { if (!from_binlog) { - auto log_event = ChannelLogEvent(channel_id, *c); + auto log_event = ChannelLogEvent(channel_id, c); auto storer = get_log_event_storer(log_event); if (c->log_event_id == 0) { c->log_event_id = binlog_add(G()->td_db()->get_binlog(), LogEvent::HandlerType::Channels, storer); @@ -9368,8 +9371,8 @@ void ContactsManager::on_binlog_channel_event(BinlogEvent &&event) { } LOG(INFO) << "Add " << channel_id << " from binlog"; - Channel *c = add_channel(channel_id, "on_binlog_channel_event"); - *c = std::move(log_event.c); // channels come from binlog before all other events, so just add them + Channel *c = channels_.emplace(channel_id, std::move(log_event.c_out)).first->second.get(); + CHECK(c != nullptr); c->log_event_id = event.id_; @@ -9563,23 +9566,24 @@ ContactsManager::Channel *ContactsManager::get_channel_force(ChannelId channel_i class ContactsManager::SecretChatLogEvent { public: SecretChatId secret_chat_id; - SecretChat c; + const SecretChat *c_in = nullptr; + unique_ptr c_out; SecretChatLogEvent() = default; - SecretChatLogEvent(SecretChatId secret_chat_id, const SecretChat &c) : secret_chat_id(secret_chat_id), c(c) { + SecretChatLogEvent(SecretChatId secret_chat_id, const SecretChat *c) : secret_chat_id(secret_chat_id), c_in(c) { } template void store(StorerT &storer) const { td::store(secret_chat_id, storer); - td::store(c, storer); + td::store(*c_in, storer); } template void parse(ParserT &parser) { td::parse(secret_chat_id, parser); - td::parse(c, parser); + td::parse(c_out, parser); } }; @@ -9590,7 +9594,7 @@ void ContactsManager::save_secret_chat(SecretChat *c, SecretChatId secret_chat_i CHECK(c != nullptr); if (!c->is_saved) { if (!from_binlog) { - auto log_event = SecretChatLogEvent(secret_chat_id, *c); + auto log_event = SecretChatLogEvent(secret_chat_id, c); auto storer = get_log_event_storer(log_event); if (c->log_event_id == 0) { c->log_event_id = binlog_add(G()->td_db()->get_binlog(), LogEvent::HandlerType::SecretChatInfos, storer); @@ -9621,8 +9625,8 @@ void ContactsManager::on_binlog_secret_chat_event(BinlogEvent &&event) { } LOG(INFO) << "Add " << secret_chat_id << " from binlog"; - SecretChat *c = add_secret_chat(secret_chat_id); - *c = std::move(log_event.c); // secret chats come from binlog before all other events, so just add them + SecretChat *c = secret_chats_.emplace(secret_chat_id, std::move(log_event.c_out)).first->second.get(); + CHECK(c != nullptr); c->log_event_id = event.id_; From 36a57c35849215783b6222022d33f691612d9a3b Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 23 Jul 2022 11:13:17 +0300 Subject: [PATCH 03/23] Remove unneeded FlatHashTable copy constructors. --- tdutils/td/utils/FlatHashTable.h | 37 +++----------------------------- tdutils/test/HashSet.cpp | 8 ------- 2 files changed, 3 insertions(+), 42 deletions(-) diff --git a/tdutils/td/utils/FlatHashTable.h b/tdutils/td/utils/FlatHashTable.h index 2048ea05d..079b3ede6 100644 --- a/tdutils/td/utils/FlatHashTable.h +++ b/tdutils/td/utils/FlatHashTable.h @@ -198,13 +198,8 @@ class FlatHashTable { }; FlatHashTable() = default; - FlatHashTable(const FlatHashTable &other) { - assign(other); - } - void operator=(const FlatHashTable &other) { - clear(); - assign(other); - } + FlatHashTable(const FlatHashTable &other) = delete; + FlatHashTable &operator=(const FlatHashTable &other) = delete; FlatHashTable(std::initializer_list nodes) { if (nodes.size() == 0) { @@ -249,9 +244,7 @@ class FlatHashTable { other.drop(); } ~FlatHashTable() { - if (nodes_ != nullptr) { - clear_nodes(nodes_); - } + clear_nodes(nodes_); } void swap(FlatHashTable &other) noexcept { @@ -433,30 +426,6 @@ class FlatHashTable { begin_bucket_ = 0; } - void assign(const FlatHashTable &other) { - if (other.size() == 0) { - return; - } - resize(other.bucket_count()); - auto other_nodes_end = other.nodes_ + other.bucket_count_; - for (const NodeT *other_node = other.nodes_; other_node != other_nodes_end; ++other_node) { - if (other_node->empty()) { - continue; - } - - auto bucket = calc_bucket(other_node->key()); - while (true) { - auto &node = nodes_[bucket]; - if (node.empty()) { - node.copy_from(*other_node); - break; - } - next_bucket(bucket); - } - } - used_node_count_ = other.used_node_count_; - } - NodeT *begin_impl() { if (empty()) { return nullptr; diff --git a/tdutils/test/HashSet.cpp b/tdutils/test/HashSet.cpp index 507960c4b..07bc1ce81 100644 --- a/tdutils/test/HashSet.cpp +++ b/tdutils/test/HashSet.cpp @@ -124,7 +124,6 @@ TEST(FlatHashMap, basic) { ASSERT_EQ(2, kv.second); } map.erase(map.find(1)); - auto map_copy = map; } td::FlatHashMap, 10>> x; @@ -159,9 +158,6 @@ TEST(FlatHashMap, basic) { } ASSERT_EQ(data, extract_kv(kv)); - KV copied_kv(kv); - ASSERT_EQ(data, extract_kv(copied_kv)); - KV moved_kv(std::move(kv)); ASSERT_EQ(data, extract_kv(moved_kv)); ASSERT_EQ(Data{}, extract_kv(kv)); @@ -169,10 +165,6 @@ TEST(FlatHashMap, basic) { kv = std::move(moved_kv); ASSERT_EQ(data, extract_kv(kv)); - KV assign_copied_kv; - assign_copied_kv = kv; - ASSERT_EQ(data, extract_kv(assign_copied_kv)); - KV assign_moved_kv; assign_moved_kv = std::move(kv); ASSERT_EQ(data, extract_kv(assign_moved_kv)); From 3506bc94dcf8cc7e1b314d2a41d03f58f00c2c64 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 23 Jul 2022 11:32:08 +0300 Subject: [PATCH 04/23] Check messageMediaPoll validness for just sent messages. --- td/telegram/MessagesManager.cpp | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 410c1895c..07bce0691 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -50,6 +50,7 @@ #include "td/telegram/NotificationSettingsManager.h" #include "td/telegram/NotificationSound.h" #include "td/telegram/NotificationType.h" +#include "td/telegram/PollId.h" #include "td/telegram/PublicDialogType.h" #include "td/telegram/ReplyMarkup.h" #include "td/telegram/ReplyMarkup.hpp" @@ -30928,8 +30929,29 @@ void MessagesManager::check_send_message_result(int64 random_id, DialogId dialog CHECK(source != nullptr); auto sent_messages = UpdatesManager::get_new_messages(updates_ptr); auto sent_messages_random_ids = UpdatesManager::get_sent_messages_random_ids(updates_ptr); + + auto is_invalid_poll_message = [](const telegram_api::Message *message) { + CHECK(message != nullptr); + auto constructor_id = message->get_id(); + if (constructor_id == telegram_api::messageEmpty::ID) { + return true; + } + if (constructor_id != telegram_api::message::ID) { + return false; + } + + auto media = static_cast(message)->media_.get(); + if (media == nullptr || media->get_id() != telegram_api::messageMediaPoll::ID) { + return false; + } + + auto poll = static_cast(media)->poll_.get(); + return !PollId(poll->id_).is_valid(); + }; + if (sent_messages.size() != 1u || sent_messages_random_ids.size() != 1u || - *sent_messages_random_ids.begin() != random_id || get_message_dialog_id(*sent_messages[0]) != dialog_id) { + *sent_messages_random_ids.begin() != random_id || get_message_dialog_id(*sent_messages[0]) != dialog_id || + is_invalid_poll_message(sent_messages[0]->get())) { LOG(ERROR) << "Receive wrong result for sending message with random_id " << random_id << " from " << source << " to " << dialog_id << ": " << oneline(to_string(*updates_ptr)); Dialog *d = get_dialog(dialog_id); From c6fc1bb98ea2c158dd3ee7af6d037182bebc4234 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 23 Jul 2022 12:04:15 +0300 Subject: [PATCH 05/23] Fix compilation error with old compilers. --- td/telegram/LinkManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/td/telegram/LinkManager.cpp b/td/telegram/LinkManager.cpp index eff592aee..cfbd84f0d 100644 --- a/td/telegram/LinkManager.cpp +++ b/td/telegram/LinkManager.cpp @@ -1057,7 +1057,7 @@ unique_ptr LinkManager::parse_tg_link_query(Slice que nullptr, std::move(user_link), url_query.get_arg("attach").str(), url_query.get_arg("startattach")); } // resolve?phone=12345 - return user_link; + return std::move(user_link); } } else if (path.size() == 1 && path[0] == "login") { // login?code=123456 From 8efe57fb6bda7a5cdd8d3be978946371e3e1c6ac Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 23 Jul 2022 12:06:14 +0300 Subject: [PATCH 06/23] Fix ForwardMessagesLogEvent. --- td/telegram/MessagesManager.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 07bce0691..bfbc14010 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -27924,8 +27924,8 @@ class MessagesManager::ForwardMessagesLogEvent { DialogId from_dialog_id; vector message_ids; vector messages_in; - bool drop_author = false; - bool drop_media_captions = false; + bool drop_author; + bool drop_media_captions; vector> messages_out; template @@ -27947,6 +27947,9 @@ class MessagesManager::ForwardMessagesLogEvent { PARSE_FLAG(drop_author); PARSE_FLAG(drop_media_captions); END_PARSE_FLAGS(); + } else { + drop_author = false; + drop_media_captions = false; } td::parse(to_dialog_id, parser); td::parse(from_dialog_id, parser); From ad8693801f699712eacf90d86dd7ca4ff1826cc9 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 23 Jul 2022 12:15:54 +0300 Subject: [PATCH 07/23] Fix another compilation error. --- td/telegram/LinkManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/td/telegram/LinkManager.cpp b/td/telegram/LinkManager.cpp index cfbd84f0d..28c82b793 100644 --- a/td/telegram/LinkManager.cpp +++ b/td/telegram/LinkManager.cpp @@ -1239,7 +1239,7 @@ unique_ptr LinkManager::parse_t_me_link_query(Slice q nullptr, std::move(user_link), url_query.get_arg("attach").str(), url_query.get_arg("startattach")); } // /+ - return user_link; + return std::move(user_link); } else { // /+ return td::make_unique(PSTRING() << "tg:join?invite=" From 1ff1059de8d9ef62787d957046471aff05dfe527 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 23 Jul 2022 12:59:21 +0300 Subject: [PATCH 08/23] Fix warning. --- td/telegram/ReplyMarkup.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/td/telegram/ReplyMarkup.cpp b/td/telegram/ReplyMarkup.cpp index 740bf413b..98bbbfd1c 100644 --- a/td/telegram/ReplyMarkup.cpp +++ b/td/telegram/ReplyMarkup.cpp @@ -221,13 +221,14 @@ static KeyboardButton get_keyboard_button(tl_object_ptr(keyboard_button_ptr); - button.type = KeyboardButton::Type::WebView; - button.text = std::move(keyboard_button->text_); auto r_url = LinkManager::check_link(keyboard_button->url_); if (r_url.is_error()) { LOG(ERROR) << "Keyboard Web App " << r_url.error().message(); - return {}; + break; } + + button.type = KeyboardButton::Type::WebView; + button.text = std::move(keyboard_button->text_); button.url = r_url.move_as_ok(); break; } From 03de62b29f5997604b424724ada3232427c4fb6d Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 23 Jul 2022 13:04:25 +0300 Subject: [PATCH 09/23] Check data validness before initing InlineKeyboardButton. --- td/telegram/ReplyMarkup.cpp | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/td/telegram/ReplyMarkup.cpp b/td/telegram/ReplyMarkup.cpp index 98bbbfd1c..229c99682 100644 --- a/td/telegram/ReplyMarkup.cpp +++ b/td/telegram/ReplyMarkup.cpp @@ -246,13 +246,13 @@ static InlineKeyboardButton get_inline_keyboard_button( switch (keyboard_button_ptr->get_id()) { case telegram_api::keyboardButtonUrl::ID: { auto keyboard_button = move_tl_object_as(keyboard_button_ptr); - button.type = InlineKeyboardButton::Type::Url; - button.text = std::move(keyboard_button->text_); auto r_url = LinkManager::check_link(keyboard_button->url_); if (r_url.is_error()) { LOG(ERROR) << "Inline keyboard " << r_url.error().message(); - return {}; + break; } + button.type = InlineKeyboardButton::Type::Url; + button.text = std::move(keyboard_button->text_); button.data = r_url.move_as_ok(); break; } @@ -287,38 +287,39 @@ static InlineKeyboardButton get_inline_keyboard_button( } case telegram_api::keyboardButtonUrlAuth::ID: { auto keyboard_button = move_tl_object_as(keyboard_button_ptr); + auto r_url = LinkManager::check_link(keyboard_button->url_); + if (r_url.is_error()) { + LOG(ERROR) << "Inline keyboard Login " << r_url.error().message(); + break; + } button.type = InlineKeyboardButton::Type::UrlAuth; button.id = keyboard_button->button_id_; button.text = std::move(keyboard_button->text_); button.forward_text = std::move(keyboard_button->fwd_text_); - auto r_url = LinkManager::check_link(keyboard_button->url_); - if (r_url.is_error()) { - LOG(ERROR) << "Inline keyboard Login " << r_url.error().message(); - return {}; - } button.data = r_url.move_as_ok(); break; } case telegram_api::keyboardButtonUserProfile::ID: { auto keyboard_button = move_tl_object_as(keyboard_button_ptr); + auto user_id = UserId(keyboard_button->user_id_); + if (!user_id.is_valid()) { + LOG(ERROR) << "Receive " << user_id << " in inline keyboard"; + break; + } button.type = InlineKeyboardButton::Type::User; button.text = std::move(keyboard_button->text_); - button.user_id = UserId(keyboard_button->user_id_); - if (!button.user_id.is_valid()) { - LOG(ERROR) << "Receive " << button.user_id << " in inline keyboard"; - return {}; - } + button.user_id = user_id; break; } case telegram_api::keyboardButtonWebView::ID: { auto keyboard_button = move_tl_object_as(keyboard_button_ptr); - button.type = InlineKeyboardButton::Type::WebView; - button.text = std::move(keyboard_button->text_); auto r_url = LinkManager::check_link(keyboard_button->url_); if (r_url.is_error()) { LOG(ERROR) << "Inline keyboard Web App " << r_url.error().message(); - return {}; + break; } + button.type = InlineKeyboardButton::Type::WebView; + button.text = std::move(keyboard_button->text_); button.data = r_url.move_as_ok(); break; } From 206becbb9fd308b3a86fbe0e1570f468cd495136 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 23 Jul 2022 14:52:42 +0300 Subject: [PATCH 10/23] Safely traverse map. --- td/telegram/DownloadManager.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/td/telegram/DownloadManager.cpp b/td/telegram/DownloadManager.cpp index 202a0272a..e6937b641 100644 --- a/td/telegram/DownloadManager.cpp +++ b/td/telegram/DownloadManager.cpp @@ -93,8 +93,18 @@ class DownloadManagerImpl final : public DownloadManager { Status toggle_all_is_paused(bool is_paused) final { TRY_STATUS(check_is_active()); + vector to_toggle; for (auto &it : files_) { - toggle_is_paused(*it.second, is_paused); + FileInfo &file_info = *it.second; + if (!is_completed(file_info) && is_paused != file_info.is_paused) { + to_toggle.push_back(file_info.file_id); + } + } + for (auto file_id : to_toggle) { + auto r_file_info_ptr = get_file_info(file_id); + if (r_file_info_ptr.is_ok()) { + toggle_is_paused(*r_file_info_ptr.ok(), is_paused); + } } return Status::OK(); From d55869eedb90886cb5a5f77257524ffb9bce79f4 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 23 Jul 2022 14:53:32 +0300 Subject: [PATCH 11/23] Minor improvements. --- tdutils/td/utils/FlatHashMapChunks.h | 5 +++-- tdutils/td/utils/MapNode.h | 2 +- tdutils/td/utils/SetNode.h | 11 ++++++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tdutils/td/utils/FlatHashMapChunks.h b/tdutils/td/utils/FlatHashMapChunks.h index 81a2c4819..70fc36bac 100644 --- a/tdutils/td/utils/FlatHashMapChunks.h +++ b/tdutils/td/utils/FlatHashMapChunks.h @@ -469,7 +469,8 @@ class FlatHashTableChunks { struct ChunkIt { size_t chunk_i; size_t chunk_mask; - size_t shift{}; + size_t shift; + size_t pos() const { return chunk_i; } @@ -482,7 +483,7 @@ class FlatHashTableChunks { }; ChunkIt get_chunk_it(size_t chunk_i) { - return {chunk_i, chunks_.size() - 1}; + return ChunkIt{chunk_i, chunks_.size() - 1, 0}; } HashInfo calc_hash(const KeyT &key) { diff --git a/tdutils/td/utils/MapNode.h b/tdutils/td/utils/MapNode.h index 25cc84448..cad2ae9b3 100644 --- a/tdutils/td/utils/MapNode.h +++ b/tdutils/td/utils/MapNode.h @@ -54,7 +54,7 @@ struct MapNode { DCHECK(empty()); DCHECK(!other.empty()); first = std::move(other.first); - other.first = KeyT{}; + other.first = KeyT(); new (&second) ValueT(std::move(other.second)); other.second.~ValueT(); } diff --git a/tdutils/td/utils/SetNode.h b/tdutils/td/utils/SetNode.h index 7e99cb1fb..6d7b6e5ff 100644 --- a/tdutils/td/utils/SetNode.h +++ b/tdutils/td/utils/SetNode.h @@ -20,7 +20,7 @@ struct SetNode { using public_type = const KeyT; using second_type = KeyT; // TODO: remove second_type? - KeyT first{}; + KeyT first; const KeyT &key() const { return first; @@ -30,7 +30,8 @@ struct SetNode { return first; } - SetNode() = default; + SetNode(): first() { + } explicit SetNode(KeyT key) : first(std::move(key)) { } SetNode(const SetNode &other) = delete; @@ -42,7 +43,7 @@ struct SetNode { DCHECK(empty()); DCHECK(!other.empty()); first = std::move(other.first); - other.first = KeyT{}; + other.first = KeyT(); } ~SetNode() = default; @@ -71,7 +72,7 @@ struct SetNode 28 * sizeof(void struct Impl { using second_type = KeyT; - KeyT first{}; + KeyT first; template explicit Impl(InputKeyT &&key) : first(std::forward(key)) { @@ -99,7 +100,7 @@ struct SetNode 28 * sizeof(void return impl_->first; } - SetNode() { + SetNode(): impl_() { } explicit SetNode(KeyT key) : impl_(td::make_unique(std::move(key))) { } From 061aac58db2904aefa8e515a64669d456db17325 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 23 Jul 2022 16:41:03 +0300 Subject: [PATCH 12/23] Make DownloadManager interface mostly asynchronous. --- td/telegram/DownloadManager.cpp | 60 ++++++++++++++++++++------------- td/telegram/DownloadManager.h | 14 ++++---- td/telegram/MessagesManager.cpp | 7 ++-- td/telegram/Td.cpp | 13 ++++--- td/telegram/UpdatesManager.cpp | 2 +- 5 files changed, 57 insertions(+), 39 deletions(-) diff --git a/td/telegram/DownloadManager.cpp b/td/telegram/DownloadManager.cpp index e6937b641..21fd17cdf 100644 --- a/td/telegram/DownloadManager.cpp +++ b/td/telegram/DownloadManager.cpp @@ -83,15 +83,15 @@ class DownloadManagerImpl final : public DownloadManager { load_database_files(); } - Status toggle_is_paused(FileId file_id, bool is_paused) final { - TRY_STATUS(check_is_active()); - TRY_RESULT(file_info_ptr, get_file_info(file_id)); + void toggle_is_paused(FileId file_id, bool is_paused, Promise promise) final { + TRY_STATUS_PROMISE(promise, check_is_active()); + TRY_RESULT_PROMISE(promise, file_info_ptr, get_file_info(file_id)); toggle_is_paused(*file_info_ptr, is_paused); - return Status::OK(); + promise.set_value(Unit()); } - Status toggle_all_is_paused(bool is_paused) final { - TRY_STATUS(check_is_active()); + void toggle_all_is_paused(bool is_paused, Promise promise) final { + TRY_STATUS_PROMISE(promise, check_is_active()); vector to_toggle; for (auto &it : files_) { @@ -107,10 +107,14 @@ class DownloadManagerImpl final : public DownloadManager { } } - return Status::OK(); + promise.set_value(Unit()); } - Status remove_file(FileId file_id, FileSourceId file_source_id, bool delete_from_cache) final { + void remove_file(FileId file_id, FileSourceId file_source_id, bool delete_from_cache, Promise promise) final { + promise.set_result(remove_file_impl(file_id, file_source_id, delete_from_cache)); + } + + Status remove_file_impl(FileId file_id, FileSourceId file_source_id, bool delete_from_cache) { LOG(INFO) << "Remove from downloads file " << file_id << " from " << file_source_id; TRY_STATUS(check_is_active()); TRY_RESULT(file_info_ptr, get_file_info(file_id, file_source_id)); @@ -140,17 +144,21 @@ class DownloadManagerImpl final : public DownloadManager { return Status::OK(); } - Status remove_file_if_finished(FileId file_id) { + void remove_file_if_finished(FileId file_id) final { + remove_file_if_finished_impl(file_id).ignore(); + } + + Status remove_file_if_finished_impl(FileId file_id) { TRY_STATUS(check_is_active()); TRY_RESULT(file_info_ptr, get_file_info(file_id, {})); if (!is_completed(*file_info_ptr)) { return Status::Error("File is active"); } - return remove_file(file_id, {}, false); + return remove_file_impl(file_id, {}, false); } - Status remove_all_files(bool only_active, bool only_completed, bool delete_from_cache) final { - TRY_STATUS(check_is_active()); + void remove_all_files(bool only_active, bool only_completed, bool delete_from_cache, Promise promise) final { + TRY_STATUS_PROMISE(promise, check_is_active()); vector to_remove; for (auto &it : files_) { FileInfo &file_info = *it.second; @@ -163,15 +171,15 @@ class DownloadManagerImpl final : public DownloadManager { to_remove.push_back(file_info.file_id); } for (auto file_id : to_remove) { - remove_file(file_id, {}, delete_from_cache); + remove_file_impl(file_id, {}, delete_from_cache); } - return Status::OK(); + promise.set_value(Unit()); } Status add_file(FileId file_id, FileSourceId file_source_id, string search_text, int8 priority) final { TRY_STATUS(check_is_active()); - remove_file(file_id, {}, false); + remove_file_impl(file_id, {}, false); auto download_id = next_download_id(); @@ -189,16 +197,20 @@ class DownloadManagerImpl final : public DownloadManager { return Status::OK(); } - Status change_search_text(FileId file_id, FileSourceId file_source_id, string search_text) final { + void change_search_text(FileId file_id, FileSourceId file_source_id, string search_text) final { if (!is_search_inited_) { - return Status::OK(); + return; } - TRY_STATUS(check_is_active()); - TRY_RESULT(file_info_ptr, get_file_info(file_id, file_source_id)); - auto &file_info = *file_info_ptr; + if (check_is_active().is_error()) { + return; + } + auto r_file_info_ptr = get_file_info(file_id, file_source_id); + if (r_file_info_ptr.is_error()) { + return; + } + auto &file_info = *r_file_info_ptr.ok(); hints_.add(file_info.download_id, search_text.empty() ? string(" ") : search_text); - return Status::OK(); } void hints_synchronized(Result) { @@ -338,7 +350,7 @@ class DownloadManagerImpl final : public DownloadManager { return; } auto &file_info = *r_file_info_ptr.ok(); - remove_file(file_info.file_id, {}, false); + remove_file_impl(file_info.file_id, {}, false); } void update_file_viewed(FileId file_id, FileSourceId file_source_id) final { @@ -540,7 +552,7 @@ class DownloadManagerImpl final : public DownloadManager { if (r_search_text.is_error()) { if (!G()->close_flag()) { - remove_file(it->second->file_id, {}, false); + remove_file_impl(it->second->file_id, {}, false); } } else { auto search_text = r_search_text.move_as_ok(); @@ -766,7 +778,7 @@ class DownloadManagerImpl final : public DownloadManager { while (completed_download_ids_.size() > MAX_COMPLETED_DOWNLOADS) { auto download_id = *completed_download_ids_.begin(); auto file_info = get_file_info(download_id).move_as_ok(); - remove_file(file_info->file_id, FileSourceId(), false); + remove_file_impl(file_info->file_id, FileSourceId(), false); } } diff --git a/td/telegram/DownloadManager.h b/td/telegram/DownloadManager.h index aa0790fd9..10e4c1c04 100644 --- a/td/telegram/DownloadManager.h +++ b/td/telegram/DownloadManager.h @@ -88,14 +88,16 @@ class DownloadManager : public Actor { // virtual void after_get_difference() = 0; virtual Status add_file(FileId file_id, FileSourceId file_source_id, string search_text, int8 priority) = 0; - virtual Status change_search_text(FileId file_id, FileSourceId file_source_id, string search_text) = 0; - virtual Status toggle_is_paused(FileId file_id, bool is_paused) = 0; - virtual Status toggle_all_is_paused(bool is_paused) = 0; + virtual void change_search_text(FileId file_id, FileSourceId file_source_id, string search_text) = 0; + virtual void remove_file_if_finished(FileId file_id) = 0; + virtual void toggle_is_paused(FileId file_id, bool is_paused, Promise promise) = 0; + virtual void toggle_all_is_paused(bool is_paused, Promise promise) = 0; virtual void search(string query, bool only_active, bool only_completed, string offset, int32 limit, Promise> promise) = 0; - virtual Status remove_file(FileId file_id, FileSourceId file_source_id, bool delete_from_cache) = 0; - virtual Status remove_file_if_finished(FileId file_id) = 0; - virtual Status remove_all_files(bool only_active, bool only_completed, bool delete_from_cache) = 0; + virtual void remove_file(FileId file_id, FileSourceId file_source_id, bool delete_from_cache, + Promise promise) = 0; + virtual void remove_all_files(bool only_active, bool only_completed, bool delete_from_cache, + Promise promise) = 0; // // private interface to handle all kinds of updates diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index bfbc14010..7f7c66d03 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -20698,7 +20698,8 @@ Status MessagesManager::view_messages(DialogId dialog_id, MessageId top_thread_m for (auto file_id : get_message_file_ids(m)) { auto file_view = td_->file_manager_->get_file_view(file_id); CHECK(!file_view.empty()); - td_->download_manager_->update_file_viewed(file_view.get_main_file_id(), file_source_id); + send_closure(td_->download_manager_actor_, &DownloadManager::update_file_viewed, file_view.get_main_file_id(), + file_source_id); } } @@ -22867,7 +22868,7 @@ void MessagesManager::remove_message_file_sources(DialogId dialog_id, const Mess for (auto file_id : file_ids) { auto file_view = td_->file_manager_->get_file_view(file_id); send_closure(td_->download_manager_actor_, &DownloadManager::remove_file, file_view.get_main_file_id(), - file_source_id, false); + file_source_id, false, Promise()); td_->file_manager_->remove_file_source(file_id, file_source_id); } } @@ -22894,7 +22895,7 @@ void MessagesManager::change_message_files(DialogId dialog_id, const Message *m, if (file_source_id.is_valid()) { auto file_view = td_->file_manager_->get_file_view(file_id); send_closure(td_->download_manager_actor_, &DownloadManager::remove_file, file_view.get_main_file_id(), - file_source_id, false); + file_source_id, false, Promise()); } } } diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index a1e06e94b..7baa0b1cb 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -6652,23 +6652,26 @@ void Td::on_request(uint64 id, const td_api::addFileToDownloads &request) { void Td::on_request(uint64 id, const td_api::toggleDownloadIsPaused &request) { CREATE_OK_REQUEST_PROMISE(); - promise.set_result(download_manager_->toggle_is_paused(FileId(request.file_id_, 0), request.is_paused_)); + send_closure(download_manager_actor_, &DownloadManager::toggle_is_paused, FileId(request.file_id_, 0), + request.is_paused_, std::move(promise)); } void Td::on_request(uint64 id, const td_api::toggleAllDownloadsArePaused &request) { CREATE_OK_REQUEST_PROMISE(); - promise.set_result(download_manager_->toggle_all_is_paused(request.are_paused_)); + send_closure(download_manager_actor_, &DownloadManager::toggle_all_is_paused, request.are_paused_, + std::move(promise)); } void Td::on_request(uint64 id, const td_api::removeFileFromDownloads &request) { CREATE_OK_REQUEST_PROMISE(); - promise.set_result(download_manager_->remove_file(FileId(request.file_id_, 0), {}, request.delete_from_cache_)); + send_closure(download_manager_actor_, &DownloadManager::remove_file, FileId(request.file_id_, 0), FileSourceId(), + request.delete_from_cache_, std::move(promise)); } void Td::on_request(uint64 id, const td_api::removeAllFilesFromDownloads &request) { CREATE_OK_REQUEST_PROMISE(); - promise.set_result( - download_manager_->remove_all_files(request.only_active_, request.only_completed_, request.delete_from_cache_)); + send_closure(download_manager_actor_, &DownloadManager::remove_all_files, request.only_active_, + request.only_completed_, request.delete_from_cache_, std::move(promise)); } void Td::on_request(uint64 id, td_api::searchFileDownloads &request) { diff --git a/td/telegram/UpdatesManager.cpp b/td/telegram/UpdatesManager.cpp index b0f1f28e0..a15f1ecae 100644 --- a/td/telegram/UpdatesManager.cpp +++ b/td/telegram/UpdatesManager.cpp @@ -1617,7 +1617,7 @@ void UpdatesManager::after_get_difference() { } } - td_->download_manager_->after_get_difference(); + send_closure(td_->download_manager_actor_, &DownloadManager::after_get_difference); td_->inline_queries_manager_->after_get_difference(); td_->messages_manager_->after_get_difference(); send_closure_later(td_->notification_manager_actor_, &NotificationManager::after_get_difference); From ec19eaad650823c6840d780caef8f18ac0b6db0d Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 23 Jul 2022 17:01:27 +0300 Subject: [PATCH 13/23] Call FileReferenceManager::get_file_search_text through DownloadManagerCallback. --- td/telegram/DownloadManager.cpp | 15 +++++++-------- td/telegram/DownloadManager.h | 3 ++- td/telegram/DownloadManagerCallback.cpp | 6 ++++++ td/telegram/DownloadManagerCallback.h | 4 +++- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/td/telegram/DownloadManager.cpp b/td/telegram/DownloadManager.cpp index 21fd17cdf..ca60fa40e 100644 --- a/td/telegram/DownloadManager.cpp +++ b/td/telegram/DownloadManager.cpp @@ -6,7 +6,6 @@ // #include "td/telegram/DownloadManager.h" -#include "td/telegram/FileReferenceManager.h" #include "td/telegram/files/FileId.hpp" #include "td/telegram/files/FileSourceId.hpp" #include "td/telegram/Global.h" @@ -534,13 +533,13 @@ class DownloadManagerImpl final : public DownloadManager { void prepare_hints() { for (auto &it : files_) { const auto &file_info = *it.second; - send_closure(G()->file_reference_manager(), &FileReferenceManager::get_file_search_text, file_info.file_source_id, - callback_->get_file_view(file_info.file_id).get_unique_file_id(), - [actor_id = actor_id(this), promise = load_search_text_multipromise_.get_promise(), - download_id = it.first](Result r_search_text) mutable { - send_closure(actor_id, &DownloadManagerImpl::add_download_to_hints, download_id, - std::move(r_search_text), std::move(promise)); - }); + auto promise = + PromiseCreator::lambda([actor_id = actor_id(this), promise = load_search_text_multipromise_.get_promise(), + download_id = it.first](Result r_search_text) mutable { + send_closure(actor_id, &DownloadManagerImpl::add_download_to_hints, download_id, std::move(r_search_text), + std::move(promise)); + }); + callback_->get_file_search_text(file_info.file_id, file_info.file_source_id, std::move(promise)); } } diff --git a/td/telegram/DownloadManager.h b/td/telegram/DownloadManager.h index 10e4c1c04..b34d20ae2 100644 --- a/td/telegram/DownloadManager.h +++ b/td/telegram/DownloadManager.h @@ -73,7 +73,8 @@ class DownloadManager : public Actor { virtual void delete_file(FileId file_id) = 0; virtual FileId dup_file_id(FileId file_id) = 0; - virtual FileView get_file_view(FileId file_id) = 0; + virtual void get_file_search_text(FileId file_id, FileSourceId file_source_id, Promise &&promise) = 0; + virtual FileView get_sync_file_view(FileId file_id) = 0; virtual td_api::object_ptr get_file_download_object(FileId file_id, FileSourceId file_source_id, diff --git a/td/telegram/DownloadManagerCallback.cpp b/td/telegram/DownloadManagerCallback.cpp index b41f119c4..a1eeacf50 100644 --- a/td/telegram/DownloadManagerCallback.cpp +++ b/td/telegram/DownloadManagerCallback.cpp @@ -64,6 +64,12 @@ FileId DownloadManagerCallback::dup_file_id(FileId file_id) { return td_->file_manager_->dup_file_id(file_id); } +void DownloadManagerCallback::get_file_search_text(FileId file_id, FileSourceId file_source_id, + Promise &&promise) { + send_closure(td_->file_reference_manager_actor_, &FileReferenceManager::get_file_search_text, file_source_id, + get_file_view(file_id).get_unique_file_id(), std::move(promise)); +} + FileView DownloadManagerCallback::get_file_view(FileId file_id) { return td_->file_manager_->get_file_view(file_id); } diff --git a/td/telegram/DownloadManagerCallback.h b/td/telegram/DownloadManagerCallback.h index 145ae7f92..769a17543 100644 --- a/td/telegram/DownloadManagerCallback.h +++ b/td/telegram/DownloadManagerCallback.h @@ -45,7 +45,7 @@ class DownloadManagerCallback final : public DownloadManager::Callback { FileId dup_file_id(FileId file_id) final; - FileView get_file_view(FileId file_id) final; + void get_file_search_text(FileId file_id, FileSourceId file_source_id, Promise &&promise) final; FileView get_sync_file_view(FileId file_id) final; @@ -57,6 +57,8 @@ class DownloadManagerCallback final : public DownloadManager::Callback { Td *td_; ActorShared<> parent_; + FileView get_file_view(FileId file_id); + static std::shared_ptr make_download_file_callback( Td *td, ActorShared download_manager); }; From 698b34709f82fa6d1a0d943b4e883af02d708bab Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 23 Jul 2022 17:11:25 +0300 Subject: [PATCH 14/23] Make DownloadManager::add_file asynchronous. --- td/telegram/DownloadManager.cpp | 7 ++++--- td/telegram/DownloadManager.h | 10 ++++++---- td/telegram/DownloadManagerCallback.cpp | 4 ++++ td/telegram/DownloadManagerCallback.h | 4 +++- td/telegram/MessagesManager.cpp | 5 ++--- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/td/telegram/DownloadManager.cpp b/td/telegram/DownloadManager.cpp index ca60fa40e..97b629622 100644 --- a/td/telegram/DownloadManager.cpp +++ b/td/telegram/DownloadManager.cpp @@ -175,8 +175,9 @@ class DownloadManagerImpl final : public DownloadManager { promise.set_value(Unit()); } - Status add_file(FileId file_id, FileSourceId file_source_id, string search_text, int8 priority) final { - TRY_STATUS(check_is_active()); + void add_file(FileId file_id, FileSourceId file_source_id, string search_text, int8 priority, + Promise> promise) final { + TRY_STATUS_PROMISE(promise, check_is_active()); remove_file_impl(file_id, {}, false); @@ -193,7 +194,7 @@ class DownloadManagerImpl final : public DownloadManager { add_file_info(std::move(file_info), search_text); - return Status::OK(); + promise.set_value(callback_->get_file_object(file_id)); } void change_search_text(FileId file_id, FileSourceId file_source_id, string search_text) final { diff --git a/td/telegram/DownloadManager.h b/td/telegram/DownloadManager.h index b34d20ae2..de2133944 100644 --- a/td/telegram/DownloadManager.h +++ b/td/telegram/DownloadManager.h @@ -76,6 +76,7 @@ class DownloadManager : public Actor { virtual void get_file_search_text(FileId file_id, FileSourceId file_source_id, Promise &&promise) = 0; virtual FileView get_sync_file_view(FileId file_id) = 0; + virtual td_api::object_ptr get_file_object(FileId file_id) = 0; virtual td_api::object_ptr get_file_download_object(FileId file_id, FileSourceId file_source_id, int32 add_date, int32 complete_date, @@ -87,10 +88,8 @@ class DownloadManager : public Actor { // // public interface for user // - virtual void after_get_difference() = 0; - virtual Status add_file(FileId file_id, FileSourceId file_source_id, string search_text, int8 priority) = 0; - virtual void change_search_text(FileId file_id, FileSourceId file_source_id, string search_text) = 0; - virtual void remove_file_if_finished(FileId file_id) = 0; + virtual void add_file(FileId file_id, FileSourceId file_source_id, string search_text, int8 priority, + Promise> promise) = 0; virtual void toggle_is_paused(FileId file_id, bool is_paused, Promise promise) = 0; virtual void toggle_all_is_paused(bool is_paused, Promise promise) = 0; virtual void search(string query, bool only_active, bool only_completed, string offset, int32 limit, @@ -103,6 +102,9 @@ class DownloadManager : public Actor { // // private interface to handle all kinds of updates // + virtual void after_get_difference() = 0; + virtual void change_search_text(FileId file_id, FileSourceId file_source_id, string search_text) = 0; + virtual void remove_file_if_finished(FileId file_id) = 0; virtual void update_file_download_state(FileId internal_file_id, int64 downloaded_size, int64 size, int64 expected_size, bool is_paused) = 0; virtual void update_file_deleted(FileId internal_file_id) = 0; diff --git a/td/telegram/DownloadManagerCallback.cpp b/td/telegram/DownloadManagerCallback.cpp index a1eeacf50..8b3622299 100644 --- a/td/telegram/DownloadManagerCallback.cpp +++ b/td/telegram/DownloadManagerCallback.cpp @@ -79,6 +79,10 @@ FileView DownloadManagerCallback::get_sync_file_view(FileId file_id) { return get_file_view(file_id); } +td_api::object_ptr DownloadManagerCallback::get_file_object(FileId file_id) { + return td_->file_manager_->get_file_object(file_id); +} + td_api::object_ptr DownloadManagerCallback::get_file_download_object( FileId file_id, FileSourceId file_source_id, int32 add_date, int32 complete_date, bool is_paused) { return td_api::make_object(td_->file_manager_->get_file_view(file_id).get_main_file_id().get(), diff --git a/td/telegram/DownloadManagerCallback.h b/td/telegram/DownloadManagerCallback.h index 769a17543..19da9c842 100644 --- a/td/telegram/DownloadManagerCallback.h +++ b/td/telegram/DownloadManagerCallback.h @@ -49,9 +49,11 @@ class DownloadManagerCallback final : public DownloadManager::Callback { FileView get_sync_file_view(FileId file_id) final; + td_api::object_ptr get_file_object(FileId file_id) final; + td_api::object_ptr get_file_download_object(FileId file_id, FileSourceId file_source_id, int32 add_date, int32 complete_date, - bool is_paused); + bool is_paused) final; private: Td *td_; diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 7f7c66d03..5f668f331 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -40148,9 +40148,8 @@ void MessagesManager::add_message_file_to_downloads(FullMessageId full_message_i auto search_text = get_message_search_text(m); auto file_source_id = get_message_file_source_id(full_message_id, true); CHECK(file_source_id.is_valid()); - TRY_STATUS_PROMISE(promise, td_->download_manager_->add_file(file_id, file_source_id, std::move(search_text), - static_cast(priority))); - promise.set_value(td_->file_manager_->get_file_object(file_id)); + send_closure(td_->download_manager_actor_, &DownloadManager::add_file, file_id, file_source_id, + std::move(search_text), static_cast(priority), std::move(promise)); } void MessagesManager::get_message_file_search_text(FullMessageId full_message_id, string unique_file_id, From 07cc51c1d5d3138c31a032569c65e88c333efee7 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 23 Jul 2022 17:16:21 +0300 Subject: [PATCH 15/23] Add check that download manager database isn't loaded recursively. --- td/telegram/DownloadManager.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/td/telegram/DownloadManager.cpp b/td/telegram/DownloadManager.cpp index 97b629622..ef6315780 100644 --- a/td/telegram/DownloadManager.cpp +++ b/td/telegram/DownloadManager.cpp @@ -398,6 +398,7 @@ class DownloadManagerImpl final : public DownloadManager { Counters sent_counters_; FileCounters file_counters_; bool is_inited_{false}; + bool is_database_being_loaded_{false}; bool is_database_loaded_{false}; bool is_search_inited_{false}; int64 max_download_id_{0}; @@ -510,6 +511,8 @@ class DownloadManagerImpl final : public DownloadManager { return; } CHECK(is_inited_); + CHECK(!is_database_being_loaded_); + is_database_being_loaded_ = true; LOG(INFO) << "Start Download Manager database loading"; @@ -525,6 +528,7 @@ class DownloadManagerImpl final : public DownloadManager { } is_database_loaded_ = true; + is_database_being_loaded_ = false; update_counters(); check_completed_downloads_size(); @@ -589,11 +593,8 @@ class DownloadManagerImpl final : public DownloadManager { << file_info->size << '/' << file_info->expected_size << " with downloaded_size = " << file_info->downloaded_size << " and is_paused = " << file_info->is_paused; - auto res = files_.emplace(download_id, std::move(file_info)); - auto it = res.first; + auto it = files_.emplace(download_id, std::move(file_info)).first; bool was_completed = is_completed(*it->second); - LOG_CHECK(!it->second->is_registered) - << res.second << ' ' << download_id << ' ' << max_download_id_ << ' ' << it->second->need_save_to_database; register_file_info(*it->second); // must be called before start_file, which can call update_file_download_state if (is_completed(*it->second)) { bool is_inserted = completed_download_ids_.insert(it->second->download_id).second; From 2729d44f71c76ec19d26bcba1d744bd29c8d97c0 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 23 Jul 2022 17:19:18 +0300 Subject: [PATCH 16/23] Move DownloadManagerImpl::*_impl functions to private section. --- td/telegram/DownloadManager.cpp | 78 ++++++++++++++++----------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/td/telegram/DownloadManager.cpp b/td/telegram/DownloadManager.cpp index ef6315780..ce220a1c8 100644 --- a/td/telegram/DownloadManager.cpp +++ b/td/telegram/DownloadManager.cpp @@ -113,49 +113,10 @@ class DownloadManagerImpl final : public DownloadManager { promise.set_result(remove_file_impl(file_id, file_source_id, delete_from_cache)); } - Status remove_file_impl(FileId file_id, FileSourceId file_source_id, bool delete_from_cache) { - LOG(INFO) << "Remove from downloads file " << file_id << " from " << file_source_id; - TRY_STATUS(check_is_active()); - TRY_RESULT(file_info_ptr, get_file_info(file_id, file_source_id)); - auto &file_info = *file_info_ptr; - auto download_id = file_info.download_id; - if (!is_completed(file_info) && !file_info.is_paused) { - callback_->pause_file(file_info.internal_file_id); - } - unregister_file_info(file_info); - if (delete_from_cache) { - callback_->delete_file(file_info.internal_file_id); - } - by_internal_file_id_.erase(file_info.internal_file_id); - by_file_id_.erase(file_info.file_id); - hints_.remove(download_id); - completed_download_ids_.erase(download_id); - - remove_from_database(file_info); - files_.erase(download_id); - if (is_search_inited_) { - callback_->update_file_removed(file_id, file_counters_); - } - - update_counters(); - on_file_viewed(download_id); - - return Status::OK(); - } - void remove_file_if_finished(FileId file_id) final { remove_file_if_finished_impl(file_id).ignore(); } - Status remove_file_if_finished_impl(FileId file_id) { - TRY_STATUS(check_is_active()); - TRY_RESULT(file_info_ptr, get_file_info(file_id, {})); - if (!is_completed(*file_info_ptr)) { - return Status::Error("File is active"); - } - return remove_file_impl(file_id, {}, false); - } - void remove_all_files(bool only_active, bool only_completed, bool delete_from_cache, Promise promise) final { TRY_STATUS_PROMISE(promise, check_is_active()); vector to_remove; @@ -611,6 +572,45 @@ class DownloadManagerImpl final : public DownloadManager { } } + Status remove_file_impl(FileId file_id, FileSourceId file_source_id, bool delete_from_cache) { + LOG(INFO) << "Remove from downloads file " << file_id << " from " << file_source_id; + TRY_STATUS(check_is_active()); + TRY_RESULT(file_info_ptr, get_file_info(file_id, file_source_id)); + auto &file_info = *file_info_ptr; + auto download_id = file_info.download_id; + if (!is_completed(file_info) && !file_info.is_paused) { + callback_->pause_file(file_info.internal_file_id); + } + unregister_file_info(file_info); + if (delete_from_cache) { + callback_->delete_file(file_info.internal_file_id); + } + by_internal_file_id_.erase(file_info.internal_file_id); + by_file_id_.erase(file_info.file_id); + hints_.remove(download_id); + completed_download_ids_.erase(download_id); + + remove_from_database(file_info); + files_.erase(download_id); + if (is_search_inited_) { + callback_->update_file_removed(file_id, file_counters_); + } + + update_counters(); + on_file_viewed(download_id); + + return Status::OK(); + } + + Status remove_file_if_finished_impl(FileId file_id) { + TRY_STATUS(check_is_active()); + TRY_RESULT(file_info_ptr, get_file_info(file_id, {})); + if (!is_completed(*file_info_ptr)) { + return Status::Error("File is active"); + } + return remove_file_impl(file_id, {}, false); + } + void timeout_expired() final { clear_counters(); } From 3afbce24b3d237c7a1cb86c83da50b4e97a0672f Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 23 Jul 2022 23:50:28 +0300 Subject: [PATCH 17/23] Disable getStorageStatisticsFast for bots to reduce disk usage in the main thread. --- td/telegram/Td.cpp | 5 +++++ td/telegram/files/FileManager.cpp | 12 ++++++++---- td/telegram/files/FileManager.h | 2 ++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index 7baa0b1cb..1b0d8627e 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -3982,6 +3982,10 @@ void Td::init_file_manager() { explicit FileManagerContext(Td *td) : td_(td) { } + bool need_notify_on_new_files() final { + return !td_->auth_manager_->is_bot(); + } + void on_new_file(int64 size, int64 real_size, int32 cnt) final { send_closure(G()->storage_manager(), &StorageManager::on_new_file, size, real_size, cnt); } @@ -4867,6 +4871,7 @@ void Td::on_request(uint64 id, td_api::getStorageStatistics &request) { } void Td::on_request(uint64 id, td_api::getStorageStatisticsFast &request) { + CHECK_IS_USER(); CREATE_REQUEST_PROMISE(); auto query_promise = PromiseCreator::lambda([promise = std::move(promise)](Result result) mutable { if (result.is_error()) { diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index 6dcc7cd37..3474c07dc 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -2157,7 +2157,9 @@ void FileManager::delete_file(FileId file_id, Promise promise, const char if (file_view.has_local_location()) { if (begins_with(file_view.local_location().path_, get_files_dir(file_view.get_type()))) { clear_from_pmc(node); - context_->on_new_file(-file_view.size(), -file_view.get_allocated_local_size(), -1); + if (context_->need_notify_on_new_files()) { + context_->on_new_file(-file_view.size(), -file_view.get_allocated_local_size(), -1); + } path = std::move(node->local_.full().path_); } } else { @@ -3511,7 +3513,7 @@ void FileManager::on_download_ok(QueryId query_id, FullLocalFileLocation local, if (r_new_file_id.is_error()) { status = Status::Error(PSLICE() << "Can't register local file after download: " << r_new_file_id.error().message()); } else { - if (is_new) { + if (is_new && context_->need_notify_on_new_files()) { context_->on_new_file(size, get_file_view(r_new_file_id.ok()).get_allocated_local_size(), 1); } auto r_file_id = merge(r_new_file_id.ok(), file_id); @@ -3684,8 +3686,10 @@ void FileManager::on_generate_ok(QueryId query_id, FullLocalFileLocation local) CHECK(file_node); FileView file_view(file_node); - if (!file_view.has_generate_location() || !begins_with(file_view.generate_location().conversion_, "#file_id#")) { - context_->on_new_file(file_view.size(), file_view.get_allocated_local_size(), 1); + if (context_->need_notify_on_new_files()) { + if (!file_view.has_generate_location() || !begins_with(file_view.generate_location().conversion_, "#file_id#")) { + context_->on_new_file(file_view.size(), file_view.get_allocated_local_size(), 1); + } } run_upload(file_node, {}); diff --git a/td/telegram/files/FileManager.h b/td/telegram/files/FileManager.h index e0ac997f7..eb2f00a38 100644 --- a/td/telegram/files/FileManager.h +++ b/td/telegram/files/FileManager.h @@ -382,6 +382,8 @@ class FileManager final : public FileLoadManager::Callback { class Context { public: + virtual bool need_notify_on_new_files() = 0; + virtual void on_new_file(int64 size, int64 real_size, int32 cnt) = 0; virtual void on_file_updated(FileId size) = 0; From 6c98bec686e9cb349e9322427c4e4cb3bd1e8fba Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 24 Jul 2022 12:19:51 +0300 Subject: [PATCH 18/23] Ensure that poll is removed from being_closed_polls_ after request is completed. --- td/telegram/PollManager.cpp | 17 ++++++++++++++++- td/telegram/PollManager.h | 2 ++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/td/telegram/PollManager.cpp b/td/telegram/PollManager.cpp index 16875f7b0..2fed21267 100644 --- a/td/telegram/PollManager.cpp +++ b/td/telegram/PollManager.cpp @@ -1214,11 +1214,26 @@ void PollManager::do_stop_poll(PollId poll_id, FullMessageId full_message_id, un bool is_inserted = being_closed_polls_.insert(poll_id).second; CHECK(is_inserted); - auto new_promise = get_erase_log_event_promise(log_event_id, std::move(promise)); + auto new_promise = PromiseCreator::lambda( + [actor_id = actor_id(this), poll_id, log_event_id, promise = std::move(promise)](Result result) mutable { + send_closure(actor_id, &PollManager::on_stop_poll_finished, poll_id, log_event_id, std::move(result), + std::move(promise)); + }); td_->create_handler(std::move(new_promise))->send(full_message_id, std::move(reply_markup), poll_id); } +void PollManager::on_stop_poll_finished(PollId poll_id, uint64 log_event_id, Result &&result, + Promise &&promise) { + being_closed_polls_.erase(poll_id); + + if (log_event_id != 0 && !G()->close_flag()) { + binlog_erase(G()->td_db()->get_binlog(), log_event_id); + } + + promise.set_result(std::move(result)); +} + void PollManager::stop_local_poll(PollId poll_id) { CHECK(is_local_poll_id(poll_id)); auto poll = get_poll_editable(poll_id); diff --git a/td/telegram/PollManager.h b/td/telegram/PollManager.h index 4a47c2ad8..cca7e714a 100644 --- a/td/telegram/PollManager.h +++ b/td/telegram/PollManager.h @@ -213,6 +213,8 @@ class PollManager final : public Actor { void do_stop_poll(PollId poll_id, FullMessageId full_message_id, unique_ptr &&reply_markup, uint64 log_event_id, Promise &&promise); + void on_stop_poll_finished(PollId poll_id, uint64 log_event_id, Result &&result, Promise &&promise); + void forget_local_poll(PollId poll_id); MultiTimeout update_poll_timeout_{"UpdatePollTimeout"}; From a63642c8bf0393303e2d5ad2b0b575449bf5eeca Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 24 Jul 2022 12:54:49 +0300 Subject: [PATCH 19/23] Avoid broken server copy for some quizes. --- td/telegram/MessagesManager.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 5f668f331..b02fdfed9 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -28242,9 +28242,20 @@ Result MessagesManager::get_forwarded_messag continue; } + bool is_broken_server_copy = [&] { + switch (forwarded_message->content->get_type()) { + case MessageContentType::Poll: + return get_message_content_poll_is_closed(td_, forwarded_message->content.get()) || + td_->auth_manager_->is_bot(); + case MessageContentType::Dice: + return true; + default: + return false; + } + }(); + bool need_copy = !message_id.is_server() || to_secret || copy_options[i].send_copy; - bool is_local_copy = need_copy && !(message_id.is_server() && can_use_server_forward && - forwarded_message->content->get_type() != MessageContentType::Dice); + bool is_local_copy = need_copy && !(message_id.is_server() && can_use_server_forward && !is_broken_server_copy); if (!(need_copy && td_->auth_manager_->is_bot()) && !can_save_message(from_dialog_id, forwarded_message)) { LOG(INFO) << "Forward of " << message_id << " is restricted"; continue; From 262ea91d9a65e5310a5f30ebb2e577daf7741fc0 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 25 Jul 2022 16:28:46 +0300 Subject: [PATCH 20/23] Drop via_bot from forwarded Audio. --- td/telegram/MessagesManager.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index b02fdfed9..ece5d8a30 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -28086,6 +28086,9 @@ unique_ptr MessagesManager::create_message_ void MessagesManager::fix_forwarded_message(Message *m, DialogId to_dialog_id, const Message *forwarded_message, int64 media_album_id, bool drop_author) const { + if (m->content->get_type() == MessageContentType::Audio) { + drop_author = true; + } bool is_game = m->content->get_type() == MessageContentType::Game; if (!drop_author || is_game) { m->via_bot_user_id = forwarded_message->via_bot_user_id; From 9c6f4577151972aac47769c915cdc1ea8419e866 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 26 Jul 2022 10:33:25 +0300 Subject: [PATCH 21/23] Explicitly define _GNU_SOURCE. --- CMake/TdSetUpCompiler.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMake/TdSetUpCompiler.cmake b/CMake/TdSetUpCompiler.cmake index d9d67da33..40c667e32 100644 --- a/CMake/TdSetUpCompiler.cmake +++ b/CMake/TdSetUpCompiler.cmake @@ -88,6 +88,9 @@ function(td_set_up_compiler) # _FILE_OFFSET_BITS is broken in Android NDK r15, r15b and r17 and doesn't work prior to Android 7.0 add_definitions(-D_FILE_OFFSET_BITS=64) + # _GNU_SOURCE might not be defined by g++ + add_definitions(-D_GNU_SOURCE) + if (CMAKE_SYSTEM_NAME STREQUAL "SunOS") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lsocket -lnsl") if (ILLUMOS) From 6283dce8b75cb7055ee47d3db65710e3b87517b0 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 27 Jul 2022 11:41:16 +0300 Subject: [PATCH 22/23] Remove superflous inline specifiers. --- tdactor/td/actor/impl/Scheduler.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdactor/td/actor/impl/Scheduler.h b/tdactor/td/actor/impl/Scheduler.h index 4319dc2c2..74910ff53 100644 --- a/tdactor/td/actor/impl/Scheduler.h +++ b/tdactor/td/actor/impl/Scheduler.h @@ -171,7 +171,7 @@ inline void Scheduler::send_to_scheduler(int32 sched_id, const ActorId &a } template -inline void Scheduler::destroy_on_scheduler(int32 sched_id, T &value) { +void Scheduler::destroy_on_scheduler(int32 sched_id, T &value) { if (!value.empty()) { destroy_on_scheduler_impl(sched_id, PromiseCreator::lambda([value = std::move(value)](Unit) { // destroy value @@ -180,7 +180,7 @@ inline void Scheduler::destroy_on_scheduler(int32 sched_id, T &value) { } template -inline void Scheduler::destroy_on_scheduler(int32 sched_id, ArgsT &...values) { +void Scheduler::destroy_on_scheduler(int32 sched_id, ArgsT &...values) { destroy_on_scheduler_impl(sched_id, PromiseCreator::lambda([values = std::make_tuple(std::move(values)...)](Unit) { // destroy values })); From 07b7faf66c1cc252d3ab0d3e540f46a15d432354 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 28 Jul 2022 00:15:18 +0300 Subject: [PATCH 23/23] Reduce run time of some tests. --- tdactor/test/actors_main.cpp | 4 ++-- tdactor/test/actors_workers.cpp | 6 +++--- test/db.cpp | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tdactor/test/actors_main.cpp b/tdactor/test/actors_main.cpp index 2e795d40a..4bb9ac54b 100644 --- a/tdactor/test/actors_main.cpp +++ b/tdactor/test/actors_main.cpp @@ -253,7 +253,7 @@ class MainQueryActor final : public td::Actor { } void wakeup() final { - int cnt = 100000; + int cnt = 10000; while (out_cnt_ < in_cnt_ + 100 && out_cnt_ < cnt) { if (td::Random::fast_bool()) { send_closure(rand_elem(actors_), &QueryActor::query, create_query()); @@ -294,7 +294,7 @@ class SimpleActor final : public td::Actor { } void wakeup() final { - if (q_ == 100000) { + if (q_ == 10000) { td::Scheduler::instance()->finish(); stop(); return; diff --git a/tdactor/test/actors_workers.cpp b/tdactor/test/actors_workers.cpp index 47b560113..748edd4e8 100644 --- a/tdactor/test/actors_workers.cpp +++ b/tdactor/test/actors_workers.cpp @@ -138,13 +138,13 @@ TEST(Actors, workers_big_query_nine_threads) { } TEST(Actors, workers_small_query_one_thread) { - test_workers(0, 10, 1000000, 1); + test_workers(0, 10, 100000, 1); } TEST(Actors, workers_small_query_two_threads) { - test_workers(2, 10, 1000000, 1); + test_workers(2, 10, 100000, 1); } TEST(Actors, workers_small_query_nine_threads) { - test_workers(9, 10, 1000000, 1); + test_workers(9, 10, 10000, 1); } diff --git a/test/db.cpp b/test/db.cpp index 7dd80fbe0..b51a8cf20 100644 --- a/test/db.cpp +++ b/test/db.cpp @@ -377,7 +377,7 @@ TEST(DB, key_value) { values.push_back(td::rand_string('a', 'b', td::Random::fast(1, 10))); } - int queries_n = 3000; + int queries_n = 1000; td::vector queries(queries_n); for (auto &q : queries) { int op = td::Random::fast(0, 2); @@ -427,7 +427,7 @@ TEST(DB, key_value) { ASSERT_EQ(a.value, c.value); ASSERT_EQ(a.value, d.value); ASSERT_EQ(a.value, e.value); - if (cnt++ % 500 == 0) { + if (cnt++ % 200 == 0) { new_kv.impl().init(new_kv_name.str()).ensure(); } }