From 90910f6ded58516b622d24ad35f9bcad9ccec996 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 21 Aug 2022 14:44:57 +0300 Subject: [PATCH 01/48] Update TDLib and use td::WaitFreeHashMap if appropriate. --- td | 2 +- telegram-bot-api/Client.cpp | 80 +++++++++------------------ telegram-bot-api/Client.h | 15 ++--- telegram-bot-api/telegram-bot-api.cpp | 6 +- 4 files changed, 37 insertions(+), 66 deletions(-) diff --git a/td b/td index d9cfcf8..96fc91e 160000 --- a/td +++ b/td @@ -1 +1 @@ -Subproject commit d9cfcf88fe4ad06dae1716ce8f66bbeb7f9491d9 +Subproject commit 96fc91e239f25a123df2e0405f58e24bc963cab7 diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 752e92f..07cc592 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -9097,19 +9097,15 @@ void Client::add_user(UserInfo *user_info, object_ptr &&user) { } Client::UserInfo *Client::add_user_info(int64 user_id) { - auto emplace_result = users_.emplace(user_id, nullptr); - auto &user_info = emplace_result.first->second; - if (emplace_result.second) { + auto &user_info = users_[user_id]; + if (user_info == nullptr) { user_info = td::make_unique(); - } else { - CHECK(user_info != nullptr); } return user_info.get(); } const Client::UserInfo *Client::get_user_info(int64 user_id) const { - auto it = users_.find(user_id); - return it == users_.end() ? nullptr : it->second.get(); + return users_.get_pointer(user_id); } void Client::set_user_photo(int64 user_id, object_ptr &&photo) { @@ -9141,19 +9137,15 @@ void Client::add_group(GroupInfo *group_info, object_ptr &&g } Client::GroupInfo *Client::add_group_info(int64 group_id) { - auto emplace_result = groups_.emplace(group_id, nullptr); - auto &group_info = emplace_result.first->second; - if (emplace_result.second) { + auto &group_info = groups_[group_id]; + if (group_info == nullptr) { group_info = td::make_unique(); - } else { - CHECK(group_info != nullptr); } return group_info.get(); } const Client::GroupInfo *Client::get_group_info(int64 group_id) const { - auto it = groups_.find(group_id); - return it == groups_.end() ? nullptr : it->second.get(); + return groups_.get_pointer(group_id); } void Client::set_group_photo(int64 group_id, object_ptr &&photo) { @@ -9211,39 +9203,27 @@ void Client::set_supergroup_location(int64 supergroup_id, object_ptrsecond; - if (emplace_result.second) { + auto &supergroup_info = supergroups_[supergroup_id]; + if (supergroup_info == nullptr) { supergroup_info = td::make_unique(); - } else { - CHECK(supergroup_info != nullptr); } return supergroup_info.get(); } const Client::SupergroupInfo *Client::get_supergroup_info(int64 supergroup_id) const { - auto it = supergroups_.find(supergroup_id); - return it == supergroups_.end() ? nullptr : it->second.get(); + return supergroups_.get_pointer(supergroup_id); } Client::ChatInfo *Client::add_chat(int64 chat_id) { - LOG(DEBUG) << "Update chat " << chat_id; - auto emplace_result = chats_.emplace(chat_id, nullptr); - auto &chat_info = emplace_result.first->second; - if (emplace_result.second) { + auto &chat_info = chats_[chat_id]; + if (chat_info == nullptr) { chat_info = td::make_unique(); - } else { - CHECK(chat_info != nullptr); } return chat_info.get(); } const Client::ChatInfo *Client::get_chat(int64 chat_id) const { - auto it = chats_.find(chat_id); - if (it == chats_.end()) { - return nullptr; - } - return it->second.get(); + return chats_.get_pointer(chat_id); } Client::ChatType Client::get_chat_type(int64 chat_id) const { @@ -10054,12 +10034,8 @@ bool Client::have_sticker_set_name(int64 sticker_set_id) const { return sticker_set_id == 0 || sticker_set_names_.count(sticker_set_id) > 0; } -Client::Slice Client::get_sticker_set_name(int64 sticker_set_id) const { - auto it = sticker_set_names_.find(sticker_set_id); - if (it == sticker_set_names_.end()) { - return Slice(); - } - return it->second; +td::string Client::get_sticker_set_name(int64 sticker_set_id) const { + return sticker_set_names_.get(sticker_set_id); } void Client::process_new_message_queue(int64 chat_id) { @@ -10169,8 +10145,8 @@ void Client::remove_replies_to_message(int64 chat_id, int64 reply_to_message_id, void Client::delete_message(int64 chat_id, int64 message_id, bool only_from_cache) { remove_replies_to_message(chat_id, message_id, only_from_cache); - auto it = messages_.find({chat_id, message_id}); - if (it == messages_.end()) { + auto message_info = messages_.get_pointer({chat_id, message_id}); + if (message_info == nullptr) { if (yet_unsent_messages_.count({chat_id, message_id}) > 0) { // yet unsent message is deleted, possible only if we are trying to write to inaccessible supergroup or // sent message was deleted before added to the chat @@ -10197,11 +10173,9 @@ void Client::delete_message(int64 chat_id, int64 message_id, bool only_from_cach return; } - auto message_info = it->second.get(); - set_message_reply_to_message_id(message_info, 0); - messages_.erase(it); + messages_.erase({chat_id, message_id}); } Client::FullMessageId Client::add_message(object_ptr &&message, bool force_update_content) { @@ -10212,12 +10186,9 @@ Client::FullMessageId Client::add_message(object_ptr &&message, int64 message_id = message->id_; LOG(DEBUG) << "Add message " << message_id << " to chat " << chat_id; - td::unique_ptr message_info; - auto it = messages_.find({chat_id, message_id}); - if (it == messages_.end()) { + auto &message_info = messages_[{chat_id, message_id}]; + if (message_info == nullptr) { message_info = td::make_unique(); - } else { - message_info = std::move(it->second); } message_info->id = message_id; @@ -10326,7 +10297,6 @@ Client::FullMessageId Client::add_message(object_ptr &&message, } set_message_reply_markup(message_info.get(), std::move(message->reply_markup_)); - messages_[{chat_id, message_id}] = std::move(message_info); message = nullptr; return {chat_id, message_id}; @@ -10354,25 +10324,25 @@ void Client::on_update_message_edited(int64 chat_id, int64 message_id, int32 edi } const Client::MessageInfo *Client::get_message(int64 chat_id, int64 message_id) const { - auto it = messages_.find({chat_id, message_id}); - if (it == messages_.end()) { + auto message_info = messages_.get_pointer({chat_id, message_id}); + if (message_info == nullptr) { LOG(DEBUG) << "Not found message " << message_id << " from chat " << chat_id; return nullptr; } LOG(DEBUG) << "Found message " << message_id << " from chat " << chat_id; - return it->second.get(); + return message_info; } Client::MessageInfo *Client::get_message_editable(int64 chat_id, int64 message_id) { - auto it = messages_.find({chat_id, message_id}); - if (it == messages_.end()) { + auto message_info = messages_.get_pointer({chat_id, message_id}); + if (message_info == nullptr) { LOG(DEBUG) << "Not found message " << message_id << " from chat " << chat_id; return nullptr; } LOG(DEBUG) << "Found message " << message_id << " from chat " << chat_id; - return it->second.get(); + return message_info; } td::string Client::get_chat_member_status(const object_ptr &status) { diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index f1f96a3..3d5bf7b 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -26,6 +26,7 @@ #include "td/utils/Promise.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" +#include "td/utils/WaitFreeHashMap.h" #include #include @@ -764,7 +765,7 @@ class Client final : public WebhookActor::Callback { bool have_sticker_set_name(int64 sticker_set_id) const; - Slice get_sticker_set_name(int64 sticker_set_id) const; + td::string get_sticker_set_name(int64 sticker_set_id) const; int64 choose_added_member_id(const td_api::messageChatAddMembers *message_add_members) const; @@ -932,11 +933,11 @@ class Client final : public WebhookActor::Callback { static td::FlatHashMap methods_; - td::FlatHashMap, FullMessageIdHash> messages_; // message cache - td::FlatHashMap> users_; // user info cache - td::FlatHashMap> groups_; // group info cache - td::FlatHashMap> supergroups_; // supergroup info cache - td::FlatHashMap> chats_; // chat info cache + td::WaitFreeHashMap, FullMessageIdHash> messages_; + td::WaitFreeHashMap> users_; + td::WaitFreeHashMap> groups_; + td::WaitFreeHashMap> supergroups_; + td::WaitFreeHashMap> chats_; td::FlatHashMap, FullMessageIdHash> reply_message_ids_; // message -> replies to it @@ -987,7 +988,7 @@ class Client final : public WebhookActor::Callback { }; td::FlatHashMap new_callback_query_queues_; // sender_user_id -> queue - td::FlatHashMap sticker_set_names_; + td::WaitFreeHashMap sticker_set_names_; int64 cur_temp_bot_user_id_ = 1; td::FlatHashMap bot_user_ids_; diff --git a/telegram-bot-api/telegram-bot-api.cpp b/telegram-bot-api/telegram-bot-api.cpp index 1777beb..6228a59 100644 --- a/telegram-bot-api/telegram-bot-api.cpp +++ b/telegram-bot-api/telegram-bot-api.cpp @@ -191,9 +191,9 @@ int main(int argc, char *argv[]) { td::set_signal_handler(td::SignalType::Other, fail_signal_handler).ensure(); td::set_extended_signal_handler(td::SignalType::Error, sigsegv_signal_handler).ensure(); - td::set_runtime_signal_handler(0, change_verbosity_level_signal_handler).ensure(); - td::set_runtime_signal_handler(1, dump_log_signal_handler).ensure(); - td::set_runtime_signal_handler(2, dump_stacktrace_signal_handler).ensure(); + td::set_real_time_signal_handler(0, change_verbosity_level_signal_handler).ensure(); + td::set_real_time_signal_handler(1, dump_log_signal_handler).ensure(); + td::set_real_time_signal_handler(2, dump_stacktrace_signal_handler).ensure(); td::init_openssl_threads(); From aa9eff357c6ad3c254a3e4b53df37e2b82aa4b30 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 22 Aug 2022 02:26:36 +0300 Subject: [PATCH 02/48] Add watchdog for main thread hanging. --- CMakeLists.txt | 2 ++ telegram-bot-api/Watchdog.cpp | 27 +++++++++++++++++++++++ telegram-bot-api/Watchdog.h | 31 +++++++++++++++++++++++++++ telegram-bot-api/telegram-bot-api.cpp | 20 +++++++++++++++-- 4 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 telegram-bot-api/Watchdog.cpp create mode 100644 telegram-bot-api/Watchdog.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 45d5df6..5f1bad6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,6 +85,7 @@ set(TELEGRAM_BOT_API_SOURCE telegram-bot-api/HttpStatConnection.cpp telegram-bot-api/Query.cpp telegram-bot-api/Stats.cpp + telegram-bot-api/Watchdog.cpp telegram-bot-api/WebhookActor.cpp telegram-bot-api/Client.h @@ -95,6 +96,7 @@ set(TELEGRAM_BOT_API_SOURCE telegram-bot-api/HttpStatConnection.h telegram-bot-api/Query.h telegram-bot-api/Stats.h + telegram-bot-api/Watchdog.h telegram-bot-api/WebhookActor.h ) diff --git a/telegram-bot-api/Watchdog.cpp b/telegram-bot-api/Watchdog.cpp new file mode 100644 index 0000000..d715574 --- /dev/null +++ b/telegram-bot-api/Watchdog.cpp @@ -0,0 +1,27 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2021 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "telegram-bot-api/Watchdog.h" + +#include "td/utils/Time.h" + +namespace telegram_bot_api { + +void Watchdog::kick() { + auto now = td::Time::now(); + if (now >= last_kick_time_ + timeout_ && last_kick_time_ > 0) { + LOG(ERROR) << "Watchdog timeout expired after " << now - last_kick_time_ << " seconds"; + td::thread::send_real_time_signal(main_thread_id_, 2); + } + last_kick_time_ = now; + set_timeout_in(timeout_); +} + +void Watchdog::timeout_expired() { + kick(); +} + +} // namespace telegram_bot_api diff --git a/telegram-bot-api/Watchdog.h b/telegram-bot-api/Watchdog.h new file mode 100644 index 0000000..5ab0115 --- /dev/null +++ b/telegram-bot-api/Watchdog.h @@ -0,0 +1,31 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2021 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/actor.h" + +#include "td/utils/port/thread.h" + +namespace telegram_bot_api { + +class Watchdog final : public td::Actor { + public: + Watchdog(td::thread::id main_thread_id, double timeout) : main_thread_id_(main_thread_id), timeout_(timeout) { + // watchdog is disabled until it is kicked for the first time + } + + void kick(); + + private: + void timeout_expired() final; + + td::thread::id main_thread_id_; + double timeout_; + double last_kick_time_ = 0.0; +}; + +} // namespace telegram_bot_api diff --git a/telegram-bot-api/telegram-bot-api.cpp b/telegram-bot-api/telegram-bot-api.cpp index 6228a59..419dfcf 100644 --- a/telegram-bot-api/telegram-bot-api.cpp +++ b/telegram-bot-api/telegram-bot-api.cpp @@ -11,6 +11,7 @@ #include "telegram-bot-api/HttpStatConnection.h" #include "telegram-bot-api/Query.h" #include "telegram-bot-api/Stats.h" +#include "telegram-bot-api/Watchdog.h" #include "td/telegram/ClientActor.h" @@ -462,7 +463,7 @@ int main(int argc, char *argv[]) { // << (td::GitInfo::is_dirty() ? "(dirty)" : "") << " started"; LOG(WARNING) << "Bot API " << parameters->version_ << " server started"; - const int threads_n = 5; // +3 for Td, one for slow HTTP connections and one for DNS resolving + const int threads_n = 6; // +3 for Td, one for watchdog, one for slow HTTP connections, one for DNS resolving td::ConcurrentScheduler sched; sched.init(threads_n); @@ -474,6 +475,7 @@ int main(int argc, char *argv[]) { auto client_manager = sched.create_actor_unsafe(0, "ClientManager", std::move(parameters), token_range).release(); + sched .create_actor_unsafe( 0, "HttpServer", http_ip_address, http_port, @@ -482,6 +484,7 @@ int main(int argc, char *argv[]) { td::create_actor("HttpConnection", client_manager, shared_data)); }) .release(); + if (http_stat_port != 0) { sched .create_actor_unsafe( @@ -492,8 +495,14 @@ int main(int argc, char *argv[]) { }) .release(); } + + constexpr double WATCHDOG_TIMEOUT = 0.5; + auto watchdog_id = + sched.create_actor_unsafe(threads_n - 2, "Watchdog", td::this_thread::get_id(), WATCHDOG_TIMEOUT); + sched.start(); + double next_watchdog_kick_time = start_time; double next_cron_time = start_time; double last_dump_time = start_time - 1000.0; double last_tqueue_gc_time = start_time - 1000.0; @@ -503,7 +512,7 @@ int main(int argc, char *argv[]) { std::atomic_bool can_quit{false}; ServerCpuStat::instance(); // create ServerCpuStat instance while (true) { - sched.run_main(next_cron_time - td::Time::now()); + sched.run_main(td::min(next_cron_time, next_watchdog_kick_time) - td::Time::now()); if (!need_reopen_log.test_and_set()) { td::log_interface->after_rotation(); @@ -519,6 +528,7 @@ int main(int argc, char *argv[]) { dump_statistics(shared_data, net_query_stats); close_flag = true; auto guard = sched.get_main_guard(); + watchdog_id.reset(); send_closure(client_manager, &ClientManager::close, td::PromiseCreator::lambda([&can_quit](td::Unit) { can_quit.store(true); td::Scheduler::instance()->yield(); @@ -557,6 +567,12 @@ int main(int argc, char *argv[]) { ServerCpuStat::update(now); } + if (now >= start_time + 600) { + auto guard = sched.get_main_guard(); + send_closure(watchdog_id, &Watchdog::kick); + next_watchdog_kick_time = now + WATCHDOG_TIMEOUT / 2; + } + if (now > last_tqueue_gc_time + 60.0) { auto unix_time = shared_data->get_unix_time(now); LOG(INFO) << "Run TQueue GC at " << unix_time; From 5eb24c7e63db35baf3515d8d953ce502570fefc1 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 24 Aug 2022 15:48:27 +0300 Subject: [PATCH 03/48] Improve statistics retrieval. --- telegram-bot-api/ClientManager.cpp | 21 ++++++++++++--------- telegram-bot-api/Stats.cpp | 9 +++++++++ telegram-bot-api/Stats.h | 2 ++ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/telegram-bot-api/ClientManager.cpp b/telegram-bot-api/ClientManager.cpp index 8a58b22..582549c 100644 --- a/telegram-bot-api/ClientManager.cpp +++ b/telegram-bot-api/ClientManager.cpp @@ -36,7 +36,7 @@ #include "td/utils/StringBuilder.h" #include "td/utils/Time.h" -#include +#include #include namespace telegram_bot_api { @@ -182,7 +182,8 @@ void ClientManager::get_stats(td::Promise promise, auto now = td::Time::now(); td::int32 active_bot_count = 0; - std::multimap top_bot_ids; + td::vector> top_bot_ids; + size_t max_bots = 50; for (auto id : clients_.ids()) { auto *client_info = clients_.get(id); CHECK(client_info); @@ -195,15 +196,17 @@ void ClientManager::get_stats(td::Promise promise, continue; } - auto stats = client_info->stat_.as_vector(now); - double score = 0.0; - for (auto &stat : stats) { - if (stat.key_ == "update_count" || stat.key_ == "request_count") { - score -= td::to_double(stat.value_); - } + auto score = static_cast(client_info->stat_.get_score(now) * -1e9); + if (score == 0 && top_bot_ids.size() >= max_bots) { + continue; } - top_bot_ids.emplace(static_cast(score * 1e9), id); + top_bot_ids.emplace_back(score, id); } + if (top_bot_ids.size() < max_bots) { + max_bots = top_bot_ids.size(); + } + std::partial_sort(top_bot_ids.begin(), top_bot_ids.begin() + max_bots, top_bot_ids.end()); + top_bot_ids.resize(max_bots); sb << stat_.get_description() << '\n'; if (id_filter.empty()) { diff --git a/telegram-bot-api/Stats.cpp b/telegram-bot-api/Stats.cpp index 59ca63e..52d0f0b 100644 --- a/telegram-bot-api/Stats.cpp +++ b/telegram-bot-api/Stats.cpp @@ -149,6 +149,15 @@ td::string BotStatActor::get_description() const { return res; } +double BotStatActor::get_score(double now) { + auto minute_stat = stat_[2].stat_duration(now); + double result = minute_stat.first.request_count_ + minute_stat.first.update_count_; + if (minute_stat.second != 0) { + result /= minute_stat.second; + } + return result; +} + bool BotStatActor::is_active(double now) const { return last_activity_timestamp_ > now - 86400; } diff --git a/telegram-bot-api/Stats.h b/telegram-bot-api/Stats.h index 3b317fc..efc6287 100644 --- a/telegram-bot-api/Stats.h +++ b/telegram-bot-api/Stats.h @@ -181,6 +181,8 @@ class BotStatActor final : public td::Actor { td::vector as_vector(double now); td::string get_description() const; + double get_score(double now); + bool is_active(double now) const; private: From 3b5cf48e5b572bec75f7bede3f1385bef0f69a21 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 28 Aug 2022 22:30:49 +0300 Subject: [PATCH 04/48] Improve errors returned by get_input_message_contents. --- telegram-bot-api/Client.cpp | 22 ++++++++++++---------- telegram-bot-api/Client.h | 3 +-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 07cc592..c1721a1 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -6616,7 +6616,7 @@ td::Result> Client::get_input_me JsonValue &&input_media, bool for_album) const { if (input_media.type() != JsonValue::Type::Object) { - return Status::Error(400, "expected an Object"); + return Status::Error("expected an Object"); } auto &object = input_media.get_object(); @@ -6631,7 +6631,7 @@ td::Result> Client::get_input_me auto input_file = get_input_file(query, Slice(), media, false); if (input_file == nullptr) { - return Status::Error(400, "media not found"); + return Status::Error("media not found"); } TRY_RESULT(thumbnail, get_json_object_string_field(object, "thumb")); @@ -6660,7 +6660,7 @@ td::Result> Client::get_input_me std::move(caption), ttl); } if (for_album && type == "animation") { - return Status::Error(400, PSLICE() << "type \"" << type << "\" can't be used in sendMediaGroup"); + return Status::Error(PSLICE() << "type \"" << type << "\" can't be used in sendMediaGroup"); } if (type == "animation") { TRY_RESULT(width, get_json_object_int_field(object, "width")); @@ -6686,12 +6686,11 @@ td::Result> Client::get_input_me disable_content_type_detection || for_album, std::move(caption)); } - return Status::Error(400, PSLICE() << "type \"" << type << "\" is unsupported"); + return Status::Error(PSLICE() << "type \"" << type << "\" is unsupported"); } td::Result> Client::get_input_media(const Query *query, - Slice field_name, - bool for_album) const { + Slice field_name) const { TRY_RESULT(media, get_required_string_arg(query, field_name)); LOG(INFO) << "Parsing JSON object: " << media; @@ -6701,7 +6700,7 @@ td::Result> Client::get_input_me return Status::Error(400, "Can't parse input media JSON object"); } - auto r_input_message_content = get_input_media(query, r_value.move_as_ok(), for_album); + auto r_input_message_content = get_input_media(query, r_value.move_as_ok(), false); if (r_input_message_content.is_error()) { return Status::Error(400, PSLICE() << "Can't parse InputMedia: " << r_input_message_content.error().message()); } @@ -6730,8 +6729,11 @@ td::Result>> Client:: td::vector> contents; for (auto &input_media : value.get_array()) { - TRY_RESULT(input_message_content, get_input_media(query, std::move(input_media), true)); - contents.push_back(std::move(input_message_content)); + auto r_input_message_content = get_input_media(query, std::move(input_media), true); + if (r_input_message_content.is_error()) { + return Status::Error(400, PSLICE() << "Can't parse InputMedia: " << r_input_message_content.error().message()); + } + contents.push_back(r_input_message_content.move_as_ok()); } return std::move(contents); } @@ -7522,7 +7524,7 @@ td::Status Client::process_edit_message_media_query(PromisedQueryPtr &query) { auto chat_id = query->arg("chat_id"); auto message_id = get_message_id(query.get()); TRY_RESULT(reply_markup, get_reply_markup(query.get())); - TRY_RESULT(input_media, get_input_media(query.get(), "media", false)); + TRY_RESULT(input_media, get_input_media(query.get(), "media")); if (chat_id.empty() && message_id == 0) { TRY_RESULT(inline_message_id, get_inline_message_id(query.get())); diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index 3d5bf7b..6d0fc54 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -437,8 +437,7 @@ class Client final : public WebhookActor::Callback { td::Result> get_input_media(const Query *query, td::JsonValue &&input_media, bool for_album) const; - td::Result> get_input_media(const Query *query, Slice field_name, - bool for_album) const; + td::Result> get_input_media(const Query *query, Slice field_name) const; td::Result>> get_input_message_contents(const Query *query, Slice field_name) const; From ffecd115fe777601228ff701d133075c5d6fd292 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 6 Sep 2022 19:02:57 +0300 Subject: [PATCH 05/48] Improve NetBSD build instructions. --- build.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.html b/build.html index 035a176..38bf606 100644 --- a/build.html +++ b/build.html @@ -454,7 +454,7 @@ function onOptionsChanged() { pre_text.push('Note that building requires a lot of memory, so you may need to increase allowed per-process memory usage in /etc/login.conf or build from root.'); } if (os_netbsd) { - pre_text.push('Note that the following instruction is for NetBSD 8.0 and default SH shell.'); + pre_text.push('Note that the following instruction is for NetBSD 8+ and default SH shell.'); } var terminal_name = (function () { @@ -586,8 +586,8 @@ function onOptionsChanged() { if (!use_root) { commands.push('su -'); } - commands.push('export PKG_PATH=ftp://ftp.netbsd.org/pub/pkgsrc/packages/NetBSD/i386/8.0_2019Q2/All'); - var packages = 'git gperf cmake openssl gcc5-libs'; + commands.push('export PKG_PATH=http://cdn.netbsd.org/pub/pkgsrc/packages/NetBSD/$(uname -p)/$(uname -r)/All'); + var packages = 'git gperf cmake openssl gcc12-libs mozilla-rootcerts-openssl'; commands.push('pkg_add ' + packages); if (!use_root) { commands.push('exit'); From c35bbf1bd231fa562ab403cb7c2c2eccf6aadcbf Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 18 Sep 2022 00:20:41 +0300 Subject: [PATCH 06/48] Update TDLib to 1.8.6 and add options for changing CPU affinity. --- td | 2 +- telegram-bot-api/Client.cpp | 36 ++++++++++------------ telegram-bot-api/telegram-bot-api.cpp | 43 +++++++++++++++++++++++---- 3 files changed, 55 insertions(+), 26 deletions(-) diff --git a/td b/td index 96fc91e..cb70993 160000 --- a/td +++ b/td @@ -1 +1 @@ -Subproject commit 96fc91e239f25a123df2e0405f58e24bc963cab7 +Subproject commit cb70993b90ce84c16dea07c347835addbff46a2c diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index c1721a1..00608f4 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -4472,27 +4472,23 @@ void Client::on_update_authorization_state() { make_object(true)), td::make_unique()); - auto parameters = make_object(); + auto request = make_object(); + request->use_test_dc_ = is_test_dc_; + request->database_directory_ = dir_; + //request->use_file_database_ = false; + //request->use_chat_info_database_ = false; + //request->use_secret_chats_ = false; + request->use_message_database_ = USE_MESSAGE_DATABASE; + request->api_id_ = parameters_->api_id_; + request->api_hash_ = parameters_->api_hash_; + request->system_language_code_ = "en"; + request->device_model_ = "server"; + request->application_version_ = parameters_->version_; + request->enable_storage_optimizer_ = true; + request->ignore_file_names_ = true; - parameters->use_test_dc_ = is_test_dc_; - parameters->database_directory_ = dir_; - //parameters->use_file_database_ = false; - //parameters->use_chat_info_database_ = false; - //parameters->use_secret_chats_ = false; - parameters->use_message_database_ = USE_MESSAGE_DATABASE; - parameters->api_id_ = parameters_->api_id_; - parameters->api_hash_ = parameters_->api_hash_; - parameters->system_language_code_ = "en"; - parameters->device_model_ = "server"; - parameters->application_version_ = parameters_->version_; - parameters->enable_storage_optimizer_ = true; - parameters->ignore_file_names_ = true; - - return send_request(make_object(std::move(parameters)), - td::make_unique(this)); + return send_request(std::move(request), td::make_unique(this)); } - case td_api::authorizationStateWaitEncryptionKey::ID: - return send_request(make_object(), td::make_unique(this)); case td_api::authorizationStateWaitPhoneNumber::ID: send_request(make_object("online", make_object(true)), td::make_unique()); @@ -5618,7 +5614,7 @@ td::Result> Client::get_input_me td_api::object_ptr Client::get_message_send_options(bool disable_notification, bool protect_content) { - return make_object(disable_notification, false, protect_content, nullptr); + return make_object(disable_notification, false, protect_content, false, nullptr); } td::Result>> Client::get_inline_query_results( diff --git a/telegram-bot-api/telegram-bot-api.cpp b/telegram-bot-api/telegram-bot-api.cpp index 419dfcf..f5cc937 100644 --- a/telegram-bot-api/telegram-bot-api.cpp +++ b/telegram-bot-api/telegram-bot-api.cpp @@ -44,6 +44,7 @@ #include "td/utils/port/signals.h" #include "td/utils/port/stacktrace.h" #include "td/utils/port/Stat.h" +#include "td/utils/port/thread.h" #include "td/utils/port/user.h" #include "td/utils/Promise.h" #include "td/utils/Slice.h" @@ -223,6 +224,8 @@ int main(int argc, char *argv[]) { td::string username; td::string groupname; td::uint64 max_connections = 0; + td::uint64 cpu_affinity = 0; + td::uint64 main_thread_affinity = 0; ClientManager::TokenRange token_range{0, 1}; parameters->api_id_ = [](auto x) -> td::int32 { @@ -310,6 +313,17 @@ int main(int argc, char *argv[]) { options.add_option('g', "groupname", "effective group name to switch to", td::OptionParser::parse_string(groupname)); options.add_checked_option('c', "max-connections", "maximum number of open file descriptors", td::OptionParser::parse_integer(max_connections)); +#if TD_HAVE_THREAD_AFFINITY + options.add_checked_option('\0', "cpu-affinity", "CPU affinity as 64-bit mask (defaults to all available CPUs)", + td::OptionParser::parse_integer(cpu_affinity)); + options.add_checked_option( + '\0', "main-thread-affinity", + "CPU affinity of the main thread as 64-bit mask (defaults to the value of the option --cpu-affinity)", + td::OptionParser::parse_integer(main_thread_affinity)); +#else + (void)cpu_affinity; + (void)main_thread_affinity; +#endif options.add_checked_option('\0', "proxy", "HTTP proxy server for outgoing webhook requests in the format http://host:port", @@ -363,6 +377,26 @@ int main(int argc, char *argv[]) { td::TsLog ts_log(&file_log); auto init_status = [&] { +#if TD_HAVE_THREAD_AFFINITY + if (main_thread_affinity == 0) { + main_thread_affinity = cpu_affinity; + } + if (main_thread_affinity != 0) { + auto initial_mask = td::thread::get_affinity_mask(td::this_thread::get_id()); + if (initial_mask == 0) { + return td::Status::Error("Failed to get current thread affinity"); + } + if (cpu_affinity != 0) { + TRY_STATUS_PREFIX(td::thread::set_affinity_mask(td::this_thread::get_id(), cpu_affinity), + "Can't set CPU affinity mask: "); + } else { + cpu_affinity = initial_mask; + } + TRY_STATUS_PREFIX(td::thread::set_affinity_mask(td::this_thread::get_id(), main_thread_affinity), + "Can't set main thread CPU affinity mask: "); + } +#endif + if (max_connections != 0) { TRY_STATUS_PREFIX(td::set_resource_limit(td::ResourceLimitType::NoFile, max_connections), "Can't set file descriptor limit: "); @@ -463,12 +497,11 @@ int main(int argc, char *argv[]) { // << (td::GitInfo::is_dirty() ? "(dirty)" : "") << " started"; LOG(WARNING) << "Bot API " << parameters->version_ << " server started"; - const int threads_n = 6; // +3 for Td, one for watchdog, one for slow HTTP connections, one for DNS resolving - td::ConcurrentScheduler sched; - sched.init(threads_n); + const int thread_count = 6; // +3 for Td, one for watchdog, one for slow HTTP connections, one for DNS resolving + td::ConcurrentScheduler sched(thread_count, cpu_affinity); td::GetHostByNameActor::Options get_host_by_name_options; - get_host_by_name_options.scheduler_id = threads_n; + get_host_by_name_options.scheduler_id = thread_count; parameters->get_host_by_name_actor_id_ = sched.create_actor_unsafe(0, "GetHostByName", std::move(get_host_by_name_options)) .release(); @@ -498,7 +531,7 @@ int main(int argc, char *argv[]) { constexpr double WATCHDOG_TIMEOUT = 0.5; auto watchdog_id = - sched.create_actor_unsafe(threads_n - 2, "Watchdog", td::this_thread::get_id(), WATCHDOG_TIMEOUT); + sched.create_actor_unsafe(thread_count - 2, "Watchdog", td::this_thread::get_id(), WATCHDOG_TIMEOUT); sched.start(); From c20f0c3d9710ae66ba3e612bfa5d51c7e4130517 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 18 Sep 2022 01:17:09 +0300 Subject: [PATCH 07/48] Improve dump of short MemoryLog. --- telegram-bot-api/telegram-bot-api.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/telegram-bot-api/telegram-bot-api.cpp b/telegram-bot-api/telegram-bot-api.cpp index f5cc937..ac91d78 100644 --- a/telegram-bot-api/telegram-bot-api.cpp +++ b/telegram-bot-api/telegram-bot-api.cpp @@ -80,8 +80,15 @@ static td::MemoryLog<1 << 20> memory_log; void print_log() { auto buf = memory_log.get_buffer(); auto pos = memory_log.get_pos(); + size_t tail_length = buf.size() - pos; + while (tail_length > 0 && buf[pos + tail_length - 1] == ' ') { + tail_length--; + } + if (tail_length + 100 >= buf.size() - pos) { + tail_length = buf.size() - pos; + } td::signal_safe_write("------- Log dump -------\n"); - td::signal_safe_write(buf.substr(pos), false); + td::signal_safe_write(buf.substr(pos, tail_length), false); td::signal_safe_write(buf.substr(0, pos), false); td::signal_safe_write("\n", false); td::signal_safe_write("------------------------\n"); From 11d27e3e52bed5cde93cee196234859638ba4275 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 18 Sep 2022 02:15:53 +0300 Subject: [PATCH 08/48] Don't output empty fields in bot statistics. --- telegram-bot-api/ClientManager.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/telegram-bot-api/ClientManager.cpp b/telegram-bot-api/ClientManager.cpp index 582549c..69dba8f 100644 --- a/telegram-bot-api/ClientManager.cpp +++ b/telegram-bot-api/ClientManager.cpp @@ -250,13 +250,20 @@ void ClientManager::get_stats(td::Promise promise, sb << "uptime\t" << now - bot_info.start_time_ << '\n'; sb << "token\t" << bot_info.token_ << '\n'; sb << "username\t" << bot_info.username_ << '\n'; - sb << "is_active\t" << client_info->stat_.is_active(now) << '\n'; - sb << "webhook\t" << bot_info.webhook_ << '\n'; - sb << "has_custom_certificate\t" << bot_info.has_webhook_certificate_ << '\n'; + if (!bot_info.webhook_.empty()) { + sb << "webhook\t" << bot_info.webhook_ << '\n'; + if (bot_info.has_webhook_certificate_) { + sb << "has_custom_certificate\t" << bot_info.has_webhook_certificate_ << '\n'; + } + if (bot_info.webhook_max_connections_ != parameters_->default_max_webhook_connections_) { + sb << "webhook_max_connections\t" << bot_info.webhook_max_connections_ << '\n'; + } + } sb << "head_update_id\t" << bot_info.head_update_id_ << '\n'; - sb << "tail_update_id\t" << bot_info.tail_update_id_ << '\n'; - sb << "pending_update_count\t" << bot_info.pending_update_count_ << '\n'; - sb << "webhook_max_connections\t" << bot_info.webhook_max_connections_ << '\n'; + if (bot_info.pending_update_count_ != 0) { + sb << "tail_update_id\t" << bot_info.tail_update_id_ << '\n'; + sb << "pending_update_count\t" << bot_info.pending_update_count_ << '\n'; + } auto stats = client_info->stat_.as_vector(now); for (auto &stat : stats) { From 27bb4831f6b0e7371a0ea5b450fb6c7173b57854 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 18 Sep 2022 02:54:22 +0300 Subject: [PATCH 09/48] Add active_request_count and active_file_upload_bytes to bot statistics. --- telegram-bot-api/ClientManager.cpp | 8 ++++++++ telegram-bot-api/Query.cpp | 2 +- telegram-bot-api/Stats.cpp | 10 ++++++++++ telegram-bot-api/Stats.h | 29 ++++++++++++++++++++++++++--- 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/telegram-bot-api/ClientManager.cpp b/telegram-bot-api/ClientManager.cpp index 69dba8f..074d874 100644 --- a/telegram-bot-api/ClientManager.cpp +++ b/telegram-bot-api/ClientManager.cpp @@ -245,11 +245,19 @@ void ClientManager::get_stats(td::Promise promise, CHECK(client_info); auto bot_info = client_info->client_->get_actor_unsafe()->get_bot_info(); + auto active_request_count = client_info->stat_.get_active_request_count(); + auto active_file_upload_bytes = client_info->stat_.get_active_file_upload_bytes(); sb << '\n'; sb << "id\t" << bot_info.id_ << '\n'; sb << "uptime\t" << now - bot_info.start_time_ << '\n'; sb << "token\t" << bot_info.token_ << '\n'; sb << "username\t" << bot_info.username_ << '\n'; + if (active_request_count != 0) { + sb << "active_request_count\t" << active_request_count << '\n'; + } + if (active_file_upload_bytes != 0) { + sb << "active_file_upload_bytes\t" << active_file_upload_bytes << '\n'; + } if (!bot_info.webhook_.empty()) { sb << "webhook\t" << bot_info.webhook_ << '\n'; if (bot_info.has_webhook_certificate_) { diff --git a/telegram-bot-api/Query.cpp b/telegram-bot-api/Query.cpp index e2a7b08..32ff8b4 100644 --- a/telegram-bot-api/Query.cpp +++ b/telegram-bot-api/Query.cpp @@ -135,7 +135,7 @@ void Query::send_response_stat() const { return; } send_closure(stat_actor_, &BotStatActor::add_event, - ServerBotStat::Response{state_ == State::OK, answer_.size()}, now); + ServerBotStat::Response{state_ == State::OK, answer_.size(), files_size()}, now); } } // namespace telegram_bot_api diff --git a/telegram-bot-api/Stats.cpp b/telegram-bot-api/Stats.cpp index 52d0f0b..54a6236 100644 --- a/telegram-bot-api/Stats.cpp +++ b/telegram-bot-api/Stats.cpp @@ -155,9 +155,19 @@ double BotStatActor::get_score(double now) { if (minute_stat.second != 0) { result /= minute_stat.second; } + result += td::max(static_cast(get_active_request_count() - 10), static_cast(0)); + result += static_cast(get_active_file_upload_bytes()) * 1e-8; return result; } +td::int64 BotStatActor::get_active_request_count() const { + return active_request_count_; +} + +td::int64 BotStatActor::get_active_file_upload_bytes() const { + return active_file_upload_bytes_; +} + bool BotStatActor::is_active(double now) const { return last_activity_timestamp_ > now - 86400; } diff --git a/telegram-bot-api/Stats.h b/telegram-bot-api/Stats.h index efc6287..a4d7a07 100644 --- a/telegram-bot-api/Stats.h +++ b/telegram-bot-api/Stats.h @@ -115,15 +115,16 @@ struct ServerBotStat { struct Response { bool ok_; size_t size_; + td::int64 files_size_; }; - void on_event(const Response &answer) { + void on_event(const Response &response) { response_count_++; - if (answer.ok_) { + if (response.ok_) { response_count_ok_++; } else { response_count_error_++; } - response_bytes_ += static_cast(answer.size_); + response_bytes_ += static_cast(response.size_); } struct Request { @@ -173,6 +174,7 @@ class BotStatActor final : public td::Actor { for (auto &stat : stat_) { stat.add_event(event, now); } + on_event(event); if (!parent_.empty()) { send_closure(parent_, &BotStatActor::add_event, event, now); } @@ -183,6 +185,10 @@ class BotStatActor final : public td::Actor { double get_score(double now); + td::int64 get_active_request_count() const; + + td::int64 get_active_file_upload_bytes() const; + bool is_active(double now) const; private: @@ -193,6 +199,23 @@ class BotStatActor final : public td::Actor { td::TimedStat stat_[SIZE]; td::ActorId parent_; double last_activity_timestamp_ = -1e9; + td::int64 active_request_count_ = 0; + td::int64 active_file_upload_bytes_ = 0; + + void on_event(const ServerBotStat::Update &update) { + } + + void on_event(const ServerBotStat::Response &response) { + active_request_count_--; + active_file_upload_bytes_ -= response.files_size_; + CHECK(active_request_count_ >= 0); + CHECK(active_file_upload_bytes_ >= 0); + } + + void on_event(const ServerBotStat::Request &request) { + active_request_count_++; + active_file_upload_bytes_ += request.files_size_; + } }; } // namespace telegram_bot_api From 834caf09bffd3c780628696ca517b96455cda732 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 18 Sep 2022 10:19:58 +0300 Subject: [PATCH 10/48] Fail queries immediately if there are too many active queries already. --- telegram-bot-api/Client.cpp | 18 +++++++++++++++++- telegram-bot-api/Client.h | 2 ++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 00608f4..bfb52a4 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -287,6 +287,11 @@ bool Client::init_methods() { return true; } +bool Client::is_local_method(Slice method) { + return method == "close" || method == "logout" || method == "getme" || method == "getupdates" || + method == "setwebhook" || method == "deletewebhook" || method == "getwebhookinfo"; +} + class Client::JsonFile final : public Jsonable { public: JsonFile(const td_api::file *file, const Client *client, bool with_path) @@ -3895,6 +3900,17 @@ void Client::start_up() { void Client::send(PromisedQueryPtr query) { if (!query->is_internal()) { query->set_stat_actor(stat_actor_); + if (!parameters_->local_mode_ && !is_local_method(query->method())) { + const BotStatActor *stat = stat_actor_.get_actor_unsafe(); + if (stat->get_active_request_count() > 10000) { + LOG(INFO) << "Fail a query, because there are too many active queries: " << *query; + return query->set_retry_after_error(60); + } + if (stat->get_active_file_upload_bytes() > (static_cast(1) << 33) && !query->files().empty()) { + LOG(INFO) << "Fail a query, because there are too many active file uploads: " << *query; + return query->set_retry_after_error(60); + } + } } cmd_queue_.emplace(std::move(query)); loop(); @@ -10457,7 +10473,7 @@ td::int32 Client::as_client_message_id(int64 message_id) { } td::int64 Client::get_supergroup_chat_id(int64 supergroup_id) { - return static_cast(-1000000000000ll) - supergroup_id; + return static_cast(-1000000000000ll) - supergroup_id; } td::int64 Client::get_basic_group_chat_id(int64 basic_group_id) { diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index 6d0fc54..4a3fe09 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -471,6 +471,8 @@ class Client final : public WebhookActor::Callback { static bool init_methods(); + static bool is_local_method(Slice method); + void on_cmd(PromisedQueryPtr query); Status process_get_me_query(PromisedQueryPtr &query); From 254ad97805431a8006230304d62146c52c718f9d Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 23 Sep 2022 20:29:42 +0300 Subject: [PATCH 11/48] Update TDLib to 1.8.7 and support extended_media in sendInvoice. --- td | 2 +- telegram-bot-api/Client.cpp | 11 ++++++++--- telegram-bot-api/Client.h | 2 +- telegram-bot-api/ClientManager.cpp | 2 +- telegram-bot-api/HttpConnection.cpp | 2 +- telegram-bot-api/HttpStatConnection.cpp | 2 +- 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/td b/td index cb70993..a7a17b3 160000 --- a/td +++ b/td @@ -1 +1 @@ -Subproject commit cb70993b90ce84c16dea07c347835addbff46a2c +Subproject commit a7a17b34b3c8fd3f7f6295f152746beb68f34d83 diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index bfb52a4..5877149 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -5619,7 +5619,7 @@ td::Result> Client::get_input_me need_shipping_address, send_phone_number_to_provider, send_email_address_to_provider, is_flexible), title, description, photo_url, photo_size, photo_width, photo_height, payload, provider_token, provider_data, - td::string()); + td::string(), nullptr); } if (is_input_message_content_required) { @@ -6750,7 +6750,7 @@ td::Result>> Client:: return std::move(contents); } -td::Result> Client::get_input_message_invoice(const Query *query) { +td::Result> Client::get_input_message_invoice(const Query *query) const { TRY_RESULT(title, get_required_string_arg(query, "title")); TRY_RESULT(description, get_required_string_arg(query, "description")); TRY_RESULT(payload, get_required_string_arg(query, "payload")); @@ -6806,13 +6806,18 @@ td::Result> Client::get_input_me auto send_email_address_to_provider = to_bool(query->arg("send_email_to_provider")); auto is_flexible = to_bool(query->arg("is_flexible")); + td_api::object_ptr extended_media; + if (!query->arg("extended_media").empty()) { + TRY_RESULT_ASSIGN(extended_media, get_input_media(query, "extended_media")); + } + return make_object( make_object(currency.str(), std::move(prices), max_tip_amount, std::move(suggested_tip_amounts), td::string(), false, need_name, need_phone_number, need_email_address, need_shipping_address, send_phone_number_to_provider, send_email_address_to_provider, is_flexible), title.str(), description.str(), photo_url.str(), photo_size, photo_width, photo_height, payload.str(), - provider_token.str(), provider_data.str(), start_parameter.str()); + provider_token.str(), provider_data.str(), start_parameter.str(), std::move(extended_media)); } td::Result> Client::get_poll_options(const Query *query) { diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index 4a3fe09..cc0fa4c 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -445,7 +445,7 @@ class Client final : public WebhookActor::Callback { td::Result>> get_input_message_contents( const Query *query, td::JsonValue &&value) const; - static td::Result> get_input_message_invoice(const Query *query); + td::Result> get_input_message_invoice(const Query *query) const; static object_ptr get_message_send_options(bool disable_notification, bool protect_content); diff --git a/telegram-bot-api/ClientManager.cpp b/telegram-bot-api/ClientManager.cpp index 074d874..bbc7ecf 100644 --- a/telegram-bot-api/ClientManager.cpp +++ b/telegram-bot-api/ClientManager.cpp @@ -244,7 +244,7 @@ void ClientManager::get_stats(td::Promise promise, auto *client_info = clients_.get(top_bot_id.second); CHECK(client_info); - auto bot_info = client_info->client_->get_actor_unsafe()->get_bot_info(); + auto bot_info = client_info->client_.get_actor_unsafe()->get_bot_info(); auto active_request_count = client_info->stat_.get_active_request_count(); auto active_file_upload_bytes = client_info->stat_.get_active_file_upload_bytes(); sb << '\n'; diff --git a/telegram-bot-api/HttpConnection.cpp b/telegram-bot-api/HttpConnection.cpp index b3fd1cf..a52d113 100644 --- a/telegram-bot-api/HttpConnection.cpp +++ b/telegram-bot-api/HttpConnection.cpp @@ -21,7 +21,7 @@ namespace telegram_bot_api { void HttpConnection::handle(td::unique_ptr http_query, td::ActorOwn connection) { - CHECK(connection_->empty()); + CHECK(connection_.empty()); connection_ = std::move(connection); LOG(DEBUG) << "Handle " << *http_query; diff --git a/telegram-bot-api/HttpStatConnection.cpp b/telegram-bot-api/HttpStatConnection.cpp index 9314a64..2606c89 100644 --- a/telegram-bot-api/HttpStatConnection.cpp +++ b/telegram-bot-api/HttpStatConnection.cpp @@ -15,7 +15,7 @@ namespace telegram_bot_api { void HttpStatConnection::handle(td::unique_ptr http_query, td::ActorOwn connection) { - CHECK(connection_->empty()); + CHECK(connection_.empty()); connection_ = std::move(connection); auto promise = td::PromiseCreator::lambda([actor_id = actor_id(this)](td::Result result) { From 2224b715be6877f528a7c4ffe8da2d281b786e41 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 29 Sep 2022 20:28:44 +0300 Subject: [PATCH 12/48] Improve logging for delayed message updates. --- telegram-bot-api/Client.cpp | 33 +++++++++++++++++++++++++++++---- telegram-bot-api/Client.h | 7 +++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 5877149..04da384 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -6750,7 +6750,8 @@ td::Result>> Client:: return std::move(contents); } -td::Result> Client::get_input_message_invoice(const Query *query) const { +td::Result> Client::get_input_message_invoice( + const Query *query) const { TRY_RESULT(title, get_required_string_arg(query, "title")); TRY_RESULT(description, get_required_string_arg(query, "description")); TRY_RESULT(payload, get_required_string_arg(query, "payload")); @@ -10114,12 +10115,36 @@ void Client::process_new_message_queue(int64 chat_id) { } int32 message_date = message->edit_date_ == 0 ? message->date_ : message->edit_date_; + if (delayed_update_count_ > 0 && (update_type != delayed_update_type_ || chat_id != delayed_chat_id_)) { + if (delayed_update_count_ == 1) { + LOG(ERROR) << "Receive very old update " << get_update_type_name(delayed_update_type_) << " sent at " + << delayed_min_date_ << " in chat " << delayed_chat_id_ << " with a delay of " << delayed_max_time_ + << " seconds"; + } else { + LOG(ERROR) << "Receive " << delayed_update_count_ << " very old updates " + << get_update_type_name(delayed_update_type_) << " sent from " << delayed_min_date_ << " to " + << delayed_max_date_ << " in chat " << delayed_chat_id_ << " with a delay up to " + << delayed_max_time_ << " seconds"; + } + delayed_update_count_ = 0; + } auto now = get_unix_time(); auto update_delay_time = now - td::max(message_date, parameters_->shared_data_->get_unix_time(webhook_set_time_)); const auto UPDATE_DELAY_WARNING_TIME = 10 * 60; - LOG_IF(ERROR, update_delay_time > UPDATE_DELAY_WARNING_TIME) - << "Receive very old update " << get_update_type_name(update_type) << " sent at " << message_date << " to chat " - << chat_id << " with a delay of " << update_delay_time << " seconds: " << to_string(message); + if (update_delay_time > UPDATE_DELAY_WARNING_TIME && message_date > last_synchronization_error_date_ + 60) { + if (delayed_update_count_ == 0) { + delayed_update_type_ = update_type; + delayed_chat_id_ = chat_id; + delayed_min_date_ = message_date; + delayed_max_date_ = message_date; + delayed_max_time_ = update_delay_time; + } else { + delayed_min_date_ = td::min(message_date, delayed_min_date_); + delayed_max_date_ = td::max(message_date, delayed_max_date_); + delayed_max_time_ = td::max(update_delay_time, delayed_max_time_); + } + delayed_update_count_++; + } auto left_time = message_date + 86400 - now; add_message(std::move(message)); diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index cc0fa4c..1e31abc 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -1059,6 +1059,13 @@ class Client final : public WebhookActor::Callback { td::uint64 webhook_generation_ = 1; + UpdateType delayed_update_type_ = UpdateType::Size; + int64 delayed_chat_id_ = 0; + int32 delayed_min_date_ = 0; + int32 delayed_max_date_ = 0; + int32 delayed_max_time_ = 0; + size_t delayed_update_count_ = 0; + std::shared_ptr parameters_; td::ActorId stat_actor_; From e4324ead54d9418c6649db1c40d95f4d8addcaca Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 4 Oct 2022 16:26:51 +0300 Subject: [PATCH 13/48] Copy webhook certificate in another thread. --- telegram-bot-api/Client.cpp | 114 ++++++++++++++++++++++++------------ telegram-bot-api/Client.h | 3 + 2 files changed, 79 insertions(+), 38 deletions(-) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 04da384..bd43527 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -4921,6 +4921,9 @@ void Client::on_closed() { if (webhook_set_query_) { fail_query(http_status_code, description, std::move(webhook_set_query_)); } + if (active_webhook_set_query_) { + fail_query(http_status_code, description, std::move(active_webhook_set_query_)); + } if (!webhook_url_.empty()) { webhook_id_.reset(); } @@ -8506,7 +8509,7 @@ td::Status Client::process_answer_custom_query_query(PromisedQueryPtr &query) { } td::Status Client::process_get_updates_query(PromisedQueryPtr &query) { - if (!webhook_url_.empty() || webhook_set_query_) { + if (!webhook_url_.empty() || webhook_set_query_ || active_webhook_set_query_) { fail_query_conflict( "Conflict: can't use getUpdates method while webhook is active; use deleteWebhook to delete the webhook first", std::move(query)); @@ -8557,6 +8560,9 @@ td::Status Client::process_set_webhook_query(PromisedQueryPtr &query) { if (webhook_set_query_) { // already updating webhook. Cancel previous request fail_query_conflict("Conflict: terminated by other setWebhook", std::move(webhook_set_query_)); + } else if (active_webhook_set_query_) { + query->set_retry_after_error(1); + return Status::OK(); } else if (webhook_url_ == new_url && !has_webhook_certificate_ && !new_has_certificate && new_max_connections == webhook_max_connections_ && new_fix_ip_address == webhook_fix_ip_address_ && new_secret_token == webhook_secret_token_ && @@ -8591,6 +8597,7 @@ td::Status Client::process_set_webhook_query(PromisedQueryPtr &query) { // wait for webhook_close callback webhook_query_type_ = WebhookQueryType::Cancel; + CHECK(!active_webhook_set_query_); webhook_set_query_ = std::move(query); return Status::OK(); } @@ -8804,51 +8811,82 @@ void Client::do_set_webhook(PromisedQueryPtr query, bool was_deleted) { return fail_query(400, "Bad Request: secret token contains unallowed characters", std::move(query)); } - has_webhook_certificate_ = false; - auto *cert_file_ptr = get_webhook_certificate(query.get()); - if (cert_file_ptr != nullptr) { - auto size = cert_file_ptr->size; - if (size > MAX_CERTIFICATE_FILE_SIZE) { - return fail_query(400, PSLICE() << "Bad Request: certificate size is too big (" << size << " bytes)", - std::move(query)); - } - auto from_path = cert_file_ptr->temp_file_name; - auto to_path = get_webhook_certificate_path(); - auto status = td::copy_file(from_path, to_path, size); - if (status.is_error()) { - return fail_query(500, "Internal Server Error: failed to save certificate", std::move(query)); - } - has_webhook_certificate_ = true; - } else if (query->is_internal() && query->arg("certificate") == "previous") { - has_webhook_certificate_ = true; + if (active_webhook_set_query_) { + // shouldn't happen, unless the active setWebhook request took more than 1 second + return query->set_retry_after_error(1); } - webhook_url_ = new_url.str(); - webhook_set_time_ = td::Time::now(); - webhook_max_connections_ = get_webhook_max_connections(query.get()); - webhook_secret_token_ = secret_token.str(); - webhook_ip_address_ = query->arg("ip_address").str(); - webhook_fix_ip_address_ = get_webhook_fix_ip_address(query.get()); - last_webhook_error_date_ = 0; - last_webhook_error_ = Status::OK(); - - update_allowed_update_types(query.get()); - - LOG(WARNING) << "Create " << (has_webhook_certificate_ ? "" : "not ") << "self signed webhook: " << url.ok(); - auto webhook_actor_name = PSTRING() << "Webhook " << url.ok(); - webhook_id_ = td::create_actor( - webhook_actor_name, actor_shared(this, webhook_generation_), tqueue_id_, url.move_as_ok(), - has_webhook_certificate_ ? get_webhook_certificate_path() : "", webhook_max_connections_, query->is_internal(), - webhook_ip_address_, webhook_fix_ip_address_, webhook_secret_token_, parameters_); - // wait for webhook verified or webhook callback - webhook_query_type_ = WebhookQueryType::Verify; - webhook_set_query_ = std::move(query); + CHECK(!has_webhook_certificate_); + if (query->is_internal()) { + has_webhook_certificate_ = query->arg("certificate") == "previous"; + } else { + auto *cert_file_ptr = get_webhook_certificate(query.get()); + if (cert_file_ptr != nullptr) { + auto size = cert_file_ptr->size; + if (size > MAX_CERTIFICATE_FILE_SIZE) { + return fail_query(400, PSLICE() << "Bad Request: certificate size is too big (" << size << " bytes)", + std::move(query)); + } + CHECK(!webhook_set_query_); + active_webhook_set_query_ = std::move(query); + td::Scheduler::instance()->run_on_scheduler( + get_database_scheduler_id(), [actor_id = actor_id(this), from_path = cert_file_ptr->temp_file_name, + to_path = get_webhook_certificate_path(), size](td::Unit) mutable { + LOG(INFO) << "Copy certificate to " << to_path; + auto status = td::copy_file(from_path, to_path, size); + send_closure(actor_id, &Client::on_webhook_certificate_copied, std::move(status)); + }); + return; + } + } + finish_set_webhook(std::move(query)); } else { answer_query(td::JsonTrue(), std::move(query), was_deleted ? Slice("Webhook was deleted") : Slice("Webhook is already deleted")); } } +void Client::on_webhook_certificate_copied(Status status) { + CHECK(active_webhook_set_query_); + if (status.is_error()) { + return fail_query(500, "Internal Server Error: failed to save certificate", std::move(active_webhook_set_query_)); + } + has_webhook_certificate_ = true; + finish_set_webhook(std::move(active_webhook_set_query_)); +} + +void Client::finish_set_webhook(PromisedQueryPtr query) { + CHECK(!active_webhook_set_query_); + CHECK(!webhook_set_query_); + CHECK(webhook_url_.empty()); + Slice new_url = query->arg("url"); + CHECK(!new_url.empty()); + webhook_url_ = new_url.str(); + webhook_set_time_ = td::Time::now(); + webhook_max_connections_ = get_webhook_max_connections(query.get()); + webhook_secret_token_ = query->arg("secret_token").str(); + webhook_ip_address_ = query->arg("ip_address").str(); + webhook_fix_ip_address_ = get_webhook_fix_ip_address(query.get()); + last_webhook_error_date_ = 0; + last_webhook_error_ = Status::OK(); + + update_allowed_update_types(query.get()); + + auto url = td::parse_url(new_url, td::HttpUrl::Protocol::Https); + CHECK(url.is_ok()); + + LOG(WARNING) << "Create " << (has_webhook_certificate_ ? "self-signed " : "") << "webhook: " << new_url; + auto webhook_actor_name = PSTRING() << "Webhook " << url.ok(); + webhook_id_ = td::create_actor( + webhook_actor_name, actor_shared(this, webhook_generation_), tqueue_id_, url.move_as_ok(), + has_webhook_certificate_ ? get_webhook_certificate_path() : td::string(), webhook_max_connections_, + query->is_internal(), webhook_ip_address_, webhook_fix_ip_address_, webhook_secret_token_, parameters_); + // wait for webhook verified or webhook callback + webhook_query_type_ = WebhookQueryType::Verify; + CHECK(!active_webhook_set_query_); + webhook_set_query_ = std::move(query); +} + void Client::do_send_message(object_ptr input_message_content, PromisedQueryPtr query) { auto chat_id = query->arg("chat_id"); auto reply_to_message_id = get_message_id(query.get(), "reply_to_message_id"); diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index 1e31abc..a5f8874 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -572,6 +572,8 @@ class Client final : public WebhookActor::Callback { int32 get_webhook_max_connections(const Query *query) const; static bool get_webhook_fix_ip_address(const Query *query); void do_set_webhook(PromisedQueryPtr query, bool was_deleted); + void on_webhook_certificate_copied(Status status); + void finish_set_webhook(PromisedQueryPtr query); void save_webhook() const; td::string get_webhook_certificate_path() const; @@ -1033,6 +1035,7 @@ class Client final : public WebhookActor::Callback { WebhookQueryType webhook_query_type_ = WebhookQueryType::Cancel; td::ActorOwn webhook_id_; PromisedQueryPtr webhook_set_query_; + PromisedQueryPtr active_webhook_set_query_; td::string webhook_url_; double webhook_set_time_ = 0; int32 webhook_max_connections_ = 0; From 2cca516445d1201b23b578ce5639bb850a0b9c90 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 4 Oct 2022 17:05:35 +0300 Subject: [PATCH 14/48] Fail pending setWebhook queries during closing. --- telegram-bot-api/Client.cpp | 23 ++++++++++++++++++----- telegram-bot-api/Client.h | 2 ++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index bd43527..efc40af 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -7014,11 +7014,8 @@ void Client::on_cmd(PromisedQueryPtr query) { } } - if (logging_out_) { - return fail_query(LOGGING_OUT_ERROR_CODE, get_logging_out_error_description(), std::move(query)); - } - if (closing_) { - return fail_query(CLOSING_ERROR_CODE, CLOSING_ERROR_DESCRIPTION, std::move(query)); + if (logging_out_ || closing_) { + return fail_query_closing(std::move(query)); } CHECK(was_authorized_); @@ -8791,6 +8788,9 @@ bool Client::get_webhook_fix_ip_address(const Query *query) { void Client::do_set_webhook(PromisedQueryPtr query, bool was_deleted) { CHECK(webhook_url_.empty()); + if (logging_out_ || closing_) { + return fail_query_closing(std::move(query)); + } if (to_bool(query->arg("drop_pending_updates"))) { clear_tqueue(); } @@ -8859,6 +8859,9 @@ void Client::finish_set_webhook(PromisedQueryPtr query) { CHECK(!active_webhook_set_query_); CHECK(!webhook_set_query_); CHECK(webhook_url_.empty()); + if (logging_out_ || closing_) { + return fail_query_closing(std::move(query)); + } Slice new_url = query->arg("url"); CHECK(!new_url.empty()); webhook_url_ = new_url.str(); @@ -8985,6 +8988,16 @@ void Client::fail_query_conflict(Slice message, PromisedQueryPtr &&query) { } } +void Client::fail_query_closing(PromisedQueryPtr &&query) const { + if (logging_out_) { + return fail_query(LOGGING_OUT_ERROR_CODE, get_logging_out_error_description(), std::move(query)); + } + if (closing_) { + return fail_query(CLOSING_ERROR_CODE, CLOSING_ERROR_DESCRIPTION, std::move(query)); + } + UNREACHABLE(); +} + class Client::JsonUpdates final : public Jsonable { public: explicit JsonUpdates(td::Span updates) : updates_(updates) { diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index a5f8874..dac3c0a 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -600,6 +600,8 @@ class Client final : public WebhookActor::Callback { void abort_long_poll(bool from_set_webhook); + void fail_query_closing(PromisedQueryPtr &&query) const; + void fail_query_conflict(Slice message, PromisedQueryPtr &&query); static void fail_query_with_error(PromisedQueryPtr query, int32 error_code, Slice error_message, From 04825c4b70a27887b1e0fd7b6e333147c0ba23e8 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 4 Oct 2022 18:27:50 +0300 Subject: [PATCH 15/48] Move ClientManager to a separate thread. --- telegram-bot-api/Client.cpp | 12 ++++-------- telegram-bot-api/ClientManager.cpp | 6 ++---- telegram-bot-api/ClientParameters.h | 4 ++-- telegram-bot-api/Query.cpp | 4 ++-- telegram-bot-api/Query.h | 4 ++-- telegram-bot-api/telegram-bot-api.cpp | 24 +++++++++++++++--------- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index efc40af..6a34008 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -3893,8 +3893,8 @@ void Client::start_up() { }; td::ClientActor::Options options; options.net_query_stats = parameters_->net_query_stats_; - td_client_ = td::create_actor("TdClientActor", td::make_unique(actor_id(this)), - std::move(options)); + td_client_ = td::create_actor_on_scheduler( + "TdClientActor", 0, td::make_unique(actor_id(this)), std::move(options)); } void Client::send(PromisedQueryPtr query) { @@ -4993,16 +4993,12 @@ void Client::timeout_expired() { td::int32 Client::get_database_scheduler_id() { // the same scheduler as for database in Td - auto current_scheduler_id = td::Scheduler::instance()->sched_id(); - auto scheduler_count = td::Scheduler::instance()->sched_count(); - return td::min(current_scheduler_id + 1, scheduler_count - 1); + return 1; } td::int32 Client::get_file_gc_scheduler_id() { // the same scheduler as for file GC in Td - auto current_scheduler_id = td::Scheduler::instance()->sched_id(); - auto scheduler_count = td::Scheduler::instance()->sched_count(); - return td::min(current_scheduler_id + 2, scheduler_count - 1); + return 2; } void Client::clear_tqueue() { diff --git a/telegram-bot-api/ClientManager.cpp b/telegram-bot-api/ClientManager.cpp index bbc7ecf..b7174e1 100644 --- a/telegram-bot-api/ClientManager.cpp +++ b/telegram-bot-api/ClientManager.cpp @@ -232,7 +232,7 @@ void ClientManager::get_stats(td::Promise promise, sb << "buffer_memory\t" << td::format::as_size(td::BufferAllocator::get_buffer_mem()) << '\n'; sb << "active_webhook_connections\t" << WebhookActor::get_total_connections_count() << '\n'; - sb << "active_requests\t" << parameters_->shared_data_->query_count_.load() << '\n'; + sb << "active_requests\t" << parameters_->shared_data_->query_count_.load(std::memory_order_relaxed) << '\n'; sb << "active_network_queries\t" << td::get_pending_network_query_count(*parameters_->net_query_stats_) << '\n'; auto stats = stat_.as_vector(now); for (auto &stat : stats) { @@ -294,9 +294,7 @@ td::int64 ClientManager::get_tqueue_id(td::int64 user_id, bool is_test_dc) { void ClientManager::start_up() { //NB: the same scheduler as for database in Td - auto current_scheduler_id = td::Scheduler::instance()->sched_id(); - auto scheduler_count = td::Scheduler::instance()->sched_count(); - auto scheduler_id = td::min(current_scheduler_id + 1, scheduler_count - 1); + auto scheduler_id = 1; // init tqueue { diff --git a/telegram-bot-api/ClientParameters.h b/telegram-bot-api/ClientParameters.h index 455e72f..b4d8f93 100644 --- a/telegram-bot-api/ClientParameters.h +++ b/telegram-bot-api/ClientParameters.h @@ -29,10 +29,10 @@ namespace telegram_bot_api { struct SharedData { std::atomic query_count_{0}; + std::atomic query_list_size_{0}; std::atomic next_verbosity_level_{-1}; - // not thread-safe - size_t query_list_size_ = 0; + // not thread-safe, must be used from a single thread td::ListNode query_list_; td::unique_ptr webhook_db_; td::unique_ptr tqueue_; diff --git a/telegram-bot-api/Query.cpp b/telegram-bot-api/Query.cpp index 32ff8b4..e917ba3 100644 --- a/telegram-bot-api/Query.cpp +++ b/telegram-bot-api/Query.cpp @@ -45,9 +45,9 @@ Query::Query(td::vector &&container, td::Slice token, bool is_t start_timestamp_ = td::Time::now(); LOG(INFO) << "QUERY: create " << td::tag("ptr", this) << *this; if (shared_data_) { - shared_data_->query_count_++; + shared_data_->query_count_.fetch_add(1, std::memory_order_relaxed); if (method_ != "getupdates") { - shared_data_->query_list_size_++; + shared_data_->query_list_size_.fetch_add(1, std::memory_order_relaxed); shared_data_->query_list_.put(this); } } diff --git a/telegram-bot-api/Query.h b/telegram-bot-api/Query.h index c315fd5..ca67fa3 100644 --- a/telegram-bot-api/Query.h +++ b/telegram-bot-api/Query.h @@ -109,9 +109,9 @@ class Query final : public td::ListNode { Query &operator=(Query &&) = delete; ~Query() { if (shared_data_) { - shared_data_->query_count_--; + shared_data_->query_count_.fetch_sub(1, std::memory_order_relaxed); if (!empty()) { - shared_data_->query_list_size_--; + shared_data_->query_list_size_.fetch_sub(1, std::memory_order_relaxed); } } } diff --git a/telegram-bot-api/telegram-bot-api.cpp b/telegram-bot-api/telegram-bot-api.cpp index ac91d78..0737655 100644 --- a/telegram-bot-api/telegram-bot-api.cpp +++ b/telegram-bot-api/telegram-bot-api.cpp @@ -158,10 +158,10 @@ static void dump_statistics(const std::shared_ptr &shared_data, LOG(WARNING) << td::tag("buffer_mem", td::format::as_size(td::BufferAllocator::get_buffer_mem())); LOG(WARNING) << td::tag("buffer_slice_size", td::format::as_size(td::BufferAllocator::get_buffer_slice_size())); - auto query_list_size = shared_data->query_list_size_; - auto query_count = shared_data->query_count_.load(); + auto query_list_size = shared_data->query_list_size_.load(std::memory_order_relaxed); + auto query_count = shared_data->query_count_.load(std::memory_order_relaxed); LOG(WARNING) << td::tag("pending queries", query_count) << td::tag("pending requests", query_list_size); - +/* td::uint64 i = 0; bool was_gap = false; for (auto end = &shared_data->query_list_, cur = end->prev; cur != end; cur = cur->prev, i++) { @@ -175,7 +175,7 @@ static void dump_statistics(const std::shared_ptr &shared_data, was_gap = true; } } - +*/ td::dump_pending_network_queries(*net_query_stats); } @@ -504,7 +504,12 @@ int main(int argc, char *argv[]) { // << (td::GitInfo::is_dirty() ? "(dirty)" : "") << " started"; LOG(WARNING) << "Bot API " << parameters->version_ << " server started"; - const int thread_count = 6; // +3 for Td, one for watchdog, one for slow HTTP connections, one for DNS resolving + // +3 threads for Td + // one thread for ClientManager and all Clients + // one thread for watchdog + // one thread for slow HTTP connections + // one thread for DNS resolving + const int thread_count = 7; td::ConcurrentScheduler sched(thread_count, cpu_affinity); td::GetHostByNameActor::Options get_host_by_name_options; @@ -514,11 +519,12 @@ int main(int argc, char *argv[]) { .release(); auto client_manager = - sched.create_actor_unsafe(0, "ClientManager", std::move(parameters), token_range).release(); + sched.create_actor_unsafe(thread_count - 3, "ClientManager", std::move(parameters), token_range) + .release(); sched .create_actor_unsafe( - 0, "HttpServer", http_ip_address, http_port, + thread_count - 3, "HttpServer", http_ip_address, http_port, [client_manager, shared_data] { return td::ActorOwn( td::create_actor("HttpConnection", client_manager, shared_data)); @@ -528,7 +534,7 @@ int main(int argc, char *argv[]) { if (http_stat_port != 0) { sched .create_actor_unsafe( - 0, "HttpStatsServer", http_stat_ip_address, http_stat_port, + thread_count - 3, "HttpStatsServer", http_stat_ip_address, http_stat_port, [client_manager] { return td::ActorOwn( td::create_actor("HttpStatConnection", client_manager)); @@ -613,7 +619,7 @@ int main(int argc, char *argv[]) { next_watchdog_kick_time = now + WATCHDOG_TIMEOUT / 2; } - if (now > last_tqueue_gc_time + 60.0) { + if (now > last_tqueue_gc_time + 60.0 && false) { auto unix_time = shared_data->get_unix_time(now); LOG(INFO) << "Run TQueue GC at " << unix_time; last_tqueue_gc_time = now; From 337b657f9cee6046a2da41832a8931d717d15da7 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 5 Oct 2022 00:06:48 +0300 Subject: [PATCH 16/48] Add watchdog for ClientManager's thread. --- telegram-bot-api/ClientManager.cpp | 13 +++++++++++++ telegram-bot-api/ClientManager.h | 6 ++++++ telegram-bot-api/Watchdog.cpp | 2 +- telegram-bot-api/telegram-bot-api.cpp | 2 +- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/telegram-bot-api/ClientManager.cpp b/telegram-bot-api/ClientManager.cpp index b7174e1..a7fcf9c 100644 --- a/telegram-bot-api/ClientManager.cpp +++ b/telegram-bot-api/ClientManager.cpp @@ -352,6 +352,11 @@ void ClientManager::start_up() { auto query = get_webhook_restore_query(key_value.first, key_value.second, parameters_->shared_data_); send_closure_later(actor_id(this), &ClientManager::send, std::move(query)); } + + // launch watchdog + watchdog_id_ = td::create_actor_on_scheduler( + "ManagerWatchdog", td::Scheduler::instance()->sched_count() - 3, td::this_thread::get_id(), WATCHDOG_TIMEOUT); + set_timeout_in(600.0); } PromisedQueryPtr ClientManager::get_webhook_restore_query(td::Slice token, td::Slice webhook_info, @@ -429,6 +434,11 @@ void ClientManager::raw_event(const td::Event::Raw &event) { } } +void ClientManager::timeout_expired() { + send_closure(watchdog_id_, &Watchdog::kick); + set_timeout_in(WATCHDOG_TIMEOUT / 2); +} + void ClientManager::hangup_shared() { auto id = get_link_token(); auto *info = clients_.get(id); @@ -458,6 +468,7 @@ void ClientManager::close_db() { void ClientManager::finish_close() { LOG(WARNING) << "Stop ClientManager"; + watchdog_id_.reset(); auto promises = std::move(close_promises_); for (auto &promise : promises) { promise.set_value(td::Unit()); @@ -465,4 +476,6 @@ void ClientManager::finish_close() { stop(); } +constexpr double ClientManager::WATCHDOG_TIMEOUT; + } // namespace telegram_bot_api diff --git a/telegram-bot-api/ClientManager.h b/telegram-bot-api/ClientManager.h index 921aaf0..21ae1cd 100644 --- a/telegram-bot-api/ClientManager.h +++ b/telegram-bot-api/ClientManager.h @@ -9,6 +9,7 @@ #include "telegram-bot-api/Client.h" #include "telegram-bot-api/Query.h" #include "telegram-bot-api/Stats.h" +#include "telegram-bot-api/Watchdog.h" #include "td/actor/actor.h" @@ -68,6 +69,10 @@ class ClientManager final : public td::Actor { bool close_flag_ = false; td::vector> close_promises_; + td::ActorOwn watchdog_id_; + + static constexpr double WATCHDOG_TIMEOUT = 0.5; + static td::int64 get_tqueue_id(td::int64 user_id, bool is_test_dc); static PromisedQueryPtr get_webhook_restore_query(td::Slice token, td::Slice webhook_info, @@ -75,6 +80,7 @@ class ClientManager final : public td::Actor { void start_up() final; void raw_event(const td::Event::Raw &event) final; + void timeout_expired() final; void hangup_shared() final; void close_db(); void finish_close(); diff --git a/telegram-bot-api/Watchdog.cpp b/telegram-bot-api/Watchdog.cpp index d715574..362b5f9 100644 --- a/telegram-bot-api/Watchdog.cpp +++ b/telegram-bot-api/Watchdog.cpp @@ -13,7 +13,7 @@ namespace telegram_bot_api { void Watchdog::kick() { auto now = td::Time::now(); if (now >= last_kick_time_ + timeout_ && last_kick_time_ > 0) { - LOG(ERROR) << "Watchdog timeout expired after " << now - last_kick_time_ << " seconds"; + LOG(ERROR) << get_name() << " timeout expired after " << now - last_kick_time_ << " seconds"; td::thread::send_real_time_signal(main_thread_id_, 2); } last_kick_time_ = now; diff --git a/telegram-bot-api/telegram-bot-api.cpp b/telegram-bot-api/telegram-bot-api.cpp index 0737655..76a0257 100644 --- a/telegram-bot-api/telegram-bot-api.cpp +++ b/telegram-bot-api/telegram-bot-api.cpp @@ -506,7 +506,7 @@ int main(int argc, char *argv[]) { // +3 threads for Td // one thread for ClientManager and all Clients - // one thread for watchdog + // one thread for watchdogs // one thread for slow HTTP connections // one thread for DNS resolving const int thread_count = 7; From 022bed651c874c844ae3ee4da46a6224b8695855 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 6 Oct 2022 21:42:33 +0300 Subject: [PATCH 17/48] Move TQueue::run_gc to ClientManager. --- telegram-bot-api/ClientManager.cpp | 16 ++++++++++++++++ telegram-bot-api/ClientManager.h | 3 +++ telegram-bot-api/telegram-bot-api.cpp | 21 +-------------------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/telegram-bot-api/ClientManager.cpp b/telegram-bot-api/ClientManager.cpp index a7fcf9c..96a6b11 100644 --- a/telegram-bot-api/ClientManager.cpp +++ b/telegram-bot-api/ClientManager.cpp @@ -332,6 +332,7 @@ void ClientManager::start_up() { LOG(WARNING) << "Loaded " << loaded_event_count << " TQueue events in " << (td::Time::now() - load_start_time) << " seconds"; + last_tqueue_gc_time_ = td::Time::now(); } // init webhook_db @@ -437,6 +438,21 @@ void ClientManager::raw_event(const td::Event::Raw &event) { void ClientManager::timeout_expired() { send_closure(watchdog_id_, &Watchdog::kick); set_timeout_in(WATCHDOG_TIMEOUT / 2); + + double now = td::Time::now(); + if (now > last_tqueue_gc_time_ + 60.0) { + auto unix_time = parameters_->shared_data_->get_unix_time(now); + LOG(INFO) << "Run TQueue GC at " << unix_time; + last_tqueue_gc_time_ = now; + auto deleted_events = parameters_->shared_data_->tqueue_->run_gc(unix_time); + LOG(INFO) << "TQueue GC deleted " << deleted_events << " events"; + + tqueue_deleted_events_ += deleted_events; + if (tqueue_deleted_events_ > last_tqueue_deleted_events_ + 10000) { + LOG(WARNING) << "TQueue GC already deleted " << tqueue_deleted_events_ << " events since the start"; + last_tqueue_deleted_events_ = tqueue_deleted_events_; + } + } } void ClientManager::hangup_shared() { diff --git a/telegram-bot-api/ClientManager.h b/telegram-bot-api/ClientManager.h index 21ae1cd..0014622 100644 --- a/telegram-bot-api/ClientManager.h +++ b/telegram-bot-api/ClientManager.h @@ -70,6 +70,9 @@ class ClientManager final : public td::Actor { td::vector> close_promises_; td::ActorOwn watchdog_id_; + double last_tqueue_gc_time_ = 0.0; + td::int64 tqueue_deleted_events_ = 0; + td::int64 last_tqueue_deleted_events_ = 0; static constexpr double WATCHDOG_TIMEOUT = 0.5; diff --git a/telegram-bot-api/telegram-bot-api.cpp b/telegram-bot-api/telegram-bot-api.cpp index 76a0257..dbf5d69 100644 --- a/telegram-bot-api/telegram-bot-api.cpp +++ b/telegram-bot-api/telegram-bot-api.cpp @@ -16,7 +16,6 @@ #include "td/telegram/ClientActor.h" #include "td/db/binlog/Binlog.h" -#include "td/db/TQueue.h" #include "td/net/GetHostByNameActor.h" #include "td/net/HttpInboundConnection.h" @@ -161,7 +160,7 @@ static void dump_statistics(const std::shared_ptr &shared_data, auto query_list_size = shared_data->query_list_size_.load(std::memory_order_relaxed); auto query_count = shared_data->query_count_.load(std::memory_order_relaxed); LOG(WARNING) << td::tag("pending queries", query_count) << td::tag("pending requests", query_list_size); -/* + /* td::uint64 i = 0; bool was_gap = false; for (auto end = &shared_data->query_list_, cur = end->prev; cur != end; cur = cur->prev, i++) { @@ -551,9 +550,6 @@ int main(int argc, char *argv[]) { double next_watchdog_kick_time = start_time; double next_cron_time = start_time; double last_dump_time = start_time - 1000.0; - double last_tqueue_gc_time = start_time - 1000.0; - td::int64 tqueue_deleted_events = 0; - td::int64 last_tqueue_deleted_events = 0; bool close_flag = false; std::atomic_bool can_quit{false}; ServerCpuStat::instance(); // create ServerCpuStat instance @@ -619,21 +615,6 @@ int main(int argc, char *argv[]) { next_watchdog_kick_time = now + WATCHDOG_TIMEOUT / 2; } - if (now > last_tqueue_gc_time + 60.0 && false) { - auto unix_time = shared_data->get_unix_time(now); - LOG(INFO) << "Run TQueue GC at " << unix_time; - last_tqueue_gc_time = now; - auto guard = sched.get_main_guard(); - auto deleted_events = shared_data->tqueue_->run_gc(unix_time); - LOG(INFO) << "TQueue GC deleted " << deleted_events << " events"; - - tqueue_deleted_events += deleted_events; - if (tqueue_deleted_events > last_tqueue_deleted_events + 10000) { - LOG(WARNING) << "TQueue GC already deleted " << tqueue_deleted_events << " events since the start"; - last_tqueue_deleted_events = tqueue_deleted_events; - } - } - if (now > last_dump_time + 300.0) { last_dump_time = now; dump_statistics(shared_data, net_query_stats); From 3be8cb63233b495033ef098cd4112b97335fdbdf Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 6 Oct 2022 22:18:36 +0300 Subject: [PATCH 18/48] Move dump_statistics to ClientManager. --- telegram-bot-api/Client.cpp | 2 +- telegram-bot-api/ClientManager.cpp | 59 +++++++++++++++++++- telegram-bot-api/ClientManager.h | 2 + telegram-bot-api/telegram-bot-api.cpp | 77 +++------------------------ 4 files changed, 69 insertions(+), 71 deletions(-) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 6a34008..717e391 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -6806,7 +6806,7 @@ td::Result> Client::get_input_me auto send_email_address_to_provider = to_bool(query->arg("send_email_to_provider")); auto is_flexible = to_bool(query->arg("is_flexible")); - td_api::object_ptr extended_media; + object_ptr extended_media; if (!query->arg("extended_media").empty()) { TRY_RESULT_ASSIGN(extended_media, get_input_media(query, "extended_media")); } diff --git a/telegram-bot-api/ClientManager.cpp b/telegram-bot-api/ClientManager.cpp index 96a6b11..53412b8 100644 --- a/telegram-bot-api/ClientManager.cpp +++ b/telegram-bot-api/ClientManager.cpp @@ -6,7 +6,6 @@ // #include "telegram-bot-api/ClientManager.h" -#include "telegram-bot-api/Client.h" #include "telegram-bot-api/ClientParameters.h" #include "telegram-bot-api/WebhookActor.h" @@ -36,6 +35,8 @@ #include "td/utils/StringBuilder.h" #include "td/utils/Time.h" +#include "memprof/memprof.h" + #include #include @@ -48,6 +49,7 @@ void ClientManager::close(td::Promise &&promise) { } close_flag_ = true; + dump_statistics(); auto ids = clients_.ids(); for (auto id : ids) { auto *client_info = clients_.get(id); @@ -419,6 +421,61 @@ PromisedQueryPtr ClientManager::get_webhook_restore_query(td::Slice token, td::S return PromisedQueryPtr(query.release(), PromiseDeleter(td::Promise>())); } +void ClientManager::dump_statistics() { + if (is_memprof_on()) { + LOG(WARNING) << "Memory dump:"; + td::vector v; + dump_alloc([&](const AllocInfo &info) { v.push_back(info); }); + std::sort(v.begin(), v.end(), [](const AllocInfo &a, const AllocInfo &b) { return a.size > b.size; }); + size_t total_size = 0; + size_t other_size = 0; + int count = 0; + for (auto &info : v) { + if (count++ < 50) { + LOG(WARNING) << td::format::as_size(info.size) << td::format::as_array(info.backtrace); + } else { + other_size += info.size; + } + total_size += info.size; + } + LOG(WARNING) << td::tag("other", td::format::as_size(other_size)); + LOG(WARNING) << td::tag("total size", td::format::as_size(total_size)); + LOG(WARNING) << td::tag("total traces", get_ht_size()); + LOG(WARNING) << td::tag("fast_backtrace_success_rate", get_fast_backtrace_success_rate()); + } + auto r_mem_stat = td::mem_stat(); + if (r_mem_stat.is_ok()) { + auto mem_stat = r_mem_stat.move_as_ok(); + LOG(WARNING) << td::tag("rss", td::format::as_size(mem_stat.resident_size_)); + LOG(WARNING) << td::tag("vm", td::format::as_size(mem_stat.virtual_size_)); + LOG(WARNING) << td::tag("rss_peak", td::format::as_size(mem_stat.resident_size_peak_)); + LOG(WARNING) << td::tag("vm_peak", td::format::as_size(mem_stat.virtual_size_peak_)); + } + LOG(WARNING) << td::tag("buffer_mem", td::format::as_size(td::BufferAllocator::get_buffer_mem())); + LOG(WARNING) << td::tag("buffer_slice_size", td::format::as_size(td::BufferAllocator::get_buffer_slice_size())); + + const auto &shared_data = parameters_->shared_data_; + auto query_list_size = shared_data->query_list_size_.load(std::memory_order_relaxed); + auto query_count = shared_data->query_count_.load(std::memory_order_relaxed); + LOG(WARNING) << td::tag("pending queries", query_count) << td::tag("pending requests", query_list_size); + + td::uint64 i = 0; + bool was_gap = false; + for (auto end = &shared_data->query_list_, cur = end->prev; cur != end; cur = cur->prev, i++) { + if (i < 20 || i > query_list_size - 20 || i % (query_list_size / 50 + 1) == 0) { + if (was_gap) { + LOG(WARNING) << "..."; + was_gap = false; + } + LOG(WARNING) << static_cast(*cur); + } else { + was_gap = true; + } + } + + td::dump_pending_network_queries(*parameters_->net_query_stats_); +} + void ClientManager::raw_event(const td::Event::Raw &event) { auto id = get_link_token(); auto *info = clients_.get(id); diff --git a/telegram-bot-api/ClientManager.h b/telegram-bot-api/ClientManager.h index 0014622..a5c29e8 100644 --- a/telegram-bot-api/ClientManager.h +++ b/telegram-bot-api/ClientManager.h @@ -42,6 +42,8 @@ class ClientManager final : public td::Actor { : parameters_(std::move(parameters)), token_range_(token_range) { } + void dump_statistics(); + void send(PromisedQueryPtr query); void get_stats(td::Promise promise, td::vector> args); diff --git a/telegram-bot-api/telegram-bot-api.cpp b/telegram-bot-api/telegram-bot-api.cpp index dbf5d69..6d99c07 100644 --- a/telegram-bot-api/telegram-bot-api.cpp +++ b/telegram-bot-api/telegram-bot-api.cpp @@ -9,12 +9,9 @@ #include "telegram-bot-api/HttpConnection.h" #include "telegram-bot-api/HttpServer.h" #include "telegram-bot-api/HttpStatConnection.h" -#include "telegram-bot-api/Query.h" #include "telegram-bot-api/Stats.h" #include "telegram-bot-api/Watchdog.h" -#include "td/telegram/ClientActor.h" - #include "td/db/binlog/Binlog.h" #include "td/net/GetHostByNameActor.h" @@ -23,13 +20,11 @@ #include "td/actor/actor.h" #include "td/actor/ConcurrentScheduler.h" -#include "td/utils/buffer.h" #include "td/utils/CombinedLog.h" #include "td/utils/common.h" #include "td/utils/crypto.h" #include "td/utils/ExitGuard.h" #include "td/utils/FileLog.h" -#include "td/utils/format.h" //#include "td/utils/GitInfo.h" #include "td/utils/logging.h" #include "td/utils/MemoryLog.h" @@ -42,7 +37,6 @@ #include "td/utils/port/rlimit.h" #include "td/utils/port/signals.h" #include "td/utils/port/stacktrace.h" -#include "td/utils/port/Stat.h" #include "td/utils/port/thread.h" #include "td/utils/port/user.h" #include "td/utils/Promise.h" @@ -52,9 +46,6 @@ #include "td/utils/Time.h" #include "td/utils/TsLog.h" -#include "memprof/memprof.h" - -#include #include #include #include @@ -123,61 +114,6 @@ static void sigsegv_signal_handler(int signum, void *addr) { fail_signal_handler(signum); } -static void dump_statistics(const std::shared_ptr &shared_data, - const std::shared_ptr &net_query_stats) { - if (is_memprof_on()) { - LOG(WARNING) << "Memory dump:"; - td::vector v; - dump_alloc([&](const AllocInfo &info) { v.push_back(info); }); - std::sort(v.begin(), v.end(), [](const AllocInfo &a, const AllocInfo &b) { return a.size > b.size; }); - size_t total_size = 0; - size_t other_size = 0; - int count = 0; - for (auto &info : v) { - if (count++ < 50) { - LOG(WARNING) << td::format::as_size(info.size) << td::format::as_array(info.backtrace); - } else { - other_size += info.size; - } - total_size += info.size; - } - LOG(WARNING) << td::tag("other", td::format::as_size(other_size)); - LOG(WARNING) << td::tag("total size", td::format::as_size(total_size)); - LOG(WARNING) << td::tag("total traces", get_ht_size()); - LOG(WARNING) << td::tag("fast_backtrace_success_rate", get_fast_backtrace_success_rate()); - } - auto r_mem_stat = td::mem_stat(); - if (r_mem_stat.is_ok()) { - auto mem_stat = r_mem_stat.move_as_ok(); - LOG(WARNING) << td::tag("rss", td::format::as_size(mem_stat.resident_size_)); - LOG(WARNING) << td::tag("vm", td::format::as_size(mem_stat.virtual_size_)); - LOG(WARNING) << td::tag("rss_peak", td::format::as_size(mem_stat.resident_size_peak_)); - LOG(WARNING) << td::tag("vm_peak", td::format::as_size(mem_stat.virtual_size_peak_)); - } - LOG(WARNING) << td::tag("buffer_mem", td::format::as_size(td::BufferAllocator::get_buffer_mem())); - LOG(WARNING) << td::tag("buffer_slice_size", td::format::as_size(td::BufferAllocator::get_buffer_slice_size())); - - auto query_list_size = shared_data->query_list_size_.load(std::memory_order_relaxed); - auto query_count = shared_data->query_count_.load(std::memory_order_relaxed); - LOG(WARNING) << td::tag("pending queries", query_count) << td::tag("pending requests", query_list_size); - /* - td::uint64 i = 0; - bool was_gap = false; - for (auto end = &shared_data->query_list_, cur = end->prev; cur != end; cur = cur->prev, i++) { - if (i < 20 || i > query_list_size - 20 || i % (query_list_size / 50 + 1) == 0) { - if (was_gap) { - LOG(WARNING) << "..."; - was_gap = false; - } - LOG(WARNING) << static_cast(*cur); - } else { - was_gap = true; - } - } -*/ - td::dump_pending_network_queries(*net_query_stats); -} - int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(VERBOSITY_NAME(FATAL)); td::ExitGuard exit_guard; @@ -416,7 +352,7 @@ int main(int argc, char *argv[]) { TRY_RESULT_PREFIX_ASSIGN(working_directory, td::realpath(working_directory, true), "Invalid working directory specified: "); if (working_directory.empty()) { - return td::Status::Error("Working directory can't be empty"); + return td::Status::Error("Empty path specified as working directory"); } if (working_directory.back() != TD_DIR_SLASH) { working_directory += TD_DIR_SLASH; @@ -567,7 +503,6 @@ int main(int argc, char *argv[]) { } LOG(WARNING) << "Stopping engine with uptime " << (td::Time::now() - start_time) << " seconds by a signal"; - dump_statistics(shared_data, net_query_stats); close_flag = true; auto guard = sched.get_main_guard(); watchdog_id.reset(); @@ -597,7 +532,8 @@ int main(int argc, char *argv[]) { if (!need_dump_log.test_and_set()) { print_log(); - dump_statistics(shared_data, net_query_stats); + auto guard = sched.get_main_guard(); + send_closure(client_manager, &ClientManager::dump_statistics); } double now = td::Time::now(); @@ -617,12 +553,15 @@ int main(int argc, char *argv[]) { if (now > last_dump_time + 300.0) { last_dump_time = now; - dump_statistics(shared_data, net_query_stats); + auto guard = sched.get_main_guard(); + send_closure(client_manager, &ClientManager::dump_statistics); } } LOG(WARNING) << "--------------------FINISH ENGINE--------------------"; - CHECK(net_query_stats.use_count() == 1); + if (net_query_stats.use_count() != 1) { + LOG(ERROR) << "NetQueryStats have leaked"; + } net_query_stats = nullptr; sched.finish(); SET_VERBOSITY_LEVEL(VERBOSITY_NAME(FATAL)); From b44bc1cabde48c77d65407d282c67dd087acde38 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 9 Oct 2022 20:16:45 +0300 Subject: [PATCH 19/48] Update TDLib and use AsyncFileLog instead of FileLog. --- td | 2 +- telegram-bot-api/Client.cpp | 2 +- telegram-bot-api/ClientManager.cpp | 4 +++- telegram-bot-api/Query.h | 1 + telegram-bot-api/Watchdog.cpp | 1 + telegram-bot-api/telegram-bot-api.cpp | 10 ++++------ 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/td b/td index a7a17b3..c1a3fa6 160000 --- a/td +++ b/td @@ -1 +1 @@ -Subproject commit a7a17b34b3c8fd3f7f6295f152746beb68f34d83 +Subproject commit c1a3fa633fbce67b8b89fee93130498db8adc039 diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 717e391..6329455 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -6593,7 +6593,7 @@ td::Result> Client::get_chat_permiss }(); if (status.is_error()) { - return Status::Error(400, PSLICE() << "Can't parse chat permissions: " << status.error().message()); + return Status::Error(400, PSLICE() << "Can't parse chat permissions: " << status.message()); } } else if (allow_legacy) { allow_legacy = false; diff --git a/telegram-bot-api/ClientManager.cpp b/telegram-bot-api/ClientManager.cpp index 53412b8..c836181 100644 --- a/telegram-bot-api/ClientManager.cpp +++ b/telegram-bot-api/ClientManager.cpp @@ -29,6 +29,7 @@ #include "td/utils/Parser.h" #include "td/utils/port/IPAddress.h" #include "td/utils/port/Stat.h" +#include "td/utils/port/thread.h" #include "td/utils/Slice.h" #include "td/utils/SliceBuilder.h" #include "td/utils/StackAllocator.h" @@ -38,6 +39,7 @@ #include "memprof/memprof.h" #include +#include #include namespace telegram_bot_api { @@ -341,7 +343,7 @@ void ClientManager::start_up() { auto concurrent_webhook_db = td::make_unique>(); auto status = concurrent_webhook_db->init(parameters_->working_directory_ + "webhooks_db.binlog", td::DbKey::empty(), scheduler_id); - LOG_IF(FATAL, status.is_error()) << "Can't open webhooks_db.binlog " << status.error(); + LOG_IF(FATAL, status.is_error()) << "Can't open webhooks_db.binlog " << status; parameters_->shared_data_->webhook_db_ = std::move(concurrent_webhook_db); auto &webhook_db = *parameters_->shared_data_->webhook_db_; diff --git a/telegram-bot-api/Query.h b/telegram-bot-api/Query.h index ca67fa3..7fdad09 100644 --- a/telegram-bot-api/Query.h +++ b/telegram-bot-api/Query.h @@ -23,6 +23,7 @@ #include "td/utils/StringBuilder.h" #include +#include #include #include diff --git a/telegram-bot-api/Watchdog.cpp b/telegram-bot-api/Watchdog.cpp index 362b5f9..8b8105e 100644 --- a/telegram-bot-api/Watchdog.cpp +++ b/telegram-bot-api/Watchdog.cpp @@ -6,6 +6,7 @@ // #include "telegram-bot-api/Watchdog.h" +#include "td/utils/logging.h" #include "td/utils/Time.h" namespace telegram_bot_api { diff --git a/telegram-bot-api/telegram-bot-api.cpp b/telegram-bot-api/telegram-bot-api.cpp index 6d99c07..f41cf25 100644 --- a/telegram-bot-api/telegram-bot-api.cpp +++ b/telegram-bot-api/telegram-bot-api.cpp @@ -20,11 +20,11 @@ #include "td/actor/actor.h" #include "td/actor/ConcurrentScheduler.h" +#include "td/utils/AsyncFileLog.h" #include "td/utils/CombinedLog.h" #include "td/utils/common.h" #include "td/utils/crypto.h" #include "td/utils/ExitGuard.h" -#include "td/utils/FileLog.h" //#include "td/utils/GitInfo.h" #include "td/utils/logging.h" #include "td/utils/MemoryLog.h" @@ -44,7 +44,6 @@ #include "td/utils/SliceBuilder.h" #include "td/utils/Status.h" #include "td/utils/Time.h" -#include "td/utils/TsLog.h" #include #include @@ -315,8 +314,7 @@ int main(int argc, char *argv[]) { log.set_second(&memory_log); td::log_interface = &log; - td::FileLog file_log; - td::TsLog ts_log(&file_log); + td::AsyncFileLog file_log; auto init_status = [&] { #if TD_HAVE_THREAD_AFFINITY @@ -408,13 +406,13 @@ int main(int argc, char *argv[]) { log_file_path = working_directory + log_file_path; } TRY_STATUS_PREFIX(file_log.init(log_file_path, log_max_file_size), "Can't open log file: "); - log.set_first(&ts_log); + log.set_first(&file_log); } return td::Status::OK(); }(); if (init_status.is_error()) { - LOG(PLAIN) << init_status.error().message(); + LOG(PLAIN) << init_status.message(); LOG(PLAIN) << options; return 1; } From 018b4fc6f71b2f5a956374bbaa0fb92444497128 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 13 Oct 2022 01:23:05 +0300 Subject: [PATCH 20/48] Stop ClientManager watchdog at the beginning of closing. --- telegram-bot-api/ClientManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telegram-bot-api/ClientManager.cpp b/telegram-bot-api/ClientManager.cpp index c836181..3a5399a 100644 --- a/telegram-bot-api/ClientManager.cpp +++ b/telegram-bot-api/ClientManager.cpp @@ -51,6 +51,7 @@ void ClientManager::close(td::Promise &&promise) { } close_flag_ = true; + watchdog_id_.reset(); dump_statistics(); auto ids = clients_.ids(); for (auto id : ids) { @@ -543,7 +544,6 @@ void ClientManager::close_db() { void ClientManager::finish_close() { LOG(WARNING) << "Stop ClientManager"; - watchdog_id_.reset(); auto promises = std::move(close_promises_); for (auto &promise : promises) { promise.set_value(td::Unit()); From aa05180325617633672682ee018c4078af97c2ca Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 14 Oct 2022 00:46:54 +0300 Subject: [PATCH 21/48] Update TDLib and improve TQueue GC. --- td | 2 +- telegram-bot-api/ClientManager.cpp | 10 ++++++---- telegram-bot-api/ClientManager.h | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/td b/td index c1a3fa6..ec86b34 160000 --- a/td +++ b/td @@ -1 +1 @@ -Subproject commit c1a3fa633fbce67b8b89fee93130498db8adc039 +Subproject commit ec86b34d557c031b0b07929734decc561315eac1 diff --git a/telegram-bot-api/ClientManager.cpp b/telegram-bot-api/ClientManager.cpp index 3a5399a..29efa83 100644 --- a/telegram-bot-api/ClientManager.cpp +++ b/telegram-bot-api/ClientManager.cpp @@ -337,7 +337,7 @@ void ClientManager::start_up() { LOG(WARNING) << "Loaded " << loaded_event_count << " TQueue events in " << (td::Time::now() - load_start_time) << " seconds"; - last_tqueue_gc_time_ = td::Time::now(); + next_tqueue_gc_time_ = td::Time::now() + 600; } // init webhook_db @@ -500,12 +500,14 @@ void ClientManager::timeout_expired() { set_timeout_in(WATCHDOG_TIMEOUT / 2); double now = td::Time::now(); - if (now > last_tqueue_gc_time_ + 60.0) { + if (now > next_tqueue_gc_time_) { auto unix_time = parameters_->shared_data_->get_unix_time(now); LOG(INFO) << "Run TQueue GC at " << unix_time; - last_tqueue_gc_time_ = now; - auto deleted_events = parameters_->shared_data_->tqueue_->run_gc(unix_time); + td::int64 deleted_events; + bool is_finished; + std::tie(deleted_events, is_finished) = parameters_->shared_data_->tqueue_->run_gc(unix_time); LOG(INFO) << "TQueue GC deleted " << deleted_events << " events"; + next_tqueue_gc_time_ = td::Time::now() + (is_finished ? 60.0 : 1.0); tqueue_deleted_events_ += deleted_events; if (tqueue_deleted_events_ > last_tqueue_deleted_events_ + 10000) { diff --git a/telegram-bot-api/ClientManager.h b/telegram-bot-api/ClientManager.h index a5c29e8..143744a 100644 --- a/telegram-bot-api/ClientManager.h +++ b/telegram-bot-api/ClientManager.h @@ -72,7 +72,7 @@ class ClientManager final : public td::Actor { td::vector> close_promises_; td::ActorOwn watchdog_id_; - double last_tqueue_gc_time_ = 0.0; + double next_tqueue_gc_time_ = 0.0; td::int64 tqueue_deleted_events_ = 0; td::int64 last_tqueue_deleted_events_ = 0; From bb8b7a67026e7ae69cc61ff88f4563cc4675e3cc Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 21 Oct 2022 13:39:32 +0300 Subject: [PATCH 22/48] Ignore dump-log and dump-stacktrace signals after crash. --- telegram-bot-api/telegram-bot-api.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/telegram-bot-api/telegram-bot-api.cpp b/telegram-bot-api/telegram-bot-api.cpp index f41cf25..55b0c57 100644 --- a/telegram-bot-api/telegram-bot-api.cpp +++ b/telegram-bot-api/telegram-bot-api.cpp @@ -83,11 +83,17 @@ void print_log() { td::signal_safe_write("------------------------\n"); } +static std::atomic_bool has_failed{false}; + static void dump_stacktrace_signal_handler(int sig) { + if (has_failed) { + return; + } td::Stacktrace::print_to_stderr(); } static void fail_signal_handler(int sig) { + has_failed = true; td::signal_safe_write_signal_number(sig); td::Stacktrace::PrintOptions options; options.use_gdb = true; @@ -105,6 +111,9 @@ static void change_verbosity_level_signal_handler(int sig) { static std::atomic_flag need_dump_log; static void dump_log_signal_handler(int sig) { + if (has_failed) { + return; + } need_dump_log.clear(); } From c1db2380bda0a7df45d1bc348bcfad3b3639dfd6 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 23 Oct 2022 11:34:24 +0300 Subject: [PATCH 23/48] Send less updates repeatedly in getUpdates. --- telegram-bot-api/Client.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 6329455..73eedef 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -8518,6 +8518,9 @@ td::Status Client::process_get_updates_query(PromisedQueryPtr &query) { if (offset == previous_get_updates_offset_ && timeout < 3 && now < previous_get_updates_start_time_ + 3.0) { timeout = 3; } + if (offset == previous_get_updates_offset_ && now < previous_get_updates_start_time_ + 0.5) { + limit = 1; + } previous_get_updates_offset_ = offset; previous_get_updates_start_time_ = now; do_get_updates(offset, limit, timeout, std::move(query)); From 90d30d9a6343aaa924ac42f2fb9c456d7ac44603 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 31 Oct 2022 12:30:11 +0300 Subject: [PATCH 24/48] Update TDLib to 1.8.8. --- td | 2 +- telegram-bot-api/Client.cpp | 56 ++++++++++++++++++++++++++----------- telegram-bot-api/Client.h | 6 ++-- 3 files changed, 45 insertions(+), 19 deletions(-) diff --git a/td b/td index ec86b34..bbe37ee 160000 --- a/td +++ b/td @@ -1 +1 @@ -Subproject commit ec86b34d557c031b0b07929734decc561315eac1 +Subproject commit bbe37ee594d97f3c7820dd23ebcd9c9b8dac51a0 diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 73eedef..d22bb98 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -355,8 +355,8 @@ class Client::JsonUser final : public Jsonable { if (user_info != nullptr && !user_info->last_name.empty()) { object("last_name", user_info->last_name); } - if (user_info != nullptr && !user_info->username.empty()) { - object("username", user_info->username); + if (user_info != nullptr && !user_info->active_usernames.empty()) { + object("username", user_info->active_usernames[0]); } if (user_info != nullptr && !user_info->language_code.empty()) { object("language_code", user_info->language_code); @@ -659,8 +659,8 @@ class Client::JsonChat final : public Jsonable { if (!user_info->last_name.empty()) { object("last_name", user_info->last_name); } - if (!user_info->username.empty()) { - object("username", user_info->username); + if (!user_info->active_usernames.empty()) { + object("username", user_info->active_usernames[0]); } object("type", "private"); if (is_full_) { @@ -707,8 +707,8 @@ class Client::JsonChat final : public Jsonable { auto supergroup_info = client_->get_supergroup_info(chat_info->supergroup_id); CHECK(supergroup_info != nullptr); - if (!supergroup_info->username.empty()) { - object("username", supergroup_info->username); + if (!supergroup_info->active_usernames.empty()) { + object("username", supergroup_info->active_usernames[0]); } if (supergroup_info->is_supergroup) { @@ -1941,6 +1941,12 @@ void Client::JsonMessage::store(JsonValueScope *scope) const { object("migrate_from_chat_id", td::JsonLong(chat_id)); break; } + case td_api::messageForumTopicCreated::ID: + break; + case td_api::messageForumTopicEdited::ID: + break; + case td_api::messageForumTopicIsClosedToggled::ID: + break; case td_api::messagePinMessage::ID: { auto content = static_cast(message_->content.get()); auto message_id = content->message_id_; @@ -3817,7 +3823,7 @@ ServerBotInfo Client::get_bot_info() const { res.token_ = bot_token_; auto user_info = get_user_info(my_id_); if (user_info != nullptr) { - res.username_ = user_info->username; + res.username_ = user_info->editable_username; } else if (!was_authorized_) { res.username_ = ""; } else { @@ -4114,7 +4120,7 @@ void Client::check_chat_access(int64 chat_id, AccessRights access_rights, const case ChatInfo::Type::Supergroup: { auto supergroup_info = get_supergroup_info(chat_info->supergroup_id); CHECK(supergroup_info != nullptr); - bool is_public = !supergroup_info->username.empty() || supergroup_info->has_location; + bool is_public = !supergroup_info->active_usernames.empty() || supergroup_info->has_location; if (supergroup_info->status->get_id() == td_api::chatMemberStatusBanned::ID) { if (supergroup_info->is_supergroup) { return fail_query(403, "Forbidden: bot was kicked from the supergroup chat", std::move(query)); @@ -4518,7 +4524,7 @@ void Client::on_update_authorization_state() { } if (!was_authorized_) { - LOG(WARNING) << "Logged in as @" << user_info->username; + LOG(WARNING) << "Logged in as @" << user_info->editable_username; was_authorized_ = true; td::send_event(parent_, td::Event::raw(static_cast(this))); update_shared_unix_time_difference(); @@ -5139,7 +5145,7 @@ td::Result> Client::get_inline_ } } if (cur_temp_bot_user_id_ >= 100000) { - return Status::Error(400, "Too much different LoginUrl bot usernames"); + return Status::Error(400, "Too many different LoginUrl bot usernames"); } auto &user_id = bot_user_ids_[bot_username]; if (user_id == 0) { @@ -6140,7 +6146,7 @@ td::Result> Client::get_chat TRY_RESULT(is_anonymous, get_json_object_bool_field(object, "is_anonymous")); return make_object( can_manage_chat, can_change_info, can_post_messages, can_edit_messages, can_delete_messages, can_invite_users, - can_restrict_members, can_pin_messages, can_promote_members, can_manage_video_chats, is_anonymous); + can_restrict_members, can_pin_messages, false, can_promote_members, can_manage_video_chats, is_anonymous); } td::Result> Client::get_chat_administrator_rights( @@ -6620,7 +6626,7 @@ td::Result> Client::get_chat_permiss } return make_object(can_send_messages, can_send_media_messages, can_send_polls, can_send_other_messages, can_add_web_page_previews, can_change_info, - can_invite_users, can_pin_messages); + can_invite_users, can_pin_messages, false); } td::Result> Client::get_input_media(const Query *query, @@ -8127,7 +8133,7 @@ td::Status Client::process_promote_chat_member_query(PromisedQueryPtr &query) { td::string(), true, make_object( can_manage_chat, can_change_info, can_post_messages, can_edit_messages, can_delete_messages, can_invite_users, - can_restrict_members, can_pin_messages, can_promote_members, can_manage_video_chats, is_anonymous)); + can_restrict_members, can_pin_messages, false, can_promote_members, can_manage_video_chats, is_anonymous)); check_chat(chat_id, AccessRights::Write, std::move(query), [this, user_id, status = std::move(status)](int64 chat_id, PromisedQueryPtr query) mutable { auto chat_info = get_chat(chat_id); @@ -9135,7 +9141,13 @@ void Client::long_poll_wakeup(bool force_flag) { void Client::add_user(UserInfo *user_info, object_ptr &&user) { user_info->first_name = std::move(user->first_name_); user_info->last_name = std::move(user->last_name_); - user_info->username = std::move(user->username_); + if (user->usernames_ == nullptr) { + user_info->active_usernames.clear(); + user_info->editable_username.clear(); + } else { + user_info->active_usernames = std::move(user->usernames_->active_usernames_); + user_info->editable_username = std::move(user->usernames_->editable_username_); + } user_info->language_code = std::move(user->language_code_); user_info->have_access = user->have_access_; @@ -9231,7 +9243,13 @@ void Client::set_group_invite_link(int64 group_id, td::string &&invite_link) { } void Client::add_supergroup(SupergroupInfo *supergroup_info, object_ptr &&supergroup) { - supergroup_info->username = std::move(supergroup->username_); + if (supergroup->usernames_ == nullptr) { + supergroup_info->active_usernames.clear(); + supergroup_info->editable_username.clear(); + } else { + supergroup_info->active_usernames = std::move(supergroup->usernames_->active_usernames_); + supergroup_info->editable_username = std::move(supergroup->usernames_->editable_username_); + } supergroup_info->date = supergroup->date_; supergroup_info->status = std::move(supergroup->status_); supergroup_info->is_supergroup = !supergroup->is_channel_; @@ -9351,7 +9369,7 @@ td::string Client::get_chat_description(int64 chat_id) const { } return PSTRING() << (supergroup_info->is_supergroup ? "supergroup" : "channel") << " chat " << chat_id << ", chat status = " << to_string(supergroup_info->status) - << ", username = " << supergroup_info->username; + << ", usernames = " << supergroup_info->active_usernames; } case ChatInfo::Type::Unknown: return PSTRING() << "unknown chat " << chat_id; @@ -9909,6 +9927,12 @@ bool Client::need_skip_update_message(int64 chat_id, const object_ptr active_usernames; + td::string editable_username; td::string language_code; object_ptr photo; @@ -670,7 +671,8 @@ class Client final : public WebhookActor::Callback { const GroupInfo *get_group_info(int64 group_id) const; struct SupergroupInfo { - td::string username; + td::vector active_usernames; + td::string editable_username; object_ptr photo; td::string description; td::string invite_link; From 1270c704994eeabe68ffcce194b7021035157eeb Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 31 Oct 2022 15:36:42 +0300 Subject: [PATCH 25/48] Add Chat.is_forum. --- telegram-bot-api/Client.cpp | 4 ++++ telegram-bot-api/Client.h | 1 + 2 files changed, 5 insertions(+) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index d22bb98..e9d2d64 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -710,6 +710,9 @@ class Client::JsonChat final : public Jsonable { if (!supergroup_info->active_usernames.empty()) { object("username", supergroup_info->active_usernames[0]); } + if (supergroup_info->is_supergroup && supergroup_info->is_forum) { + object("is_forum", td::JsonTrue()); + } if (supergroup_info->is_supergroup) { object("type", "supergroup"); @@ -9253,6 +9256,7 @@ void Client::add_supergroup(SupergroupInfo *supergroup_info, object_ptrdate = supergroup->date_; supergroup_info->status = std::move(supergroup->status_); supergroup_info->is_supergroup = !supergroup->is_channel_; + supergroup_info->is_forum = supergroup->is_forum_; supergroup_info->has_location = supergroup->has_location_; supergroup_info->join_to_send_messages = supergroup->join_to_send_messages_; supergroup_info->join_by_request = supergroup->join_by_request_; diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index caeb0d3..9b040cb 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -683,6 +683,7 @@ class Client final : public WebhookActor::Callback { object_ptr location; object_ptr status; bool is_supergroup = false; + bool is_forum = false; bool can_set_sticker_set = false; bool has_location = false; bool join_to_send_messages = false; From 1ba439264860d1899b3af7af7cc600f1a6ab494d Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 31 Oct 2022 15:49:53 +0300 Subject: [PATCH 26/48] Add Chat.active_usernames. --- telegram-bot-api/Client.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index e9d2d64..0d18143 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -664,6 +664,10 @@ class Client::JsonChat final : public Jsonable { } object("type", "private"); if (is_full_) { + if (!user_info->active_usernames.empty()) { + object("active_usernames", td::json_array(user_info->active_usernames, + [](Slice username) { return td::JsonString(username); })); + } if (!user_info->bio.empty()) { object("bio", user_info->bio); } @@ -720,6 +724,10 @@ class Client::JsonChat final : public Jsonable { object("type", "channel"); } if (is_full_) { + if (!supergroup_info->active_usernames.empty()) { + object("active_usernames", td::json_array(supergroup_info->active_usernames, + [](Slice username) { return td::JsonString(username); })); + } if (!supergroup_info->description.empty()) { object("description", supergroup_info->description); } From 5fb80085fb0883830e5465537b9d40e59c721e27 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 31 Oct 2022 15:59:15 +0300 Subject: [PATCH 27/48] Add Chat.emoji_status_custom_emoji_id. --- telegram-bot-api/Client.cpp | 4 ++++ telegram-bot-api/Client.h | 1 + 2 files changed, 5 insertions(+) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 0d18143..2666d9b 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -668,6 +668,9 @@ class Client::JsonChat final : public Jsonable { object("active_usernames", td::json_array(user_info->active_usernames, [](Slice username) { return td::JsonString(username); })); } + if (user_info->emoji_status_custom_emoji_id != 0) { + object("emoji_status_custom_emoji_id", user_info->emoji_status_custom_emoji_id); + } if (!user_info->bio.empty()) { object("bio", user_info->bio); } @@ -9160,6 +9163,7 @@ void Client::add_user(UserInfo *user_info, object_ptr &&user) { user_info->editable_username = std::move(user->usernames_->editable_username_); } user_info->language_code = std::move(user->language_code_); + user_info->emoji_status_custom_emoji_id = user->emoji_status_ != nullptr ? user->emoji_status_->custom_emoji_id_ : 0; user_info->have_access = user->have_access_; user_info->is_premium = user->is_premium_; diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index 9b040cb..281d118 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -632,6 +632,7 @@ class Client final : public WebhookActor::Callback { td::vector active_usernames; td::string editable_username; td::string language_code; + int64 emoji_status_custom_emoji_id; object_ptr photo; td::string bio; From ea978c5770e58e59c017ef8bb74ba9d50caa28fa Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 31 Oct 2022 16:12:13 +0300 Subject: [PATCH 28/48] Add can_manage_topics administrator right. --- telegram-bot-api/Client.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 2666d9b..84d34b3 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -6155,12 +6155,14 @@ td::Result> Client::get_chat TRY_RESULT(can_invite_users, get_json_object_bool_field(object, "can_invite_users")); TRY_RESULT(can_restrict_members, get_json_object_bool_field(object, "can_restrict_members")); TRY_RESULT(can_pin_messages, get_json_object_bool_field(object, "can_pin_messages")); + TRY_RESULT(can_manage_topics, get_json_object_bool_field(object, "can_manage_topics")); TRY_RESULT(can_promote_members, get_json_object_bool_field(object, "can_promote_members")); TRY_RESULT(can_manage_video_chats, get_json_object_bool_field(object, "can_manage_video_chats")); TRY_RESULT(is_anonymous, get_json_object_bool_field(object, "is_anonymous")); - return make_object( - can_manage_chat, can_change_info, can_post_messages, can_edit_messages, can_delete_messages, can_invite_users, - can_restrict_members, can_pin_messages, false, can_promote_members, can_manage_video_chats, is_anonymous); + return make_object(can_manage_chat, can_change_info, can_post_messages, + can_edit_messages, can_delete_messages, can_invite_users, + can_restrict_members, can_pin_messages, can_manage_topics, + can_promote_members, can_manage_video_chats, is_anonymous); } td::Result> Client::get_chat_administrator_rights( @@ -8139,15 +8141,17 @@ td::Status Client::process_promote_chat_member_query(PromisedQueryPtr &query) { auto can_invite_users = to_bool(query->arg("can_invite_users")); auto can_restrict_members = to_bool(query->arg("can_restrict_members")); auto can_pin_messages = to_bool(query->arg("can_pin_messages")); + auto can_manage_topics = to_bool(query->arg("can_manage_topics")); auto can_promote_members = to_bool(query->arg("can_promote_members")); auto can_manage_video_chats = to_bool(query->arg("can_manage_voice_chats")) || to_bool(query->arg("can_manage_video_chats")); auto is_anonymous = to_bool(query->arg("is_anonymous")); auto status = make_object( td::string(), true, - make_object( - can_manage_chat, can_change_info, can_post_messages, can_edit_messages, can_delete_messages, can_invite_users, - can_restrict_members, can_pin_messages, false, can_promote_members, can_manage_video_chats, is_anonymous)); + make_object(can_manage_chat, can_change_info, can_post_messages, + can_edit_messages, can_delete_messages, can_invite_users, + can_restrict_members, can_pin_messages, can_manage_topics, + can_promote_members, can_manage_video_chats, is_anonymous)); check_chat(chat_id, AccessRights::Write, std::move(query), [this, user_id, status = std::move(status)](int64 chat_id, PromisedQueryPtr query) mutable { auto chat_info = get_chat(chat_id); @@ -9471,6 +9475,9 @@ void Client::json_store_administrator_rights(td::JsonObjectScope &object, const if (chat_type == ChatType::Group || chat_type == ChatType::Supergroup) { object("can_pin_messages", td::JsonBool(rights->can_pin_messages_)); } + if (chat_type == ChatType::Supergroup) { + object("can_manage_topics", td::JsonBool(rights->can_manage_topics_)); + } object("can_promote_members", td::JsonBool(rights->can_promote_members_)); object("can_manage_video_chats", td::JsonBool(rights->can_manage_video_chats_)); object("is_anonymous", td::JsonBool(rights->is_anonymous_)); From 342d59a19d36ca1e6c6d64dad53fc47570fcd727 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 31 Oct 2022 16:15:46 +0300 Subject: [PATCH 29/48] Add can_manage_topics chat permission. --- telegram-bot-api/Client.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 84d34b3..ed60f05 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -6586,6 +6586,7 @@ td::Result> Client::get_chat_permiss auto can_change_info = false; auto can_invite_users = false; auto can_pin_messages = false; + auto can_manage_topics = false; if (query->has_arg("permissions")) { allow_legacy = false; @@ -6611,6 +6612,11 @@ td::Result> Client::get_chat_permiss TRY_RESULT_ASSIGN(can_change_info, get_json_object_bool_field(object, "can_change_info")); TRY_RESULT_ASSIGN(can_invite_users, get_json_object_bool_field(object, "can_invite_users")); TRY_RESULT_ASSIGN(can_pin_messages, get_json_object_bool_field(object, "can_pin_messages")); + if (has_json_object_field(object, "can_manage_topics")) { + TRY_RESULT_ASSIGN(can_manage_topics, get_json_object_bool_field(object, "can_manage_topics")); + } else { + can_manage_topics = can_pin_messages; + } return Status::OK(); }(); @@ -6631,6 +6637,7 @@ td::Result> Client::get_chat_permiss can_change_info = true; can_invite_users = true; can_pin_messages = true; + can_manage_topics = true; } else if (query->has_arg("can_send_messages") || query->has_arg("can_send_media_messages") || query->has_arg("can_send_other_messages") || query->has_arg("can_add_web_page_previews")) { allow_legacy = true; @@ -6642,7 +6649,7 @@ td::Result> Client::get_chat_permiss } return make_object(can_send_messages, can_send_media_messages, can_send_polls, can_send_other_messages, can_add_web_page_previews, can_change_info, - can_invite_users, can_pin_messages, false); + can_invite_users, can_pin_messages, can_manage_topics); } td::Result> Client::get_input_media(const Query *query, @@ -8257,6 +8264,7 @@ td::Status Client::process_restrict_chat_member_query(PromisedQueryPtr &query) { permissions->can_change_info_ = old_permissions->can_change_info_; permissions->can_invite_users_ = old_permissions->can_invite_users_; permissions->can_pin_messages_ = old_permissions->can_pin_messages_; + permissions->can_manage_topics_ = old_permissions->can_manage_topics_; } send_request(make_object( @@ -9492,6 +9500,7 @@ void Client::json_store_permissions(td::JsonObjectScope &object, const td_api::c object("can_change_info", td::JsonBool(permissions->can_change_info_)); object("can_invite_users", td::JsonBool(permissions->can_invite_users_)); object("can_pin_messages", td::JsonBool(permissions->can_pin_messages_)); + object("can_manage_topics", td::JsonBool(permissions->can_manage_topics_)); } Client::Slice Client::get_update_type_name(UpdateType update_type) { From b0fec0b09de963626e62ea804ed414667780e35e Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 31 Oct 2022 17:09:34 +0300 Subject: [PATCH 30/48] Add Message.message_thread_id. --- telegram-bot-api/Client.cpp | 4 ++++ telegram-bot-api/Client.h | 1 + 2 files changed, 5 insertions(+) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index ed60f05..f6d8ebb 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -1746,6 +1746,9 @@ void Client::JsonMessage::store(JsonValueScope *scope) const { if (message_->edit_date > 0) { object("edit_date", message_->edit_date); } + if (message_->message_thread_id != 0) { + object("message_thread_id", as_client_message_id(message_->message_thread_id)); + } if (message_->initial_send_date > 0) { if (message_->initial_sender_user_id != 0) { object("forward_from", JsonUser(message_->initial_sender_user_id, client_)); @@ -10343,6 +10346,7 @@ Client::FullMessageId Client::add_message(object_ptr &&message, message_info->id = message_id; message_info->chat_id = chat_id; + message_info->message_thread_id = message->message_thread_id_; message_info->date = message->date_; message_info->edit_date = message->edit_date_; message_info->media_album_id = message->media_album_id_; diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index 281d118..5e79e27 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -730,6 +730,7 @@ class Client final : public WebhookActor::Callback { int64 sender_user_id = 0; int64 sender_chat_id = 0; int64 chat_id = 0; + int64 message_thread_id = 0; int32 date = 0; int32 edit_date = 0; int64 initial_chat_id = 0; From 1c3235b402b858968f72e25c592919cce35cdf0a Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 31 Oct 2022 17:14:23 +0300 Subject: [PATCH 31/48] Add Message.is_topic_message. --- telegram-bot-api/Client.cpp | 4 ++++ telegram-bot-api/Client.h | 1 + 2 files changed, 5 insertions(+) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index f6d8ebb..65df636 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -2077,6 +2077,9 @@ void Client::JsonMessage::store(JsonValueScope *scope) const { if (!message_->can_be_saved) { object("has_protected_content", td::JsonTrue()); } + if (message_->is_topic_message) { + object("is_topic_message", td::JsonTrue()); + } } class Client::JsonDeletedMessage final : public Jsonable { @@ -10430,6 +10433,7 @@ Client::FullMessageId Client::add_message(object_ptr &&message, } message_info->can_be_saved = message->can_be_saved_; + message_info->is_topic_message = message->is_topic_message_; message_info->author_signature = std::move(message->author_signature_); if (message->reply_in_chat_id_ != chat_id && message->reply_to_message_id_ != 0) { diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index 5e79e27..fbf98b8 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -749,6 +749,7 @@ class Client final : public WebhookActor::Callback { bool can_be_saved = false; bool is_automatic_forward = false; + bool is_topic_message = false; mutable bool is_reply_to_message_deleted = false; mutable bool is_content_changed = false; }; From 8fa63c21f98b80b412e114c12967fa0fd89c233f Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 31 Oct 2022 17:35:54 +0300 Subject: [PATCH 32/48] Add "forum_topic_created" messages. --- telegram-bot-api/Client.cpp | 26 +++++++++++++++++++++++--- telegram-bot-api/Client.h | 1 + 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 65df636..8e1cc84 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -1346,6 +1346,24 @@ class Client::JsonPollAnswer final : public Jsonable { const Client *client_; }; +class Client::JsonForumTopicCreated final : public Jsonable { + public: + explicit JsonForumTopicCreated(const td_api::messageForumTopicCreated *forum_topic_created) + : forum_topic_created_(forum_topic_created) { + } + void store(JsonValueScope *scope) const { + auto object = scope->enter_object(); + object("name", forum_topic_created_->name_); + object("icon_color", forum_topic_created_->icon_->color_); + if (forum_topic_created_->icon_->custom_emoji_id_ != 0) { + object("icon_custom_emoji_id", forum_topic_created_->icon_->custom_emoji_id_); + } + } + + private: + const td_api::messageForumTopicCreated *forum_topic_created_; +}; + class Client::JsonAddress final : public Jsonable { public: explicit JsonAddress(const td_api::address *address) : address_(address) { @@ -1958,8 +1976,11 @@ void Client::JsonMessage::store(JsonValueScope *scope) const { object("migrate_from_chat_id", td::JsonLong(chat_id)); break; } - case td_api::messageForumTopicCreated::ID: + case td_api::messageForumTopicCreated::ID: { + auto content = static_cast(message_->content.get()); + object("forum_topic_created", JsonForumTopicCreated(content)); break; + } case td_api::messageForumTopicEdited::ID: break; case td_api::messageForumTopicIsClosedToggled::ID: @@ -9867,6 +9888,7 @@ bool Client::need_skip_update_message(int64 chat_id, const object_ptr Date: Mon, 31 Oct 2022 17:42:34 +0300 Subject: [PATCH 33/48] Add "forum_topic_closed"/"forum_topic_reopened" messages. --- telegram-bot-api/Client.cpp | 19 ++++++++++++++++--- telegram-bot-api/Client.h | 1 + 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 8e1cc84..39ff34c 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -1364,6 +1364,13 @@ class Client::JsonForumTopicCreated final : public Jsonable { const td_api::messageForumTopicCreated *forum_topic_created_; }; +class Client::JsonForumTopicIsClosedToggled final : public Jsonable { + public: + void store(JsonValueScope *scope) const { + auto object = scope->enter_object(); + } +}; + class Client::JsonAddress final : public Jsonable { public: explicit JsonAddress(const td_api::address *address) : address_(address) { @@ -1983,8 +1990,15 @@ void Client::JsonMessage::store(JsonValueScope *scope) const { } case td_api::messageForumTopicEdited::ID: break; - case td_api::messageForumTopicIsClosedToggled::ID: + case td_api::messageForumTopicIsClosedToggled::ID: { + auto content = static_cast(message_->content.get()); + if (content->is_closed_) { + object("forum_topic_closed", JsonForumTopicIsClosedToggled()); + } else { + object("forum_topic_reopened", JsonForumTopicIsClosedToggled()); + } break; + } case td_api::messagePinMessage::ID: { auto content = static_cast(message_->content.get()); auto message_id = content->message_id_; @@ -9889,6 +9903,7 @@ bool Client::need_skip_update_message(int64 chat_id, const object_ptr Date: Mon, 31 Oct 2022 18:16:51 +0300 Subject: [PATCH 34/48] Allow to specify message_thread_id while sending messages. --- telegram-bot-api/Client.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 39ff34c..4ead621 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -7480,6 +7480,7 @@ td::Status Client::process_forward_message_query(PromisedQueryPtr &query) { td::Status Client::process_send_media_group_query(PromisedQueryPtr &query) { auto chat_id = query->arg("chat_id"); + auto message_thread_id = get_message_id(query.get(), "message_thread_id"); auto reply_to_message_id = get_message_id(query.get(), "reply_to_message_id"); auto allow_sending_without_reply = to_bool(query->arg("allow_sending_without_reply")); auto disable_notification = to_bool(query->arg("disable_notification")); @@ -7490,10 +7491,10 @@ td::Status Client::process_send_media_group_query(PromisedQueryPtr &query) { resolve_reply_markup_bot_usernames( std::move(reply_markup), std::move(query), - [this, chat_id = chat_id.str(), reply_to_message_id, allow_sending_without_reply, disable_notification, - protect_content, input_message_contents = std::move(input_message_contents)]( + [this, chat_id = chat_id.str(), message_thread_id, reply_to_message_id, allow_sending_without_reply, + disable_notification, protect_content, input_message_contents = std::move(input_message_contents)]( object_ptr reply_markup, PromisedQueryPtr query) mutable { - auto on_success = [this, disable_notification, protect_content, + auto on_success = [this, message_thread_id, disable_notification, protect_content, input_message_contents = std::move(input_message_contents), reply_markup = std::move(reply_markup)](int64 chat_id, int64 reply_to_message_id, PromisedQueryPtr query) mutable { @@ -7503,7 +7504,7 @@ td::Status Client::process_send_media_group_query(PromisedQueryPtr &query) { } send_request( - make_object(chat_id, 0, reply_to_message_id, + make_object(chat_id, message_thread_id, reply_to_message_id, get_message_send_options(disable_notification, protect_content), std::move(input_message_contents), false), td::make_unique(this, std::move(query))); @@ -8964,6 +8965,7 @@ void Client::finish_set_webhook(PromisedQueryPtr query) { void Client::do_send_message(object_ptr input_message_content, PromisedQueryPtr query) { auto chat_id = query->arg("chat_id"); + auto message_thread_id = get_message_id(query.get(), "message_thread_id"); auto reply_to_message_id = get_message_id(query.get(), "reply_to_message_id"); auto allow_sending_without_reply = to_bool(query->arg("allow_sending_without_reply")); auto disable_notification = to_bool(query->arg("disable_notification")); @@ -8976,10 +8978,10 @@ void Client::do_send_message(object_ptr input_messa resolve_reply_markup_bot_usernames( std::move(reply_markup), std::move(query), - [this, chat_id = chat_id.str(), reply_to_message_id, allow_sending_without_reply, disable_notification, - protect_content, input_message_content = std::move(input_message_content)]( + [this, chat_id = chat_id.str(), message_thread_id, reply_to_message_id, allow_sending_without_reply, + disable_notification, protect_content, input_message_content = std::move(input_message_content)]( object_ptr reply_markup, PromisedQueryPtr query) mutable { - auto on_success = [this, disable_notification, protect_content, + auto on_success = [this, message_thread_id, disable_notification, protect_content, input_message_content = std::move(input_message_content), reply_markup = std::move(reply_markup)](int64 chat_id, int64 reply_to_message_id, PromisedQueryPtr query) mutable { @@ -8988,7 +8990,7 @@ void Client::do_send_message(object_ptr input_messa return query->set_retry_after_error(60); } - send_request(make_object(chat_id, 0, reply_to_message_id, + send_request(make_object(chat_id, message_thread_id, reply_to_message_id, get_message_send_options(disable_notification, protect_content), std::move(reply_markup), std::move(input_message_content)), td::make_unique(this, std::move(query))); From 83bcb31994de09a7a34c3a8580bc86da8260e21d Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 31 Oct 2022 18:27:17 +0300 Subject: [PATCH 35/48] Add getForumTopicIconStickers method. --- telegram-bot-api/Client.cpp | 7 +++++++ telegram-bot-api/Client.h | 1 + 2 files changed, 8 insertions(+) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 4ead621..566bfac 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -253,6 +253,7 @@ bool Client::init_methods() { methods_.emplace("unpinallchatmessages", &Client::process_unpin_all_chat_messages_query); methods_.emplace("setchatstickerset", &Client::process_set_chat_sticker_set_query); methods_.emplace("deletechatstickerset", &Client::process_delete_chat_sticker_set_query); + methods_.emplace("getforumtopiciconstickers", &Client::process_get_forum_topic_icon_stickers_query); methods_.emplace("getchatmember", &Client::process_get_chat_member_query); methods_.emplace("getchatadministrators", &Client::process_get_chat_administrators_query); methods_.emplace("getchatmembercount", &Client::process_get_chat_member_count_query); @@ -8105,6 +8106,12 @@ td::Status Client::process_delete_chat_sticker_set_query(PromisedQueryPtr &query return Status::OK(); } +td::Status Client::process_get_forum_topic_icon_stickers_query(PromisedQueryPtr &query) { + send_request(make_object(), + td::make_unique(this, std::move(query))); + return Status::OK(); +} + td::Status Client::process_get_chat_member_query(PromisedQueryPtr &query) { auto chat_id = query->arg("chat_id"); TRY_RESULT(user_id, get_user_id(query.get())); diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index e8649c0..3b8e2ef 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -536,6 +536,7 @@ class Client final : public WebhookActor::Callback { Status process_unpin_all_chat_messages_query(PromisedQueryPtr &query); Status process_set_chat_sticker_set_query(PromisedQueryPtr &query); Status process_delete_chat_sticker_set_query(PromisedQueryPtr &query); + Status process_get_forum_topic_icon_stickers_query(PromisedQueryPtr &query); Status process_get_chat_member_query(PromisedQueryPtr &query); Status process_get_chat_administrators_query(PromisedQueryPtr &query); Status process_get_chat_member_count_query(PromisedQueryPtr &query); From 1e68d491935e6235f6cc0355cdcf3e0311082b3d Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 31 Oct 2022 18:44:29 +0300 Subject: [PATCH 36/48] Add createForumTopic method. --- telegram-bot-api/Client.cpp | 53 +++++++++++++++++++++++++++++++++++++ telegram-bot-api/Client.h | 3 +++ 2 files changed, 56 insertions(+) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 566bfac..2398fc9 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -254,6 +254,7 @@ bool Client::init_methods() { methods_.emplace("setchatstickerset", &Client::process_set_chat_sticker_set_query); methods_.emplace("deletechatstickerset", &Client::process_delete_chat_sticker_set_query); methods_.emplace("getforumtopiciconstickers", &Client::process_get_forum_topic_icon_stickers_query); + methods_.emplace("createforumtopic", &Client::process_create_forum_topic_query); methods_.emplace("getchatmember", &Client::process_get_chat_member_query); methods_.emplace("getchatadministrators", &Client::process_get_chat_administrators_query); methods_.emplace("getchatmembercount", &Client::process_get_chat_member_count_query); @@ -1372,6 +1373,24 @@ class Client::JsonForumTopicIsClosedToggled final : public Jsonable { } }; +class Client::JsonForumTopicInfo final : public Jsonable { + public: + explicit JsonForumTopicInfo(const td_api::forumTopicInfo *forum_topic_info) : forum_topic_info_(forum_topic_info) { + } + void store(JsonValueScope *scope) const { + auto object = scope->enter_object(); + object("message_thread_id", as_client_message_id(forum_topic_info_->message_thread_id_)); + object("name", forum_topic_info_->name_); + object("icon_color", forum_topic_info_->icon_->color_); + if (forum_topic_info_->icon_->custom_emoji_id_ != 0) { + object("icon_custom_emoji_id", forum_topic_info_->icon_->custom_emoji_id_); + } + } + + private: + const td_api::forumTopicInfo *forum_topic_info_; +}; + class Client::JsonAddress final : public Jsonable { public: explicit JsonAddress(const td_api::address *address) : address_(address) { @@ -3500,6 +3519,25 @@ class Client::TdOnGetMyDefaultAdministratorRightsCallback final : public TdQuery PromisedQueryPtr query_; }; +class Client::TdOnGetForumTopicInfoCallback final : public TdQueryCallback { + public: + explicit TdOnGetForumTopicInfoCallback(PromisedQueryPtr query) : query_(std::move(query)) { + } + + void on_result(object_ptr result) final { + if (result->get_id() == td_api::error::ID) { + return fail_query_with_error(std::move(query_), move_object_as(result)); + } + + CHECK(result->get_id() == td_api::forumTopicInfo::ID); + auto forum_topic_info = move_object_as(result); + answer_query(JsonForumTopicInfo(forum_topic_info.get()), std::move(query_)); + } + + private: + PromisedQueryPtr query_; +}; + class Client::TdOnGetMenuButtonCallback final : public TdQueryCallback { public: explicit TdOnGetMenuButtonCallback(PromisedQueryPtr query) : query_(std::move(query)) { @@ -8112,6 +8150,21 @@ td::Status Client::process_get_forum_topic_icon_stickers_query(PromisedQueryPtr return Status::OK(); } +td::Status Client::process_create_forum_topic_query(PromisedQueryPtr &query) { + auto chat_id = query->arg("chat_id"); + auto name = query->arg("name"); + int32 icon_color = get_integer_arg(query.get(), "icon_color", 0); + auto icon_custom_emoji_id = td::to_integer(query->arg("icon_custom_emoji_id")); + + check_chat(chat_id, AccessRights::Write, std::move(query), + [this, name = name.str(), icon_color, icon_custom_emoji_id](int64 chat_id, PromisedQueryPtr query) { + send_request(make_object( + chat_id, name, make_object(icon_color, icon_custom_emoji_id)), + td::make_unique(std::move(query))); + }); + return Status::OK(); +} + td::Status Client::process_get_chat_member_query(PromisedQueryPtr &query) { auto chat_id = query->arg("chat_id"); TRY_RESULT(user_id, get_user_id(query.get())); diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index 3b8e2ef..b69f852 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -151,6 +151,7 @@ class Client final : public WebhookActor::Callback { class JsonChatJoinRequest; class JsonForumTopicCreated; class JsonForumTopicIsClosedToggled; + class JsonForumTopicInfo; class JsonGameHighScore; class JsonAddress; class JsonOrderInfo; @@ -186,6 +187,7 @@ class Client final : public WebhookActor::Callback { class TdOnGetEditedMessageCallback; class TdOnGetCallbackQueryMessageCallback; class TdOnGetStickerSetCallback; + class TdOnGetForumTopicInfoCallback; class TdOnGetMenuButtonCallback; class TdOnGetMyCommandsCallback; class TdOnGetMyDefaultAdministratorRightsCallback; @@ -537,6 +539,7 @@ class Client final : public WebhookActor::Callback { Status process_set_chat_sticker_set_query(PromisedQueryPtr &query); Status process_delete_chat_sticker_set_query(PromisedQueryPtr &query); Status process_get_forum_topic_icon_stickers_query(PromisedQueryPtr &query); + Status process_create_forum_topic_query(PromisedQueryPtr &query); Status process_get_chat_member_query(PromisedQueryPtr &query); Status process_get_chat_administrators_query(PromisedQueryPtr &query); Status process_get_chat_member_count_query(PromisedQueryPtr &query); From 1f7cd2cba5063eafbf324b8b4caee08cb6b2f17a Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 31 Oct 2022 18:55:34 +0300 Subject: [PATCH 37/48] Add editForumTopic method. --- telegram-bot-api/Client.cpp | 15 +++++++++++++++ telegram-bot-api/Client.h | 1 + 2 files changed, 16 insertions(+) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 2398fc9..64b0e42 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -255,6 +255,7 @@ bool Client::init_methods() { methods_.emplace("deletechatstickerset", &Client::process_delete_chat_sticker_set_query); methods_.emplace("getforumtopiciconstickers", &Client::process_get_forum_topic_icon_stickers_query); methods_.emplace("createforumtopic", &Client::process_create_forum_topic_query); + methods_.emplace("editforumtopic", &Client::process_edit_forum_topic_query); methods_.emplace("getchatmember", &Client::process_get_chat_member_query); methods_.emplace("getchatadministrators", &Client::process_get_chat_administrators_query); methods_.emplace("getchatmembercount", &Client::process_get_chat_member_count_query); @@ -8165,6 +8166,20 @@ td::Status Client::process_create_forum_topic_query(PromisedQueryPtr &query) { return Status::OK(); } +td::Status Client::process_edit_forum_topic_query(PromisedQueryPtr &query) { + auto chat_id = query->arg("chat_id"); + auto message_thread_id = get_message_id(query.get(), "message_thread_id"); + auto name = query->arg("name"); + auto icon_custom_emoji_id = td::to_integer(query->arg("icon_custom_emoji_id")); + + check_chat(chat_id, AccessRights::Write, std::move(query), + [this, message_thread_id, name = name.str(), icon_custom_emoji_id](int64 chat_id, PromisedQueryPtr query) { + send_request(make_object(chat_id, message_thread_id, name, icon_custom_emoji_id), + td::make_unique(std::move(query))); + }); + return Status::OK(); +} + td::Status Client::process_get_chat_member_query(PromisedQueryPtr &query) { auto chat_id = query->arg("chat_id"); TRY_RESULT(user_id, get_user_id(query.get())); diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index b69f852..e3a1c33 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -540,6 +540,7 @@ class Client final : public WebhookActor::Callback { Status process_delete_chat_sticker_set_query(PromisedQueryPtr &query); Status process_get_forum_topic_icon_stickers_query(PromisedQueryPtr &query); Status process_create_forum_topic_query(PromisedQueryPtr &query); + Status process_edit_forum_topic_query(PromisedQueryPtr &query); Status process_get_chat_member_query(PromisedQueryPtr &query); Status process_get_chat_administrators_query(PromisedQueryPtr &query); Status process_get_chat_member_count_query(PromisedQueryPtr &query); From 82ec9dc608b8198695e5a75edf010dd8bab13e8b Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 31 Oct 2022 19:11:29 +0300 Subject: [PATCH 38/48] Add closeTopicForum method. --- telegram-bot-api/Client.cpp | 13 +++++++++++++ telegram-bot-api/Client.h | 1 + 2 files changed, 14 insertions(+) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 64b0e42..d3c6e7a 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -256,6 +256,7 @@ bool Client::init_methods() { methods_.emplace("getforumtopiciconstickers", &Client::process_get_forum_topic_icon_stickers_query); methods_.emplace("createforumtopic", &Client::process_create_forum_topic_query); methods_.emplace("editforumtopic", &Client::process_edit_forum_topic_query); + methods_.emplace("closeforumtopic", &Client::process_close_forum_topic_query); methods_.emplace("getchatmember", &Client::process_get_chat_member_query); methods_.emplace("getchatadministrators", &Client::process_get_chat_administrators_query); methods_.emplace("getchatmembercount", &Client::process_get_chat_member_count_query); @@ -8180,6 +8181,18 @@ td::Status Client::process_edit_forum_topic_query(PromisedQueryPtr &query) { return Status::OK(); } +td::Status Client::process_close_forum_topic_query(PromisedQueryPtr &query) { + auto chat_id = query->arg("chat_id"); + auto message_thread_id = get_message_id(query.get(), "message_thread_id"); + + check_chat(chat_id, AccessRights::Write, std::move(query), + [this, message_thread_id](int64 chat_id, PromisedQueryPtr query) { + send_request(make_object(chat_id, message_thread_id, true), + td::make_unique(std::move(query))); + }); + return Status::OK(); +} + td::Status Client::process_get_chat_member_query(PromisedQueryPtr &query) { auto chat_id = query->arg("chat_id"); TRY_RESULT(user_id, get_user_id(query.get())); diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index e3a1c33..1278189 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -541,6 +541,7 @@ class Client final : public WebhookActor::Callback { Status process_get_forum_topic_icon_stickers_query(PromisedQueryPtr &query); Status process_create_forum_topic_query(PromisedQueryPtr &query); Status process_edit_forum_topic_query(PromisedQueryPtr &query); + Status process_close_forum_topic_query(PromisedQueryPtr &query); Status process_get_chat_member_query(PromisedQueryPtr &query); Status process_get_chat_administrators_query(PromisedQueryPtr &query); Status process_get_chat_member_count_query(PromisedQueryPtr &query); From 807c353fc02c9c86243ebc5656f4601a9f9ee234 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 31 Oct 2022 19:13:57 +0300 Subject: [PATCH 39/48] Add reopenForumTopic method. --- telegram-bot-api/Client.cpp | 13 +++++++++++++ telegram-bot-api/Client.h | 1 + 2 files changed, 14 insertions(+) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index d3c6e7a..a8d90d9 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -257,6 +257,7 @@ bool Client::init_methods() { methods_.emplace("createforumtopic", &Client::process_create_forum_topic_query); methods_.emplace("editforumtopic", &Client::process_edit_forum_topic_query); methods_.emplace("closeforumtopic", &Client::process_close_forum_topic_query); + methods_.emplace("reopenforumtopic", &Client::process_reopen_forum_topic_query); methods_.emplace("getchatmember", &Client::process_get_chat_member_query); methods_.emplace("getchatadministrators", &Client::process_get_chat_administrators_query); methods_.emplace("getchatmembercount", &Client::process_get_chat_member_count_query); @@ -8193,6 +8194,18 @@ td::Status Client::process_close_forum_topic_query(PromisedQueryPtr &query) { return Status::OK(); } +td::Status Client::process_reopen_forum_topic_query(PromisedQueryPtr &query) { + auto chat_id = query->arg("chat_id"); + auto message_thread_id = get_message_id(query.get(), "message_thread_id"); + + check_chat(chat_id, AccessRights::Write, std::move(query), + [this, message_thread_id](int64 chat_id, PromisedQueryPtr query) { + send_request(make_object(chat_id, message_thread_id, false), + td::make_unique(std::move(query))); + }); + return Status::OK(); +} + td::Status Client::process_get_chat_member_query(PromisedQueryPtr &query) { auto chat_id = query->arg("chat_id"); TRY_RESULT(user_id, get_user_id(query.get())); diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index 1278189..2a44c54 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -542,6 +542,7 @@ class Client final : public WebhookActor::Callback { Status process_create_forum_topic_query(PromisedQueryPtr &query); Status process_edit_forum_topic_query(PromisedQueryPtr &query); Status process_close_forum_topic_query(PromisedQueryPtr &query); + Status process_reopen_forum_topic_query(PromisedQueryPtr &query); Status process_get_chat_member_query(PromisedQueryPtr &query); Status process_get_chat_administrators_query(PromisedQueryPtr &query); Status process_get_chat_member_count_query(PromisedQueryPtr &query); From 88748a50f394aac64a07e348f9fd27186de273d2 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 31 Oct 2022 19:16:14 +0300 Subject: [PATCH 40/48] Add deleteForumTopic method. --- telegram-bot-api/Client.cpp | 13 +++++++++++++ telegram-bot-api/Client.h | 1 + 2 files changed, 14 insertions(+) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index a8d90d9..db4f63d 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -258,6 +258,7 @@ bool Client::init_methods() { methods_.emplace("editforumtopic", &Client::process_edit_forum_topic_query); methods_.emplace("closeforumtopic", &Client::process_close_forum_topic_query); methods_.emplace("reopenforumtopic", &Client::process_reopen_forum_topic_query); + methods_.emplace("deleteforumtopic", &Client::process_delete_forum_topic_query); methods_.emplace("getchatmember", &Client::process_get_chat_member_query); methods_.emplace("getchatadministrators", &Client::process_get_chat_administrators_query); methods_.emplace("getchatmembercount", &Client::process_get_chat_member_count_query); @@ -8206,6 +8207,18 @@ td::Status Client::process_reopen_forum_topic_query(PromisedQueryPtr &query) { return Status::OK(); } +td::Status Client::process_delete_forum_topic_query(PromisedQueryPtr &query) { + auto chat_id = query->arg("chat_id"); + auto message_thread_id = get_message_id(query.get(), "message_thread_id"); + + check_chat(chat_id, AccessRights::Write, std::move(query), + [this, message_thread_id](int64 chat_id, PromisedQueryPtr query) { + send_request(make_object(chat_id, message_thread_id), + td::make_unique(std::move(query))); + }); + return Status::OK(); +} + td::Status Client::process_get_chat_member_query(PromisedQueryPtr &query) { auto chat_id = query->arg("chat_id"); TRY_RESULT(user_id, get_user_id(query.get())); diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index 2a44c54..e283a69 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -543,6 +543,7 @@ class Client final : public WebhookActor::Callback { Status process_edit_forum_topic_query(PromisedQueryPtr &query); Status process_close_forum_topic_query(PromisedQueryPtr &query); Status process_reopen_forum_topic_query(PromisedQueryPtr &query); + Status process_delete_forum_topic_query(PromisedQueryPtr &query); Status process_get_chat_member_query(PromisedQueryPtr &query); Status process_get_chat_administrators_query(PromisedQueryPtr &query); Status process_get_chat_member_count_query(PromisedQueryPtr &query); From 36e41d6b7fb083be074cedf0ca42714b210dd5f2 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 31 Oct 2022 20:29:38 +0300 Subject: [PATCH 41/48] Add unpinAllForumTopicMessages method. --- telegram-bot-api/Client.cpp | 13 +++++++++++++ telegram-bot-api/Client.h | 1 + 2 files changed, 14 insertions(+) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index db4f63d..192ae0b 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -259,6 +259,7 @@ bool Client::init_methods() { methods_.emplace("closeforumtopic", &Client::process_close_forum_topic_query); methods_.emplace("reopenforumtopic", &Client::process_reopen_forum_topic_query); methods_.emplace("deleteforumtopic", &Client::process_delete_forum_topic_query); + methods_.emplace("unpinallforumtopicmessages", &Client::process_unpin_all_forum_topic_messages_query); methods_.emplace("getchatmember", &Client::process_get_chat_member_query); methods_.emplace("getchatadministrators", &Client::process_get_chat_administrators_query); methods_.emplace("getchatmembercount", &Client::process_get_chat_member_count_query); @@ -8219,6 +8220,18 @@ td::Status Client::process_delete_forum_topic_query(PromisedQueryPtr &query) { return Status::OK(); } +td::Status Client::process_unpin_all_forum_topic_messages_query(PromisedQueryPtr &query) { + auto chat_id = query->arg("chat_id"); + auto message_thread_id = get_message_id(query.get(), "message_thread_id"); + + check_chat(chat_id, AccessRights::Write, std::move(query), + [this, message_thread_id](int64 chat_id, PromisedQueryPtr query) { + send_request(make_object(chat_id, message_thread_id), + td::make_unique(std::move(query))); + }); + return Status::OK(); +} + td::Status Client::process_get_chat_member_query(PromisedQueryPtr &query) { auto chat_id = query->arg("chat_id"); TRY_RESULT(user_id, get_user_id(query.get())); diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index e283a69..555eb8f 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -544,6 +544,7 @@ class Client final : public WebhookActor::Callback { Status process_close_forum_topic_query(PromisedQueryPtr &query); Status process_reopen_forum_topic_query(PromisedQueryPtr &query); Status process_delete_forum_topic_query(PromisedQueryPtr &query); + Status process_unpin_all_forum_topic_messages_query(PromisedQueryPtr &query); Status process_get_chat_member_query(PromisedQueryPtr &query); Status process_get_chat_administrators_query(PromisedQueryPtr &query); Status process_get_chat_member_count_query(PromisedQueryPtr &query); From 8aac13eb4d546cfbe97a816e4db68e3100487b25 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 2 Nov 2022 14:05:23 +0300 Subject: [PATCH 42/48] Check message_thread_id parameter before using it. --- telegram-bot-api/Client.cpp | 122 +++++++++++++++++++++++++++++++----- telegram-bot-api/Client.h | 6 ++ 2 files changed, 111 insertions(+), 17 deletions(-) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 192ae0b..fdfaf42 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -3194,6 +3194,55 @@ class Client::TdOnCheckMessageCallback final : public TdQueryCallback { OnSuccess on_success_; }; +template +class Client::TdOnCheckMessageThreadCallback final : public TdQueryCallback { + public: + TdOnCheckMessageThreadCallback(Client *client, int64 chat_id, int64 message_thread_id, int64 reply_to_message_id, + PromisedQueryPtr query, OnSuccess on_success) + : client_(client) + , chat_id_(chat_id) + , message_thread_id_(message_thread_id) + , reply_to_message_id_(reply_to_message_id) + , query_(std::move(query)) + , on_success_(std::move(on_success)) { + } + + void on_result(object_ptr result) final { + if (result->get_id() == td_api::error::ID) { + auto error = move_object_as(result); + if (error->code_ == 429) { + LOG(WARNING) << "Failed to get message thread " << message_thread_id_ << " in " << chat_id_; + } + return fail_query_with_error(std::move(query_), std::move(error), "Message thread not found"); + } + + CHECK(result->get_id() == td_api::message::ID); + auto full_message_id = client_->add_message(move_object_as(result)); + CHECK(full_message_id.chat_id == chat_id_); + CHECK(full_message_id.message_id == message_thread_id_); + + const MessageInfo *message_info = client_->get_message(chat_id_, message_thread_id_); + CHECK(message_info != nullptr); + if (message_info->message_thread_id != message_thread_id_) { + return fail_query_with_error(std::move(query_), 400, "MESSAGE_THREAD_INVALID", "Message thread not found"); + } + if (!message_info->is_topic_message) { + return fail_query_with_error(std::move(query_), 400, "MESSAGE_THREAD_INVALID", + "Message thread is not a forum topic thread"); + } + + on_success_(chat_id_, message_thread_id_, reply_to_message_id_, std::move(query_)); + } + + private: + Client *client_; + int64 chat_id_; + int64 message_thread_id_; + int64 reply_to_message_id_; + PromisedQueryPtr query_; + OnSuccess on_success_; +}; + template class Client::TdOnCheckRemoteFileIdCallback final : public TdQueryCallback { public: @@ -4381,6 +4430,30 @@ void Client::check_message(Slice chat_id_str, int64 message_id, bool allow_empty }); } +template +void Client::check_message_thread(int64 chat_id, int64 message_thread_id, int64 reply_to_message_id, + PromisedQueryPtr query, OnSuccess on_success) { + if (message_thread_id <= 0) { + return on_success(chat_id, 0, reply_to_message_id, std::move(query)); + } + + if (reply_to_message_id != 0) { + const MessageInfo *message_info = get_message(chat_id, reply_to_message_id); + CHECK(message_info != nullptr); + if (message_info->message_thread_id != message_thread_id) { + return fail_query_with_error(std::move(query), 400, "MESSAGE_THREAD_INVALID", + "Replied message is not in the specified message thread"); + } + } + if (reply_to_message_id == message_thread_id) { + return on_success(chat_id, message_thread_id, reply_to_message_id, std::move(query)); + } + + send_request(make_object(chat_id, message_thread_id), + td::make_unique>( + this, chat_id, message_thread_id, reply_to_message_id, std::move(query), std::move(on_success))); +} + template void Client::resolve_sticker_set(const td::string &sticker_set_name, PromisedQueryPtr query, OnSuccess on_success) { if (sticker_set_name.empty()) { @@ -7542,16 +7615,23 @@ td::Status Client::process_send_media_group_query(PromisedQueryPtr &query) { input_message_contents = std::move(input_message_contents), reply_markup = std::move(reply_markup)](int64 chat_id, int64 reply_to_message_id, PromisedQueryPtr query) mutable { - auto it = yet_unsent_message_count_.find(chat_id); - if (it != yet_unsent_message_count_.end() && it->second > MAX_CONCURRENTLY_SENT_CHAT_MESSAGES) { - return query->set_retry_after_error(60); - } + auto on_message_thread_checked = + [this, disable_notification, protect_content, input_message_contents = std::move(input_message_contents), + reply_markup = std::move(reply_markup)](int64 chat_id, int64 message_thread_id, + int64 reply_to_message_id, PromisedQueryPtr query) mutable { + auto it = yet_unsent_message_count_.find(chat_id); + if (it != yet_unsent_message_count_.end() && it->second > MAX_CONCURRENTLY_SENT_CHAT_MESSAGES) { + return query->set_retry_after_error(60); + } - send_request( - make_object(chat_id, message_thread_id, reply_to_message_id, - get_message_send_options(disable_notification, protect_content), - std::move(input_message_contents), false), - td::make_unique(this, std::move(query))); + send_request(make_object( + chat_id, message_thread_id, reply_to_message_id, + get_message_send_options(disable_notification, protect_content), + std::move(input_message_contents), false), + td::make_unique(this, std::move(query))); + }; + check_message_thread(chat_id, message_thread_id, reply_to_message_id, std::move(query), + std::move(on_message_thread_checked)); }; check_message(chat_id, reply_to_message_id, reply_to_message_id <= 0 || allow_sending_without_reply, AccessRights::Write, "replied message", std::move(query), std::move(on_success)); @@ -9112,15 +9192,23 @@ void Client::do_send_message(object_ptr input_messa input_message_content = std::move(input_message_content), reply_markup = std::move(reply_markup)](int64 chat_id, int64 reply_to_message_id, PromisedQueryPtr query) mutable { - auto it = yet_unsent_message_count_.find(chat_id); - if (it != yet_unsent_message_count_.end() && it->second > MAX_CONCURRENTLY_SENT_CHAT_MESSAGES) { - return query->set_retry_after_error(60); - } + auto on_message_thread_checked = + [this, disable_notification, protect_content, input_message_content = std::move(input_message_content), + reply_markup = std::move(reply_markup)](int64 chat_id, int64 message_thread_id, + int64 reply_to_message_id, PromisedQueryPtr query) mutable { + auto it = yet_unsent_message_count_.find(chat_id); + if (it != yet_unsent_message_count_.end() && it->second > MAX_CONCURRENTLY_SENT_CHAT_MESSAGES) { + return query->set_retry_after_error(60); + } - send_request(make_object(chat_id, message_thread_id, reply_to_message_id, - get_message_send_options(disable_notification, protect_content), - std::move(reply_markup), std::move(input_message_content)), - td::make_unique(this, std::move(query))); + send_request( + make_object(chat_id, message_thread_id, reply_to_message_id, + get_message_send_options(disable_notification, protect_content), + std::move(reply_markup), std::move(input_message_content)), + td::make_unique(this, std::move(query))); + }; + check_message_thread(chat_id, message_thread_id, reply_to_message_id, std::move(query), + std::move(on_message_thread_checked)); }; check_message(chat_id, reply_to_message_id, reply_to_message_id <= 0 || allow_sending_without_reply, AccessRights::Write, "replied message", std::move(query), std::move(on_success)); diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index 555eb8f..bf5f57c 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -250,6 +250,8 @@ class Client final : public WebhookActor::Callback { template class TdOnCheckMessageCallback; template + class TdOnCheckMessageThreadCallback; + template class TdOnCheckRemoteFileIdCallback; template class TdOnGetChatMemberCallback; @@ -288,6 +290,10 @@ class Client final : public WebhookActor::Callback { void check_message(Slice chat_id_str, int64 message_id, bool allow_empty, AccessRights access_rights, Slice message_type, PromisedQueryPtr query, OnSuccess on_success); + template + void check_message_thread(int64 chat_id, int64 message_thread_id, int64 reply_to_message_id, PromisedQueryPtr query, + OnSuccess on_success); + template void resolve_sticker_set(const td::string &sticker_set_name, PromisedQueryPtr query, OnSuccess on_success); From daf986972b26be3cc7fee1f298ef90f956ff5ed6 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 3 Nov 2022 20:30:14 +0300 Subject: [PATCH 43/48] Destroy deleted messages on another thread. --- telegram-bot-api/Client.cpp | 20 ++++++++++++-------- telegram-bot-api/Client.h | 3 ++- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index fdfaf42..a8d29b8 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -4806,9 +4806,14 @@ void Client::on_update(object_ptr result) { } case td_api::updateDeleteMessages::ID: { auto update = move_object_as(result); + td::vector> deleted_messages; for (auto message_id : update->message_ids_) { - delete_message(update->chat_id_, message_id, update->from_cache_); + auto deleted_message = delete_message(update->chat_id_, message_id, update->from_cache_); + if (deleted_message != nullptr) { + deleted_messages.push_back(std::move(deleted_message)); + } } + td::Scheduler::instance()->destroy_on_scheduler(get_file_gc_scheduler_id(), deleted_messages); break; } case td_api::updateFile::ID: { @@ -10548,10 +10553,10 @@ void Client::remove_replies_to_message(int64 chat_id, int64 reply_to_message_id, reply_message_ids_.erase(it); } -void Client::delete_message(int64 chat_id, int64 message_id, bool only_from_cache) { +td::unique_ptr Client::delete_message(int64 chat_id, int64 message_id, bool only_from_cache) { remove_replies_to_message(chat_id, message_id, only_from_cache); - auto message_info = messages_.get_pointer({chat_id, message_id}); + auto message_info = std::move(messages_[{chat_id, message_id}]); if (message_info == nullptr) { if (yet_unsent_messages_.count({chat_id, message_id}) > 0) { // yet unsent message is deleted, possible only if we are trying to write to inaccessible supergroup or @@ -10576,12 +10581,11 @@ void Client::delete_message(int64 chat_id, int64 message_id, bool only_from_cach on_message_send_failed(chat_id, message_id, 0, std::move(error)); } - return; + } else { + set_message_reply_to_message_id(message_info.get(), 0); + messages_.erase({chat_id, message_id}); } - - set_message_reply_to_message_id(message_info, 0); - - messages_.erase({chat_id, message_id}); + return message_info; } Client::FullMessageId Client::add_message(object_ptr &&message, bool force_update_content) { diff --git a/telegram-bot-api/Client.h b/telegram-bot-api/Client.h index bf5f57c..a814939 100644 --- a/telegram-bot-api/Client.h +++ b/telegram-bot-api/Client.h @@ -812,7 +812,8 @@ class Client final : public WebhookActor::Callback { static void json_store_permissions(td::JsonObjectScope &object, const td_api::chatPermissions *permissions); void remove_replies_to_message(int64 chat_id, int64 reply_to_message_id, bool only_from_cache); - void delete_message(int64 chat_id, int64 message_id, bool only_from_cache); + + td::unique_ptr delete_message(int64 chat_id, int64 message_id, bool only_from_cache); void add_new_message(object_ptr &&message, bool is_edited); void process_new_message_queue(int64 chat_id); From 571baeead026fd6c671e00d6c5cf76d2bdf8eeb1 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 4 Nov 2022 18:01:15 +0300 Subject: [PATCH 44/48] Update version to 6.3. --- CMakeLists.txt | 2 +- telegram-bot-api/telegram-bot-api.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f1bad6..c9804ea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ if (POLICY CMP0065) cmake_policy(SET CMP0065 NEW) endif() -project(TelegramBotApi VERSION 6.2 LANGUAGES CXX) +project(TelegramBotApi VERSION 6.3 LANGUAGES CXX) if (POLICY CMP0069) option(TELEGRAM_BOT_API_ENABLE_LTO "Use \"ON\" to enable Link Time Optimization.") diff --git a/telegram-bot-api/telegram-bot-api.cpp b/telegram-bot-api/telegram-bot-api.cpp index 55b0c57..3dd29bc 100644 --- a/telegram-bot-api/telegram-bot-api.cpp +++ b/telegram-bot-api/telegram-bot-api.cpp @@ -152,7 +152,7 @@ int main(int argc, char *argv[]) { auto start_time = td::Time::now(); auto shared_data = std::make_shared(); auto parameters = std::make_unique(); - parameters->version_ = "6.2"; + parameters->version_ = "6.3"; parameters->shared_data_ = shared_data; parameters->start_time_ = start_time; auto net_query_stats = td::create_net_query_stats(); From 0ddf9a460df65d19747e0f4b4d2bcb72be56f1e8 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 7 Nov 2022 00:59:20 +0300 Subject: [PATCH 45/48] Fix type of emoji_status_custom_emoji_id. --- telegram-bot-api/Client.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index a8d29b8..11c3210 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -676,7 +676,7 @@ class Client::JsonChat final : public Jsonable { [](Slice username) { return td::JsonString(username); })); } if (user_info->emoji_status_custom_emoji_id != 0) { - object("emoji_status_custom_emoji_id", user_info->emoji_status_custom_emoji_id); + object("emoji_status_custom_emoji_id", td::to_string(user_info->emoji_status_custom_emoji_id)); } if (!user_info->bio.empty()) { object("bio", user_info->bio); From ab2f0f0bbcdc88f60c2d15fba5249e881a676d69 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 7 Nov 2022 00:59:44 +0300 Subject: [PATCH 46/48] Update version to 6.3.1. --- CMakeLists.txt | 2 +- telegram-bot-api/telegram-bot-api.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c9804ea..6a01c45 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ if (POLICY CMP0065) cmake_policy(SET CMP0065 NEW) endif() -project(TelegramBotApi VERSION 6.3 LANGUAGES CXX) +project(TelegramBotApi VERSION 6.3.1 LANGUAGES CXX) if (POLICY CMP0069) option(TELEGRAM_BOT_API_ENABLE_LTO "Use \"ON\" to enable Link Time Optimization.") diff --git a/telegram-bot-api/telegram-bot-api.cpp b/telegram-bot-api/telegram-bot-api.cpp index 3dd29bc..46bd3b7 100644 --- a/telegram-bot-api/telegram-bot-api.cpp +++ b/telegram-bot-api/telegram-bot-api.cpp @@ -152,7 +152,7 @@ int main(int argc, char *argv[]) { auto start_time = td::Time::now(); auto shared_data = std::make_shared(); auto parameters = std::make_unique(); - parameters->version_ = "6.3"; + parameters->version_ = "6.3.1"; parameters->shared_data_ = shared_data; parameters->start_time_ = start_time; auto net_query_stats = td::create_net_query_stats(); From 29cc000c08e7156e1a56beefee9e4760334107c7 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 8 Nov 2022 01:28:57 +0300 Subject: [PATCH 47/48] Fix type of icon_custom_emoji_id. --- telegram-bot-api/Client.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/telegram-bot-api/Client.cpp b/telegram-bot-api/Client.cpp index 11c3210..1392038 100644 --- a/telegram-bot-api/Client.cpp +++ b/telegram-bot-api/Client.cpp @@ -1363,7 +1363,7 @@ class Client::JsonForumTopicCreated final : public Jsonable { object("name", forum_topic_created_->name_); object("icon_color", forum_topic_created_->icon_->color_); if (forum_topic_created_->icon_->custom_emoji_id_ != 0) { - object("icon_custom_emoji_id", forum_topic_created_->icon_->custom_emoji_id_); + object("icon_custom_emoji_id", td::to_string(forum_topic_created_->icon_->custom_emoji_id_)); } } @@ -1388,7 +1388,7 @@ class Client::JsonForumTopicInfo final : public Jsonable { object("name", forum_topic_info_->name_); object("icon_color", forum_topic_info_->icon_->color_); if (forum_topic_info_->icon_->custom_emoji_id_ != 0) { - object("icon_custom_emoji_id", forum_topic_info_->icon_->custom_emoji_id_); + object("icon_custom_emoji_id", td::to_string(forum_topic_info_->icon_->custom_emoji_id_)); } } @@ -7158,7 +7158,7 @@ void Client::on_message_send_failed(int64 chat_id, int64 old_message_id, int64 n auto &query = *pending_send_message_queries_[query_id]; if (query.is_multisend) { if (query.error == nullptr || query.error->message_ == "Group send failed") { - if (error->code_ == 429 || error->message_ == "Group send failed") { + if (error->code_ == 429 || error->code_ >= 500 || error->message_ == "Group send failed") { query.error = std::move(error); } else { auto pos = (query.total_message_count - query.awaited_message_count + 1); From a822b35d8737c4aab01d702c788a28dc75aa0f55 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 8 Nov 2022 01:35:39 +0300 Subject: [PATCH 48/48] Update TDLib and version to 6.3.2. --- CMakeLists.txt | 2 +- td | 2 +- telegram-bot-api/telegram-bot-api.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a01c45..41128e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ if (POLICY CMP0065) cmake_policy(SET CMP0065 NEW) endif() -project(TelegramBotApi VERSION 6.3.1 LANGUAGES CXX) +project(TelegramBotApi VERSION 6.3.2 LANGUAGES CXX) if (POLICY CMP0069) option(TELEGRAM_BOT_API_ENABLE_LTO "Use \"ON\" to enable Link Time Optimization.") diff --git a/td b/td index bbe37ee..7eba198 160000 --- a/td +++ b/td @@ -1 +1 @@ -Subproject commit bbe37ee594d97f3c7820dd23ebcd9c9b8dac51a0 +Subproject commit 7eba19887ad834fd731b6b07b53c2426fe4beb59 diff --git a/telegram-bot-api/telegram-bot-api.cpp b/telegram-bot-api/telegram-bot-api.cpp index 46bd3b7..a8e0717 100644 --- a/telegram-bot-api/telegram-bot-api.cpp +++ b/telegram-bot-api/telegram-bot-api.cpp @@ -152,7 +152,7 @@ int main(int argc, char *argv[]) { auto start_time = td::Time::now(); auto shared_data = std::make_shared(); auto parameters = std::make_unique(); - parameters->version_ = "6.3.1"; + parameters->version_ = "6.3.2"; parameters->shared_data_ = shared_data; parameters->start_time_ = start_time; auto net_query_stats = td::create_net_query_stats();