From 119fc9563ad96b147ac55d5697afe6312a77b02b Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 5 Oct 2020 15:48:37 +0300 Subject: [PATCH 01/13] Return raw pointer as ClientJson response. GitOrigin-RevId: 6b309681539b3ee9b2c2e87c57ed4cc8f94d798f --- td/telegram/ClientJson.cpp | 10 +++++----- td/telegram/ClientJson.h | 4 ++-- td/telegram/td_json_client.cpp | 14 ++------------ td/telegram/td_json_client.h | 2 +- 4 files changed, 10 insertions(+), 20 deletions(-) diff --git a/td/telegram/ClientJson.cpp b/td/telegram/ClientJson.cpp index 94358660c..71d424252 100644 --- a/td/telegram/ClientJson.cpp +++ b/td/telegram/ClientJson.cpp @@ -67,10 +67,10 @@ static std::string from_response(const td_api::Object &object, const string &ext static TD_THREAD_LOCAL std::string *current_output; -static CSlice store_string(std::string str) { +static const char *store_string(std::string str) { init_thread_local(current_output); *current_output = std::move(str); - return *current_output; + return current_output->c_str(); } void ClientJson::send(Slice request) { @@ -83,10 +83,10 @@ void ClientJson::send(Slice request) { client_.send(Client::Request{extra_id, std::move(parsed_request.first)}); } -CSlice ClientJson::receive(double timeout) { +const char *ClientJson::receive(double timeout) { auto response = client_.receive(timeout); if (!response.object) { - return {}; + return nullptr; } std::string extra; @@ -101,7 +101,7 @@ CSlice ClientJson::receive(double timeout) { return store_string(from_response(*response.object, extra)); } -CSlice ClientJson::execute(Slice request) { +const char *ClientJson::execute(Slice request) { auto parsed_request = to_request(request); return store_string(from_response(*Client::execute(Client::Request{0, std::move(parsed_request.first)}).object, parsed_request.second)); diff --git a/td/telegram/ClientJson.h b/td/telegram/ClientJson.h index 3b08cd28b..62df510b7 100644 --- a/td/telegram/ClientJson.h +++ b/td/telegram/ClientJson.h @@ -22,9 +22,9 @@ class ClientJson final { public: void send(Slice request); - CSlice receive(double timeout); + const char *receive(double timeout); - static CSlice execute(Slice request); + static const char *execute(Slice request); private: Client client_; diff --git a/td/telegram/td_json_client.cpp b/td/telegram/td_json_client.cpp index 8637ee2e6..33d29d4b0 100644 --- a/td/telegram/td_json_client.cpp +++ b/td/telegram/td_json_client.cpp @@ -27,19 +27,9 @@ void td_json_client_send(void *client, const char *request) { } const char *td_json_client_receive(void *client, double timeout) { - auto slice = static_cast(client)->receive(timeout); - if (slice.empty()) { - return nullptr; - } else { - return slice.c_str(); - } + return static_cast(client)->receive(timeout); } const char *td_json_client_execute(void *client, const char *request) { - auto slice = td::ClientJson::execute(td::Slice(request == nullptr ? "" : request)); - if (slice.empty()) { - return nullptr; - } else { - return slice.c_str(); - } + return td::ClientJson::execute(td::Slice(request == nullptr ? "" : request)); } diff --git a/td/telegram/td_json_client.h b/td/telegram/td_json_client.h index 2d030ecdb..d713c532c 100644 --- a/td/telegram/td_json_client.h +++ b/td/telegram/td_json_client.h @@ -84,7 +84,7 @@ TDJSON_EXPORT const char *td_json_client_receive(void *client, double timeout); * in the same thread, so it can't be used after that. * \param[in] client The client. Currently ignored for all requests, so NULL can be passed. * \param[in] request JSON-serialized null-terminated request to TDLib. - * \return JSON-serialized null-terminated request response. May be NULL if the request can't be parsed. + * \return JSON-serialized null-terminated request response. */ TDJSON_EXPORT const char *td_json_client_execute(void *client, const char *request); From 7d6f14db10e49c359be541cfc2a961bae4d612a5 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 5 Oct 2020 16:08:07 +0300 Subject: [PATCH 02/13] Add new JSON interface. GitOrigin-RevId: aaf756de59e72f949c1150d99e1277047f25fac9 --- td/telegram/ClientJson.cpp | 72 ++++++++++++++++++++++++++++----- td/telegram/ClientJson.h | 8 ++++ td/telegram/td_json_client.cpp | 20 ++++++++-- td/telegram/td_json_client.h | 73 +++++++++++++++++++++++++++++++--- 4 files changed, 155 insertions(+), 18 deletions(-) diff --git a/td/telegram/ClientJson.cpp b/td/telegram/ClientJson.cpp index 71d424252..d5beab12d 100644 --- a/td/telegram/ClientJson.cpp +++ b/td/telegram/ClientJson.cpp @@ -13,6 +13,7 @@ #include "td/utils/JsonBuilder.h" #include "td/utils/logging.h" #include "td/utils/port/thread_local.h" +#include "td/utils/StringBuilder.h" #include @@ -52,23 +53,29 @@ static std::pair, string> to_request(Slice return std::make_pair(std::move(func), std::move(extra)); } -static std::string from_response(const td_api::Object &object, const string &extra) { +static string from_response(const td_api::Object &object, const string &extra, int client_id) { auto str = json_encode(ToJson(object)); CHECK(!str.empty() && str.back() == '}'); + str.reserve(str.size() + (extra.empty() ? 0 : 10 + extra.size()) + (client_id == 0 ? 0 : 14 + 10)); if (!extra.empty()) { str.pop_back(); - str.reserve(str.size() + 11 + extra.size()); str += ",\"@extra\":"; str += extra; str += '}'; } + if (client_id != 0) { + str.pop_back(); + str += ",\"@client_id\":"; + str += to_string(client_id); + str += '}'; + } return str; } -static TD_THREAD_LOCAL std::string *current_output; +static TD_THREAD_LOCAL string *current_output; -static const char *store_string(std::string str) { - init_thread_local(current_output); +static const char *store_string(string str) { + init_thread_local(current_output); *current_output = std::move(str); return current_output->c_str(); } @@ -85,11 +92,11 @@ void ClientJson::send(Slice request) { const char *ClientJson::receive(double timeout) { auto response = client_.receive(timeout); - if (!response.object) { + if (response.object == nullptr) { return nullptr; } - std::string extra; + string extra; if (response.id != 0) { std::lock_guard guard(mutex_); auto it = extra_.find(response.id); @@ -98,13 +105,60 @@ const char *ClientJson::receive(double timeout) { extra_.erase(it); } } - return store_string(from_response(*response.object, extra)); + return store_string(from_response(*response.object, extra, 0)); } const char *ClientJson::execute(Slice request) { auto parsed_request = to_request(request); return store_string(from_response(*Client::execute(Client::Request{0, std::move(parsed_request.first)}).object, - parsed_request.second)); + parsed_request.second, 0)); +} + +static ClientManager *get_manager() { + static ClientManager client_manager; + return &client_manager; +} + +static std::mutex extra_mutex; +static std::unordered_map extra; +static std::atomic extra_id{1}; + +int td_json_create_client() { + return static_cast(get_manager()->create_client()); +} + +void td_json_send(int client_id, Slice request) { + auto parsed_request = to_request(request); + auto request_id = extra_id.fetch_add(1, std::memory_order_relaxed); + if (!parsed_request.second.empty()) { + std::lock_guard guard(extra_mutex); + extra[request_id] = std::move(parsed_request.second); + } + get_manager()->send(client_id, request_id, std::move(parsed_request.first)); +} + +const char *td_json_receive(double timeout) { + auto response = get_manager()->receive(timeout); + if (!response.object) { + return nullptr; + } + + string extra_str; + if (response.request_id != 0) { + std::lock_guard guard(extra_mutex); + auto it = extra.find(response.request_id); + if (it != extra.end()) { + extra_str = std::move(it->second); + extra.erase(it); + } + } + return store_string(from_response(*response.object, extra_str, response.client_id)); +} + +const char *td_json_execute(Slice request) { + auto parsed_request = to_request(request); + return store_string( + from_response(*ClientManager::execute(std::move(parsed_request.first)), parsed_request.second, 0)); } } // namespace td diff --git a/td/telegram/ClientJson.h b/td/telegram/ClientJson.h index 62df510b7..9421b1d33 100644 --- a/td/telegram/ClientJson.h +++ b/td/telegram/ClientJson.h @@ -33,4 +33,12 @@ class ClientJson final { std::atomic extra_id_{1}; }; +int td_json_create_client(); + +void td_json_send(int client_id, Slice request); + +const char *td_json_receive(double timeout); + +const char *td_json_execute(Slice request); + } // namespace td diff --git a/td/telegram/td_json_client.cpp b/td/telegram/td_json_client.cpp index 33d29d4b0..3ddb5945c 100644 --- a/td/telegram/td_json_client.cpp +++ b/td/telegram/td_json_client.cpp @@ -10,10 +10,6 @@ #include "td/utils/Slice.h" -extern "C" int td_json_client_square(int x, const char *str) { - return x * x; -} - void *td_json_client_create() { return new td::ClientJson(); } @@ -33,3 +29,19 @@ const char *td_json_client_receive(void *client, double timeout) { const char *td_json_client_execute(void *client, const char *request) { return td::ClientJson::execute(td::Slice(request == nullptr ? "" : request)); } + +int td_create_client() { + return td::td_json_create_client(); +} + +void td_send(int client_id, const char *request) { + td::td_json_send(client_id, td::Slice(request == nullptr ? "" : request)); +} + +const char *td_receive(double timeout) { + return td::td_json_receive(timeout); +} + +const char *td_execute(const char *request) { + return td::td_json_execute(td::Slice(request == nullptr ? "" : request)); +} diff --git a/td/telegram/td_json_client.h b/td/telegram/td_json_client.h index d713c532c..918e40522 100644 --- a/td/telegram/td_json_client.h +++ b/td/telegram/td_json_client.h @@ -21,11 +21,11 @@ * The main TDLib interface is asynchronous. To match requests with a corresponding response a field "@extra" can * be added to the request object. The corresponding response will have an "@extra" field with exactly the same value. * - * A TDLib client instance should be created through td_json_client_create. + * A TDLib client instance can be created through td_json_client_create. * Requests then can be sent using td_json_client_send from any thread. * New updates and request responses can be received through td_json_client_receive from any thread. This function - * shouldn't be called simultaneously from two different threads. Also note that all updates and request responses - * should be applied in the order they were received to ensure consistency. + * must not be called simultaneously from two different threads. Also note that all updates and request responses + * must be applied in the order they were received to ensure consistency. * Given this information, it's advisable to call this function from a dedicated thread. * Some service TDLib requests can be executed synchronously from any thread by using td_json_client_execute. * The TDLib client instance can be destroyed via td_json_client_destroy. @@ -68,7 +68,7 @@ TDJSON_EXPORT void td_json_client_send(void *client, const char *request); /** * Receives incoming updates and request responses from the TDLib client. May be called from any thread, but - * shouldn't be called simultaneously from two different threads. + * must not be called simultaneously from two different threads. * Returned pointer will be deallocated by TDLib during next call to td_json_client_receive or td_json_client_execute * in the same thread, so it can't be used after that. * \param[in] client The client. @@ -89,11 +89,74 @@ TDJSON_EXPORT const char *td_json_client_receive(void *client, double timeout); TDJSON_EXPORT const char *td_json_client_execute(void *client, const char *request); /** - * Destroys the TDLib client instance. After this is called the client instance shouldn't be used anymore. + * Destroys the TDLib client instance. After this is called the client instance must not be used anymore. * \param[in] client The client. */ TDJSON_EXPORT void td_json_client_destroy(void *client); +/* + * New TDLib JSON interface. + * + * The main TDLib interface is asynchronous. To match requests with a corresponding response a field "@extra" can + * be added to the request object. The corresponding response will have an "@extra" field with exactly the same value. + * Each returned object will have an "@client_id" field, containing and identifier of the client for which + * a response or an update is received. + * + * A TDLib client instance can be created through td_create_client. + * Requests then can be sent using td_send from any thread and the received client identifier. + * New updates and request responses can be received through td_receive from any thread. This function + * must not be called simultaneously from two different threads. Also note that all updates and request responses + * must be applied in the order they were received to ensure consistency. + * Some TDLib requests can be executed synchronously from any thread by using td_execute. + * The TDLib client instances are destroyed automatically after they are closed. + * + * General pattern of usage: + * \code + * int client_id = td_create_client(); + * // share the client_id with other threads, which will be able to send requests via td_send + * + * const double WAIT_TIMEOUT = 10.0; // seconds + * while (true) { + * const char *result = td_receive(WAIT_TIMEOUT); + * if (result) { + * // parse the result as JSON object and process it as an incoming update or an answer to a previously sent request + * } + * } + * \endcode + */ + +/** + * Creates a new instance of TDLib. + * \return Opaque indentifier of the created TDLib client. + */ +TDJSON_EXPORT int td_create_client(); + +/** + * Sends request to the TDLib client. May be called from any thread. + * \param[in] client_id The TDLib client identifier. + * \param[in] request JSON-serialized null-terminated request to TDLib. + */ +TDJSON_EXPORT void td_send(int client_id, const char *request); + +/** + * Receives incoming updates and request responses. Must not be called simultaneously from two different threads. + * Returned pointer will be deallocated by TDLib during next call to td_receive or td_execute + * in the same thread, so it can't be used after that. + * \param[in] timeout The maximum number of seconds allowed for this function to wait for new data. + * \return JSON-serialized null-terminated incoming update or request response. May be NULL if the timeout expires. + */ +TDJSON_EXPORT const char *td_receive(double timeout); + +/** + * Synchronously executes TDLib request. May be called from any thread. + * Only a few requests can be executed synchronously. + * Returned pointer will be deallocated by TDLib during next call to td_receive or td_execute + * in the same thread, so it can't be used after that. + * \param[in] request JSON-serialized null-terminated request to TDLib. + * \return JSON-serialized null-terminated request response. + */ +TDJSON_EXPORT const char *td_execute(const char *request); + #ifdef __cplusplus } // extern "C" #endif From c9c19f3399e385d5e83de6bbc5a95790ab42a381 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 5 Oct 2020 16:41:52 +0300 Subject: [PATCH 03/13] Add TdApi.UpdateChatIsBlocked handling in Java example. GitOrigin-RevId: 95ddae8fe77e29bac6cd3846f78c42a4290b7b8f --- example/java/org/drinkless/tdlib/example/Example.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/example/java/org/drinkless/tdlib/example/Example.java b/example/java/org/drinkless/tdlib/example/Example.java index 484c90208..c2d3598dc 100644 --- a/example/java/org/drinkless/tdlib/example/Example.java +++ b/example/java/org/drinkless/tdlib/example/Example.java @@ -549,6 +549,14 @@ public final class Example { } break; } + case TdApi.UpdateChatIsBlocked.CONSTRUCTOR: { + TdApi.UpdateChatIsBlocked update = (TdApi.UpdateChatIsBlocked) object; + TdApi.Chat chat = chats.get(update.chatId); + synchronized (chat) { + chat.isBlocked = update.isBlocked; + } + break; + } case TdApi.UpdateChatHasScheduledMessages.CONSTRUCTOR: { TdApi.UpdateChatHasScheduledMessages update = (TdApi.UpdateChatHasScheduledMessages) object; TdApi.Chat chat = chats.get(update.chatId); From d2d0cb5cc02a85b21e6210cf1bf11b829924ce1d Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 5 Oct 2020 18:07:23 +0300 Subject: [PATCH 04/13] Move log tags definitions out of logging.cpp. GitOrigin-RevId: 0e74b94ec194f8ee678bb77a04d35df46702b330 --- td/mtproto/SessionConnection.cpp | 3 +++ td/mtproto/SessionConnection.h | 3 +++ td/mtproto/Transport.cpp | 3 +++ td/mtproto/Transport.h | 3 +++ td/telegram/Logging.cpp | 10 ++++++++++ td/telegram/Td.cpp | 5 +++-- td/telegram/Td.h | 1 + td/telegram/cli.cpp | 3 ++- td/telegram/files/FileLoader.cpp | 2 +- td/telegram/files/FileLoaderActor.h | 3 +-- td/telegram/files/FileLoaderUtils.cpp | 2 ++ td/telegram/files/FileLoaderUtils.h | 2 ++ td/telegram/files/PartsManager.cpp | 8 ++++++++ td/telegram/files/ResourceManager.cpp | 4 ++++ td/telegram/files/ResourceManager.h | 2 ++ td/telegram/net/DcAuthManager.cpp | 2 ++ td/telegram/net/DcAuthManager.h | 2 ++ td/telegram/net/NetQuery.cpp | 2 ++ td/telegram/net/NetQuery.h | 2 ++ td/telegram/net/Session.cpp | 1 + tdactor/td/actor/impl/Scheduler-decl.h | 2 ++ tdactor/td/actor/impl/Scheduler.cpp | 2 ++ tddb/td/db/SqliteStatement.cpp | 2 ++ tddb/td/db/SqliteStatement.h | 2 ++ tdutils/td/utils/logging.cpp | 10 ---------- tdutils/td/utils/logging.h | 10 ---------- tdutils/td/utils/port/detail/NativeFd.cpp | 3 ++- tdutils/td/utils/port/detail/NativeFd.h | 3 +++ 28 files changed, 70 insertions(+), 27 deletions(-) diff --git a/td/mtproto/SessionConnection.cpp b/td/mtproto/SessionConnection.cpp index 8c94fb8d7..7ee267f5e 100644 --- a/td/mtproto/SessionConnection.cpp +++ b/td/mtproto/SessionConnection.cpp @@ -32,6 +32,9 @@ #include namespace td { + +int VERBOSITY_NAME(mtproto) = VERBOSITY_NAME(DEBUG) + 7; + namespace mtproto_api { const int32 msg_container::ID; diff --git a/td/mtproto/SessionConnection.h b/td/mtproto/SessionConnection.h index 05a147056..7aa6e6b65 100644 --- a/td/mtproto/SessionConnection.h +++ b/td/mtproto/SessionConnection.h @@ -25,6 +25,9 @@ #include namespace td { + +extern int VERBOSITY_NAME(mtproto); + namespace mtproto_api { class rpc_error; diff --git a/td/mtproto/Transport.cpp b/td/mtproto/Transport.cpp index a2dadbc02..62db5190e 100644 --- a/td/mtproto/Transport.cpp +++ b/td/mtproto/Transport.cpp @@ -21,6 +21,9 @@ #include namespace td { + +int VERBOSITY_NAME(raw_mtproto) = VERBOSITY_NAME(DEBUG) + 10; + namespace mtproto { #pragma pack(push, 4) diff --git a/td/mtproto/Transport.h b/td/mtproto/Transport.h index 5daca4892..aeb1e95ea 100644 --- a/td/mtproto/Transport.h +++ b/td/mtproto/Transport.h @@ -17,6 +17,9 @@ #include namespace td { + +extern int VERBOSITY_NAME(raw_mtproto); + namespace mtproto { class AuthKey; diff --git a/td/telegram/Logging.cpp b/td/telegram/Logging.cpp index 921e79d20..97c57fa78 100644 --- a/td/telegram/Logging.cpp +++ b/td/telegram/Logging.cpp @@ -6,23 +6,33 @@ // #include "td/telegram/Logging.h" +#include "td/mtproto/SessionConnection.h" +#include "td/mtproto/Transport.h" + #include "td/telegram/ConfigManager.h" #include "td/telegram/FileReferenceManager.h" #include "td/telegram/files/FileGcWorker.h" +#include "td/telegram/files/FileLoaderUtils.h" #include "td/telegram/files/FileManager.h" #include "td/telegram/net/ConnectionCreator.h" +#include "td/telegram/net/DcAuthManager.h" +#include "td/telegram/net/NetQuery.h" #include "td/telegram/NotificationManager.h" #include "td/telegram/Td.h" #include "td/telegram/UpdatesManager.h" #include "td/db/binlog/BinlogEvent.h" +#include "td/db/SqliteStatement.h" #include "td/net/GetHostByNameActor.h" #include "td/net/TransparentProxy.h" +#include "td/actor/actor.h" + #include "td/utils/FileLog.h" #include "td/utils/logging.h" #include "td/utils/misc.h" +#include "td/utils/port/detail/NativeFd.h" #include #include diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index befd94d47..86e63719a 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -138,6 +138,9 @@ namespace td { +int VERBOSITY_NAME(td_init) = VERBOSITY_NAME(DEBUG) + 3; +int VERBOSITY_NAME(td_requests) = VERBOSITY_NAME(INFO); + void Td::ResultHandler::set_td(Td *new_td) { CHECK(td == nullptr); td = new_td; @@ -4132,8 +4135,6 @@ class Td::UploadFileCallback : public FileManager::UploadCallback { } }; -int VERBOSITY_NAME(td_init) = VERBOSITY_NAME(DEBUG) + 3; - template void Td::complete_pending_preauthentication_requests(const T &func) { for (auto &request : pending_preauthentication_requests_) { diff --git a/td/telegram/Td.h b/td/telegram/Td.h index 923f0c693..11102e3d8 100644 --- a/td/telegram/Td.h +++ b/td/telegram/Td.h @@ -77,6 +77,7 @@ class WebPagesManager; namespace td { extern int VERBOSITY_NAME(td_init); +extern int VERBOSITY_NAME(td_requests); // Td may start closing after explicit "close" or "destroy" query. // Or it may start closing by itself, because authorization is lost. diff --git a/td/telegram/cli.cpp b/td/telegram/cli.cpp index 4b829fcc0..95533a81c 100644 --- a/td/telegram/cli.cpp +++ b/td/telegram/cli.cpp @@ -14,6 +14,7 @@ #include "td/telegram/ClientActor.h" #include "td/telegram/Log.h" #include "td/telegram/td_api_json.h" +#include "td/telegram/Td.h" // for VERBOSITY_NAME(td_requests) #include "td/utils/base64.h" #include "td/utils/buffer.h" @@ -4229,7 +4230,7 @@ class CliClient final : public Actor { LOG(ERROR) << r_cpu_stats.error(); } else { auto stats = r_cpu_stats.move_as_ok(); - LOG(ERROR) << cpu_counter_ << ", total ticks = " << stats.total_ticks_ + LOG(ERROR) << cpu_counter_.load() << ", total ticks = " << stats.total_ticks_ << ", user ticks = " << stats.process_user_ticks_ << ", system ticks = " << stats.process_system_ticks_; } diff --git a/td/telegram/files/FileLoader.cpp b/td/telegram/files/FileLoader.cpp index f4ed1f865..c93c29644 100644 --- a/td/telegram/files/FileLoader.cpp +++ b/td/telegram/files/FileLoader.cpp @@ -6,6 +6,7 @@ // #include "td/telegram/files/FileLoader.h" +#include "td/telegram/files/FileLoaderUtils.h" #include "td/telegram/files/ResourceManager.h" #include "td/telegram/Global.h" #include "td/telegram/net/NetQueryDispatcher.h" @@ -18,7 +19,6 @@ #include "td/utils/ScopeGuard.h" #include -#include namespace td { diff --git a/td/telegram/files/FileLoaderActor.h b/td/telegram/files/FileLoaderActor.h index 3164e133b..a9a707970 100644 --- a/td/telegram/files/FileLoaderActor.h +++ b/td/telegram/files/FileLoaderActor.h @@ -14,12 +14,11 @@ namespace td { -class LocalFileLocation; class ResourceManager; class FileLoaderActor : public NetQueryCallback { public: - virtual void set_resource_manager(ActorShared) = 0; + virtual void set_resource_manager(ActorShared resource_manager) = 0; virtual void update_priority(int8 priority) = 0; virtual void update_resources(const ResourceState &other) = 0; diff --git a/td/telegram/files/FileLoaderUtils.cpp b/td/telegram/files/FileLoaderUtils.cpp index 17b8c3e86..143479960 100644 --- a/td/telegram/files/FileLoaderUtils.cpp +++ b/td/telegram/files/FileLoaderUtils.cpp @@ -25,6 +25,8 @@ namespace td { +int VERBOSITY_NAME(files) = VERBOSITY_NAME(DEBUG) + 2; + namespace { Result> try_create_new_file(Result result_name) { TRY_RESULT(name, std::move(result_name)); diff --git a/td/telegram/files/FileLoaderUtils.h b/td/telegram/files/FileLoaderUtils.h index 32a10cf4c..526b37dc3 100644 --- a/td/telegram/files/FileLoaderUtils.h +++ b/td/telegram/files/FileLoaderUtils.h @@ -19,6 +19,8 @@ namespace td { +extern int VERBOSITY_NAME(files); + Result> open_temp_file(FileType file_type) TD_WARN_UNUSED_RESULT; Result create_from_temp(CSlice temp_path, CSlice dir, CSlice name) TD_WARN_UNUSED_RESULT; diff --git a/td/telegram/files/PartsManager.cpp b/td/telegram/files/PartsManager.cpp index 78e764f92..3777fd729 100644 --- a/td/telegram/files/PartsManager.cpp +++ b/td/telegram/files/PartsManager.cpp @@ -6,6 +6,8 @@ // #include "td/telegram/files/PartsManager.h" +#include "td/telegram/files/FileLoaderUtils.h" + #include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/misc.h" @@ -150,12 +152,14 @@ bool PartsManager::unchecked_ready() { << ", checked_prefix_size = " << checked_prefix_size_; return !unknown_size_flag_ && ready_size_ == size_; } + bool PartsManager::may_finish() { if (is_streaming_limit_reached()) { return true; } return ready(); } + bool PartsManager::ready() { return unchecked_ready() && (!need_check_ || checked_prefix_size_ == size_); } @@ -213,9 +217,11 @@ int32 PartsManager::get_ready_prefix_count() { } return res; } + int64 PartsManager::get_streaming_offset() const { return streaming_offset_; } + string PartsManager::get_bitmask() { int32 prefix_count = -1; if (need_check_) { @@ -399,6 +405,7 @@ int64 PartsManager::get_size() const { CHECK(!unknown_size_flag_); return size_; } + int64 PartsManager::get_size_or_zero() const { return size_; } @@ -511,6 +518,7 @@ void PartsManager::set_checked_prefix_size(int64 size) { int64 PartsManager::get_checked_prefix_size() const { return checked_prefix_size_; } + int64 PartsManager::get_unchecked_ready_prefix_size() { update_first_not_ready_part(); auto count = first_not_ready_part_; diff --git a/td/telegram/files/ResourceManager.cpp b/td/telegram/files/ResourceManager.cpp index a66b0fc81..641403c3b 100644 --- a/td/telegram/files/ResourceManager.cpp +++ b/td/telegram/files/ResourceManager.cpp @@ -6,6 +6,8 @@ // #include "td/telegram/files/ResourceManager.h" +#include "td/telegram/files/FileLoaderUtils.h" + #include "td/utils/common.h" #include "td/utils/format.h" #include "td/utils/logging.h" @@ -159,6 +161,7 @@ void ResourceManager::loop() { } } } + void ResourceManager::add_node(NodeId node_id, int8 priority) { if (priority >= 0) { auto it = std::find_if(to_xload_.begin(), to_xload_.end(), [&](auto &x) { return x.first <= priority; }); @@ -168,6 +171,7 @@ void ResourceManager::add_node(NodeId node_id, int8 priority) { to_xload_.insert(it, std::make_pair(narrow_cast(-priority), node_id)); } } + bool ResourceManager::remove_node(NodeId node_id) { auto it = std::find_if(to_xload_.begin(), to_xload_.end(), [&](auto &x) { return x.second == node_id; }); if (it != to_xload_.end()) { diff --git a/td/telegram/files/ResourceManager.h b/td/telegram/files/ResourceManager.h index eaeba97c2..47884b172 100644 --- a/td/telegram/files/ResourceManager.h +++ b/td/telegram/files/ResourceManager.h @@ -17,6 +17,7 @@ #include namespace td { + class ResourceManager : public Actor { public: enum class Mode : int32 { Baseline, Greedy }; @@ -64,4 +65,5 @@ class ResourceManager : public Actor { void add_node(NodeId node_id, int8 priority); bool remove_node(NodeId node_id); }; + } // namespace td diff --git a/td/telegram/net/DcAuthManager.cpp b/td/telegram/net/DcAuthManager.cpp index c370fcf92..147d358d5 100644 --- a/td/telegram/net/DcAuthManager.cpp +++ b/td/telegram/net/DcAuthManager.cpp @@ -26,6 +26,8 @@ namespace td { +int VERBOSITY_NAME(dc) = VERBOSITY_NAME(DEBUG) + 2; + DcAuthManager::DcAuthManager(ActorShared<> parent) { parent_ = std::move(parent); auto s_main_dc_id = G()->td_db()->get_binlog_pmc()->get("main_dc_id"); diff --git a/td/telegram/net/DcAuthManager.h b/td/telegram/net/DcAuthManager.h index bec44bc28..71f0ab159 100644 --- a/td/telegram/net/DcAuthManager.h +++ b/td/telegram/net/DcAuthManager.h @@ -20,6 +20,8 @@ namespace td { +extern int VERBOSITY_NAME(dc); + class DcAuthManager : public NetQueryCallback { public: explicit DcAuthManager(ActorShared<> parent); diff --git a/td/telegram/net/NetQuery.cpp b/td/telegram/net/NetQuery.cpp index e02565093..df426042d 100644 --- a/td/telegram/net/NetQuery.cpp +++ b/td/telegram/net/NetQuery.cpp @@ -15,6 +15,8 @@ namespace td { +int VERBOSITY_NAME(net_query) = VERBOSITY_NAME(INFO); + int32 NetQuery::get_my_id() { return G()->get_my_id(); } diff --git a/td/telegram/net/NetQuery.h b/td/telegram/net/NetQuery.h index 246c1d497..db0c60238 100644 --- a/td/telegram/net/NetQuery.h +++ b/td/telegram/net/NetQuery.h @@ -31,6 +31,8 @@ namespace td { +extern int VERBOSITY_NAME(net_query); + class NetQuery; using NetQueryPtr = ObjectPool::OwnerPtr; using NetQueryRef = ObjectPool::WeakPtr; diff --git a/td/telegram/net/Session.cpp b/td/telegram/net/Session.cpp index cdf7075b1..8498839c0 100644 --- a/td/telegram/net/Session.cpp +++ b/td/telegram/net/Session.cpp @@ -10,6 +10,7 @@ #include "td/telegram/DhCache.h" #include "td/telegram/Global.h" +#include "td/telegram/net/DcAuthManager.h" #include "td/telegram/net/DcId.h" #include "td/telegram/net/MtprotoHeader.h" #include "td/telegram/net/NetQuery.h" diff --git a/tdactor/td/actor/impl/Scheduler-decl.h b/tdactor/td/actor/impl/Scheduler-decl.h index f9e80861a..73b94debd 100644 --- a/tdactor/td/actor/impl/Scheduler-decl.h +++ b/tdactor/td/actor/impl/Scheduler-decl.h @@ -32,6 +32,8 @@ namespace td { +extern int VERBOSITY_NAME(actor); + class ActorInfo; enum class ActorSendType { Immediate, Later, LaterWeak }; diff --git a/tdactor/td/actor/impl/Scheduler.cpp b/tdactor/td/actor/impl/Scheduler.cpp index 69c48a2f4..39cb4661e 100644 --- a/tdactor/td/actor/impl/Scheduler.cpp +++ b/tdactor/td/actor/impl/Scheduler.cpp @@ -27,6 +27,8 @@ namespace td { +int VERBOSITY_NAME(actor) = VERBOSITY_NAME(DEBUG) + 10; + TD_THREAD_LOCAL Scheduler *Scheduler::scheduler_; // static zero-initialized TD_THREAD_LOCAL ActorContext *Scheduler::context_; // static zero-initialized diff --git a/tddb/td/db/SqliteStatement.cpp b/tddb/td/db/SqliteStatement.cpp index 0547a6c55..f13218870 100644 --- a/tddb/td/db/SqliteStatement.cpp +++ b/tddb/td/db/SqliteStatement.cpp @@ -15,6 +15,8 @@ namespace td { +int VERBOSITY_NAME(sqlite) = VERBOSITY_NAME(DEBUG) + 10; + namespace { int printExplainQueryPlan(StringBuilder &sb, sqlite3_stmt *pStmt) { const char *zSql = sqlite3_sql(pStmt); diff --git a/tddb/td/db/SqliteStatement.h b/tddb/td/db/SqliteStatement.h index 7182b1409..6612fcfca 100644 --- a/tddb/td/db/SqliteStatement.h +++ b/tddb/td/db/SqliteStatement.h @@ -20,6 +20,8 @@ struct sqlite3_stmt; namespace td { +extern int VERBOSITY_NAME(sqlite); + class SqliteStatement { public: SqliteStatement() = default; diff --git a/tdutils/td/utils/logging.cpp b/tdutils/td/utils/logging.cpp index 46a5dbc60..7fac06f33 100644 --- a/tdutils/td/utils/logging.cpp +++ b/tdutils/td/utils/logging.cpp @@ -29,16 +29,6 @@ namespace td { -int VERBOSITY_NAME(net_query) = VERBOSITY_NAME(INFO); -int VERBOSITY_NAME(td_requests) = VERBOSITY_NAME(INFO); -int VERBOSITY_NAME(dc) = VERBOSITY_NAME(DEBUG) + 2; -int VERBOSITY_NAME(files) = VERBOSITY_NAME(DEBUG) + 2; -int VERBOSITY_NAME(mtproto) = VERBOSITY_NAME(DEBUG) + 7; -int VERBOSITY_NAME(raw_mtproto) = VERBOSITY_NAME(DEBUG) + 10; -int VERBOSITY_NAME(fd) = VERBOSITY_NAME(DEBUG) + 9; -int VERBOSITY_NAME(actor) = VERBOSITY_NAME(DEBUG) + 10; -int VERBOSITY_NAME(sqlite) = VERBOSITY_NAME(DEBUG) + 10; - LogOptions log_options; TD_THREAD_LOCAL const char *Logger::tag_ = nullptr; diff --git a/tdutils/td/utils/logging.h b/tdutils/td/utils/logging.h index 07d37c384..1202f0083 100644 --- a/tdutils/td/utils/logging.h +++ b/tdutils/td/utils/logging.h @@ -109,16 +109,6 @@ constexpr int VERBOSITY_NAME(DEBUG) = 4; constexpr int VERBOSITY_NAME(NEVER) = 1024; namespace td { -// TODO Not part of utils. Should be in some separate file -extern int VERBOSITY_NAME(mtproto); -extern int VERBOSITY_NAME(raw_mtproto); -extern int VERBOSITY_NAME(dc); -extern int VERBOSITY_NAME(fd); -extern int VERBOSITY_NAME(net_query); -extern int VERBOSITY_NAME(td_requests); -extern int VERBOSITY_NAME(actor); -extern int VERBOSITY_NAME(files); -extern int VERBOSITY_NAME(sqlite); struct LogOptions { std::atomic level{VERBOSITY_NAME(DEBUG) + 1}; diff --git a/tdutils/td/utils/port/detail/NativeFd.cpp b/tdutils/td/utils/port/detail/NativeFd.cpp index 3899b4a71..9c8b4f880 100644 --- a/tdutils/td/utils/port/detail/NativeFd.cpp +++ b/tdutils/td/utils/port/detail/NativeFd.cpp @@ -8,7 +8,6 @@ #include "td/utils/format.h" #include "td/utils/logging.h" -#include "td/utils/Status.h" #if TD_PORT_POSIX #include @@ -22,6 +21,8 @@ namespace td { +int VERBOSITY_NAME(fd) = VERBOSITY_NAME(DEBUG) + 9; + #if TD_FD_DEBUG class FdSet { public: diff --git a/tdutils/td/utils/port/detail/NativeFd.h b/tdutils/td/utils/port/detail/NativeFd.h index f6dcce9b6..4ecd99cbb 100644 --- a/tdutils/td/utils/port/detail/NativeFd.h +++ b/tdutils/td/utils/port/detail/NativeFd.h @@ -9,11 +9,14 @@ #include "td/utils/port/config.h" #include "td/utils/common.h" +#include "td/utils/logging.h" #include "td/utils/Status.h" #include "td/utils/StringBuilder.h" namespace td { +extern int VERBOSITY_NAME(fd); + class NativeFd { public: #if TD_PORT_POSIX From c72d18bcd3f163c3f30fba8eb2f5761690248c0d Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 5 Oct 2020 18:58:38 +0300 Subject: [PATCH 05/13] Java example: store update handlers separately. GitOrigin-RevId: dc035dc72756db604731ae7549b172ba54b5d78e --- example/java/org/drinkless/tdlib/Client.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/example/java/org/drinkless/tdlib/Client.java b/example/java/org/drinkless/tdlib/Client.java index 8b022db04..9baef5a04 100644 --- a/example/java/org/drinkless/tdlib/Client.java +++ b/example/java/org/drinkless/tdlib/Client.java @@ -112,7 +112,7 @@ public final class Client implements Runnable { * exception thrown from updatesHandler, if it is null, defaultExceptionHandler will be invoked. */ public void setUpdatesHandler(ResultHandler updatesHandler, ExceptionHandler exceptionHandler) { - handlers.put(0L, new Handler(updatesHandler, exceptionHandler)); + updateHandlers.put(nativeClientId, new Handler(updatesHandler, exceptionHandler)); } /** @@ -174,9 +174,10 @@ public final class Client implements Runnable { while (!stopFlag) { Thread.yield(); } - while (handlers.size() != 1) { + while (!handlers.isEmpty()) { receiveQueries(300.0); } + updateHandlers.remove(nativeClientId); destroyNativeClient(nativeClientId); } finally { writeLock.unlock(); @@ -191,6 +192,8 @@ public final class Client implements Runnable { private volatile boolean isClientDestroyed = false; private final long nativeClientId; + private static final ConcurrentHashMap updateHandlers = new ConcurrentHashMap(); + private final ConcurrentHashMap handlers = new ConcurrentHashMap(); private final AtomicLong currentQueryId = new AtomicLong(); @@ -212,7 +215,7 @@ public final class Client implements Runnable { private Client(ResultHandler updatesHandler, ExceptionHandler updateExceptionHandler, ExceptionHandler defaultExceptionHandler) { nativeClientId = createNativeClient(); - handlers.put(0L, new Handler(updatesHandler, updateExceptionHandler)); + updateHandlers.put(nativeClientId, new Handler(updatesHandler, updateExceptionHandler)); this.defaultExceptionHandler = defaultExceptionHandler; } @@ -234,7 +237,7 @@ public final class Client implements Runnable { Handler handler; if (id == 0) { // update handler stays forever - handler = handlers.get(id); + handler = updateHandlers.get(nativeClientId); } else { handler = handlers.remove(id); } From 5f605d2dd2e11349e9477909b622db2eff5aa723 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 5 Oct 2020 20:13:15 +0300 Subject: [PATCH 06/13] Fix spelling of "update handler". GitOrigin-RevId: 2137852d967606e909e82396b4103817b1b39386 --- example/csharp/TdExample.cs | 4 +-- example/java/org/drinkless/tdlib/Client.java | 30 +++++++++---------- .../org/drinkless/tdlib/example/Example.java | 6 ++-- td/telegram/ClientDotNet.cpp | 16 +++++----- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/example/csharp/TdExample.cs b/example/csharp/TdExample.cs index 960e81d8a..eb5c0c0f5 100644 --- a/example/csharp/TdExample.cs +++ b/example/csharp/TdExample.cs @@ -33,7 +33,7 @@ namespace TdExample private static Td.Client CreateTdClient() { - Td.Client result = Td.Client.Create(new UpdatesHandler()); + Td.Client result = Td.Client.Create(new UpdateHandler()); new Thread(() => { Thread.CurrentThread.IsBackground = true; @@ -250,7 +250,7 @@ namespace TdExample } } - private class UpdatesHandler : Td.ClientResultHandler + private class UpdateHandler : Td.ClientResultHandler { void Td.ClientResultHandler.OnResult(TdApi.BaseObject @object) { diff --git a/example/java/org/drinkless/tdlib/Client.java b/example/java/org/drinkless/tdlib/Client.java index 9baef5a04..dc5a4bfb0 100644 --- a/example/java/org/drinkless/tdlib/Client.java +++ b/example/java/org/drinkless/tdlib/Client.java @@ -106,27 +106,27 @@ public final class Client implements Runnable { /** * Replaces handler for incoming updates from the TDLib. * - * @param updatesHandler Handler with onResult method which will be called for every incoming + * @param updateHandler Handler with onResult method which will be called for every incoming * update from the TDLib. * @param exceptionHandler Exception handler with onException method which will be called on - * exception thrown from updatesHandler, if it is null, defaultExceptionHandler will be invoked. + * exception thrown from updateHandler, if it is null, defaultExceptionHandler will be invoked. */ - public void setUpdatesHandler(ResultHandler updatesHandler, ExceptionHandler exceptionHandler) { - updateHandlers.put(nativeClientId, new Handler(updatesHandler, exceptionHandler)); + public void setUpdateHandler(ResultHandler updateHandler, ExceptionHandler exceptionHandler) { + updateHandlers.put(nativeClientId, new Handler(updateHandler, exceptionHandler)); } /** * Replaces handler for incoming updates from the TDLib. Sets empty ExceptionHandler. * - * @param updatesHandler Handler with onResult method which will be called for every incoming - * update from the TDLib. + * @param updateHandler Handler with onResult method which will be called for every incoming + * update from the TDLib. */ - public void setUpdatesHandler(ResultHandler updatesHandler) { - setUpdatesHandler(updatesHandler, null); + public void setUpdateHandler(ResultHandler updateHandler) { + setUpdateHandler(updateHandler, null); } /** - * Replaces default exception handler to be invoked on exceptions thrown from updatesHandler and all other ResultHandler. + * Replaces default exception handler to be invoked on exceptions thrown from updateHandler and all other ResultHandler. * * @param defaultExceptionHandler Default exception handler. If null Exceptions are ignored. */ @@ -147,13 +147,13 @@ public final class Client implements Runnable { /** * Creates new Client. * - * @param updatesHandler Handler for incoming updates. - * @param updatesExceptionHandler Handler for exceptions thrown from updatesHandler. If it is null, exceptions will be iggnored. + * @param updateHandler Handler for incoming updates. + * @param updateExceptionHandler Handler for exceptions thrown from updateHandler. If it is null, exceptions will be iggnored. * @param defaultExceptionHandler Default handler for exceptions thrown from all ResultHandler. If it is null, exceptions will be iggnored. * @return created Client */ - public static Client create(ResultHandler updatesHandler, ExceptionHandler updatesExceptionHandler, ExceptionHandler defaultExceptionHandler) { - Client client = new Client(updatesHandler, updatesExceptionHandler, defaultExceptionHandler); + public static Client create(ResultHandler updateHandler, ExceptionHandler updateExceptionHandler, ExceptionHandler defaultExceptionHandler) { + Client client = new Client(updateHandler, updateExceptionHandler, defaultExceptionHandler); new Thread(client, "TDLib thread").start(); return client; } @@ -213,9 +213,9 @@ public final class Client implements Runnable { } } - private Client(ResultHandler updatesHandler, ExceptionHandler updateExceptionHandler, ExceptionHandler defaultExceptionHandler) { + private Client(ResultHandler updateHandler, ExceptionHandler updateExceptionHandler, ExceptionHandler defaultExceptionHandler) { nativeClientId = createNativeClient(); - updateHandlers.put(nativeClientId, new Handler(updatesHandler, updateExceptionHandler)); + updateHandlers.put(nativeClientId, new Handler(updateHandler, updateExceptionHandler)); this.defaultExceptionHandler = defaultExceptionHandler; } diff --git a/example/java/org/drinkless/tdlib/example/Example.java b/example/java/org/drinkless/tdlib/example/Example.java index c2d3598dc..8e52fda92 100644 --- a/example/java/org/drinkless/tdlib/example/Example.java +++ b/example/java/org/drinkless/tdlib/example/Example.java @@ -161,7 +161,7 @@ public final class Example { case TdApi.AuthorizationStateClosed.CONSTRUCTOR: print("Closed"); if (!quiting) { - client = Client.create(new UpdatesHandler(), null, null); // recreate client after previous has closed + client = Client.create(new UpdateHandler(), null, null); // recreate client after previous has closed } break; default: @@ -310,7 +310,7 @@ public final class Example { } // create client - client = Client.create(new UpdatesHandler(), null, null); + client = Client.create(new UpdateHandler(), null, null); // test Client.execute defaultHandler.onResult(Client.execute(new TdApi.GetTextEntities("@telegram /test_command https://telegram.org telegram.me @gif @test"))); @@ -367,7 +367,7 @@ public final class Example { } } - private static class UpdatesHandler implements Client.ResultHandler { + private static class UpdateHandler implements Client.ResultHandler { @Override public void onResult(TdApi.Object object) { switch (object.getConstructor()) { diff --git a/td/telegram/ClientDotNet.cpp b/td/telegram/ClientDotNet.cpp index 080f72464..98d35aeeb 100644 --- a/td/telegram/ClientDotNet.cpp +++ b/td/telegram/ClientDotNet.cpp @@ -79,9 +79,9 @@ public: /// /// Replaces handler for incoming updates from the TDLib. /// - /// Handler with OnResult method which will be called for every incoming update from the TDLib. - void SetUpdatesHandler(ClientResultHandler^ updatesHandler) { - handlers[0] = updatesHandler; + /// Handler with OnResult method which will be called for every incoming update from the TDLib. + void SetUpdateHandler(ClientResultHandler^ updateHandler) { + handlers[0] = updateHandler; } /// @@ -107,16 +107,16 @@ public: /// /// Creates new Client. /// - /// Handler for incoming updates. + /// Handler for incoming updates. /// Returns created Client. - static Client^ Create(ClientResultHandler^ updatesHandler) { - return REF_NEW Client(updatesHandler); + static Client^ Create(ClientResultHandler^ updateHandler) { + return REF_NEW Client(updateHandler); } private: - Client(ClientResultHandler^ updatesHandler) { + Client(ClientResultHandler^ updateHandler) { client = new td::Client(); - handlers[0] = updatesHandler; + handlers[0] = updateHandler; } ~Client() { From da55a34afc53a42cb0b0d899f79159771c628980 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 5 Oct 2020 20:18:59 +0300 Subject: [PATCH 07/13] Remove ability to change update handler after client creation. GitOrigin-RevId: 917737a440afaeb99f8a8ca809cda2d267bb4b02 --- example/java/org/drinkless/tdlib/Client.java | 22 -------------------- td/telegram/ClientDotNet.cpp | 8 ------- 2 files changed, 30 deletions(-) diff --git a/example/java/org/drinkless/tdlib/Client.java b/example/java/org/drinkless/tdlib/Client.java index dc5a4bfb0..9050e467b 100644 --- a/example/java/org/drinkless/tdlib/Client.java +++ b/example/java/org/drinkless/tdlib/Client.java @@ -103,28 +103,6 @@ public final class Client implements Runnable { return nativeClientExecute(query); } - /** - * Replaces handler for incoming updates from the TDLib. - * - * @param updateHandler Handler with onResult method which will be called for every incoming - * update from the TDLib. - * @param exceptionHandler Exception handler with onException method which will be called on - * exception thrown from updateHandler, if it is null, defaultExceptionHandler will be invoked. - */ - public void setUpdateHandler(ResultHandler updateHandler, ExceptionHandler exceptionHandler) { - updateHandlers.put(nativeClientId, new Handler(updateHandler, exceptionHandler)); - } - - /** - * Replaces handler for incoming updates from the TDLib. Sets empty ExceptionHandler. - * - * @param updateHandler Handler with onResult method which will be called for every incoming - * update from the TDLib. - */ - public void setUpdateHandler(ResultHandler updateHandler) { - setUpdateHandler(updateHandler, null); - } - /** * Replaces default exception handler to be invoked on exceptions thrown from updateHandler and all other ResultHandler. * diff --git a/td/telegram/ClientDotNet.cpp b/td/telegram/ClientDotNet.cpp index 98d35aeeb..4d5cf4612 100644 --- a/td/telegram/ClientDotNet.cpp +++ b/td/telegram/ClientDotNet.cpp @@ -76,14 +76,6 @@ public: return Api::FromUnmanaged(*td::Client::execute(std::move(request)).object); } - /// - /// Replaces handler for incoming updates from the TDLib. - /// - /// Handler with OnResult method which will be called for every incoming update from the TDLib. - void SetUpdateHandler(ClientResultHandler^ updateHandler) { - handlers[0] = updateHandler; - } - /// /// Launches a cycle which will fetch all results of queries to TDLib and incoming updates from TDLib. /// Must be called once on a separate dedicated thread, on which all updates and query results will be handled. From e57669363c08ce3e4caf80fe68d5bef026857437 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 5 Oct 2020 21:18:38 +0300 Subject: [PATCH 08/13] Minor binding name fix. GitOrigin-RevId: db092245555fb9da70980ecf9c087beae04bb5ae --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a0210500..bb3eb774a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -670,7 +670,7 @@ Changes in 1.3.0 (5 Sep 2018): `venue_search_bot_username` containing usernames of bots which can be used in inline mode for animations, photos and venues search respectively. * Numerous optimizations and bug fixes: - - Fixed string encoding for C# binding. + - Fixed string encoding for .NET binding. - Fixed building TDLib SDK for Universal Windows Platform for ARM with MSVC 2017. - Fixed the Swift example project. - Fixed the syntax error in the Python example. From b204ad4ee15cc4937cb99de9823747e86dc01d10 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 5 Oct 2020 21:31:27 +0300 Subject: [PATCH 09/13] Store default exception handlers as static map in Java example. GitOrigin-RevId: 5745859a70250d104a21edad473b655967060609 --- example/java/org/drinkless/tdlib/Client.java | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/example/java/org/drinkless/tdlib/Client.java b/example/java/org/drinkless/tdlib/Client.java index 9050e467b..986d5b3c7 100644 --- a/example/java/org/drinkless/tdlib/Client.java +++ b/example/java/org/drinkless/tdlib/Client.java @@ -103,15 +103,6 @@ public final class Client implements Runnable { return nativeClientExecute(query); } - /** - * Replaces default exception handler to be invoked on exceptions thrown from updateHandler and all other ResultHandler. - * - * @param defaultExceptionHandler Default exception handler. If null Exceptions are ignored. - */ - public void setDefaultExceptionHandler(Client.ExceptionHandler defaultExceptionHandler) { - this.defaultExceptionHandler = defaultExceptionHandler; - } - /** * Overridden method from Runnable, do not call it directly. */ @@ -156,6 +147,7 @@ public final class Client implements Runnable { receiveQueries(300.0); } updateHandlers.remove(nativeClientId); + defaultExceptionHandlers.remove(nativeClientId); destroyNativeClient(nativeClientId); } finally { writeLock.unlock(); @@ -170,13 +162,12 @@ public final class Client implements Runnable { private volatile boolean isClientDestroyed = false; private final long nativeClientId; + private static final ConcurrentHashMap defaultExceptionHandlers = new ConcurrentHashMap(); private static final ConcurrentHashMap updateHandlers = new ConcurrentHashMap(); private final ConcurrentHashMap handlers = new ConcurrentHashMap(); private final AtomicLong currentQueryId = new AtomicLong(); - private volatile ExceptionHandler defaultExceptionHandler = null; - private static final int MAX_EVENTS = 1000; private final long[] eventIds = new long[MAX_EVENTS]; private final TdApi.Object[] events = new TdApi.Object[MAX_EVENTS]; @@ -194,7 +185,9 @@ public final class Client implements Runnable { private Client(ResultHandler updateHandler, ExceptionHandler updateExceptionHandler, ExceptionHandler defaultExceptionHandler) { nativeClientId = createNativeClient(); updateHandlers.put(nativeClientId, new Handler(updateHandler, updateExceptionHandler)); - this.defaultExceptionHandler = defaultExceptionHandler; + if (defaultExceptionHandler != null) { + defaultExceptionHandlers.put(nativeClientId, defaultExceptionHandler); + } } @Override @@ -235,7 +228,7 @@ public final class Client implements Runnable { resultHandler.onResult(object); } catch (Throwable cause) { if (exceptionHandler == null) { - exceptionHandler = defaultExceptionHandler; + exceptionHandler = defaultExceptionHandlers.get(nativeClientId); } if (exceptionHandler != null) { try { From 8a607933a706091318b0a5cffa2af06dbf15b519 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 5 Oct 2020 22:08:24 +0300 Subject: [PATCH 10/13] Rename "files" log tag to "file_loader". GitOrigin-RevId: 24fca707a99eeb32adabcca46131123d584f2c1d --- td/telegram/Logging.cpp | 10 +++++----- td/telegram/files/FileLoader.cpp | 16 ++++++++-------- td/telegram/files/FileLoaderUtils.cpp | 2 +- td/telegram/files/FileLoaderUtils.h | 2 +- td/telegram/files/FileManager.cpp | 6 +++--- td/telegram/files/PartsManager.cpp | 8 ++++---- td/telegram/files/ResourceManager.cpp | 12 +++++------- 7 files changed, 27 insertions(+), 29 deletions(-) diff --git a/td/telegram/Logging.cpp b/td/telegram/Logging.cpp index 97c57fa78..8694ab498 100644 --- a/td/telegram/Logging.cpp +++ b/td/telegram/Logging.cpp @@ -48,11 +48,11 @@ static NullLog null_log; #define ADD_TAG(tag) \ { #tag, &VERBOSITY_NAME(tag) } static const std::map log_tags{ - ADD_TAG(td_init), ADD_TAG(update_file), ADD_TAG(connections), ADD_TAG(binlog), - ADD_TAG(proxy), ADD_TAG(net_query), ADD_TAG(td_requests), ADD_TAG(dc), - ADD_TAG(files), ADD_TAG(mtproto), ADD_TAG(raw_mtproto), ADD_TAG(fd), - ADD_TAG(actor), ADD_TAG(sqlite), ADD_TAG(notifications), ADD_TAG(get_difference), - ADD_TAG(file_gc), ADD_TAG(config_recoverer), ADD_TAG(dns_resolver), ADD_TAG(file_references)}; + ADD_TAG(td_init), ADD_TAG(update_file), ADD_TAG(connections), ADD_TAG(binlog), + ADD_TAG(proxy), ADD_TAG(net_query), ADD_TAG(td_requests), ADD_TAG(dc), + ADD_TAG(file_loader), ADD_TAG(mtproto), ADD_TAG(raw_mtproto), ADD_TAG(fd), + ADD_TAG(actor), ADD_TAG(sqlite), ADD_TAG(notifications), ADD_TAG(get_difference), + ADD_TAG(file_gc), ADD_TAG(config_recoverer), ADD_TAG(dns_resolver), ADD_TAG(file_references)}; #undef ADD_TAG Status Logging::set_current_stream(td_api::object_ptr stream) { diff --git a/td/telegram/files/FileLoader.cpp b/td/telegram/files/FileLoader.cpp index c93c29644..a515e766a 100644 --- a/td/telegram/files/FileLoader.cpp +++ b/td/telegram/files/FileLoader.cpp @@ -31,7 +31,7 @@ void FileLoader::update_priority(int8 priority) { } void FileLoader::update_resources(const ResourceState &other) { resource_state_.update_slave(other); - VLOG(files) << "Update resources " << resource_state_; + VLOG(file_loader) << "Update resources " << resource_state_; loop(); } void FileLoader::set_ordered_flag(bool flag) { @@ -79,10 +79,10 @@ void FileLoader::update_downloaded_part(int64 offset, int64 limit) { : static_cast((offset + limit - 1) / parts_manager_.get_part_size()) + 1; auto max_parts = static_cast(ResourceManager::MAX_RESOURCE_LIMIT / parts_manager_.get_part_size()); auto end_part_id = begin_part_id + td::min(max_parts, new_end_part_id - begin_part_id); - VLOG(files) << "Protect parts " << begin_part_id << " ... " << end_part_id - 1; + VLOG(file_loader) << "Protect parts " << begin_part_id << " ... " << end_part_id - 1; for (auto &it : part_map_) { if (!it.second.second.empty() && !(begin_part_id <= it.second.first.id && it.second.first.id < end_part_id)) { - VLOG(files) << "Cancel part " << it.second.first.id; + VLOG(file_loader) << "Cancel part " << it.second.first.id; it.second.second.reset(); // cancel_query(it.second.second); } } @@ -194,14 +194,14 @@ Status FileLoader::do_loop() { break; } if (resource_state_.unused() < static_cast(parts_manager_.get_part_size())) { - VLOG(files) << "Got only " << resource_state_.unused() << " resource"; + VLOG(file_loader) << "Got only " << resource_state_.unused() << " resource"; break; } TRY_RESULT(part, parts_manager_.start_part()); if (part.size == 0) { break; } - VLOG(files) << "Start part " << tag("id", part.id) << tag("size", part.size); + VLOG(file_loader) << "Start part " << tag("id", part.id) << tag("size", part.size); resource_state_.start_use(static_cast(part.size)); TRY_RESULT(query_flag, start_part(part, parts_manager_.get_part_count(), parts_manager_.get_streaming_offset())); @@ -245,7 +245,7 @@ void FileLoader::update_estimated_limit() { } auto estimated_extra = parts_manager_.get_estimated_extra(); resource_state_.update_estimated_limit(estimated_extra); - VLOG(files) << "Update estimated limit " << estimated_extra; + VLOG(file_loader) << "Update estimated limit " << estimated_extra; if (!resource_manager_.empty()) { keep_fd_flag(narrow_cast(resource_state_.active_limit()) >= parts_manager_.get_part_size()); send_closure(resource_manager_, &ResourceManager::update_resources, resource_state_); @@ -282,7 +282,7 @@ void FileLoader::on_result(NetQueryPtr query) { should_restart = true; } if (should_restart) { - VLOG(files) << "Restart part " << tag("id", part.id) << tag("size", part.size); + VLOG(file_loader) << "Restart part " << tag("id", part.id) << tag("size", part.size); resource_state_.stop_use(static_cast(part.size)); parts_manager_.on_part_failed(part.id); } else { @@ -331,7 +331,7 @@ void FileLoader::on_common_query(NetQueryPtr query) { Status FileLoader::try_on_part_query(Part part, NetQueryPtr query) { TRY_RESULT(size, process_part(part, std::move(query))); - VLOG(files) << "Ok part " << tag("id", part.id) << tag("size", part.size); + VLOG(file_loader) << "Ok part " << tag("id", part.id) << tag("size", part.size); resource_state_.stop_use(static_cast(part.size)); auto old_ready_prefix_count = parts_manager_.get_unchecked_ready_prefix_count(); TRY_STATUS(parts_manager_.on_part_ok(part.id, part.size, size)); diff --git a/td/telegram/files/FileLoaderUtils.cpp b/td/telegram/files/FileLoaderUtils.cpp index 143479960..08729cc29 100644 --- a/td/telegram/files/FileLoaderUtils.cpp +++ b/td/telegram/files/FileLoaderUtils.cpp @@ -25,7 +25,7 @@ namespace td { -int VERBOSITY_NAME(files) = VERBOSITY_NAME(DEBUG) + 2; +int VERBOSITY_NAME(file_loader) = VERBOSITY_NAME(DEBUG) + 2; namespace { Result> try_create_new_file(Result result_name) { diff --git a/td/telegram/files/FileLoaderUtils.h b/td/telegram/files/FileLoaderUtils.h index 526b37dc3..742343dbf 100644 --- a/td/telegram/files/FileLoaderUtils.h +++ b/td/telegram/files/FileLoaderUtils.h @@ -19,7 +19,7 @@ namespace td { -extern int VERBOSITY_NAME(files); +extern int VERBOSITY_NAME(file_loader); Result> open_temp_file(FileType file_type) TD_WARN_UNUSED_RESULT; diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index 5aa606d1d..55108dd09 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -943,11 +943,11 @@ Status FileManager::check_local_location(FullLocalFileLocation &location, int64 size = stat.size_; } if (location.mtime_nsec_ == 0) { - VLOG(files) << "Set file \"" << location.path_ << "\" modification time to " << stat.mtime_nsec_; + VLOG(file_loader) << "Set file \"" << location.path_ << "\" modification time to " << stat.mtime_nsec_; location.mtime_nsec_ = stat.mtime_nsec_; } else if (!are_modification_times_equal(location.mtime_nsec_, stat.mtime_nsec_)) { - VLOG(files) << "File \"" << location.path_ << "\" was modified: old mtime = " << location.mtime_nsec_ - << ", new mtime = " << stat.mtime_nsec_; + VLOG(file_loader) << "File \"" << location.path_ << "\" was modified: old mtime = " << location.mtime_nsec_ + << ", new mtime = " << stat.mtime_nsec_; return Status::Error(PSLICE() << "File \"" << location.path_ << "\" was modified"); } if (skip_file_size_checks) { diff --git a/td/telegram/files/PartsManager.cpp b/td/telegram/files/PartsManager.cpp index 3777fd729..b986c57ca 100644 --- a/td/telegram/files/PartsManager.cpp +++ b/td/telegram/files/PartsManager.cpp @@ -147,9 +147,9 @@ Status PartsManager::init(int64 size, int64 expected_size, bool is_size_final, s } bool PartsManager::unchecked_ready() { - VLOG(files) << "Check readiness. Ready size is " << ready_size_ << ", total size is " << size_ - << ", unknown_size_flag = " << unknown_size_flag_ << ", need_check = " << need_check_ - << ", checked_prefix_size = " << checked_prefix_size_; + VLOG(file_loader) << "Check readiness. Ready size is " << ready_size_ << ", total size is " << size_ + << ", unknown_size_flag = " << unknown_size_flag_ << ", need_check = " << need_check_ + << ", checked_prefix_size = " << checked_prefix_size_; return !unknown_size_flag_ && ready_size_ == size_; } @@ -352,7 +352,7 @@ Status PartsManager::on_part_ok(int32 id, size_t part_size, size_t actual_size) streaming_ready_size_ += narrow_cast(actual_size); } - VLOG(files) << "Transferred part " << id << " of size " << part_size << ", total ready size = " << ready_size_; + VLOG(file_loader) << "Transferred part " << id << " of size " << part_size << ", total ready size = " << ready_size_; int64 offset = narrow_cast(part_size_) * id; int64 end_offset = offset + narrow_cast(actual_size); diff --git a/td/telegram/files/ResourceManager.cpp b/td/telegram/files/ResourceManager.cpp index 641403c3b..551fba7a0 100644 --- a/td/telegram/files/ResourceManager.cpp +++ b/td/telegram/files/ResourceManager.cpp @@ -53,13 +53,11 @@ void ResourceManager::update_resources(const ResourceState &resource_state) { } auto node = (*node_ptr).get(); CHECK(node); - VLOG(files) << "Before total: " << resource_state_; - VLOG(files) << "Before " << tag("node_id", node_id) << ": " << node->resource_state_; + VLOG(file_loader) << "Before total: " << resource_state_ << "; node " << node_id << ": " << node->resource_state_; resource_state_ -= node->resource_state_; node->resource_state_.update_master(resource_state); resource_state_ += node->resource_state_; - VLOG(files) << "After total: " << resource_state_; - VLOG(files) << "After " << tag("node_id", node_id) << ": " << node->resource_state_; + VLOG(file_loader) << "After total: " << resource_state_ << "; node " << node_id << ": " << node->resource_state_; if (mode_ == Mode::Greedy) { add_to_heap(node); @@ -107,16 +105,16 @@ bool ResourceManager::satisfy_node(NodeId file_node_id) { CHECK(file_node); auto part_size = narrow_cast(file_node->resource_state_.unit_size()); auto need = file_node->resource_state_.estimated_extra(); - VLOG(files) << tag("need", need) << tag("part_size", part_size); + VLOG(file_loader) << tag("need", need) << tag("part_size", part_size); need = (need + part_size - 1) / part_size * part_size; - VLOG(files) << tag("need", need); + VLOG(file_loader) << tag("need", need); if (need == 0) { return true; } auto give = resource_state_.unused(); give = min(need, give); give -= give % part_size; - VLOG(files) << tag("give", give); + VLOG(file_loader) << tag("give", give); if (give == 0) { return false; } From f7393d27175d5908068bc04939d36efd56722771 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 5 Oct 2020 23:05:16 +0300 Subject: [PATCH 11/13] Fix handling of requests to unexisting Client. GitOrigin-RevId: 5de9709abe73ae73dc52c9ca04c480a60efbf437 --- td/telegram/Client.cpp | 14 +++++++++++++- test/tdclient.cpp | 11 +++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/td/telegram/Client.cpp b/td/telegram/Client.cpp index 68ae138ec..e2ba2acb2 100644 --- a/td/telegram/Client.cpp +++ b/td/telegram/Client.cpp @@ -100,6 +100,10 @@ class TdReceiver { return td::make_unique(client_id, this); } + void add_response(ClientManager::ClientId client_id, uint64 id, td_api::object_ptr result) { + responses_.push({client_id, id, std::move(result)}); + } + private: std::queue responses_; }; @@ -256,6 +260,10 @@ class TdReceiver { return td::make_unique(client_id, output_queue_); } + void add_response(ClientManager::ClientId client_id, uint64 id, td_api::object_ptr result) { + output_queue_->writer_put({client_id, id, std::move(result)}); + } + private: using OutputQueue = MpscPollableQueue; std::shared_ptr output_queue_; @@ -385,7 +393,11 @@ class ClientManager::Impl final { void send(ClientId client_id, RequestId request_id, td_api::object_ptr &&request) { auto lock = impls_mutex_.lock_read().move_as_ok(); auto it = impls_.find(client_id); - CHECK(it != impls_.end()); + if (it == impls_.end()) { + receiver_->add_response(client_id, request_id, + td_api::make_object(400, "Invalid TDLib instance specified")); + return; + } it->second->send(client_id, request_id, std::move(request)); } diff --git a/test/tdclient.cpp b/test/tdclient.cpp index 340a0d9b5..ac9257093 100644 --- a/test/tdclient.cpp +++ b/test/tdclient.cpp @@ -922,6 +922,8 @@ TEST(Client, Manager) { td::ClientManager client; int threads_n = 4; int clients_n = 1000; + client.send(0, 3, td::make_tl_object(3)); + client.send(-1, 3, td::make_tl_object(3)); for (int i = 0; i < threads_n; i++) { threads.emplace_back([&] { for (int i = 0; i < clients_n; i++) { @@ -937,8 +939,13 @@ TEST(Client, Manager) { std::set ids; while (ids.size() != static_cast(threads_n) * clients_n) { auto event = client.receive(10); - if (event.client_id != 0 && event.request_id == 3) { - ids.insert(event.client_id); + if (event.client_id == 0 || event.client_id == -1) { + ASSERT_EQ(td_api::error::ID, event.object->get_id()); + continue; + } + if (event.request_id == 3) { + ASSERT_EQ(td_api::testInt::ID, event.object->get_id()); + ASSERT_TRUE(ids.insert(event.client_id).second); } } } From 29f3c91596da4369318b5b9a20b31577e3ff7924 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 6 Oct 2020 02:12:50 +0300 Subject: [PATCH 12/13] Fix request errors during closing. GitOrigin-RevId: 6587b35bcb7aef082686a93e75ae8b891113da0b --- td/telegram/Td.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index 86e63719a..acda48bbf 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -3457,7 +3457,7 @@ void Td::request(uint64 id, tl_object_ptr function) { pending_preauthentication_requests_.emplace_back(id, std::move(function)); return; } - return send_error_raw(id, 401, "Initialization parameters are needed: call setTdlibParameters first"); + return send_error_raw(id, 400, "Initialization parameters are needed: call setTdlibParameters first"); } break; } @@ -3486,12 +3486,16 @@ void Td::request(uint64 id, tl_object_ptr function) { pending_preauthentication_requests_.emplace_back(id, std::move(function)); return; } - return send_error_raw(id, 401, "Database encryption key is needed: call checkDatabaseEncryptionKey first"); + return send_error_raw(id, 400, "Database encryption key is needed: call checkDatabaseEncryptionKey first"); } return answer_ok_query(id, init(as_db_key(encryption_key))); } case State::Close: - return send_error_raw(id, 401, "Unauthorized"); + if (destroy_flag_) { + return send_error_raw(id, 401, "Unauthorized"); + } else { + return send_error_raw(id, 500, "Request aborted"); + } case State::Run: break; } From d87558177241862f7db1b934f8a211b94180f28b Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 6 Oct 2020 20:46:54 +0300 Subject: [PATCH 13/13] Simplify Client closing. GitOrigin-RevId: 9a35d7ef66cea495d376a98df81a568478b0bb35 --- td/telegram/Client.cpp | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/td/telegram/Client.cpp b/td/telegram/Client.cpp index e2ba2acb2..9f6be8035 100644 --- a/td/telegram/Client.cpp +++ b/td/telegram/Client.cpp @@ -53,8 +53,8 @@ class MultiTd : public Actor { } void close(int32 td_id) { - auto size = tds_.erase(td_id); - CHECK(size == 1); + // no check that td_id hasn't been deleted before + tds_.erase(td_id); } private: @@ -151,7 +151,7 @@ class ClientManager::Impl final { } else { ConcurrentScheduler::emscripten_clear_main_timeout(); } - if (response.client_id != 0 && !response.object) { + if (response.object == nullptr && response.client_id != 0 && response.request_id == 0) { auto guard = concurrent_scheduler_->get_main_guard(); tds_.erase(response.client_id); } @@ -322,9 +322,9 @@ class MultiImpl { send_closure(multi_td_, &MultiTd::send, client_id, request_id, std::move(request)); } - void close(int32 td_id) { + void close(ClientManager::ClientId client_id) { auto guard = concurrent_scheduler_->get_send_guard(); - send_closure(multi_td_, &MultiTd::close, td_id); + send_closure(multi_td_, &MultiTd::close, client_id); } ~MultiImpl() { @@ -402,12 +402,12 @@ class ClientManager::Impl final { } Response receive(double timeout) { - auto res = receiver_->receive(timeout); - if (res.client_id != 0 && !res.object) { + auto response = receiver_->receive(timeout); + if (response.object == nullptr && response.client_id != 0 && response.request_id == 0) { auto lock = impls_mutex_.lock_write().move_as_ok(); - impls_.erase(res.client_id); + impls_.erase(response.client_id); } - return res; + return response; } Impl() = default; @@ -452,10 +452,6 @@ class Client::Impl final { Client::Response receive(double timeout) { auto res = receiver_->receive(timeout); - if (res.client_id != 0 && !res.object) { - is_closed_ = true; - } - Client::Response old_res; old_res.id = res.request_id; old_res.object = std::move(res.object); @@ -468,8 +464,11 @@ class Client::Impl final { Impl &operator=(Impl &&) = delete; ~Impl() { multi_impl_->close(td_id_); - while (!is_closed_) { - receive(10); + while (true) { + auto response = receiver_->receive(10.0); + if (response.object == nullptr && response.client_id != 0 && response.request_id == 0) { + break; + } } } @@ -477,7 +476,6 @@ class Client::Impl final { std::shared_ptr multi_impl_; unique_ptr receiver_; - bool is_closed_{false}; int32 td_id_; }; #endif