From c8e0c2fedc81f0530947298bf68be8868d93ec67 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 2 Jan 2023 13:10:12 +0300 Subject: [PATCH 01/44] Improve internalLinkTypePassportDataRequest documentation. --- td/generate/scheme/td_api.tl | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index e286c2ec7..d39b37acd 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -649,9 +649,12 @@ user id:int53 first_name:string last_name:string usernames:usernames phone_numbe botInfo share_text:string description:string photo:photo animation:animation menu_button:botMenuButton commands:vector default_group_administrator_rights:chatAdministratorRights default_channel_administrator_rights:chatAdministratorRights = BotInfo; //@description Contains full information about a user -//@personal_photo User profile photo set by the current user for the contact; may be null. If null and user.profile_photo is null, then the photo is empty, otherwise unknown. If non-null, then it is the same photo as in user.profile_photo and chat.photo. This photo isn't returned in the list of user photos -//@photo User profile photo; may be null. If null and user.profile_photo is null, then the photo is empty, otherwise unknown. If non-null and personal_photo is null, then it is the same photo as in user.profile_photo and chat.photo -//@public_photo User profile photo visible if the main photo is hidden by privacy settings; may be null. If null and user.profile_photo is null, then the photo is empty, otherwise unknown. If non-null and both photo and personal_photo are null, then it is the same photo as in user.profile_photo and chat.photo. This photo isn't returned in the list of user photos +//@personal_photo User profile photo set by the current user for the contact; may be null. If null and user.profile_photo is null, then the photo is empty, otherwise unknown. +//-If non-null, then it is the same photo as in user.profile_photo and chat.photo. This photo isn't returned in the list of user photos +//@photo User profile photo; may be null. If null and user.profile_photo is null, then the photo is empty, otherwise unknown. +//-If non-null and personal_photo is null, then it is the same photo as in user.profile_photo and chat.photo +//@public_photo User profile photo visible if the main photo is hidden by privacy settings; may be null. If null and user.profile_photo is null, then the photo is empty, otherwise unknown. +//-If non-null and both photo and personal_photo are null, then it is the same photo as in user.profile_photo and chat.photo. This photo isn't returned in the list of user photos //@is_blocked True, if the user is blocked by the current user //@can_be_called True, if the user can be called //@supports_video_calls True, if a video call can be created with the user @@ -3764,7 +3767,8 @@ storePaymentPurposePremiumSubscription is_restore:Bool = StorePaymentPurpose; storePaymentPurposeGiftedPremium user_id:int53 currency:string amount:int53 = StorePaymentPurpose; -//@class DeviceToken @description Represents a data needed to subscribe for push notifications through registerDevice method. To use specific push notification service, the correct application platform must be specified and a valid server authentication data must be uploaded at https://my.telegram.org +//@class DeviceToken @description Represents a data needed to subscribe for push notifications through registerDevice method. +//-To use specific push notification service, the correct application platform must be specified and a valid server authentication data must be uploaded at https://my.telegram.org //@description A token for Firebase Cloud Messaging @token Device registration token; may be empty to deregister a device @encrypt True, if push notifications must be additionally encrypted deviceTokenFirebaseCloudMessaging token:string encrypt:Bool = DeviceToken; @@ -4477,7 +4481,8 @@ internalLinkTypeMessageDraft text:formattedText contains_link:Bool = InternalLin //@scope Telegram Passport element types requested by the service //@public_key Service's public key //@nonce Unique request identifier provided by the service -//@callback_url An HTTP URL to open once the request is finished or canceled with the parameter tg_passport=success or tg_passport=cancel respectively. If empty, then the link tgbot{bot_user_id}://passport/success or tgbot{bot_user_id}://passport/cancel needs to be opened instead +//@callback_url An HTTP URL to open once the request is finished, canceled, or failed with the parameters tg_passport=success, tg_passport=cancel, or tg_passport=error&error=... respectively. +//-If empty, then onActivityResult method must be used to return response on Android, or the link tgbot{bot_user_id}://passport/success or tgbot{bot_user_id}://passport/cancel must be opened otherwise internalLinkTypePassportDataRequest bot_user_id:int53 scope:string public_key:string nonce:string callback_url:string = InternalLinkType; //@description The link can be used to confirm ownership of a phone number to prevent account deletion. Call sendPhoneNumberConfirmationCode with the given hash and phone number to process the link @@ -5768,7 +5773,9 @@ getMessageThreadHistory chat_id:int53 message_id:int53 from_message_id:int53 off //@revoke Pass true to delete chat history for all users deleteChatHistory chat_id:int53 remove_from_chat_list:Bool revoke:Bool = Ok; -//@description Deletes a chat along with all messages in the corresponding chat for all chat members. For group chats this will release the usernames and remove all members. Use the field chat.can_be_deleted_for_all_users to find whether the method can be applied to the chat @chat_id Chat identifier +//@description Deletes a chat along with all messages in the corresponding chat for all chat members. For group chats this will release the usernames and remove all members. +//-Use the field chat.can_be_deleted_for_all_users to find whether the method can be applied to the chat +//@chat_id Chat identifier deleteChat chat_id:int53 = Ok; //@description Searches for messages with given words in the chat. Returns the results in reverse chronological order, i.e. in order of decreasing message_id. Cannot be used in secret chats with a non-empty query @@ -7350,7 +7357,9 @@ toggleSupergroupJoinByRequest supergroup_id:int53 join_by_request:Bool = Ok; //@description Toggles whether the message history of a supergroup is available to new members; requires can_change_info administrator right @supergroup_id The identifier of the supergroup @is_all_history_available The new value of is_all_history_available toggleSupergroupIsAllHistoryAvailable supergroup_id:int53 is_all_history_available:Bool = Ok; -//@description Toggles whether non-administrators can receive only administrators and bots using getSupergroupMembers or searchChatMembers. Can be called only if supergroupFullInfo.can_hide_members == true @supergroup_id Identifier of the supergroup @has_hidden_members New value of has_hidden_members +//@description Toggles whether non-administrators can receive only administrators and bots using getSupergroupMembers or searchChatMembers. Can be called only if supergroupFullInfo.can_hide_members == true +//@supergroup_id Identifier of the supergroup +//@has_hidden_members New value of has_hidden_members toggleSupergroupHasHiddenMembers supergroup_id:int53 has_hidden_members:Bool = Ok; //@description Toggles whether aggressive anti-spam checks are enabled in the supergroup. Can be called only if supergroupFullInfo.can_toggle_aggressive_anti_spam == true From 4258030967030bccac7368e9432b0ba036fe67ba Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 2 Jan 2023 13:51:04 +0300 Subject: [PATCH 02/44] Drop cached attachment menu bots when database is deleted. --- td/telegram/AttachMenuManager.h | 4 ++-- td/telegram/TdDb.cpp | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/td/telegram/AttachMenuManager.h b/td/telegram/AttachMenuManager.h index ca49b07e3..b7e38dd2f 100644 --- a/td/telegram/AttachMenuManager.h +++ b/td/telegram/AttachMenuManager.h @@ -55,6 +55,8 @@ class AttachMenuManager final : public Actor { void get_current_state(vector> &updates) const; + static string get_attach_menu_bots_database_key(); + private: static const int32 PING_WEB_VIEW_TIMEOUT = 60; @@ -135,8 +137,6 @@ class AttachMenuManager final : public Actor { void send_update_attach_menu_bots() const; - static string get_attach_menu_bots_database_key(); - void save_attach_menu_bots(); void on_reload_attach_menu_bots(Result> &&result); diff --git a/td/telegram/TdDb.cpp b/td/telegram/TdDb.cpp index c7c5f99d8..5f836d429 100644 --- a/td/telegram/TdDb.cpp +++ b/td/telegram/TdDb.cpp @@ -6,6 +6,7 @@ // #include "td/telegram/TdDb.h" +#include "td/telegram/AttachMenuManager.h" #include "td/telegram/DialogDb.h" #include "td/telegram/files/FileDb.h" #include "td/telegram/Global.h" @@ -390,6 +391,7 @@ Status TdDb::init_sqlite(const TdParameters ¶meters, const DbKey &key, const binlog_pmc.erase("saved_contact_count"); binlog_pmc.erase("old_featured_sticker_set_count"); binlog_pmc.erase("invalidate_old_featured_sticker_sets"); + binlog_pmc.erase(AttachMenuManager::get_attach_menu_bots_database_key()); } binlog_pmc.force_sync({}); From 8b763bedafb3919040af6ce95906c4f6d80f2655 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 2 Jan 2023 14:29:03 +0300 Subject: [PATCH 03/44] Move random_shuffle to tests.h. --- tdutils/td/utils/Random.h | 12 ------------ tdutils/td/utils/tests.h | 10 ++++++++++ tdutils/test/ChainScheduler.cpp | 2 +- tdutils/test/hashset_benchmark.cpp | 2 +- tdutils/test/heap.cpp | 2 +- test/online.cpp | 3 ++- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/tdutils/td/utils/Random.h b/tdutils/td/utils/Random.h index 749e3f3fe..ee0d159ae 100644 --- a/tdutils/td/utils/Random.h +++ b/tdutils/td/utils/Random.h @@ -8,9 +8,6 @@ #include "td/utils/common.h" #include "td/utils/Slice.h" -#include "td/utils/Span.h" - -#include namespace td { @@ -57,13 +54,4 @@ class Random { }; }; -template -void random_shuffle(MutableSpan v, R &rnd) { - for (size_t i = 1; i < v.size(); i++) { - auto pos = static_cast(rnd()) % (i + 1); - using std::swap; - swap(v[i], v[pos]); - } -} - } // namespace td diff --git a/tdutils/td/utils/tests.h b/tdutils/td/utils/tests.h index 781571e78..8e2e518ee 100644 --- a/tdutils/td/utils/tests.h +++ b/tdutils/td/utils/tests.h @@ -12,6 +12,7 @@ #include "td/utils/logging.h" #include "td/utils/port/sleep.h" #include "td/utils/Slice.h" +#include "td/utils/Span.h" #include "td/utils/Status.h" #include @@ -165,6 +166,15 @@ string rand_string(int from, int to, size_t len); vector rand_split(Slice str); +template +void rand_shuffle(MutableSpan v, R &rnd) { + for (size_t i = 1; i < v.size(); i++) { + auto pos = static_cast(rnd()) % (i + 1); + using std::swap; + swap(v[i], v[pos]); + } +} + template void assert_eq_impl(const T1 &expected, const T2 &got, const char *file, int line) { LOG_CHECK(expected == got) << tag("expected", expected) << tag("got", got) << " in " << file << " at line " << line; diff --git a/tdutils/test/ChainScheduler.cpp b/tdutils/test/ChainScheduler.cpp index 3b04d73a1..029a4ff36 100644 --- a/tdutils/test/ChainScheduler.cpp +++ b/tdutils/test/ChainScheduler.cpp @@ -139,7 +139,7 @@ TEST(ChainScheduler, Stress) { int chain_n = rnd.fast(1, ChainsN); td::vector chain_ids(ChainsN); std::iota(chain_ids.begin(), chain_ids.end(), 0); - td::random_shuffle(td::as_mutable_span(chain_ids), rnd); + td::rand_shuffle(td::as_mutable_span(chain_ids), rnd); chain_ids.resize(chain_n); for (auto chain_id : chain_ids) { chains[td::narrow_cast(chain_id)].push_back(query); diff --git a/tdutils/test/hashset_benchmark.cpp b/tdutils/test/hashset_benchmark.cpp index a1df84465..026211d12 100644 --- a/tdutils/test/hashset_benchmark.cpp +++ b/tdutils/test/hashset_benchmark.cpp @@ -204,7 +204,7 @@ static void BM_Get(benchmark::State &state) { } std::size_t key_i = 0; - td::random_shuffle(td::as_mutable_span(keys), rnd); + td::rand_shuffle(td::as_mutable_span(keys), rnd); auto next_key = [&] { key_i++; if (key_i == data.size()) { diff --git a/tdutils/test/heap.cpp b/tdutils/test/heap.cpp index e78cc4d5e..bc317152d 100644 --- a/tdutils/test/heap.cpp +++ b/tdutils/test/heap.cpp @@ -23,7 +23,7 @@ TEST(Heap, sort_random_perm) { v[i] = i; } td::Random::Xorshift128plus rnd(123); - td::random_shuffle(td::as_mutable_span(v), rnd); + td::rand_shuffle(td::as_mutable_span(v), rnd); td::vector nodes(n); td::KHeap kheap; for (int i = 0; i < n; i++) { diff --git a/test/online.cpp b/test/online.cpp index 7f6210c89..1ffc5523f 100644 --- a/test/online.cpp +++ b/test/online.cpp @@ -23,6 +23,7 @@ #include "td/utils/port/signals.h" #include "td/utils/Promise.h" #include "td/utils/Random.h" +#include "td/utils/tests.h" #include #include @@ -437,7 +438,7 @@ class TestDownloadFile : public Task { begin = end; } - random_shuffle(as_mutable_span(ranges_), rnd); + rand_shuffle(as_mutable_span(ranges_), rnd); start_chunk(); } From 903d9947999f6298ff7f83987a4b6cc816f92e3a Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 2 Jan 2023 15:38:04 +0300 Subject: [PATCH 04/44] Improve punctuation. --- CHANGELOG.md | 2 +- CMake/GetGitRevisionDescription.cmake | 2 +- README.md | 2 +- benchmark/bench_queue.cpp | 1 - example/README.md | 6 ++-- example/cpp/README.md | 4 +-- example/csharp/README.md | 2 +- example/java/README.md | 4 +-- example/python/README.md | 2 +- example/swift/README.md | 2 +- example/uwp/README.md | 3 +- td/generate/scheme/td_api.tl | 42 +++++++++++++-------------- td/mtproto/SessionConnection.cpp | 3 +- td/telegram/Client.h | 4 +-- td/telegram/SecretChatActor.cpp | 4 +-- td/telegram/SecretChatActor.h | 4 +-- td/telegram/UpdatesManager.cpp | 3 +- td/telegram/files/FileManager.h | 2 +- td/telegram/net/Session.cpp | 2 +- td/telegram/td_json_client.h | 4 +-- tdutils/td/utils/MpmcWaiter.h | 5 ++-- tdutils/td/utils/ObjectPool.h | 7 ++--- tdutils/td/utils/port/UdpSocketFd.cpp | 4 +-- tdutils/td/utils/port/platform.h | 2 +- 24 files changed, 56 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c526968..3543b504f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1435,7 +1435,7 @@ Changes in 1.2.0 (20 Mar 2018): See [using in .NET projects](README.md#using-dotnet) for more details. * Added a C# example. See [README](example/csharp/README.md) for build and usage instructions. * Added a build and usage example of TDLib SDK for Universal Windows Platform. See [README](example/uwp/README.md) - for detailed build and usage instructions. Also see [Unigram](https://github.com/UnigramDev/Unigram), which is + for detailed build and usage instructions. Also, see [Unigram](https://github.com/UnigramDev/Unigram), which is a full-featured client rewritten from scratch using TDLib SDK for Universal Windows Platform in less than 2 months. * Added a Swift example. See [README](example/swift/README.md) for build and usage instructions. * Added an example of building TDLib for iOS, watchOS, tvOS, and also macOS. See [README](example/ios/README.md) for diff --git a/CMake/GetGitRevisionDescription.cmake b/CMake/GetGitRevisionDescription.cmake index b56558209..420772966 100644 --- a/CMake/GetGitRevisionDescription.cmake +++ b/CMake/GetGitRevisionDescription.cmake @@ -80,7 +80,7 @@ function(get_git_head_revision _refspecvar _hashvar) # The following git command will return a non empty string that # points to the super project working tree if the current # source dir is inside a git submodule. - # Otherwise the command will return an empty string. + # Otherwise, the command will return an empty string. # execute_process( COMMAND "${GIT_EXECUTABLE}" rev-parse --show-superproject-working-tree diff --git a/README.md b/README.md index 4d3120b2f..52b0f8120 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ TDLib (Telegram Database library) is a cross-platform library for building [Tele `TDLib` has many advantages. Notably `TDLib` is: * **Cross-platform**: `TDLib` can be used on Android, iOS, Windows, macOS, Linux, FreeBSD, OpenBSD, NetBSD, illumos, Windows Phone, WebAssembly, watchOS, tvOS, Tizen, Cygwin. It should also work on other *nix systems with or without minimal effort. -* **Multilanguage**: `TDLib` can be easily used with any programming language that is able to execute C functions. Additionally it already has native Java (using `JNI`) bindings and .NET (using `C++/CLI` and `C++/CX`) bindings. +* **Multilanguage**: `TDLib` can be easily used with any programming language that is able to execute C functions. Additionally, it already has native Java (using `JNI`) bindings and .NET (using `C++/CLI` and `C++/CX`) bindings. * **Easy to use**: `TDLib` takes care of all network implementation details, encryption and local data storage. * **High-performance**: in the [Telegram Bot API](https://core.telegram.org/bots/api), each `TDLib` instance handles more than 24000 active bots simultaneously. * **Well-documented**: all `TDLib` API methods and public interfaces are fully documented. diff --git a/benchmark/bench_queue.cpp b/benchmark/bench_queue.cpp index a9313e087..5eb1f898a 100644 --- a/benchmark/bench_queue.cpp +++ b/benchmark/bench_queue.cpp @@ -888,7 +888,6 @@ int main() { #endif #if TD_PORT_POSIX - // TODO: yield makes it extremely slow. Yet some backoff may be necessary. // td::bench(RingBenchmark()); // td::bench(RingBenchmark>()); diff --git a/example/README.md b/example/README.md index c2c57642e..137f80e97 100644 --- a/example/README.md +++ b/example/README.md @@ -4,7 +4,7 @@ This directory contains basic examples of TDLib usage from different programming If you are looking for documentation of all available TDLib methods, see the [td_api.tl](https://github.com/tdlib/td/blob/master/td/generate/scheme/td_api.tl) scheme or the automatically generated [HTML documentation](https://core.telegram.org/tdlib/docs/td__api_8h.html) for a list of all available TDLib [methods](https://core.telegram.org/tdlib/docs/classtd_1_1td__api_1_1_function.html) and [classes](https://core.telegram.org/tdlib/docs/classtd_1_1td__api_1_1_object.html). -Also take a look at our [Getting Started](https://core.telegram.org/tdlib/getting-started) tutorial for a description of basic TDLib concepts. +Also, take a look at our [Getting Started](https://core.telegram.org/tdlib/getting-started) tutorial for a description of basic TDLib concepts. TDLib can be easily used from almost any programming language on any platform. See a [TDLib build instructions generator](https://tdlib.github.io/td/build.html) for detailed instructions on how to build TDLib. Choose your preferred programming language to see examples of usage and a detailed description: @@ -119,7 +119,7 @@ provides an asynchronous interface for interaction with TDLib, automatically gen You can also use [TDLibCore](https://github.com/ph09nix/TDLibCore) library. -Also see [Unigram](https://github.com/UnigramDev/Unigram), which is a full-featured client rewritten from scratch in C# using TDLib SDK for Universal Windows Platform in less than 2 months, +Also, see [Unigram](https://github.com/UnigramDev/Unigram), which is a full-featured client rewritten from scratch in C# using TDLib SDK for Universal Windows Platform in less than 2 months, [egram.tel](https://github.com/egramtel/egram.tel) – a cross-platform Telegram client written in C#, .NET Core, ReactiveUI and Avalonia, or [telewear](https://github.com/telewear/telewear) - a Telegram client for Samsung watches. @@ -189,7 +189,7 @@ See [rust-tdlib](https://github.com/aCLr/rust-tdlib), or [tdlib](https://github. See [rtdlib](https://github.com/fewensa/rtdlib), [tdlib-rs](https://github.com/d653/tdlib-rs), [tdlib-futures](https://github.com/yuri91/tdlib-futures), [tdlib-sys](https://github.com/nuxeh/tdlib-sys), [tdjson-rs](https://github.com/mersinvald/tdjson-rs), [rust-tdlib](https://github.com/vhaoran/rust-tdlib), or [tdlib-json-sys](https://github.com/aykxt/tdlib-json-sys) for examples of TDLib Rust bindings. -Also see [Telegrand](https://github.com/melix99/telegrand) – a Telegram client optimized for the GNOME desktop. +Also, see [Telegrand](https://github.com/melix99/telegrand) – a Telegram client optimized for the GNOME desktop. ## Using TDLib in Erlang projects diff --git a/example/cpp/README.md b/example/cpp/README.md index 1658ade6f..c8ee34d4f 100644 --- a/example/cpp/README.md +++ b/example/cpp/README.md @@ -8,9 +8,9 @@ cd build cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX:PATH=../example/cpp/td .. cmake --build . --target install ``` -Also see [building](https://github.com/tdlib/td#building) for additional details on TDLib building. +Also, see [building](https://github.com/tdlib/td#building) for additional details on TDLib building. -Then you can build the examples: +After this you can build the examples: ``` cd /example/cpp mkdir build diff --git a/example/csharp/README.md b/example/csharp/README.md index b20d65947..c4b35afe1 100644 --- a/example/csharp/README.md +++ b/example/csharp/README.md @@ -35,4 +35,4 @@ After `TDLib` is built you can open and run TdExample project. It contains a simple console C# application with implementation of authorization and message sending. Just open it with Visual Studio 2015 or later and run. -Also see TdExample.csproj for example of including TDLib in C# project with all native shared library dependencies. +Also, see TdExample.csproj for example of including TDLib in C# project with all native shared library dependencies. diff --git a/example/java/README.md b/example/java/README.md index 4c7df1916..bca3e7ad9 100644 --- a/example/java/README.md +++ b/example/java/README.md @@ -19,7 +19,7 @@ If you want to compile TDLib for 32-bit/64-bit Java on Windows using MSVC, you w In Windows, use vcpkg toolchain file by adding parameter -DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake -Then you can build this example: +After this you can compile the example source code: ``` cd /example/java mkdir build @@ -30,7 +30,7 @@ cmake --build . --target install Compiled TDLib shared library and Java example after that will be placed in bin/ and Javadoc documentation in `docs/`. -Now you can run Java example: +After this you can run the Java example: ``` cd /example/java/bin java '-Djava.library.path=.' org/drinkless/tdlib/example/Example diff --git a/example/python/README.md b/example/python/README.md index c3c171d90..0afb4a70b 100644 --- a/example/python/README.md +++ b/example/python/README.md @@ -2,7 +2,7 @@ To run this example you need to [build](https://github.com/tdlib/td#building) TDLib and copy built tdjson shared library to this directory. -Then you can run the example: +After this you can run the example: ``` python tdjson_example.py ``` diff --git a/example/swift/README.md b/example/swift/README.md index 7c57dcf64..ab7d85500 100644 --- a/example/swift/README.md +++ b/example/swift/README.md @@ -9,7 +9,7 @@ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX:PATH=../example/swift/td cmake --build . --target install ``` -Then you can open and build the example with the latest Xcode. +After this you can open and build the example with the latest Xcode. Description of all available classes and methods can be found at [td_json_client](https://core.telegram.org/tdlib/docs/td__json__client_8h.html) and [td_api](https://core.telegram.org/tdlib/docs/td__api_8h.html) documentation. diff --git a/example/uwp/README.md b/example/uwp/README.md index c0b4de723..3a78deedf 100644 --- a/example/uwp/README.md +++ b/example/uwp/README.md @@ -22,7 +22,8 @@ powershell -ExecutionPolicy ByPass .\build.ps1 -vcpkg_root C:\vcpkg If you need to restart the build from scratch, call `.\build.ps1 -vcpkg_root C:\vcpkg -mode clean` first. * Install Visual Studio Extension "TDLib for Universal Windows Platform" located at `build-uwp\vsix\tdlib.vsix`, which was created on the previous step by `build.ps1` script. -Now `TDLib` can be used from any UWP project, built in Visual Studio. +After this `TDLib` can be used from any UWP project, built in Visual Studio. ## Example of usage + The `app/` directory contains a simple example of a C# application for Universal Windows Platform. Just open it with Visual Studio 2015 or later and run. diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index d39b37acd..71f1d033b 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -432,7 +432,7 @@ animatedEmoji sticker:sticker sticker_width:int32 sticker_height:int32 fitzpatri //@first_name First name of the user; 1-255 characters in length //@last_name Last name of the user //@vcard Additional data about the user in a form of vCard; 0-2048 bytes in length -//@user_id Identifier of the user, if known; otherwise 0 +//@user_id Identifier of the user, if known; 0 otherwise contact phone_number:string first_name:string last_name:string vcard:string user_id:int53 = Contact; //@description Describes a location on planet Earth @@ -649,11 +649,11 @@ user id:int53 first_name:string last_name:string usernames:usernames phone_numbe botInfo share_text:string description:string photo:photo animation:animation menu_button:botMenuButton commands:vector default_group_administrator_rights:chatAdministratorRights default_channel_administrator_rights:chatAdministratorRights = BotInfo; //@description Contains full information about a user -//@personal_photo User profile photo set by the current user for the contact; may be null. If null and user.profile_photo is null, then the photo is empty, otherwise unknown. +//@personal_photo User profile photo set by the current user for the contact; may be null. If null and user.profile_photo is null, then the photo is empty; otherwise, it is unknown. //-If non-null, then it is the same photo as in user.profile_photo and chat.photo. This photo isn't returned in the list of user photos -//@photo User profile photo; may be null. If null and user.profile_photo is null, then the photo is empty, otherwise unknown. +//@photo User profile photo; may be null. If null and user.profile_photo is null, then the photo is empty; otherwise, it is unknown. //-If non-null and personal_photo is null, then it is the same photo as in user.profile_photo and chat.photo -//@public_photo User profile photo visible if the main photo is hidden by privacy settings; may be null. If null and user.profile_photo is null, then the photo is empty, otherwise unknown. +//@public_photo User profile photo visible if the main photo is hidden by privacy settings; may be null. If null and user.profile_photo is null, then the photo is empty; otherwise, it is unknown. //-If non-null and both photo and personal_photo are null, then it is the same photo as in user.profile_photo and chat.photo. This photo isn't returned in the list of user photos //@is_blocked True, if the user is blocked by the current user //@can_be_called True, if the user can be called @@ -919,7 +919,7 @@ secretChatStateClosed = SecretChatState; //@id Secret chat identifier //@user_id Identifier of the chat partner //@state State of the secret chat -//@is_outbound True, if the chat was created by the current user; otherwise false +//@is_outbound True, if the chat was created by the current user; false otherwise //@key_hash Hash of the currently used key for comparison with the hash of the chat partner's key. This is a string of 36 little-endian bytes, which must be split into groups of 2 bits, each denoting a pixel of one of 4 colors FFFFFF, D5E6F3, 2D5775, and 2F99C9. //-The pixels must be used to make a 12x12 square image filled from left to right, top to bottom. Alternatively, the first 32 bytes of the hash can be converted to the hexadecimal format and printed as 32 2-digit hex numbers //@layer Secret chat layer; determines features supported by the chat partner's application. Nested text entities and underline and strikethrough entities are supported if the layer >= 101, @@ -1419,7 +1419,7 @@ replyMarkupForceReply is_personal:Bool input_field_placeholder:string = ReplyMar //@description Contains a custom keyboard layout to quickly reply to bots //@rows A list of rows of bot keyboard buttons -//@is_persistent True, if the keyboard is supposed to be always shown when the ordinary keyboard is hidden +//@is_persistent True, if the keyboard is supposed to always be shown when the ordinary keyboard is hidden //@resize_keyboard True, if the application needs to resize the keyboard vertically //@one_time True, if the application needs to hide the keyboard after use //@is_personal True, if the keyboard must automatically be shown to the current user. For outgoing messages, specify true to show the keyboard only for the mentioned users and for the target user of a reply @@ -1912,10 +1912,10 @@ paymentOption title:string url:string = PaymentOption; //@product_photo Product photo; may be null paymentForm id:int64 invoice:invoice seller_bot_user_id:int53 payment_provider_user_id:int53 payment_provider:PaymentProvider additional_payment_options:vector saved_order_info:orderInfo saved_credentials:vector can_save_credentials:Bool need_password:Bool product_title:string product_description:formattedText product_photo:photo = PaymentForm; -//@description Contains a temporary identifier of validated order information, which is stored for one hour. Also contains the available shipping options @order_info_id Temporary identifier of the order information @shipping_options Available shipping options +//@description Contains a temporary identifier of validated order information, which is stored for one hour, and the available shipping options @order_info_id Temporary identifier of the order information @shipping_options Available shipping options validatedOrderInfo order_info_id:string shipping_options:vector = ValidatedOrderInfo; -//@description Contains the result of a payment request @success True, if the payment request was successful; otherwise the verification_url will be non-empty @verification_url URL for additional payment credentials verification +//@description Contains the result of a payment request @success True, if the payment request was successful; otherwise, the verification_url will be non-empty @verification_url URL for additional payment credentials verification paymentResult success:Bool verification_url:string = PaymentResult; //@description Contains information about a successful payment @@ -2388,7 +2388,7 @@ messagePinMessage message_id:int53 = MessageContent; //@description A screenshot of a message in the chat has been taken messageScreenshotTaken = MessageContent; -//@description A theme in the chat has been changed @theme_name If non-empty, name of a new theme, set for the chat. Otherwise chat theme was reset to the default one +//@description A theme in the chat has been changed @theme_name If non-empty, name of a new theme, set for the chat. Otherwise, chat theme was reset to the default one messageChatSetTheme theme_name:string = MessageContent; //@description The auto-delete or self-destruct timer for messages in the chat has been changed @message_auto_delete_time New value auto-delete or self-destruct time, in seconds; 0 if disabled @from_user_id If not 0, a user identifier, which default setting was automatically applied @@ -2403,10 +2403,10 @@ messageForumTopicCreated name:string icon:forumTopicIcon = MessageContent; //@icon_custom_emoji_id New unique identifier of the custom emoji shown on the topic icon; 0 if none. Must be ignored if edit_icon_custom_emoji_id is false messageForumTopicEdited name:string edit_icon_custom_emoji_id:Bool icon_custom_emoji_id:int64 = MessageContent; -//@description A forum topic has been closed or opened @is_closed True, if the topic was closed, otherwise the topic was reopened +//@description A forum topic has been closed or opened @is_closed True, if the topic was closed; otherwise, the topic was reopened messageForumTopicIsClosedToggled is_closed:Bool = MessageContent; -//@description A General forum topic has been hidden or unhidden @is_hidden True, if the topic was hidden, otherwise the topic was unhidden +//@description A General forum topic has been hidden or unhidden @is_hidden True, if the topic was hidden; otherwise, the topic was unhidden messageForumTopicIsHiddenToggled is_hidden:Bool = MessageContent; //@description A profile photo was suggested to a user in a private chat @photo The suggested chat photo. Use the method setProfilePhoto with inputChatPhotoPrevious to apply the photo @@ -4044,7 +4044,7 @@ pushMessageContentChatChangePhoto = PushMessageContent; //@description A chat title was edited @title New chat title pushMessageContentChatChangeTitle title:string = PushMessageContent; -//@description A chat theme was edited @theme_name If non-empty, name of a new theme, set for the chat. Otherwise chat theme was reset to the default one +//@description A chat theme was edited @theme_name If non-empty, name of a new theme, set for the chat. Otherwise, the chat theme was reset to the default one pushMessageContentChatSetTheme theme_name:string = PushMessageContent; //@description A chat member was deleted @@ -4397,7 +4397,7 @@ targetChatInternalLink link:InternalLinkType = TargetChat; internalLinkTypeActiveSessions = InternalLinkType; //@description The link is a link to an attachment menu bot to be opened in the specified or a chosen chat. Process given target_chat to open the chat. -//-Then call searchPublicChat with the given bot username, check that the user is a bot and can be added to attachment menu. Then use getAttachmentMenuBot to receive information about the bot. +//-Then, call searchPublicChat with the given bot username, check that the user is a bot and can be added to attachment menu. Then, use getAttachmentMenuBot to receive information about the bot. //-If the bot isn't added to attachment menu, then user needs to confirm adding the bot to attachment menu. If user confirms adding, then use toggleBotIsAddedToAttachmentMenu to add it. //-If the attachment menu bot can't be used in the opened chat, show an error to the user. If the bot is added to attachment menu and can be used in the chat, then use openWebApp with the given URL //@target_chat Target chat to be opened @@ -4423,15 +4423,15 @@ internalLinkTypeBotStart bot_username:string start_parameter:string autostart:Bo //-If administrator rights are provided by the link, call getChatMember to receive the current bot rights in the chat and if the bot already is an administrator, //-check that the current user can edit its administrator rights, combine received rights with the requested administrator rights, show confirmation box to the user, //-and call setChatMemberStatus with the chosen chat and confirmed administrator rights. Before call to setChatMemberStatus it may be required to upgrade the chosen basic group chat to a supergroup chat. -//-Then if start_parameter isn't empty, call sendBotStartMessage with the given start parameter and the chosen chat, otherwise just send /start message with bot's username added to the chat. +//-Then, if start_parameter isn't empty, call sendBotStartMessage with the given start parameter and the chosen chat; otherwise, just send /start message with bot's username added to the chat. //@bot_username Username of the bot //@start_parameter The parameter to be passed to sendBotStartMessage //@administrator_rights Expected administrator rights for the bot; may be null internalLinkTypeBotStartInGroup bot_username:string start_parameter:string administrator_rights:chatAdministratorRights = InternalLinkType; //@description The link is a link to a Telegram bot, which is supposed to be added to a channel chat as an administrator. Call searchPublicChat with the given bot username and check that the user is a bot, -//-ask the current user to select a channel chat to add the bot to as an administrator. Then call getChatMember to receive the current bot rights in the chat and if the bot already is an administrator, -//-check that the current user can edit its administrator rights and combine received rights with the requested administrator rights. Then show confirmation box to the user, and call setChatMemberStatus with the chosen chat and confirmed rights +//-ask the current user to select a channel chat to add the bot to as an administrator. Then, call getChatMember to receive the current bot rights in the chat and if the bot already is an administrator, +//-check that the current user can edit its administrator rights and combine received rights with the requested administrator rights. Then, show confirmation box to the user, and call setChatMemberStatus with the chosen chat and confirmed rights //@bot_username Username of the bot //@administrator_rights Expected administrator rights for the bot internalLinkTypeBotAddToChannel bot_username:string administrator_rights:chatAdministratorRights = InternalLinkType; @@ -4476,7 +4476,7 @@ internalLinkTypeMessage url:string = InternalLinkType; //@contains_link True, if the first line of the text contains a link. If true, the input field needs to be focused and the text after the link must be selected internalLinkTypeMessageDraft text:formattedText contains_link:Bool = InternalLinkType; -//@description The link contains a request of Telegram passport data. Call getPassportAuthorizationForm with the given parameters to process the link if the link was received from outside of the application, otherwise ignore it +//@description The link contains a request of Telegram passport data. Call getPassportAuthorizationForm with the given parameters to process the link if the link was received from outside of the application; otherwise, ignore it //@bot_user_id User identifier of the service's bot //@scope Telegram Passport element types requested by the service //@public_key Service's public key @@ -5051,7 +5051,7 @@ updateChatPermissions chat_id:int53 permissions:chatPermissions = Update; //@positions The new chat positions in the chat lists updateChatLastMessage chat_id:int53 last_message:message positions:vector = Update; -//@description The position of a chat in a chat list has changed. Instead of this update updateChatLastMessage or updateChatDraftMessage might be sent +//@description The position of a chat in a chat list has changed. An updateChatLastMessage or updateChatDraftMessage update might be sent instead of the update //@chat_id Chat identifier //@position New chat position. If new order is 0, then the chat needs to be removed from the list updateChatPosition chat_id:int53 position:chatPosition = Update; @@ -5274,7 +5274,7 @@ updateInstalledStickerSets sticker_type:StickerType sticker_set_ids:vector = Update; //@description The list of favorite stickers was updated @sticker_ids The new list of file identifiers of favorite stickers @@ -5642,7 +5642,7 @@ getMessage chat_id:int53 message_id:int53 = Message; //@description Returns information about a message, if it is available without sending network request. This is an offline request @chat_id Identifier of the chat the message belongs to @message_id Identifier of the message to get getMessageLocally chat_id:int53 message_id:int53 = Message; -//@description Returns information about a message that is replied by a given message. Also returns the pinned message, the game message, the invoice message, and the topic creation message for messages +//@description Returns information about a message that is replied by a given message. Also, returns the pinned message, the game message, the invoice message, and the topic creation message for messages //-of the types messagePinMessage, messageGameScore, messagePaymentSuccessful, and topic messages without replied message respectively //@chat_id Identifier of the chat the message belongs to //@message_id Identifier of the reply message @@ -5684,7 +5684,7 @@ loadChats chat_list:ChatList limit:int32 = Ok; //@limit The maximum number of chats to be returned getChats chat_list:ChatList limit:int32 = Chats; -//@description Searches a public chat by its username. Currently, only private chats, supergroups and channels can be public. Returns the chat if found; otherwise an error is returned @username Username to be resolved +//@description Searches a public chat by its username. Currently, only private chats, supergroups and channels can be public. Returns the chat if found; otherwise, an error is returned @username Username to be resolved searchPublicChat username:string = Chat; //@description Searches public chats by looking for specified query in their username and title. Currently, only private chats, supergroups and channels can be public. Returns a meaningful number of results. diff --git a/td/mtproto/SessionConnection.cpp b/td/mtproto/SessionConnection.cpp index 3de279db3..3ea234c73 100644 --- a/td/mtproto/SessionConnection.cpp +++ b/td/mtproto/SessionConnection.cpp @@ -91,8 +91,7 @@ namespace mtproto { * A notification about new session. * It is reasonable to store unique_id with current session, in order to process duplicated notifications once. * - * Causes all older than first_msg_id to be re-sent. - * Also there is a gap in updates, so getDifference MUST be sent + * Causes all messages older than first_msg_id to be re-sent and notifies about a gap in updates * output: * - new_session_created#9ec20908 first_msg_id:long unique_id:long server_salt:long = NewSession * diff --git a/td/telegram/Client.h b/td/telegram/Client.h index 18bbc2bfb..86ec2b205 100644 --- a/td/telegram/Client.h +++ b/td/telegram/Client.h @@ -23,7 +23,7 @@ namespace td { * Requests can be sent using the method ClientManager::send from any thread. * New updates and responses to requests can be received using the method ClientManager::receive from any thread after * the first request has been sent to the client instance. ClientManager::receive must not be called simultaneously from - * two different threads. Also note that all updates and responses to requests should be applied in the same order as + * two different threads. Also, note that all updates and responses to requests should be applied in the same order as * they were received, to ensure consistency. * Some TDLib requests can be executed synchronously from any thread using the method ClientManager::execute. * @@ -177,7 +177,7 @@ class ClientManager final { * The TDLib instance is created for the lifetime of the Client object. * Requests to TDLib can be sent using the Client::send method from any thread. * New updates and responses to requests can be received using the Client::receive method from any thread, - * this function must not be called simultaneously from two different threads. Also note that all updates and + * this function must not be called simultaneously from two different threads. Also, note that all updates and * responses to requests should be applied in the same order as they were received, to ensure consistency. * Given this information, it's advisable to call this function from a dedicated thread. * Some service TDLib requests can be executed synchronously from any thread using the Client::execute method. diff --git a/td/telegram/SecretChatActor.cpp b/td/telegram/SecretChatActor.cpp index 1a658d7b4..f3b33547e 100644 --- a/td/telegram/SecretChatActor.cpp +++ b/td/telegram/SecretChatActor.cpp @@ -595,7 +595,7 @@ void SecretChatActor::run_pfs() { break; } case PfsState::SendCommit: { - // must wait till pfs_state is saved to binlog. Otherwise we may save ActionCommit to binlog without pfs_state, + // must wait till pfs_state is saved to binlog. Otherwise, we may save ActionCommit to binlog without pfs_state, // which has the new auth_key. if (saved_pfs_state_message_id_ < pfs_state_.wait_message_id) { return; @@ -1183,7 +1183,7 @@ Status SecretChatActor::do_inbound_message_decrypted(unique_ptr Add log event. [save_log_event] // 2. [save_log_event] => Save SeqNoState [save_changes] // 3. [save_log_event] => Add message to MessageManager [save_message] - // Note: if we are able to add message by random_id, we may not wait for (log event). Otherwise we should force + // Note: if we are able to add message by random_id, we may not wait for (log event). Otherwise, we should force // binlog flush. // 4. [save_log_event] => Update qts [qts] // 5. [save_changes; save_message; ?qts) => Remove log event [remove_log_event] diff --git a/td/telegram/SecretChatActor.h b/td/telegram/SecretChatActor.h index 963d2f2c5..f42779fd8 100644 --- a/td/telegram/SecretChatActor.h +++ b/td/telegram/SecretChatActor.h @@ -479,10 +479,10 @@ class SecretChatActor final : public NetQueryCallback { // This is completly flawed. // (A-start_save_to_binlog ----> B-start_save_to_binlog+change_memory ----> A-finish_save_to_binlog+surprise) // - // Instead I suggest general solution that is already used with SeqNoState and qts + // Instead, I suggest general solution that is already used with SeqNoState and qts // 1. We APPLY CHANGE to memory immediately AFTER corresponding EVENT is SENT to the binlog. // 2. We SEND CHANGE to database only after corresponding EVENT is SAVED to the binlog. - // 3. Then we are able to ERASE EVENT just AFTER the CHANGE is SAVED to the binlog. + // 3. Then, we are able to ERASE EVENT just AFTER the CHANGE is SAVED to the binlog. // // Actually the change will be saved to binlog too. // So we can do it immediatelly after EVENT is SENT to the binlog, because SEND CHANGE and ERASE EVENT will be diff --git a/td/telegram/UpdatesManager.cpp b/td/telegram/UpdatesManager.cpp index af527e5fb..f53401ff1 100644 --- a/td/telegram/UpdatesManager.cpp +++ b/td/telegram/UpdatesManager.cpp @@ -508,8 +508,7 @@ void UpdatesManager::set_date(int32 date, bool from_update, string date_source) } auto now = G()->unix_time(); if (date_ > now + 1) { - LOG(ERROR) << "Receive wrong by " << (date_ - now) << " date = " << date_ << " from " << date_source - << ". Now = " << now; + LOG(ERROR) << "Receive wrong by " << (date_ - now) << " date = " << date_ << " from " << date_source; date_ = now; if (date_ <= date) { return; diff --git a/td/telegram/files/FileManager.h b/td/telegram/files/FileManager.h index d8bc4e5c4..f72e8deb7 100644 --- a/td/telegram/files/FileManager.h +++ b/td/telegram/files/FileManager.h @@ -375,7 +375,7 @@ class FileManager final : public FileLoadManager::Callback { // After on_upload_ok all uploads of this file will be paused till merge, delete_partial_remote_location or // explicit upload request with the same file_id. - // Also upload may be resumed after some other merges. + // Also, upload may be resumed after some other merge. virtual void on_upload_ok(FileId file_id, tl_object_ptr input_file) = 0; virtual void on_upload_encrypted_ok(FileId file_id, tl_object_ptr input_file) = 0; virtual void on_upload_secure_ok(FileId file_id, tl_object_ptr input_file) = 0; diff --git a/td/telegram/net/Session.cpp b/td/telegram/net/Session.cpp index 25ba58dfe..f04baca6e 100644 --- a/td/telegram/net/Session.cpp +++ b/td/telegram/net/Session.cpp @@ -1517,7 +1517,7 @@ void Session::loop() { long_poll_connection_.wakeup_at_ = 0; // NB: order is crucial. First long_poll_connection, then main_connection - // Otherwise queries could be sent with big delay + // Otherwise, queries could be sent with big delay connection_check_mode(&main_connection_); connection_check_mode(&long_poll_connection_); diff --git a/td/telegram/td_json_client.h b/td/telegram/td_json_client.h index 529f2d72c..b302fff38 100644 --- a/td/telegram/td_json_client.h +++ b/td/telegram/td_json_client.h @@ -28,7 +28,7 @@ * Requests can be sent using td_send and the received client identifier. * New updates and responses to requests can be received through td_receive from any thread after the first request * has been sent to the client instance. This function must not be called simultaneously from two different threads. - * Also note that all updates and responses to requests must be applied in the order they were received for consistency. + * Also, note that all updates and responses to requests must be applied in the order they were received for consistency. * Some TDLib requests can be executed synchronously from any thread using td_execute. * TDLib client instances are destroyed automatically after they are closed. * All TDLib client instances must be closed before application termination to ensure data consistency. @@ -118,7 +118,7 @@ TDJSON_EXPORT void td_set_log_message_callback(int max_verbosity_level, td_log_m * A TDLib client instance can be created through td_json_client_create. * Requests then can be sent using td_json_client_send from any thread. * New updates and request responses can be received through td_json_client_receive from any thread. This function - * must not be called simultaneously from two different threads. Also note that all updates and request responses + * must not be called simultaneously from two different threads. Also, note that all updates and request responses * must be applied in the order they were received to ensure consistency. * Given this information, it's advisable to call this function from a dedicated thread. * Some service TDLib requests can be executed synchronously from any thread by using td_json_client_execute. diff --git a/tdutils/td/utils/MpmcWaiter.h b/tdutils/td/utils/MpmcWaiter.h index 3793aa235..38c3bff6d 100644 --- a/tdutils/td/utils/MpmcWaiter.h +++ b/tdutils/td/utils/MpmcWaiter.h @@ -186,9 +186,8 @@ class MpmcSleepyWaiter { // This may put it in a Sleep for some time. // After wait returns worker will be in Search state again. // - // Suppose worker found a work and ready to process it. - // Then it may call stop_wait. This will cause transition from - // Search to Work state. + // If a worker found a work and ready to process it, then it may call stop_wait. + // This will cause transition from Search to Work state. // // Main invariant: // After notify is called there should be at least on worker in Search or Work state. diff --git a/tdutils/td/utils/ObjectPool.h b/tdutils/td/utils/ObjectPool.h index bed2c0a09..791c3dcbb 100644 --- a/tdutils/td/utils/ObjectPool.h +++ b/tdutils/td/utils/ObjectPool.h @@ -43,10 +43,9 @@ class ObjectPool { // Pattern of usage: 1. Read an object 2. Check if read was valid via is_alive // // It is not very usual case of acquire/release use. - // Instead of publishing an object via some flag we do the opposite. - // We publish new generation via destruction of the data. + // We publish new generation via destruction of the data instead of publishing the object via some flag. // In usual case if we see a flag, then we are able to use an object. - // In our case if we have used an object and it is already invalid, then generation will mismatch + // In our case if we have used an object and it is already invalid, then generation will mismatch. bool is_alive() const { if (!storage_) { return false; @@ -216,7 +215,7 @@ class ObjectPool { std::atomic head_{static_cast(nullptr)}; bool check_empty_flag_ = false; - // TODO(perf): allocation Storages in chunks? Anyway we won't be able to release them. + // TODO(perf): allocation Storages in chunks? Anyway, we won't be able to release them. // TODO(perf): memory order // TODO(perf): use another non lockfree list for release on the same thread // only one thread, so no aba problem diff --git a/tdutils/td/utils/port/UdpSocketFd.cpp b/tdutils/td/utils/port/UdpSocketFd.cpp index 300d4db28..3c94e7021 100644 --- a/tdutils/td/utils/port/UdpSocketFd.cpp +++ b/tdutils/td/utils/port/UdpSocketFd.cpp @@ -574,7 +574,7 @@ class UdpSocketFdImpl { auto error = Status::PosixError(sendmsg_errno, PSLICE() << "Send from " << get_native_fd() << " has failed"); switch (sendmsg_errno) { - // Still may send some other packets, but there is no point to resend this particular message + // We still may send some other packets, but there is no point to resend this particular message case EACCES: case EMSGSIZE: case EPERM: @@ -583,7 +583,7 @@ class UdpSocketFdImpl { is_sent = true; return error; - // Some general problems, which may be fixed in future + // Some general issues, which may be fixed in the future case ENOMEM: case EDQUOT: case EFBIG: diff --git a/tdutils/td/utils/port/platform.h b/tdutils/td/utils/port/platform.h index be9de3992..577294875 100644 --- a/tdutils/td/utils/port/platform.h +++ b/tdutils/td/utils/port/platform.h @@ -99,7 +99,7 @@ #undef TD_HAVE_ATOMIC_SHARED_PTR #endif -// Also no atomic operations on std::shared_ptr when clang __has_feature(cxx_atomic) is defined and zero +// Also, no atomic operations on std::shared_ptr when clang __has_feature(cxx_atomic) is defined and zero #if defined(__has_feature) #if !__has_feature(cxx_atomic) #undef TD_HAVE_ATOMIC_SHARED_PTR From 0f05e72fa389b5c8ff3a43dca2d2070a2310065f Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 2 Jan 2023 17:06:26 +0300 Subject: [PATCH 05/44] Silently ignore PERSISTENT_TIMESTAMP_INVALID errors for channels.getDifference. --- td/telegram/MessagesManager.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 1bd94d7b7..36dfd3adb 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -4719,7 +4719,8 @@ class GetChannelDifferenceQuery final : public Td::ResultHandler { } void on_error(Status status) final { - if (!td_->messages_manager_->on_get_dialog_error(dialog_id_, status, "GetChannelDifferenceQuery")) { + if (!td_->messages_manager_->on_get_dialog_error(dialog_id_, status, "GetChannelDifferenceQuery") && + status.message() != "PERSISTENT_TIMESTAMP_INVALID") { LOG(ERROR) << "Receive error for GetChannelDifferenceQuery for " << dialog_id_ << " with pts " << pts_ << " and limit " << limit_ << ": " << status; } From 12561229fe8b04fa10cc23899e76ee840a7ba77c Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 2 Jan 2023 17:36:38 +0300 Subject: [PATCH 06/44] Improve logging in on_get_channel_difference. --- td/telegram/MessagesManager.cpp | 42 ++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 36dfd3adb..46273dd93 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -39447,8 +39447,10 @@ void MessagesManager::on_get_channel_difference( DialogId dialog_id, int32 request_pts, int32 request_limit, tl_object_ptr &&difference_ptr) { LOG(INFO) << "----- END GET CHANNEL DIFFERENCE----- for " << dialog_id; - CHECK(active_get_channel_differencies_.count(dialog_id) == 1); - active_get_channel_differencies_.erase(dialog_id); + auto it = active_get_channel_differencies_.find(dialog_id); + CHECK(it != active_get_channel_differencies_.end()); + string source = std::move(it->second); + active_get_channel_differencies_.erase(it); auto d = get_dialog_force(dialog_id, "on_get_channel_difference"); if (difference_ptr == nullptr) { @@ -39476,7 +39478,7 @@ void MessagesManager::on_get_channel_difference( channel_get_difference_retry_timeouts_.erase(dialog_id); LOG(INFO) << "Receive result of getChannelDifference for " << dialog_id << " with pts = " << request_pts - << " and limit = " << request_limit << ": " << to_string(difference_ptr); + << " and limit = " << request_limit << " from " << source << ": " << to_string(difference_ptr); bool have_new_messages = false; switch (difference_ptr->get_id()) { @@ -39518,7 +39520,7 @@ void MessagesManager::on_get_channel_difference( int32 cur_pts = d->pts <= 0 ? 1 : d->pts; LOG_IF(ERROR, cur_pts != request_pts) << "Channel PTS has changed from " << request_pts << " to " << d->pts << " in " - << dialog_id << " during getChannelDifference"; + << dialog_id << " during getChannelDifference from " << source; bool is_final = true; bool is_old = false; @@ -39528,9 +39530,9 @@ void MessagesManager::on_get_channel_difference( auto difference = move_tl_object_as(difference_ptr); int32 flags = difference->flags_; is_final = (flags & CHANNEL_DIFFERENCE_FLAG_IS_FINAL) != 0; - LOG_IF(ERROR, !is_final) << "Receive channelDifferenceEmpty as result of getChannelDifference with pts = " - << request_pts << " and limit = " << request_limit << " in " << dialog_id - << ", but it is not final"; + LOG_IF(ERROR, !is_final) << "Receive channelDifferenceEmpty as result of getChannelDifference from " << source + << " with pts = " << request_pts << " and limit = " << request_limit << " in " + << dialog_id << ", but it is not final"; if (flags & CHANNEL_DIFFERENCE_FLAG_HAS_TIMEOUT) { timeout = difference->timeout_; } @@ -39539,9 +39541,9 @@ void MessagesManager::on_get_channel_difference( // also, this can happen for deleted channels if (request_pts != difference->pts_ && !td_->auth_manager_->is_bot() && have_input_peer(dialog_id, AccessRights::Read)) { - LOG(ERROR) << "Receive channelDifferenceEmpty as result of getChannelDifference with pts = " << request_pts - << " and limit = " << request_limit << " in " << dialog_id << ", but PTS has changed to " - << difference->pts_; + LOG(ERROR) << "Receive channelDifferenceEmpty as result of getChannelDifference from " << source + << " with pts = " << request_pts << " and limit = " << request_limit << " in " << dialog_id + << ", but PTS has changed to " << difference->pts_; } set_channel_pts(d, difference->pts_, "channel difference empty"); break; @@ -39557,9 +39559,10 @@ void MessagesManager::on_get_channel_difference( auto new_pts = difference->pts_; if (request_pts >= new_pts && request_pts > 1 && (request_pts > new_pts || !td_->auth_manager_->is_bot())) { - LOG(ERROR) << "Receive channelDifference as result of getChannelDifference with pts = " << request_pts - << " and limit = " << request_limit << " in " << dialog_id << ", but PTS has changed from " << d->pts - << " to " << new_pts << ". Difference: " << oneline(to_string(difference)); + LOG(ERROR) << "Receive channelDifference as result of getChannelDifference from " << source + << " with pts = " << request_pts << " and limit = " << request_limit << " in " << dialog_id + << ", but PTS has changed from " << d->pts << " to " << new_pts + << ". Difference: " << oneline(to_string(difference)); new_pts = request_pts + 1; } @@ -39570,8 +39573,8 @@ void MessagesManager::on_get_channel_difference( auto message_id = MessageId::get_message_id(message, false); if (message_id <= cur_message_id) { LOG(ERROR) << "Receive " << cur_message_id << " after " << message_id << " in channelDifference of " - << dialog_id << " with pts " << request_pts << " and limit " << request_limit << ": " - << to_string(difference); + << dialog_id << " from " << source << " with pts " << request_pts << " and limit " + << request_limit << ": " << to_string(difference); after_get_channel_difference(dialog_id, false); return; } @@ -39620,9 +39623,10 @@ void MessagesManager::on_get_channel_difference( auto new_pts = dialog->pts_; if (request_pts > new_pts - request_limit) { - LOG(ERROR) << "Receive channelDifferenceTooLong as result of getChannelDifference with pts = " << request_pts - << " and limit = " << request_limit << " in " << dialog_id << ", but PTS has changed from " << d->pts - << " to " << new_pts << ". Difference: " << oneline(to_string(difference)); + LOG(ERROR) << "Receive channelDifferenceTooLong as result of getChannelDifference from " << source + << " with pts = " << request_pts << " and limit = " << request_limit << " in " << dialog_id + << ", but PTS has changed from " << d->pts << " to " << new_pts + << ". Difference: " << oneline(to_string(difference)); if (request_pts >= new_pts) { new_pts = request_pts + 1; } @@ -39668,7 +39672,7 @@ void MessagesManager::on_get_channel_difference( } if (!is_final) { - LOG_IF(ERROR, timeout > 0) << "Have timeout in nonfinal ChannelDifference in " << dialog_id; + LOG_IF(ERROR, timeout > 0) << "Have timeout in non-final ChannelDifference in " << dialog_id; get_channel_difference(dialog_id, d->pts, true, "on_get_channel_difference", is_old); return; } From 686fcc19336d20d780c1b0bb365a0deb2b678388 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 2 Jan 2023 18:05:35 +0300 Subject: [PATCH 07/44] Log source for failed to add awaited messages. --- td/telegram/MessagesManager.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 46273dd93..c91b85f3a 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -6624,8 +6624,11 @@ void MessagesManager::skip_old_pending_pts_update(tl_object_ptr 0) { if (new_pts == old_pts || old_pts == std::numeric_limits::max()) { // apply sent message anyway if it is definitely non-deleted or being skipped because of pts overflow - on_get_message(std::move(update_new_message->message_), true, false, false, true, true, - "updateNewMessage with an awaited message"); + auto added_full_message_id = on_get_message(std::move(update_new_message->message_), true, false, false, true, + true, "updateNewMessage with an awaited message"); + if (added_full_message_id != full_message_id) { + LOG(ERROR) << "Failed to add an awaited " << full_message_id << " from " << source; + } return; } else { LOG(ERROR) << "Receive awaited sent " << full_message_id << " from " << source << " with pts " << new_pts @@ -7927,8 +7930,12 @@ void MessagesManager::add_pending_channel_update(DialogId dialog_id, tl_object_p FullMessageId full_message_id(dialog_id, message_id); if (update_message_ids_.count(full_message_id) > 0) { // apply sent channel message - on_get_message(std::move(update_new_channel_message->message_), true, true, false, true, true, - "updateNewChannelMessage with an awaited message"); + auto added_full_message_id = + on_get_message(std::move(update_new_channel_message->message_), true, true, false, true, true, + "updateNewChannelMessage with an awaited message"); + if (added_full_message_id != full_message_id) { + LOG(ERROR) << "Failed to add an awaited " << full_message_id << " from " << source; + } promise.set_value(Unit()); return; } From 29c264df59bcf1371eb1f2f3b6441fb489d33756 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 2 Jan 2023 19:07:48 +0300 Subject: [PATCH 08/44] Log number of unprocessed updateMessageID while closing. --- td/telegram/MessagesManager.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index c91b85f3a..bfcc1344e 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -13529,6 +13529,12 @@ void MessagesManager::hangup() { while (!being_sent_messages_.empty()) { on_send_message_fail(being_sent_messages_.begin()->first, Global::request_aborted_error()); } + if (!update_message_ids_.empty()) { + LOG(ERROR) << "Have " << update_message_ids_.size() << " awaited sent messages"; + } + if (!update_scheduled_message_ids_.empty()) { + LOG(ERROR) << "Have " << update_scheduled_message_ids_.size() << " awaited sent scheduled messages"; + } } fail_promises(load_active_live_location_messages_queries_, Global::request_aborted_error()); From 5711c6c8eb76f3e239d7c82cb29737318d1b2090 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 2 Jan 2023 20:12:15 +0300 Subject: [PATCH 09/44] Add some emptiness checks. --- td/telegram/ContactsManager.cpp | 1 + td/telegram/UpdatesManager.cpp | 1 + td/telegram/VideoNotesManager.cpp | 2 ++ td/telegram/VoiceNotesManager.cpp | 2 ++ 4 files changed, 6 insertions(+) diff --git a/td/telegram/ContactsManager.cpp b/td/telegram/ContactsManager.cpp index fe2a359af..5a22922ac 100644 --- a/td/telegram/ContactsManager.cpp +++ b/td/telegram/ContactsManager.cpp @@ -12022,6 +12022,7 @@ void ContactsManager::on_get_user_full(tl_object_ptr &&u } ContactsManager::UserPhotos *ContactsManager::add_user_photos(UserId user_id) { + CHECK(user_id.is_valid()); auto &user_photos_ptr = user_photos_[user_id]; if (user_photos_ptr == nullptr) { user_photos_ptr = make_unique(); diff --git a/td/telegram/UpdatesManager.cpp b/td/telegram/UpdatesManager.cpp index f53401ff1..1de1f3a55 100644 --- a/td/telegram/UpdatesManager.cpp +++ b/td/telegram/UpdatesManager.cpp @@ -1837,6 +1837,7 @@ void UpdatesManager::try_reload_data() { } void UpdatesManager::subscribe_to_transcribed_audio_updates(int64 transcription_id, TranscribedAudioHandler on_update) { + CHECK(transcription_id != 0); if (pending_audio_transcriptions_.count(transcription_id) != 0) { on_pending_audio_transcription_failed(transcription_id, Status::Error(500, "Receive duplicate speech recognition identifier")); diff --git a/td/telegram/VideoNotesManager.cpp b/td/telegram/VideoNotesManager.cpp index b59c63d1c..928c4785f 100644 --- a/td/telegram/VideoNotesManager.cpp +++ b/td/telegram/VideoNotesManager.cpp @@ -177,6 +177,7 @@ void VideoNotesManager::register_video_note(FileId video_note_file_id, FullMessa return; } LOG(INFO) << "Register video note " << video_note_file_id << " from " << full_message_id << " from " << source; + CHECK(video_note_file_id.is_valid()); bool is_inserted = video_note_messages_[video_note_file_id].insert(full_message_id).second; LOG_CHECK(is_inserted) << source << ' ' << video_note_file_id << ' ' << full_message_id; is_inserted = message_video_notes_.emplace(full_message_id, video_note_file_id).second; @@ -190,6 +191,7 @@ void VideoNotesManager::unregister_video_note(FileId video_note_file_id, FullMes return; } LOG(INFO) << "Unregister video note " << video_note_file_id << " from " << full_message_id << " from " << source; + CHECK(video_note_file_id.is_valid()); auto &message_ids = video_note_messages_[video_note_file_id]; auto is_deleted = message_ids.erase(full_message_id) > 0; LOG_CHECK(is_deleted) << source << ' ' << video_note_file_id << ' ' << full_message_id; diff --git a/td/telegram/VoiceNotesManager.cpp b/td/telegram/VoiceNotesManager.cpp index b3d0b9c0f..f7520951f 100644 --- a/td/telegram/VoiceNotesManager.cpp +++ b/td/telegram/VoiceNotesManager.cpp @@ -141,6 +141,7 @@ void VoiceNotesManager::register_voice_note(FileId voice_note_file_id, FullMessa return; } LOG(INFO) << "Register voice note " << voice_note_file_id << " from " << full_message_id << " from " << source; + CHECK(voice_note_file_id.is_valid()); bool is_inserted = voice_note_messages_[voice_note_file_id].insert(full_message_id).second; LOG_CHECK(is_inserted) << source << ' ' << voice_note_file_id << ' ' << full_message_id; is_inserted = message_voice_notes_.emplace(full_message_id, voice_note_file_id).second; @@ -154,6 +155,7 @@ void VoiceNotesManager::unregister_voice_note(FileId voice_note_file_id, FullMes return; } LOG(INFO) << "Unregister voice note " << voice_note_file_id << " from " << full_message_id << " from " << source; + CHECK(voice_note_file_id.is_valid()); auto &message_ids = voice_note_messages_[voice_note_file_id]; auto is_deleted = message_ids.erase(full_message_id) > 0; LOG_CHECK(is_deleted) << source << ' ' << voice_note_file_id << ' ' << full_message_id; From ab735519a6d3790441a198e2aa12f2c4314deb32 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 3 Jan 2023 12:58:19 +0300 Subject: [PATCH 10/44] Log last used time for auth key. --- td/telegram/net/AuthDataShared.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/td/telegram/net/AuthDataShared.cpp b/td/telegram/net/AuthDataShared.cpp index 031158150..59e99c5d2 100644 --- a/td/telegram/net/AuthDataShared.cpp +++ b/td/telegram/net/AuthDataShared.cpp @@ -104,9 +104,14 @@ class AuthDataSharedImpl final : public AuthDataShared { } void log_auth_key(const mtproto::AuthKey &auth_key) { + auto salts = get_future_salts(); + int64 last_used = 0; + if (!salts.empty()) { + last_used = static_cast(salts[0].valid_until); + } LOG(WARNING) << dc_id_ << " " << tag("auth_key_id", auth_key.id()) << tag("state", AuthDataShared::get_auth_key_state(auth_key)) - << tag("created_at", auth_key.created_at()); + << tag("created_at", static_cast(auth_key.created_at())) << tag("last_used", last_used); } }; From c06ae49db4ee0c65ffc088cb69709694dce1ebbd Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 3 Jan 2023 13:54:49 +0300 Subject: [PATCH 11/44] Log if instance wasn't launched for more than 24 hours. --- td/telegram/TdDb.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/td/telegram/TdDb.cpp b/td/telegram/TdDb.cpp index 5f836d429..df15cf9f6 100644 --- a/td/telegram/TdDb.cpp +++ b/td/telegram/TdDb.cpp @@ -33,7 +33,9 @@ #include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/misc.h" +#include "td/utils/port/Clocks.h" #include "td/utils/port/path.h" +#include "td/utils/port/Stat.h" #include "td/utils/Random.h" #include "td/utils/SliceBuilder.h" #include "td/utils/StringBuilder.h" @@ -55,6 +57,14 @@ std::string get_sqlite_path(const TdParameters ¶meters) { Status init_binlog(Binlog &binlog, string path, BinlogKeyValue &binlog_pmc, BinlogKeyValue &config_pmc, TdDb::OpenedDatabase &events, DbKey key) { + auto r_binlog_stat = stat(path); + if (r_binlog_stat.is_ok()) { + auto since_last_open = Clocks::system() - r_binlog_stat.ok().mtime_nsec_ * 1e-9; + if (since_last_open >= 86400) { + LOG(WARNING) << "Binlog wasn't opened for " << since_last_open << " seconds"; + } + } + auto callback = [&](const BinlogEvent &event) { switch (event.type_) { case LogEvent::HandlerType::SecretChats: From 62040c8d8e168351efd934e3ff719a609f5b8ae9 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 3 Jan 2023 15:06:12 +0300 Subject: [PATCH 12/44] Fix check for use_pfs. --- td/telegram/net/Session.cpp | 6 +++--- td/telegram/net/Session.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/td/telegram/net/Session.cpp b/td/telegram/net/Session.cpp index f04baca6e..4610159fa 100644 --- a/td/telegram/net/Session.cpp +++ b/td/telegram/net/Session.cpp @@ -392,7 +392,7 @@ void Session::on_bind_result(NetQueryPtr query) { bool has_immunity = !is_server_time_reliable || auth_key_age < 60 || (auth_key_age > 86400 && last_success_time > now - 86400); auto debug = PSTRING() << ". Server time is " << server_time << ", auth key created at " << auth_key_creation_date - << ", is_server_time_reliable = " << is_server_time_reliable + << ", is_server_time_reliable = " << is_server_time_reliable << ", use_pfs = " << use_pfs_ << ", last_success_time = " << last_success_time << ", now = " << now; if (!use_pfs_) { if (has_immunity) { @@ -688,7 +688,7 @@ void Session::on_closed(Status status) { void Session::on_session_created(uint64 unique_id, uint64 first_id) { // TODO: use unique_id LOG(INFO) << "New session " << unique_id << " created with first message_id " << first_id; - if (!use_pfs_) { + if (!use_pfs_ && !auth_data_.use_pfs()) { last_success_timestamp_ = Time::now(); } if (is_main_) { @@ -846,7 +846,7 @@ Status Session::on_update(BufferSlice packet) { return Status::Error("Receive at update from CDN connection"); } - if (!use_pfs_) { + if (!use_pfs_ && !auth_data_.use_pfs()) { last_success_timestamp_ = Time::now(); } last_activity_timestamp_ = Time::now(); diff --git a/td/telegram/net/Session.h b/td/telegram/net/Session.h index adad670f6..6dba9fcfd 100644 --- a/td/telegram/net/Session.h +++ b/td/telegram/net/Session.h @@ -175,7 +175,7 @@ class Session final struct ContainerInfo { size_t ref_cnt; - std::vector message_ids; + vector message_ids; }; FlatHashMap sent_containers_; From 9834594b1634ab931ed6c26f1271c16e4e7f580e Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 3 Jan 2023 15:32:10 +0300 Subject: [PATCH 13/44] Don't send parallel ping-server requests. --- td/telegram/UpdatesManager.cpp | 5 +++++ td/telegram/UpdatesManager.h | 2 ++ 2 files changed, 7 insertions(+) diff --git a/td/telegram/UpdatesManager.cpp b/td/telegram/UpdatesManager.cpp index 1de1f3a55..34334c91b 100644 --- a/td/telegram/UpdatesManager.cpp +++ b/td/telegram/UpdatesManager.cpp @@ -1473,6 +1473,10 @@ void UpdatesManager::init_state() { } void UpdatesManager::ping_server() { + if (is_ping_sent_) { + return; + } + is_ping_sent_ = true; auto promise = PromiseCreator::lambda([](Result> result) { auto state = result.is_ok() ? result.move_as_ok() : nullptr; send_closure(G()->updates_manager(), &UpdatesManager::on_server_pong, std::move(state)); @@ -1482,6 +1486,7 @@ void UpdatesManager::ping_server() { void UpdatesManager::on_server_pong(tl_object_ptr &&state) { LOG(INFO) << "Receive " << oneline(to_string(state)); + is_ping_sent_ = false; if (state == nullptr || state->pts_ > get_pts() || state->seq_ > seq_) { get_difference("on server pong"); } diff --git a/td/telegram/UpdatesManager.h b/td/telegram/UpdatesManager.h index d1edd5d19..345645ef1 100644 --- a/td/telegram/UpdatesManager.h +++ b/td/telegram/UpdatesManager.h @@ -246,6 +246,8 @@ class UpdatesManager final : public Actor { int32 min_postponed_update_qts_ = 0; double get_difference_start_time_ = 0; // time from which we started to get difference without success + bool is_ping_sent_ = false; + FlatHashMap pending_audio_transcriptions_; MultiTimeout pending_audio_transcription_timeout_{"PendingAudioTranscriptionTimeout"}; From e6084e6e68467ac91b1e1371be02de3bd84e7390 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Jan 2023 09:09:06 +0300 Subject: [PATCH 14/44] Preload all root certificates instead of using SSL_CTX_set_default_verify_paths, which would lazily load the same certificates over and over. --- tdnet/td/net/SslCtx.cpp | 114 +++++++++++++++++++++++++--------------- 1 file changed, 72 insertions(+), 42 deletions(-) diff --git a/tdnet/td/net/SslCtx.cpp b/tdnet/td/net/SslCtx.cpp index b47397822..209fe43a9 100644 --- a/tdnet/td/net/SslCtx.cpp +++ b/tdnet/td/net/SslCtx.cpp @@ -10,7 +10,10 @@ #include "td/utils/crypto.h" #include "td/utils/FlatHashMap.h" #include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/path.h" #include "td/utils/port/wstring_convert.h" +#include "td/utils/ScopeGuard.h" #include "td/utils/SliceBuilder.h" #include "td/utils/Time.h" @@ -18,6 +21,7 @@ #include #include #include +#include #include #include @@ -56,6 +60,69 @@ int verify_callback(int preverify_ok, X509_STORE_CTX *ctx) { return preverify_ok; } +X509_STORE *load_system_certificate_store() { + int32 cert_count = 0; + LOG(DEBUG) << "Begin to load system certificate store"; + SCOPE_EXIT { + LOG(DEBUG) << "End to load " << cert_count << " certificates from system store"; + }; +#if TD_PORT_WINDOWS + auto flags = CERT_STORE_OPEN_EXISTING_FLAG | CERT_STORE_READONLY_FLAG | CERT_SYSTEM_STORE_CURRENT_USER; + HCERTSTORE system_store = + CertOpenStore(CERT_STORE_PROV_SYSTEM_W, X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, HCRYPTPROV_LEGACY(), flags, + static_cast(to_wstring("ROOT").ok().c_str())); + if (!system_store) { + return nullptr; + } + X509_STORE *store = X509_STORE_new(); + + for (PCCERT_CONTEXT cert_context = CertEnumCertificatesInStore(system_store, nullptr); cert_context != nullptr; + cert_context = CertEnumCertificatesInStore(system_store, cert_context)) { + const unsigned char *in = cert_context->pbCertEncoded; + X509 *x509 = d2i_X509(nullptr, &in, static_cast(cert_context->cbCertEncoded)); + if (x509 != nullptr) { + if (X509_STORE_add_cert(store, x509) != 1) { + auto error_code = ERR_peek_error(); + auto error = create_openssl_error(-20, "Failed to add certificate"); + if (ERR_GET_REASON(error_code) != X509_R_CERT_ALREADY_IN_HASH_TABLE) { + LOG(ERROR) << error; + } else { + LOG(INFO) << error; + } + } else { + cert_count++; + } + + X509_free(x509); + } else { + LOG(ERROR) << create_openssl_error(-21, "Failed to load X509 certificate"); + } + } + + CertCloseStore(system_store, 0); +#else + string default_cert_dir = X509_get_default_cert_dir(); + if (default_cert_dir.empty()) { + return nullptr; + } + X509_STORE *store = X509_STORE_new(); + + for (auto cert_dir : full_split(default_cert_dir, ':')) { + walk_path(cert_dir, [&](CSlice path, WalkPath::Type type) { + if (type != WalkPath::Type::NotDir) { + return WalkPath::Action::Continue; + } + if (X509_STORE_load_locations(store, path.c_str(), nullptr) == 1) { + cert_count++; + } + return WalkPath::Action::Continue; + }).ignore(); + } +#endif + + return store; +} + using SslCtxPtr = std::shared_ptr; Result do_create_ssl_ctx(CSlice cert_file, SslCtx::VerifyPeer verify_peer) { @@ -87,54 +154,17 @@ Result do_create_ssl_ctx(CSlice cert_file, SslCtx::VerifyPeer verify_ SSL_CTX_set_mode(ssl_ctx, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_ENABLE_PARTIAL_WRITE); if (cert_file.empty()) { -#if TD_PORT_WINDOWS - LOG(DEBUG) << "Begin to load system store"; - auto flags = CERT_STORE_OPEN_EXISTING_FLAG | CERT_STORE_READONLY_FLAG | CERT_SYSTEM_STORE_CURRENT_USER; - HCERTSTORE system_store = - CertOpenStore(CERT_STORE_PROV_SYSTEM_W, X509_ASN_ENCODING | PKCS_7_ASN_ENCODING, HCRYPTPROV_LEGACY(), flags, - static_cast(to_wstring("ROOT").ok().c_str())); - - if (system_store) { - X509_STORE *store = X509_STORE_new(); - - for (PCCERT_CONTEXT cert_context = CertEnumCertificatesInStore(system_store, nullptr); cert_context != nullptr; - cert_context = CertEnumCertificatesInStore(system_store, cert_context)) { - const unsigned char *in = cert_context->pbCertEncoded; - X509 *x509 = d2i_X509(nullptr, &in, static_cast(cert_context->cbCertEncoded)); - if (x509 != nullptr) { - if (X509_STORE_add_cert(store, x509) != 1) { - auto error_code = ERR_peek_error(); - auto error = create_openssl_error(-20, "Failed to add certificate"); - if (ERR_GET_REASON(error_code) != X509_R_CERT_ALREADY_IN_HASH_TABLE) { - LOG(ERROR) << error; - } else { - LOG(INFO) << error; - } - } - - X509_free(x509); - } else { - LOG(ERROR) << create_openssl_error(-21, "Failed to load X509 certificate"); - } - } - - CertCloseStore(system_store, 0); - - SSL_CTX_set_cert_store(ssl_ctx, store); - LOG(DEBUG) << "End to load system store"; - } else { - LOG(ERROR) << create_openssl_error(-22, "Failed to open system certificate store"); - } -#else - if (SSL_CTX_set_default_verify_paths(ssl_ctx) == 0) { - auto error = create_openssl_error(-8, "Failed to load default verify paths"); + auto *store = load_system_certificate_store(); + if (store == nullptr) { + auto error = create_openssl_error(-8, "Failed to load system certificate store"); if (verify_peer == SslCtx::VerifyPeer::On) { return std::move(error); } else { LOG(ERROR) << error; } + } else { + SSL_CTX_set_cert_store(ssl_ctx, store); } -#endif } else { if (SSL_CTX_load_verify_locations(ssl_ctx, cert_file.c_str(), nullptr) == 0) { return create_openssl_error(-8, "Failed to set custom certificate file"); From cde0cc4afdc8a346c64b5500b00fe9f3b066972a Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Jan 2023 10:42:12 +0300 Subject: [PATCH 15/44] Improve system certificate store loading. --- tdnet/td/net/SslCtx.cpp | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tdnet/td/net/SslCtx.cpp b/tdnet/td/net/SslCtx.cpp index 209fe43a9..e73b425d9 100644 --- a/tdnet/td/net/SslCtx.cpp +++ b/tdnet/td/net/SslCtx.cpp @@ -62,9 +62,10 @@ int verify_callback(int preverify_ok, X509_STORE_CTX *ctx) { X509_STORE *load_system_certificate_store() { int32 cert_count = 0; + int32 file_count = 0; LOG(DEBUG) << "Begin to load system certificate store"; SCOPE_EXIT { - LOG(DEBUG) << "End to load " << cert_count << " certificates from system store"; + LOG(DEBUG) << "End to load " << cert_count << " certificates in " << file_count << " files from system store"; }; #if TD_PORT_WINDOWS auto flags = CERT_STORE_OPEN_EXISTING_FLAG | CERT_STORE_READONLY_FLAG | CERT_SYSTEM_STORE_CURRENT_USER; @@ -75,6 +76,9 @@ X509_STORE *load_system_certificate_store() { return nullptr; } X509_STORE *store = X509_STORE_new(); + if (store == nullptr) { + return nullptr; + } for (PCCERT_CONTEXT cert_context = CertEnumCertificatesInStore(system_store, nullptr); cert_context != nullptr; cert_context = CertEnumCertificatesInStore(system_store, cert_context)) { @@ -106,18 +110,29 @@ X509_STORE *load_system_certificate_store() { return nullptr; } X509_STORE *store = X509_STORE_new(); + if (store == nullptr) { + return nullptr; + } for (auto cert_dir : full_split(default_cert_dir, ':')) { walk_path(cert_dir, [&](CSlice path, WalkPath::Type type) { if (type != WalkPath::Type::NotDir) { return WalkPath::Action::Continue; } - if (X509_STORE_load_locations(store, path.c_str(), nullptr) == 1) { - cert_count++; + if (X509_STORE_load_locations(store, path.c_str(), nullptr) != 1) { + LOG(INFO) << path << ": " << create_openssl_error(-20, "Failed to add certificate"); + } else { + file_count++; } return WalkPath::Action::Continue; }).ignore(); } +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + auto objects = X509_STORE_get0_objects(store); + cert_count = objects == nullptr ? 0 : sk_X509_OBJECT_num(objects); +#else + cert_count = file_count; +#endif #endif return store; From 887ddbd88e107fb3baf1ecc155eee6c940950bc1 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Jan 2023 10:49:53 +0300 Subject: [PATCH 16/44] Improve warnings about changed MessageForwardInfo. --- td/telegram/MessagesManager.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index bfcc1344e..b657190ad 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -36660,7 +36660,10 @@ bool MessagesManager::update_message(Dialog *d, Message *old_message, unique_ptr if (replace_legacy) { return false; } - if (!is_scheduled && !message_id.is_yet_unsent()) { + if (old_message->forward_info->is_imported != new_message->forward_info->is_imported) { + return true; + } + if (!is_scheduled && !message_id.is_yet_unsent() && !old_message->forward_info->is_imported) { return true; } return !is_forward_info_sender_hidden(new_message->forward_info.get()) && From 4a5b2ac7228786381f95c754b3efe5cd4e9bf47f Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Jan 2023 13:18:38 +0300 Subject: [PATCH 17/44] Skip returned by server messages, not matching the filter. --- td/telegram/MessagesManager.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index b657190ad..a46e63bc2 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -10408,6 +10408,17 @@ void MessagesManager::on_get_dialog_messages_search_result( continue; } + if (filter != MessageSearchFilter::Empty) { + const Message *m = get_message(new_full_message_id); + CHECK(m != nullptr); + auto index_mask = get_message_index_mask(new_full_message_id.get_dialog_id(), m); + if ((message_search_filter_index_mask(filter) & index_mask) == 0) { + LOG(INFO) << "Skip " << new_full_message_id << " of unexpected type"; + total_count--; + continue; + } + } + // TODO check that messages are returned in decreasing message_id order if (message_id < first_added_message_id || !first_added_message_id.is_valid()) { first_added_message_id = message_id; From c4c104384c4d5483eb413227ec6ef7ec32e491ef Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Jan 2023 13:43:15 +0300 Subject: [PATCH 18/44] Load certificate from default_cert_file. --- tdnet/td/net/SslCtx.cpp | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tdnet/td/net/SslCtx.cpp b/tdnet/td/net/SslCtx.cpp index e73b425d9..a67f8a67f 100644 --- a/tdnet/td/net/SslCtx.cpp +++ b/tdnet/td/net/SslCtx.cpp @@ -105,28 +105,35 @@ X509_STORE *load_system_certificate_store() { CertCloseStore(system_store, 0); #else - string default_cert_dir = X509_get_default_cert_dir(); - if (default_cert_dir.empty()) { - return nullptr; - } X509_STORE *store = X509_STORE_new(); if (store == nullptr) { return nullptr; } + auto add_file = [&](CSlice path) { + if (X509_STORE_load_locations(store, path.c_str(), nullptr) != 1) { + LOG(INFO) << path << ": " << create_openssl_error(-20, "Failed to add certificate"); + } else { + file_count++; + } + }; + + string default_cert_dir = X509_get_default_cert_dir(); for (auto cert_dir : full_split(default_cert_dir, ':')) { walk_path(cert_dir, [&](CSlice path, WalkPath::Type type) { if (type != WalkPath::Type::NotDir) { return WalkPath::Action::Continue; } - if (X509_STORE_load_locations(store, path.c_str(), nullptr) != 1) { - LOG(INFO) << path << ": " << create_openssl_error(-20, "Failed to add certificate"); - } else { - file_count++; - } + add_file(path); return WalkPath::Action::Continue; }).ignore(); } + + string default_cert_path = X509_get_default_cert_file(); + if (!default_cert_path.empty()) { + add_file(default_cert_path); + } + #if OPENSSL_VERSION_NUMBER >= 0x10100000L auto objects = X509_STORE_get0_objects(store); cert_count = objects == nullptr ? 0 : sk_X509_OBJECT_num(objects); From 0d35ce485f7383edcd4b879349a259137002d6fd Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Jan 2023 14:28:54 +0300 Subject: [PATCH 19/44] Skip subdirectories in system certificate store. --- tdnet/td/net/SslCtx.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tdnet/td/net/SslCtx.cpp b/tdnet/td/net/SslCtx.cpp index a67f8a67f..128b888fa 100644 --- a/tdnet/td/net/SslCtx.cpp +++ b/tdnet/td/net/SslCtx.cpp @@ -122,7 +122,8 @@ X509_STORE *load_system_certificate_store() { for (auto cert_dir : full_split(default_cert_dir, ':')) { walk_path(cert_dir, [&](CSlice path, WalkPath::Type type) { if (type != WalkPath::Type::NotDir) { - return WalkPath::Action::Continue; + return type == WalkPath::Type::EnterDir && path != cert_dir ? WalkPath::Action::SkipDir + : WalkPath::Action::Continue; } add_file(path); return WalkPath::Action::Continue; From 8df67f0c3a77fd9bb07ef05c68042cd5870986a0 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Jan 2023 16:39:33 +0300 Subject: [PATCH 20/44] Add Stat.is_symbolic_link_. --- tdutils/td/utils/port/FileFd.cpp | 18 ++++++++++++++++-- tdutils/td/utils/port/Stat.cpp | 1 + tdutils/td/utils/port/Stat.h | 1 + tdutils/td/utils/port/path.cpp | 1 + tdutils/test/port.cpp | 8 ++++++++ 5 files changed, 27 insertions(+), 2 deletions(-) diff --git a/tdutils/td/utils/port/FileFd.cpp b/tdutils/td/utils/port/FileFd.cpp index 552e1eb2e..829b6c05f 100644 --- a/tdutils/td/utils/port/FileFd.cpp +++ b/tdutils/td/utils/port/FileFd.cpp @@ -182,6 +182,8 @@ Result FileFd::open(CSlice filepath, int32 flags, int32 mode) { // TODO: share mode DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_DELETE | FILE_SHARE_WRITE; + DWORD native_flags = 0; + DWORD creation_disposition = 0; if (flags & Create) { if (flags & Truncate) { @@ -197,9 +199,9 @@ Result FileFd::open(CSlice filepath, int32 flags, int32 mode) { } else { creation_disposition = OPEN_EXISTING; } + native_flags |= FILE_FLAG_OPEN_REPARSE_POINT; } - DWORD native_flags = 0; if (flags & Direct) { native_flags |= FILE_FLAG_WRITE_THROUGH | FILE_FLAG_NO_BUFFERING; } @@ -593,7 +595,19 @@ Result FileFd::stat() const { res.atime_nsec_ = filetime_to_unix_time_nsec(basic_info.LastAccessTime.QuadPart); res.mtime_nsec_ = filetime_to_unix_time_nsec(basic_info.LastWriteTime.QuadPart); res.is_dir_ = (basic_info.FileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; - res.is_reg_ = !res.is_dir_; // TODO this is still wrong + if ((basic_info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) { + FILE_ATTRIBUTE_TAG_INFO tag_info; + status = GetFileInformationByHandleEx(get_native_fd().fd(), FileAttributeTagInfo, &tag_info, sizeof(tag_info)); + if (!status) { + return OS_ERROR("Get FileAttributeTagInfo failed"); + } + res.is_reg_ = false; + res.is_symbolic_link_ = + (tag_info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0 && tag_info.ReparseTag == IO_REPARSE_TAG_SYMLINK; + } else { + res.is_reg_ = !res.is_dir_; + res.is_symbolic_link_ = false; + } TRY_RESULT(file_size, get_file_size(*this)); res.size_ = file_size.size_; diff --git a/tdutils/td/utils/port/Stat.cpp b/tdutils/td/utils/port/Stat.cpp index 2f6ef62a6..1347360f4 100644 --- a/tdutils/td/utils/port/Stat.cpp +++ b/tdutils/td/utils/port/Stat.cpp @@ -126,6 +126,7 @@ Stat from_native_stat(const struct ::stat &buf) { res.real_size_ = buf.st_blocks * 512; res.is_dir_ = (buf.st_mode & S_IFMT) == S_IFDIR; res.is_reg_ = (buf.st_mode & S_IFMT) == S_IFREG; + res.is_symbolic_link_ = (buf.st_mode & S_IFMT) == S_IFLNK; return res; } diff --git a/tdutils/td/utils/port/Stat.h b/tdutils/td/utils/port/Stat.h index 4775ba466..179f87a35 100644 --- a/tdutils/td/utils/port/Stat.h +++ b/tdutils/td/utils/port/Stat.h @@ -17,6 +17,7 @@ namespace td { struct Stat { bool is_dir_; bool is_reg_; + bool is_symbolic_link_; int64 size_; int64 real_size_; uint64 atime_nsec_; diff --git a/tdutils/td/utils/port/path.cpp b/tdutils/td/utils/port/path.cpp index d11d962e5..d7d23aaf7 100644 --- a/tdutils/td/utils/port/path.cpp +++ b/tdutils/td/utils/port/path.cpp @@ -434,6 +434,7 @@ Result realpath(CSlice slice, bool ignore_access_denied) { if (res.empty()) { return Status::Error("Empty path"); } + // TODO GetFullPathName doesn't resolve symbolic links if (!slice.empty() && slice.end()[-1] == TD_DIR_SLASH) { if (res.back() != TD_DIR_SLASH) { res += TD_DIR_SLASH; diff --git a/tdutils/test/port.cpp b/tdutils/test/port.cpp index 8bea47bf9..af3dffabe 100644 --- a/tdutils/test/port.cpp +++ b/tdutils/test/port.cpp @@ -167,6 +167,14 @@ TEST(Port, Writev) { td::string content(expected_content.size(), '\0'); ASSERT_EQ(content.size(), fd.read(content).move_as_ok()); ASSERT_EQ(expected_content, content); + + auto stat = td::stat(test_file_path).move_as_ok(); + CHECK(!stat.is_dir_); + CHECK(stat.is_reg_); + CHECK(!stat.is_symbolic_link_); + CHECK(stat.size_ == static_cast(expected_content.size())); + + td::unlink(test_file_path).ignore(); } #if TD_PORT_POSIX && !TD_THREAD_UNSUPPORTED From 3573990d524123b861730827366f898183c998f1 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Jan 2023 17:25:30 +0300 Subject: [PATCH 21/44] Support symbolic links in walk_path. --- benchmark/bench_misc.cpp | 24 ++------------ benchmark/rmdir.cpp | 17 +++++++++- td/telegram/files/FileStatsWorker.cpp | 2 +- tdnet/td/net/SslCtx.cpp | 2 +- tdutils/td/utils/port/path.cpp | 47 ++++++++++++++++++++++----- tdutils/td/utils/port/path.h | 2 +- tdutils/test/port.cpp | 2 +- 7 files changed, 62 insertions(+), 34 deletions(-) diff --git a/benchmark/bench_misc.cpp b/benchmark/bench_misc.cpp index 18b68417f..1cbbbe170 100644 --- a/benchmark/bench_misc.cpp +++ b/benchmark/bench_misc.cpp @@ -228,13 +228,7 @@ class CreateFileBench final : public td::Benchmark { } } void tear_down() final { - td::walk_path("A/", [&](td::CSlice path, auto type) { - if (type == td::WalkPath::Type::ExitDir) { - td::rmdir(path).ignore(); - } else if (type == td::WalkPath::Type::NotDir) { - td::unlink(path).ignore(); - } - }).ignore(); + td::rmrf("A/").ignore(); } }; @@ -250,22 +244,10 @@ class WalkPathBench final : public td::Benchmark { } void run(int n) final { int cnt = 0; - td::walk_path("A/", [&](td::CSlice path, auto type) { - if (type == td::WalkPath::Type::EnterDir) { - return; - } - td::stat(path).ok(); - cnt++; - }).ignore(); + td::rmrf("A/").ignore(); } void tear_down() final { - td::walk_path("A/", [&](td::CSlice path, auto type) { - if (type == td::WalkPath::Type::ExitDir) { - td::rmdir(path).ignore(); - } else if (type == td::WalkPath::Type::NotDir) { - td::unlink(path).ignore(); - } - }).ignore(); + td::rmrf("A/").ignore(); } }; diff --git a/benchmark/rmdir.cpp b/benchmark/rmdir.cpp index 163f2986b..1a1a7133b 100644 --- a/benchmark/rmdir.cpp +++ b/benchmark/rmdir.cpp @@ -17,8 +17,23 @@ int main(int argc, char *argv[]) { auto status = td::walk_path(dir, [&](td::CSlice path, auto type) { if (type != td::WalkPath::Type::EnterDir) { cnt++; - LOG(INFO) << path << " " << (type == td::WalkPath::Type::ExitDir); } + auto type_name = [&] { + switch (type) { + case td::WalkPath::Type::EnterDir: + return td::CSlice("Open"); + case td::WalkPath::Type::ExitDir: + return td::CSlice("Exit"); + case td::WalkPath::Type::RegularFile: + return td::CSlice("File"); + case td::WalkPath::Type::Symlink: + return td::CSlice("Link"); + default: + UNREACHABLE(); + return td::CSlice(); + } + }(); + LOG(INFO) << type_name << ' ' << path; //if (is_dir) { // td::rmdir(path); //} else { diff --git a/td/telegram/files/FileStatsWorker.cpp b/td/telegram/files/FileStatsWorker.cpp index 81446eca3..1f5e001da 100644 --- a/td/telegram/files/FileStatsWorker.cpp +++ b/td/telegram/files/FileStatsWorker.cpp @@ -112,7 +112,7 @@ void scan_fs(CancellationToken &token, CallbackT &&callback) { if (token) { return WalkPath::Action::Abort; } - if (type != WalkPath::Type::NotDir) { + if (type != WalkPath::Type::RegularFile) { return WalkPath::Action::Continue; } auto r_stat = stat(path); diff --git a/tdnet/td/net/SslCtx.cpp b/tdnet/td/net/SslCtx.cpp index 128b888fa..a7737ac02 100644 --- a/tdnet/td/net/SslCtx.cpp +++ b/tdnet/td/net/SslCtx.cpp @@ -121,7 +121,7 @@ X509_STORE *load_system_certificate_store() { string default_cert_dir = X509_get_default_cert_dir(); for (auto cert_dir : full_split(default_cert_dir, ':')) { walk_path(cert_dir, [&](CSlice path, WalkPath::Type type) { - if (type != WalkPath::Type::NotDir) { + if (type != WalkPath::Type::RegularFile) { return type == WalkPath::Type::EnterDir && path != cert_dir ? WalkPath::Action::SkipDir : WalkPath::Action::Continue; } diff --git a/tdutils/td/utils/port/path.cpp b/tdutils/td/utils/port/path.cpp index d7d23aaf7..744d706d0 100644 --- a/tdutils/td/utils/port/path.cpp +++ b/tdutils/td/utils/port/path.cpp @@ -91,9 +91,12 @@ Status rmrf(CSlice path) { case WalkPath::Type::ExitDir: rmdir(path).ignore(); break; - case WalkPath::Type::NotDir: + case WalkPath::Type::RegularFile: unlink(path).ignore(); break; + case WalkPath::Type::Symlink: + // never follow symbolic links + break; } }); } @@ -263,6 +266,8 @@ Result walk_path_dir(string &path, const WalkFunction &func) TD_WARN_UNUSE Result walk_path_file(string &path, const WalkFunction &func) TD_WARN_UNUSED_RESULT; +Result walk_path_symlink(string &path, const WalkFunction &func) TD_WARN_UNUSED_RESULT; + Result walk_path(string &path, const WalkFunction &func) TD_WARN_UNUSED_RESULT; Result walk_path_subdir(string &path, DIR *dir, const WalkFunction &func) { @@ -296,6 +301,8 @@ Result walk_path_subdir(string &path, DIR *dir, const WalkFunction &func) status = walk_path_dir(path, func); } else if (entry->d_type == DT_REG) { status = walk_path_file(path, func); + } else if (entry->d_type == DT_LNK) { + status = walk_path_symlink(path, func); } #else #if !TD_SOLARIS @@ -354,7 +361,18 @@ Result walk_path_dir(string &path, const WalkFunction &func) { } Result walk_path_file(string &path, const WalkFunction &func) { - switch (func(path, WalkPath::Type::NotDir)) { + switch (func(path, WalkPath::Type::RegularFile)) { + case WalkPath::Action::Abort: + return false; + case WalkPath::Action::SkipDir: + case WalkPath::Action::Continue: + break; + } + return true; +} + +Result walk_path_symlink(string &path, const WalkFunction &func) { + switch (func(path, WalkPath::Type::Symlink)) { case WalkPath::Action::Abort: return false; case WalkPath::Action::SkipDir: @@ -368,17 +386,20 @@ Result walk_path(string &path, const WalkFunction &func) { TRY_RESULT(fd, FileFd::open(path, FileFd::Read)); TRY_RESULT(stat, fd.stat()); - bool is_dir = stat.is_dir_; - bool is_reg = stat.is_reg_; - if (is_dir) { + if (stat.is_dir_) { return walk_path_dir(path, std::move(fd), func); } fd.close(); - if (is_reg) { + + if (stat.is_reg_) { return walk_path_file(path, func); } + if (stat.is_symbolic_link_) { + return walk_path_symlink(path, func); + } + return true; } } // namespace detail @@ -589,14 +610,24 @@ static Result walk_path_dir(const std::wstring &dir_name, if (!is_ok) { return false; } - } else { - switch (func(entry_name, WalkPath::Type::NotDir)) { + } else if ((file_data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) == 0) { + switch (func(entry_name, WalkPath::Type::RegularFile)) { case WalkPath::Action::Abort: return false; case WalkPath::Action::SkipDir: case WalkPath::Action::Continue: break; } + } else if (file_data.dwReserved0 == IO_REPARSE_TAG_SYMLINK) { + switch (func(entry_name, WalkPath::Type::Symlink)) { + case WalkPath::Action::Abort: + return false; + case WalkPath::Action::SkipDir: + case WalkPath::Action::Continue: + break; + } + } else { + // skip other reparse points } } auto status = FindNextFileW(handle, &file_data); diff --git a/tdutils/td/utils/port/path.h b/tdutils/td/utils/port/path.h index 5c86ac1ae..613a5027f 100644 --- a/tdutils/td/utils/port/path.h +++ b/tdutils/td/utils/port/path.h @@ -44,7 +44,7 @@ Result mkdtemp(CSlice dir, Slice prefix) TD_WARN_UNUSED_RESULT; class WalkPath { public: enum class Action { Continue, Abort, SkipDir }; - enum class Type { EnterDir, ExitDir, NotDir }; + enum class Type { EnterDir, ExitDir, RegularFile, Symlink }; template ()("", Type::ExitDir))> static TD_WARN_UNUSED_RESULT std::enable_if_t::value, Status> run(CSlice path, F &&func) { diff --git a/tdutils/test/port.cpp b/tdutils/test/port.cpp index af3dffabe..6701efdbc 100644 --- a/tdutils/test/port.cpp +++ b/tdutils/test/port.cpp @@ -54,7 +54,7 @@ TEST(Port, files) { const int ITER_COUNT = 1000; for (int i = 0; i < ITER_COUNT; i++) { td::walk_path(main_dir, [&](td::CSlice name, td::WalkPath::Type type) { - if (type == td::WalkPath::Type::NotDir) { + if (type == td::WalkPath::Type::RegularFile) { ASSERT_TRUE(name == fd_path || name == fd2_path); } cnt++; From 7effec1abfb40c25758f2054b6a1b673bd134c18 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Jan 2023 17:27:29 +0300 Subject: [PATCH 22/44] Process symlinks in system certificate store. --- tdnet/td/net/SslCtx.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdnet/td/net/SslCtx.cpp b/tdnet/td/net/SslCtx.cpp index a7737ac02..15c870d37 100644 --- a/tdnet/td/net/SslCtx.cpp +++ b/tdnet/td/net/SslCtx.cpp @@ -121,7 +121,7 @@ X509_STORE *load_system_certificate_store() { string default_cert_dir = X509_get_default_cert_dir(); for (auto cert_dir : full_split(default_cert_dir, ':')) { walk_path(cert_dir, [&](CSlice path, WalkPath::Type type) { - if (type != WalkPath::Type::RegularFile) { + if (type != WalkPath::Type::RegularFile && type != WalkPath::Type::Symlink) { return type == WalkPath::Type::EnterDir && path != cert_dir ? WalkPath::Action::SkipDir : WalkPath::Action::Continue; } From 9f6fc348b125b7ef3b834b1be6de1108c9106af5 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Jan 2023 17:48:34 +0300 Subject: [PATCH 23/44] Fix bench_misc. --- benchmark/bench_misc.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/benchmark/bench_misc.cpp b/benchmark/bench_misc.cpp index 1cbbbe170..8345438f7 100644 --- a/benchmark/bench_misc.cpp +++ b/benchmark/bench_misc.cpp @@ -244,7 +244,13 @@ class WalkPathBench final : public td::Benchmark { } void run(int n) final { int cnt = 0; - td::rmrf("A/").ignore(); + td::walk_path("A/", [&](td::CSlice path, auto type) { + if (type == td::WalkPath::Type::EnterDir) { + return; + } + td::stat(path).ok(); + cnt++; + }).ignore(); } void tear_down() final { td::rmrf("A/").ignore(); From e3951e524c0fe44643f1ac4579d26f9d2fc2dc22 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Jan 2023 17:49:53 +0300 Subject: [PATCH 24/44] Fix warning. --- td/telegram/TdDb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/td/telegram/TdDb.cpp b/td/telegram/TdDb.cpp index df15cf9f6..afa1aea06 100644 --- a/td/telegram/TdDb.cpp +++ b/td/telegram/TdDb.cpp @@ -59,7 +59,7 @@ Status init_binlog(Binlog &binlog, string path, BinlogKeyValue &binlog_p TdDb::OpenedDatabase &events, DbKey key) { auto r_binlog_stat = stat(path); if (r_binlog_stat.is_ok()) { - auto since_last_open = Clocks::system() - r_binlog_stat.ok().mtime_nsec_ * 1e-9; + auto since_last_open = Clocks::system() - static_cast(r_binlog_stat.ok().mtime_nsec_) * 1e-9; if (since_last_open >= 86400) { LOG(WARNING) << "Binlog wasn't opened for " << since_last_open << " seconds"; } From bc7734a267488dc3dacd94ee43e3fee7767ce2e4 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 4 Jan 2023 17:55:31 +0300 Subject: [PATCH 25/44] Improve loading of system certificate store. --- tdnet/td/net/SslCtx.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tdnet/td/net/SslCtx.cpp b/tdnet/td/net/SslCtx.cpp index 15c870d37..913a0202d 100644 --- a/tdnet/td/net/SslCtx.cpp +++ b/tdnet/td/net/SslCtx.cpp @@ -65,7 +65,11 @@ X509_STORE *load_system_certificate_store() { int32 file_count = 0; LOG(DEBUG) << "Begin to load system certificate store"; SCOPE_EXIT { - LOG(DEBUG) << "End to load " << cert_count << " certificates in " << file_count << " files from system store"; + LOG(DEBUG) << "End to load " << cert_count << " certificates from " << file_count << " files from system store"; + if (ERR_peek_error() != 0) { + auto error = create_openssl_error(-22, "Have unprocessed errors"); + LOG(INFO) << error; + } }; #if TD_PORT_WINDOWS auto flags = CERT_STORE_OPEN_EXISTING_FLAG | CERT_STORE_READONLY_FLAG | CERT_SYSTEM_STORE_CURRENT_USER; @@ -112,7 +116,8 @@ X509_STORE *load_system_certificate_store() { auto add_file = [&](CSlice path) { if (X509_STORE_load_locations(store, path.c_str(), nullptr) != 1) { - LOG(INFO) << path << ": " << create_openssl_error(-20, "Failed to add certificate"); + auto error = create_openssl_error(-20, "Failed to add certificate"); + LOG(INFO) << path << ": " << error; } else { file_count++; } @@ -139,7 +144,7 @@ X509_STORE *load_system_certificate_store() { auto objects = X509_STORE_get0_objects(store); cert_count = objects == nullptr ? 0 : sk_X509_OBJECT_num(objects); #else - cert_count = file_count; + cert_count = -1; #endif #endif From 651f49fc90fd7cda121bb1d83ff62022e6ef4c65 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 5 Jan 2023 16:46:39 +0300 Subject: [PATCH 26/44] Delete symbolic links in rmrf. --- tdutils/td/utils/port/path.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tdutils/td/utils/port/path.cpp b/tdutils/td/utils/port/path.cpp index 744d706d0..1c93cf32d 100644 --- a/tdutils/td/utils/port/path.cpp +++ b/tdutils/td/utils/port/path.cpp @@ -95,7 +95,8 @@ Status rmrf(CSlice path) { unlink(path).ignore(); break; case WalkPath::Type::Symlink: - // never follow symbolic links + // never follow symbolic links, but delete the link themselves + unlink(path).ignore(); break; } }); From c4afb9283c52d38b7aaa03c23aa300764c14c65c Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 5 Jan 2023 23:17:06 +0300 Subject: [PATCH 27/44] Return all deleted events fron TQueue::clear. --- tddb/td/db/TQueue.cpp | 19 +++++++++++++------ tddb/td/db/TQueue.h | 2 +- test/tqueue.cpp | 3 ++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tddb/td/db/TQueue.cpp b/tddb/td/db/TQueue.cpp index 80f10f4ac..d539fda51 100644 --- a/tddb/td/db/TQueue.cpp +++ b/tddb/td/db/TQueue.cpp @@ -218,15 +218,15 @@ class TQueueImpl final : public TQueue { pop(q, queue_id, it, q.tail_id); } - void clear(QueueId queue_id, size_t keep_count) final { + std::map clear(QueueId queue_id, size_t keep_count) final { auto queue_it = queues_.find(queue_id); if (queue_it == queues_.end()) { - return; + return {}; } auto &q = queue_it->second; auto size = get_size(q); if (size <= keep_count) { - return; + return {}; } auto start_time = Time::now(); @@ -261,16 +261,23 @@ class TQueueImpl final : public TQueue { } auto callback_clear_time = Time::now() - start_time; - for (auto it = q.events.begin(); it != end_it;) { - remove_event(q, it); + q.total_event_length = 0; + std::map new_events; + for (auto it = end_it; it != q.events.end();) { + q.total_event_length += it->second.data.size(); + bool is_inserted = new_events.emplace(it->first, std::move(it->second)).second; + CHECK(is_inserted); + it = q.events.erase(it); } + std::swap(new_events, q.events); auto clear_time = Time::now() - start_time; - if (clear_time > 0.1) { + if (clear_time > 0.01) { LOG(WARNING) << "Cleared " << (size - keep_count) << " TQueue events with total size " << (total_event_length - q.total_event_length) << " in " << clear_time - callback_clear_time << " seconds and deleted them from callback in " << callback_clear_time << " seconds"; } + return new_events; } Result get(QueueId queue_id, EventId from_id, bool forget_previous, int32 unix_time_now, diff --git a/tddb/td/db/TQueue.h b/tddb/td/db/TQueue.h index 67ad8f815..1a587e01c 100644 --- a/tddb/td/db/TQueue.h +++ b/tddb/td/db/TQueue.h @@ -104,7 +104,7 @@ class TQueue { virtual void forget(QueueId queue_id, EventId event_id) = 0; - virtual void clear(QueueId queue_id, size_t keep_count) = 0; + virtual std::map clear(QueueId queue_id, size_t keep_count) = 0; virtual EventId get_head(QueueId queue_id) const = 0; virtual EventId get_tail(QueueId queue_id) const = 0; diff --git a/test/tqueue.cpp b/test/tqueue.cpp index aa7e1bbda..bbfc5000a 100644 --- a/test/tqueue.cpp +++ b/test/tqueue.cpp @@ -240,11 +240,12 @@ TEST(TQueue, clear) { auto tail_id = tqueue->get_tail(1); auto clear_start_time = td::Time::now(); size_t keep_count = td::Random::fast(0, 2); - tqueue->clear(1, keep_count); + auto deleted_events = tqueue->clear(1, keep_count); auto finish_time = td::Time::now(); LOG(INFO) << "Added TQueue events in " << clear_start_time - start_time << " seconds and cleared them in " << finish_time - clear_start_time << " seconds"; CHECK(tqueue->get_size(1) == keep_count); CHECK(tqueue->get_head(1).advance(keep_count).ok() == tail_id); CHECK(tqueue->get_tail(1) == tail_id); + CHECK(deleted_events.size() == 100000 - keep_count); } From 19cde80fd164f6c139283486a55d3acd8c8b00eb Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 5 Jan 2023 23:33:15 +0300 Subject: [PATCH 28/44] Reduce struct RawEvent size on some compilers. --- tddb/td/db/TQueue.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tddb/td/db/TQueue.h b/tddb/td/db/TQueue.h index 1a587e01c..694e82831 100644 --- a/tddb/td/db/TQueue.h +++ b/tddb/td/db/TQueue.h @@ -61,9 +61,9 @@ class TQueue { struct RawEvent { uint64 log_event_id{0}; EventId event_id; + int32 expires_at{0}; string data; int64 extra{0}; - int32 expires_at{0}; }; using QueueId = int64; From aa8c5ec1c9a101e1ef5af2cc79cc132b255f5c41 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 6 Jan 2023 14:19:44 +0300 Subject: [PATCH 29/44] Improve setAuthenticationPhoneNumber and requestQrCodeAuthentication documentation. --- td/generate/scheme/td_api.tl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index 71f1d033b..3980ebfdc 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -5478,7 +5478,7 @@ getAuthorizationState = AuthorizationState; setTdlibParameters use_test_dc:Bool database_directory:string files_directory:string database_encryption_key:bytes use_file_database:Bool use_chat_info_database:Bool use_message_database:Bool use_secret_chats:Bool api_id:int32 api_hash:string system_language_code:string device_model:string system_version:string application_version:string enable_storage_optimizer:Bool ignore_file_names:Bool = Ok; //@description Sets the phone number of the user and sends an authentication code to the user. Works only when the current authorization state is authorizationStateWaitPhoneNumber, -//-or if there is no pending authentication query and the current authorization state is authorizationStateWaitCode, authorizationStateWaitRegistration, or authorizationStateWaitPassword +//-or if there is no pending authentication query and the current authorization state is authorizationStateWaitEmailAddress, authorizationStateWaitEmailCode, authorizationStateWaitCode, authorizationStateWaitRegistration, or authorizationStateWaitPassword //@phone_number The phone number of the user, in international format //@settings Settings for the authentication of the user's phone number; pass null to use default settings setAuthenticationPhoneNumber phone_number:string settings:phoneNumberAuthenticationSettings = Ok; @@ -5497,7 +5497,7 @@ checkAuthenticationEmailCode code:EmailAddressAuthentication = Ok; checkAuthenticationCode code:string = Ok; //@description Requests QR code authentication by scanning a QR code on another logged in device. Works only when the current authorization state is authorizationStateWaitPhoneNumber, -//-or if there is no pending authentication query and the current authorization state is authorizationStateWaitCode, authorizationStateWaitRegistration, or authorizationStateWaitPassword +//-or if there is no pending authentication query and the current authorization state is authorizationStateWaitEmailAddress, authorizationStateWaitEmailCode, authorizationStateWaitCode, authorizationStateWaitRegistration, or authorizationStateWaitPassword //@other_user_ids List of user identifiers of other users currently using the application requestQrCodeAuthentication other_user_ids:vector = Ok; From d12ca158c4ea06b0c068504a2abf95fd23a31b08 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 6 Jan 2023 14:54:54 +0300 Subject: [PATCH 30/44] Add constant methods to td::Container. --- tdutils/td/utils/Container.h | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tdutils/td/utils/Container.h b/tdutils/td/utils/Container.h index f7bec812a..a3d62c7c4 100644 --- a/tdutils/td/utils/Container.h +++ b/tdutils/td/utils/Container.h @@ -28,6 +28,14 @@ class Container { return &slots_[slot_id].data; } + const DataT *get(Id id) const { + int32 slot_id = decode_id(id); + if (slot_id == -1) { + return nullptr; + } + return &slots_[slot_id].data; + } + void erase(Id id) { int32 slot_id = decode_id(id); if (slot_id == -1) { @@ -60,7 +68,7 @@ class Container { return static_cast(id); } - vector ids() { + vector ids() const { vector is_bad(slots_.size(), false); for (auto id : empty_slots_) { is_bad[id] = true; @@ -73,6 +81,7 @@ class Container { } return res; } + template void for_each(const F &f) { auto ids = this->ids(); @@ -80,13 +89,24 @@ class Container { f(id, *get(id)); } } + + template + void for_each(const F &f) const { + auto ids = this->ids(); + for (auto id : ids) { + f(id, *get(id)); + } + } + size_t size() const { CHECK(empty_slots_.size() <= slots_.size()); return slots_.size() - empty_slots_.size(); } + bool empty() const { return size() == 0; } + void clear() { *this = Container(); } From c8acf63e02e84f9c5053a4cef253a219dfbe870c Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 6 Jan 2023 17:54:02 +0300 Subject: [PATCH 31/44] Delete TQueue events directly if appropriate. --- tddb/td/db/TQueue.cpp | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tddb/td/db/TQueue.cpp b/tddb/td/db/TQueue.cpp index d539fda51..55e29be0c 100644 --- a/tddb/td/db/TQueue.cpp +++ b/tddb/td/db/TQueue.cpp @@ -261,15 +261,24 @@ class TQueueImpl final : public TQueue { } auto callback_clear_time = Time::now() - start_time; - q.total_event_length = 0; - std::map new_events; - for (auto it = end_it; it != q.events.end();) { - q.total_event_length += it->second.data.size(); - bool is_inserted = new_events.emplace(it->first, std::move(it->second)).second; - CHECK(is_inserted); - it = q.events.erase(it); + std::map deleted_events; + if (keep_count > size / 2) { + for (auto it = q.events.begin(); it != end_it;) { + q.total_event_length -= it->second.data.size(); + bool is_inserted = deleted_events.emplace(it->first, std::move(it->second)).second; + CHECK(is_inserted); + it = q.events.erase(it); + } + } else { + q.total_event_length = 0; + for (auto it = end_it; it != q.events.end();) { + q.total_event_length += it->second.data.size(); + bool is_inserted = deleted_events.emplace(it->first, std::move(it->second)).second; + CHECK(is_inserted); + it = q.events.erase(it); + } + std::swap(deleted_events, q.events); } - std::swap(new_events, q.events); auto clear_time = Time::now() - start_time; if (clear_time > 0.01) { @@ -277,7 +286,7 @@ class TQueueImpl final : public TQueue { << (total_event_length - q.total_event_length) << " in " << clear_time - callback_clear_time << " seconds and deleted them from callback in " << callback_clear_time << " seconds"; } - return new_events; + return deleted_events; } Result get(QueueId queue_id, EventId from_id, bool forget_previous, int32 unix_time_now, From d0500988ac39b069192458160e53e4c264e3b2eb Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 7 Jan 2023 23:36:55 +0300 Subject: [PATCH 32/44] Remove checks that status.is_error in ResultHandler::on_error. --- td/telegram/BackgroundManager.cpp | 1 - td/telegram/StickersManager.cpp | 8 -------- 2 files changed, 9 deletions(-) diff --git a/td/telegram/BackgroundManager.cpp b/td/telegram/BackgroundManager.cpp index 849aa5e54..dd510ee7a 100644 --- a/td/telegram/BackgroundManager.cpp +++ b/td/telegram/BackgroundManager.cpp @@ -157,7 +157,6 @@ class UploadBackgroundQuery final : public Td::ResultHandler { } void on_error(Status status) final { - CHECK(status.is_error()); CHECK(file_id_.is_valid()); if (begins_with(status.message(), "FILE_PART_") && ends_with(status.message(), "_MISSING")) { // TODO td_->background_manager_->on_upload_background_file_part_missing(file_id_, to_integer(status.message().substr(10))); diff --git a/td/telegram/StickersManager.cpp b/td/telegram/StickersManager.cpp index 7bf9f1a03..c5c5a58da 100644 --- a/td/telegram/StickersManager.cpp +++ b/td/telegram/StickersManager.cpp @@ -934,7 +934,6 @@ class InstallStickerSetQuery final : public Td::ResultHandler { } void on_error(Status status) final { - CHECK(status.is_error()); promise_.set_error(std::move(status)); } }; @@ -969,7 +968,6 @@ class UninstallStickerSetQuery final : public Td::ResultHandler { } void on_error(Status status) final { - CHECK(status.is_error()); promise_.set_error(std::move(status)); } }; @@ -1029,7 +1027,6 @@ class UploadStickerFileQuery final : public Td::ResultHandler { } void on_error(Status status) final { - CHECK(status.is_error()); if (was_uploaded_) { CHECK(file_id_.is_valid()); if (begins_with(status.message(), "FILE_PART_") && ends_with(status.message(), "_MISSING")) { @@ -1154,7 +1151,6 @@ class CreateNewStickerSetQuery final : public Td::ResultHandler { } void on_error(Status status) final { - CHECK(status.is_error()); promise_.set_error(std::move(status)); } }; @@ -1189,7 +1185,6 @@ class AddStickerToSetQuery final : public Td::ResultHandler { } void on_error(Status status) final { - CHECK(status.is_error()); promise_.set_error(std::move(status)); } }; @@ -1224,7 +1219,6 @@ class SetStickerSetThumbnailQuery final : public Td::ResultHandler { } void on_error(Status status) final { - CHECK(status.is_error()); promise_.set_error(std::move(status)); } }; @@ -1258,7 +1252,6 @@ class SetStickerPositionQuery final : public Td::ResultHandler { } void on_error(Status status) final { - CHECK(status.is_error()); promise_.set_error(std::move(status)); } }; @@ -1292,7 +1285,6 @@ class DeleteStickerFromSetQuery final : public Td::ResultHandler { } void on_error(Status status) final { - CHECK(status.is_error()); promise_.set_error(std::move(status)); } }; From 1c037ff0e0bf5d26fc568d4ba801f04ea023d71d Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 8 Jan 2023 23:07:57 +0300 Subject: [PATCH 33/44] Don't send more than 100 user entities to server. --- td/telegram/MessageEntity.cpp | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index 471f81782..e791820a1 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -4398,6 +4398,8 @@ vector> get_input_message_entities(co const char *source) { vector> result; vector splittable_entities; + constexpr size_t MAX_USER_ENTITY_COUNT = 100; // server-side limit + size_t user_entity_count = 0; for (auto &entity : entities) { if (!is_user_entity(entity.type)) { continue; @@ -4406,6 +4408,15 @@ vector> get_input_message_entities(co splittable_entities.push_back(entity); continue; } + if (entity.type == MessageEntity::Type::CustomEmoji) { + result.push_back(make_tl_object(entity.offset, entity.length, + entity.custom_emoji_id.get())); + continue; + } + if (user_entity_count >= MAX_USER_ENTITY_COUNT) { + continue; + } + user_entity_count++; switch (entity.type) { case MessageEntity::Type::BlockQuote: result.push_back(make_tl_object(entity.offset, entity.length)); @@ -4430,16 +4441,17 @@ vector> get_input_message_entities(co r_input_user.move_as_ok())); break; } - case MessageEntity::Type::CustomEmoji: - result.push_back(make_tl_object(entity.offset, entity.length, - entity.custom_emoji_id.get())); - break; default: UNREACHABLE(); } } split_entities(splittable_entities, vector()); for (auto &entity : splittable_entities) { + if (user_entity_count >= MAX_USER_ENTITY_COUNT) { + break; + } + user_entity_count++; + switch (entity.type) { case MessageEntity::Type::Bold: result.push_back(make_tl_object(entity.offset, entity.length)); From 4dc554bd08bef83b34c7e2b84685a043cf8c931d Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 9 Jan 2023 00:48:45 +0300 Subject: [PATCH 34/44] Avoid some minor string copying in message entity parsing. --- td/telegram/MessageEntity.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index e791820a1..377ac824e 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -1963,7 +1963,7 @@ Result> parse_markdown(string &text) { i += 2; } } - text = result; + text = std::move(result); return entities; } @@ -2228,7 +2228,7 @@ static Result> do_parse_markdown_v2(CSlice text, string &r Result> parse_markdown_v2(string &text) { string result; TRY_RESULT(entities, do_parse_markdown_v2(text, result)); - text = result; + text = std::move(result); return entities; } @@ -3000,14 +3000,14 @@ static Result> do_parse_html(CSlice text, string &result) int32 entity_offset; size_t entity_begin_pos; - EntityInfo(string tag_name, string argument, int32 entity_offset, size_t entity_begin_pos) + EntityInfo(string &&tag_name, string &&argument, int32 entity_offset, size_t entity_begin_pos) : tag_name(std::move(tag_name)) , argument(std::move(argument)) , entity_offset(entity_offset) , entity_begin_pos(entity_begin_pos) { } }; - std::vector nested_entities; + vector nested_entities; for (size_t i = 0; i < text.size(); i++) { auto c = static_cast(text[i]); @@ -3148,7 +3148,7 @@ static Result> do_parse_html(CSlice text, string &result) return Status::Error(400, PSLICE() << "Unclosed end tag at byte offset " << begin_pos); } - string tag_name = std::move(nested_entities.back().tag_name); + const string &tag_name = nested_entities.back().tag_name; if (!end_tag_name.empty() && end_tag_name != tag_name) { return Status::Error(400, PSLICE() << "Unmatched end tag at byte offset " << begin_pos << ", expected \"\", found \"\""); @@ -3237,7 +3237,7 @@ Result> parse_html(string &text) { "Text contains invalid Unicode characters after decoding HTML entities, check for unmatched " "surrogate code units"); } - text = result; + text = std::move(result); return entities; } From 2ba41ac279a46784ec0ac9aaaa255eb6b60fa16e Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 9 Jan 2023 10:46:49 +0300 Subject: [PATCH 35/44] Explicitly use std::move to return entities. --- td/telegram/MessageEntity.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index 377ac824e..015a9034b 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -1964,7 +1964,7 @@ Result> parse_markdown(string &text) { } } text = std::move(result); - return entities; + return std::move(entities); } static Result> do_parse_markdown_v2(CSlice text, string &result) { @@ -2222,14 +2222,14 @@ static Result> do_parse_markdown_v2(CSlice text, string &r sort_entities(entities); - return entities; + return std::move(entities); } Result> parse_markdown_v2(string &text) { string result; TRY_RESULT(entities, do_parse_markdown_v2(text, result)); text = std::move(result); - return entities; + return std::move(entities); } static vector find_text_url_entities_v3(Slice text) { @@ -3226,7 +3226,7 @@ static Result> do_parse_html(CSlice text, string &result) sort_entities(entities); - return entities; + return std::move(entities); } Result> parse_html(string &text) { @@ -3238,7 +3238,7 @@ Result> parse_html(string &text) { "surrogate code units"); } text = std::move(result); - return entities; + return std::move(entities); } vector> get_input_secret_message_entities( @@ -3443,7 +3443,7 @@ Result> get_message_entities(const ContactsManager *contac entities.pop_back(); } } - return entities; + return std::move(entities); } vector get_message_entities(const ContactsManager *contacts_manager, From 0fb64f97a47619b79b8bdb212e568e51f3aff9fd Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 9 Jan 2023 12:43:15 +0300 Subject: [PATCH 36/44] Use StringBuilder to create new string in parse_html. --- td/telegram/MessageEntity.cpp | 14 ++++++++++---- tdutils/td/utils/StringBuilder.h | 15 +++++++++++++++ tdutils/td/utils/utf8.cpp | 18 ------------------ tdutils/td/utils/utf8.h | 19 ++++++++++++++++++- 4 files changed, 43 insertions(+), 23 deletions(-) diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index 015a9034b..72a0eb716 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -24,6 +24,8 @@ #include "td/utils/misc.h" #include "td/utils/Promise.h" #include "td/utils/SliceBuilder.h" +#include "td/utils/StackAllocator.h" +#include "td/utils/StringBuilder.h" #include "td/utils/unicode.h" #include "td/utils/utf8.h" @@ -2994,6 +2996,9 @@ static Result> do_parse_html(CSlice text, string &result) vector entities; int32 utf16_offset = 0; + auto buf = StackAllocator::alloc(text.size() + 30); + StringBuilder new_text(buf.as_slice(), true); + struct EntityInfo { string tag_name; string argument; @@ -3016,7 +3021,7 @@ static Result> do_parse_html(CSlice text, string &result) if (ch != 0) { i--; // i will be incremented in for utf16_offset += 1 + (ch > 0xffff); - append_utf8_character(result, ch); + append_utf8_character(new_text, ch); continue; } } @@ -3024,7 +3029,7 @@ static Result> do_parse_html(CSlice text, string &result) if (is_utf8_character_first_code_unit(c)) { utf16_offset += 1 + (c >= 0xf0); // >= 4 bytes in symbol => surrogate pair } - result.push_back(text[i]); + new_text.push_back(text[i]); continue; } @@ -3130,7 +3135,7 @@ static Result> do_parse_html(CSlice text, string &result) << "Tag \"span\" must have class \"tg-spoiler\" at byte offset " << begin_pos); } - nested_entities.emplace_back(std::move(tag_name), std::move(argument), utf16_offset, result.size()); + nested_entities.emplace_back(std::move(tag_name), std::move(argument), utf16_offset, new_text.size()); } else { // end of an entity if (nested_entities.empty()) { @@ -3177,7 +3182,7 @@ static Result> do_parse_html(CSlice text, string &result) } else if (tag_name == "a") { auto url = std::move(nested_entities.back().argument); if (url.empty()) { - url = result.substr(nested_entities.back().entity_begin_pos); + url = new_text.as_cslice().substr(nested_entities.back().entity_begin_pos).str(); } auto user_id = LinkManager::get_link_user_id(url); if (user_id.is_valid()) { @@ -3226,6 +3231,7 @@ static Result> do_parse_html(CSlice text, string &result) sort_entities(entities); + result = new_text.as_cslice().str(); return std::move(entities); } diff --git a/tdutils/td/utils/StringBuilder.h b/tdutils/td/utils/StringBuilder.h index b980c0dcb..3bd40574a 100644 --- a/tdutils/td/utils/StringBuilder.h +++ b/tdutils/td/utils/StringBuilder.h @@ -32,6 +32,16 @@ class StringBuilder { current_ptr_--; } + void push_back(char c) { + if (unlikely(end_ptr_ <= current_ptr_)) { + if (!reserve_inner(RESERVED_SIZE)) { + on_error(); + return; + } + } + *current_ptr_++ = c; + } + MutableCSlice as_cslice() { if (current_ptr_ >= end_ptr_ + RESERVED_SIZE) { std::abort(); // shouldn't happen @@ -40,6 +50,10 @@ class StringBuilder { return MutableCSlice(begin_ptr_, current_ptr_); } + size_t size() { + return static_cast(current_ptr_ - begin_ptr_); + } + bool is_error() const { return error_flag_; } @@ -132,6 +146,7 @@ class StringBuilder { } return reserve_inner(RESERVED_SIZE); } + bool reserve(size_t size) { if (end_ptr_ > current_ptr_ && static_cast(end_ptr_ - current_ptr_) >= size) { return true; diff --git a/tdutils/td/utils/utf8.cpp b/tdutils/td/utils/utf8.cpp index 61c679088..c280baba4 100644 --- a/tdutils/td/utils/utf8.cpp +++ b/tdutils/td/utils/utf8.cpp @@ -62,24 +62,6 @@ bool check_utf8(CSlice str) { return false; } -void append_utf8_character(string &str, uint32 ch) { - if (ch <= 0x7f) { - str.push_back(static_cast(ch)); - } else if (ch <= 0x7ff) { - str.push_back(static_cast(0xc0 | (ch >> 6))); // implementation-defined - str.push_back(static_cast(0x80 | (ch & 0x3f))); - } else if (ch <= 0xffff) { - str.push_back(static_cast(0xe0 | (ch >> 12))); // implementation-defined - str.push_back(static_cast(0x80 | ((ch >> 6) & 0x3f))); - str.push_back(static_cast(0x80 | (ch & 0x3f))); - } else { - str.push_back(static_cast(0xf0 | (ch >> 18))); // implementation-defined - str.push_back(static_cast(0x80 | ((ch >> 12) & 0x3f))); - str.push_back(static_cast(0x80 | ((ch >> 6) & 0x3f))); - str.push_back(static_cast(0x80 | (ch & 0x3f))); - } -} - const unsigned char *next_utf8_unsafe(const unsigned char *ptr, uint32 *code) { uint32 a = ptr[0]; if ((a & 0x80) == 0) { diff --git a/tdutils/td/utils/utf8.h b/tdutils/td/utils/utf8.h index 89cb85482..13f4896b9 100644 --- a/tdutils/td/utils/utf8.h +++ b/tdutils/td/utils/utf8.h @@ -32,7 +32,24 @@ inline size_t utf8_length(Slice str) { size_t utf8_utf16_length(Slice str); /// appends a Unicode character using UTF-8 encoding -void append_utf8_character(string &str, uint32 ch); +template +void append_utf8_character(T &str, uint32 ch) { + if (ch <= 0x7f) { + str.push_back(static_cast(ch)); + } else if (ch <= 0x7ff) { + str.push_back(static_cast(0xc0 | (ch >> 6))); // implementation-defined + str.push_back(static_cast(0x80 | (ch & 0x3f))); + } else if (ch <= 0xffff) { + str.push_back(static_cast(0xe0 | (ch >> 12))); // implementation-defined + str.push_back(static_cast(0x80 | ((ch >> 6) & 0x3f))); + str.push_back(static_cast(0x80 | (ch & 0x3f))); + } else { + str.push_back(static_cast(0xf0 | (ch >> 18))); // implementation-defined + str.push_back(static_cast(0x80 | ((ch >> 12) & 0x3f))); + str.push_back(static_cast(0x80 | ((ch >> 6) & 0x3f))); + str.push_back(static_cast(0x80 | (ch & 0x3f))); + } +} /// moves pointer one UTF-8 character back inline const unsigned char *prev_utf8_unsafe(const unsigned char *ptr) { From 34226ac4adf51f5750dbf0821c653cda42afd26e Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 9 Jan 2023 12:59:14 +0300 Subject: [PATCH 37/44] Recheck UTF-8 in parse_html only if needed. --- td/telegram/MessageEntity.cpp | 37 ++++++++++++++++++----------------- td/telegram/MessageEntity.h | 2 +- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index 72a0eb716..cae87a87f 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -2992,11 +2992,15 @@ static uint32 decode_html_entity(CSlice text, size_t &pos) { return res; } -static Result> do_parse_html(CSlice text, string &result) { +Result> parse_html(string &str) { + auto str_size = str.size(); + const char *text = str.c_str(); + vector entities; int32 utf16_offset = 0; + bool need_recheck_utf8 = false; - auto buf = StackAllocator::alloc(text.size() + 30); + auto buf = StackAllocator::alloc(str_size + 30); StringBuilder new_text(buf.as_slice(), true); struct EntityInfo { @@ -3014,13 +3018,17 @@ static Result> do_parse_html(CSlice text, string &result) }; vector nested_entities; - for (size_t i = 0; i < text.size(); i++) { + for (size_t i = 0; i < str_size; i++) { auto c = static_cast(text[i]); if (c == '&') { - auto ch = decode_html_entity(text, i); + auto ch = decode_html_entity(str, i); if (ch != 0) { i--; // i will be incremented in for utf16_offset += 1 + (ch > 0xffff); + if (ch >= 0xd800 && ch <= 0xdfff) { + // half of a surrogate pair + need_recheck_utf8 = true; + } append_utf8_character(new_text, ch); continue; } @@ -3043,7 +3051,7 @@ static Result> do_parse_html(CSlice text, string &result) return Status::Error(400, PSLICE() << "Unclosed start tag at byte offset " << begin_pos); } - string tag_name = to_lower(text.substr(begin_pos + 1, i - begin_pos - 1)); + string tag_name = to_lower(Slice(text + begin_pos + 1, i - begin_pos - 1)); if (tag_name != "a" && tag_name != "b" && tag_name != "strong" && tag_name != "i" && tag_name != "em" && tag_name != "s" && tag_name != "strike" && tag_name != "del" && tag_name != "u" && tag_name != "ins" && tag_name != "tg-spoiler" && tag_name != "tg-emoji" && tag_name != "span" && tag_name != "pre" && @@ -3064,7 +3072,7 @@ static Result> do_parse_html(CSlice text, string &result) while (!is_space(text[i]) && text[i] != '=') { i++; } - Slice attribute_name = text.substr(attribute_begin_pos, i - attribute_begin_pos); + Slice attribute_name(text + attribute_begin_pos, i - attribute_begin_pos); if (attribute_name.empty()) { return Status::Error( 400, PSLICE() << "Empty attribute name in the tag \"" << tag_name << "\" at byte offset " << begin_pos); @@ -3092,7 +3100,7 @@ static Result> do_parse_html(CSlice text, string &result) while (is_alnum(text[i]) || text[i] == '.' || text[i] == '-') { i++; } - attribute_value = to_lower(text.substr(token_begin_pos, i - token_begin_pos)); + attribute_value = to_lower(Slice(text + token_begin_pos, i - token_begin_pos)); if (!is_space(text[i]) && text[i] != '>') { return Status::Error(400, PSLICE() << "Unexpected end of name token at byte offset " << token_begin_pos); @@ -3102,7 +3110,7 @@ static Result> do_parse_html(CSlice text, string &result) char end_character = text[i++]; while (text[i] != end_character && text[i] != 0) { if (text[i] == '&') { - auto ch = decode_html_entity(text, i); + auto ch = decode_html_entity(str, i); if (ch != 0) { append_utf8_character(attribute_value, ch); continue; @@ -3145,7 +3153,7 @@ static Result> do_parse_html(CSlice text, string &result) while (!is_space(text[i]) && text[i] != '>') { i++; } - string end_tag_name = to_lower(text.substr(begin_pos + 2, i - begin_pos - 2)); + string end_tag_name = to_lower(Slice(text + begin_pos + 2, i - begin_pos - 2)); while (is_space(text[i]) && text[i] != 0) { i++; } @@ -3231,19 +3239,12 @@ static Result> do_parse_html(CSlice text, string &result) sort_entities(entities); - result = new_text.as_cslice().str(); - return std::move(entities); -} - -Result> parse_html(string &text) { - string result; - TRY_RESULT(entities, do_parse_html(text, result)); - if (!check_utf8(result)) { + if (need_recheck_utf8 && !check_utf8(new_text.as_cslice())) { return Status::Error(400, "Text contains invalid Unicode characters after decoding HTML entities, check for unmatched " "surrogate code units"); } - text = std::move(result); + str = new_text.as_cslice().str(); return std::move(entities); } diff --git a/td/telegram/MessageEntity.h b/td/telegram/MessageEntity.h index 856a9124a..c392ab2cc 100644 --- a/td/telegram/MessageEntity.h +++ b/td/telegram/MessageEntity.h @@ -181,7 +181,7 @@ FormattedText parse_markdown_v3(FormattedText text); FormattedText get_markdown_v3(FormattedText text); -Result> parse_html(string &text); +Result> parse_html(string &str); vector> get_input_message_entities(const ContactsManager *contacts_manager, const vector &entities, From fd53a81603b953b9cd879406f6ddfef7b5e28608 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 9 Jan 2023 13:11:45 +0300 Subject: [PATCH 38/44] Replace text in-place in parse_markdown. --- td/telegram/MessageEntity.cpp | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index cae87a87f..8b3b31e1f 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -1851,7 +1851,7 @@ string get_first_url(const FormattedText &text) { } Result> parse_markdown(string &text) { - string result; + size_t result_size = 0; vector entities; size_t size = text.size(); int32 utf16_offset = 0; @@ -1859,7 +1859,7 @@ Result> parse_markdown(string &text) { auto c = static_cast(text[i]); if (c == '\\' && (text[i + 1] == '_' || text[i + 1] == '*' || text[i + 1] == '`' || text[i + 1] == '[')) { i++; - result.push_back(text[i]); + text[result_size++] = text[i]; utf16_offset++; continue; } @@ -1867,7 +1867,7 @@ Result> parse_markdown(string &text) { if (is_utf8_character_first_code_unit(c)) { utf16_offset += 1 + (c >= 0xf0); // >= 4 bytes in symbol => surrogate pair } - result.push_back(text[i]); + text[result_size++] = text[i]; continue; } @@ -1909,7 +1909,7 @@ Result> parse_markdown(string &text) { if (is_utf8_character_first_code_unit(cur_ch)) { utf16_offset += 1 + (cur_ch >= 0xf0); // >= 4 bytes in symbol => surrogate pair } - result.push_back(text[i++]); + text[result_size++] = text[i++]; } if (i == size) { return Status::Error(400, PSLICE() << "Can't find end of the entity starting at byte offset " << begin_pos); @@ -1965,11 +1965,12 @@ Result> parse_markdown(string &text) { i += 2; } } - text = std::move(result); + text.resize(result_size); return std::move(entities); } -static Result> do_parse_markdown_v2(CSlice text, string &result) { +Result> parse_markdown_v2(string &text) { + size_t result_size = 0; vector entities; int32 utf16_offset = 0; @@ -1996,7 +1997,7 @@ static Result> do_parse_markdown_v2(CSlice text, string &r if (c == '\\' && text[i + 1] > 0 && text[i + 1] <= 126) { i++; utf16_offset += 1; - result += text[i]; + text[result_size++] = text[i]; continue; } @@ -2017,7 +2018,7 @@ static Result> do_parse_markdown_v2(CSlice text, string &r if (is_utf8_character_first_code_unit(c)) { utf16_offset += 1 + (c >= 0xf0); // >= 4 bytes in symbol => surrogate pair } - result.push_back(text[i]); + text[result_size++] = text[i]; continue; } @@ -2093,7 +2094,7 @@ static Result> do_parse_markdown_v2(CSlice text, string &r } if (i != language_end && language_end < text.size() && text[language_end] != '`') { type = MessageEntity::Type::PreCode; - argument = text.substr(i, language_end - i).str(); + argument = text.substr(i, language_end - i); i = language_end; } // skip one new line in the beginning of the text @@ -2123,7 +2124,7 @@ static Result> do_parse_markdown_v2(CSlice text, string &r return Status::Error( 400, PSLICE() << "Character '" << text[i] << "' is reserved and must be escaped with the preceding '\\'"); } - nested_entities.emplace_back(type, std::move(argument), utf16_offset, entity_byte_offset, result.size()); + nested_entities.emplace_back(type, std::move(argument), utf16_offset, entity_byte_offset, result_size); } else { // end of an entity auto type = nested_entities.back().type; @@ -2149,7 +2150,8 @@ static Result> do_parse_markdown_v2(CSlice text, string &r string url; if (text[i + 1] != '(') { // use text as a URL - url = result.substr(nested_entities.back().entity_begin_pos); + url = text.substr(nested_entities.back().entity_begin_pos, + result_size - nested_entities.back().entity_begin_pos); } else { i += 2; auto url_begin_pos = i; @@ -2224,13 +2226,7 @@ static Result> do_parse_markdown_v2(CSlice text, string &r sort_entities(entities); - return std::move(entities); -} - -Result> parse_markdown_v2(string &text) { - string result; - TRY_RESULT(entities, do_parse_markdown_v2(text, result)); - text = std::move(result); + text.resize(result_size); return std::move(entities); } From ac0de06b5f55e7bb5c2904ed6ef3aeb8c2df8c85 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 9 Jan 2023 13:27:28 +0300 Subject: [PATCH 39/44] Ensure that PTS/QTS updates are null after processing. --- td/telegram/MessagesManager.cpp | 1 + td/telegram/UpdatesManager.cpp | 3 +++ 2 files changed, 4 insertions(+) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index a46e63bc2..ea3a9ecaa 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -8093,6 +8093,7 @@ void MessagesManager::process_pts_update(tl_object_ptr &&u default: UNREACHABLE(); } + update_ptr = nullptr; CHECK(!td_->updates_manager_->running_get_difference()); } diff --git a/td/telegram/UpdatesManager.cpp b/td/telegram/UpdatesManager.cpp index 34334c91b..fee041b45 100644 --- a/td/telegram/UpdatesManager.cpp +++ b/td/telegram/UpdatesManager.cpp @@ -2301,8 +2301,10 @@ void UpdatesManager::process_updates(vector> } process_pts_update(std::move(update)); + CHECK(update == nullptr); } else if (is_qts_update(update.get())) { process_qts_update(std::move(update), 0, get_promise()); + CHECK(update == nullptr); } else if (update->get_id() == telegram_api::updateChannelTooLong::ID) { td_->messages_manager_->on_update_channel_too_long( move_tl_object_as(update), true); @@ -2331,6 +2333,7 @@ void UpdatesManager::process_pts_update(tl_object_ptr &&up if (!check_pts_update(update)) { LOG(ERROR) << "Receive wrong pts update: " << oneline(to_string(update)); + update = nullptr; return; } From 137fd3beb981633f7fad1747baad19017f91e36e Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 9 Jan 2023 15:12:14 +0300 Subject: [PATCH 40/44] Replace text in-place in parse_html. --- td/telegram/MessageEntity.cpp | 18 +++++++++--------- tdutils/td/utils/utf8.cpp | 19 +++++++++++++++++++ tdutils/td/utils/utf8.h | 31 +++++++++++++++++-------------- 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index 8b3b31e1f..0642f106a 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -2991,14 +2991,13 @@ static uint32 decode_html_entity(CSlice text, size_t &pos) { Result> parse_html(string &str) { auto str_size = str.size(); const char *text = str.c_str(); + auto result_end = MutableSlice(str).ubegin(); + const unsigned char *result_begin = result_end; vector entities; int32 utf16_offset = 0; bool need_recheck_utf8 = false; - auto buf = StackAllocator::alloc(str_size + 30); - StringBuilder new_text(buf.as_slice(), true); - struct EntityInfo { string tag_name; string argument; @@ -3025,7 +3024,8 @@ Result> parse_html(string &str) { // half of a surrogate pair need_recheck_utf8 = true; } - append_utf8_character(new_text, ch); + result_end = append_utf8_character_unsafe(result_end, ch); + CHECK(result_end <= result_begin + i); continue; } } @@ -3033,7 +3033,7 @@ Result> parse_html(string &str) { if (is_utf8_character_first_code_unit(c)) { utf16_offset += 1 + (c >= 0xf0); // >= 4 bytes in symbol => surrogate pair } - new_text.push_back(text[i]); + *result_end++ = c; continue; } @@ -3139,7 +3139,7 @@ Result> parse_html(string &str) { << "Tag \"span\" must have class \"tg-spoiler\" at byte offset " << begin_pos); } - nested_entities.emplace_back(std::move(tag_name), std::move(argument), utf16_offset, new_text.size()); + nested_entities.emplace_back(std::move(tag_name), std::move(argument), utf16_offset, result_end - result_begin); } else { // end of an entity if (nested_entities.empty()) { @@ -3186,7 +3186,7 @@ Result> parse_html(string &str) { } else if (tag_name == "a") { auto url = std::move(nested_entities.back().argument); if (url.empty()) { - url = new_text.as_cslice().substr(nested_entities.back().entity_begin_pos).str(); + url = Slice(result_begin + nested_entities.back().entity_begin_pos, result_end).str(); } auto user_id = LinkManager::get_link_user_id(url); if (user_id.is_valid()) { @@ -3235,12 +3235,12 @@ Result> parse_html(string &str) { sort_entities(entities); - if (need_recheck_utf8 && !check_utf8(new_text.as_cslice())) { + str.resize(static_cast(result_end - result_begin)); + if (need_recheck_utf8 && !check_utf8(str)) { return Status::Error(400, "Text contains invalid Unicode characters after decoding HTML entities, check for unmatched " "surrogate code units"); } - str = new_text.as_cslice().str(); return std::move(entities); } diff --git a/tdutils/td/utils/utf8.cpp b/tdutils/td/utils/utf8.cpp index c280baba4..a2c9256f6 100644 --- a/tdutils/td/utils/utf8.cpp +++ b/tdutils/td/utils/utf8.cpp @@ -82,6 +82,25 @@ const unsigned char *next_utf8_unsafe(const unsigned char *ptr, uint32 *code) { return ptr; } +unsigned char *append_utf8_character_unsafe(unsigned char *ptr, uint32 code) { + if (code <= 0x7f) { + *ptr++ = static_cast(code); + } else if (code <= 0x7ff) { + *ptr++ = static_cast(0xc0 | (code >> 6)); + *ptr++ = static_cast(0x80 | (code & 0x3f)); + } else if (code <= 0xffff) { + *ptr++ = static_cast(0xe0 | (code >> 12)); + *ptr++ = static_cast(0x80 | ((code >> 6) & 0x3f)); + *ptr++ = static_cast(0x80 | (code & 0x3f)); + } else { + *ptr++ = static_cast(0xf0 | (code >> 18)); + *ptr++ = static_cast(0x80 | ((code >> 12) & 0x3f)); + *ptr++ = static_cast(0x80 | ((code >> 6) & 0x3f)); + *ptr++ = static_cast(0x80 | (code & 0x3f)); + } + return ptr; +} + string utf8_to_lower(Slice str) { string result; auto pos = str.ubegin(); diff --git a/tdutils/td/utils/utf8.h b/tdutils/td/utils/utf8.h index 13f4896b9..21a02eca1 100644 --- a/tdutils/td/utils/utf8.h +++ b/tdutils/td/utils/utf8.h @@ -33,21 +33,21 @@ size_t utf8_utf16_length(Slice str); /// appends a Unicode character using UTF-8 encoding template -void append_utf8_character(T &str, uint32 ch) { - if (ch <= 0x7f) { - str.push_back(static_cast(ch)); - } else if (ch <= 0x7ff) { - str.push_back(static_cast(0xc0 | (ch >> 6))); // implementation-defined - str.push_back(static_cast(0x80 | (ch & 0x3f))); - } else if (ch <= 0xffff) { - str.push_back(static_cast(0xe0 | (ch >> 12))); // implementation-defined - str.push_back(static_cast(0x80 | ((ch >> 6) & 0x3f))); - str.push_back(static_cast(0x80 | (ch & 0x3f))); +void append_utf8_character(T &str, uint32 code) { + if (code <= 0x7f) { + str.push_back(static_cast(code)); + } else if (code <= 0x7ff) { + str.push_back(static_cast(0xc0 | (code >> 6))); // implementation-defined + str.push_back(static_cast(0x80 | (code & 0x3f))); + } else if (code <= 0xffff) { + str.push_back(static_cast(0xe0 | (code >> 12))); // implementation-defined + str.push_back(static_cast(0x80 | ((code >> 6) & 0x3f))); + str.push_back(static_cast(0x80 | (code & 0x3f))); } else { - str.push_back(static_cast(0xf0 | (ch >> 18))); // implementation-defined - str.push_back(static_cast(0x80 | ((ch >> 12) & 0x3f))); - str.push_back(static_cast(0x80 | ((ch >> 6) & 0x3f))); - str.push_back(static_cast(0x80 | (ch & 0x3f))); + str.push_back(static_cast(0xf0 | (code >> 18))); // implementation-defined + str.push_back(static_cast(0x80 | ((code >> 12) & 0x3f))); + str.push_back(static_cast(0x80 | ((code >> 6) & 0x3f))); + str.push_back(static_cast(0x80 | (code & 0x3f))); } } @@ -62,6 +62,9 @@ inline const unsigned char *prev_utf8_unsafe(const unsigned char *ptr) { /// moves pointer one UTF-8 character forward and saves code of the skipped character in *code const unsigned char *next_utf8_unsafe(const unsigned char *ptr, uint32 *code); +/// appends a Unicode character using UTF-8 encoding and returns updated pointer +unsigned char *append_utf8_character_unsafe(unsigned char *ptr, uint32 code); + /// truncates UTF-8 string to the given length in Unicode characters template T utf8_truncate(T str, size_t length) { From 1919671df3dea1e9d0d218083c5ddc1b37733d4e Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 9 Jan 2023 17:07:08 +0300 Subject: [PATCH 41/44] Improve parse_html test. --- test/message_entities.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/message_entities.cpp b/test/message_entities.cpp index 459065c57..2196b6f66 100644 --- a/test/message_entities.cpp +++ b/test/message_entities.cpp @@ -1255,7 +1255,8 @@ TEST(MessageEntities, parse_html) { check_parse_html("", "", {}); check_parse_html("➡️ ➡️", "➡️ ➡️", {}); - check_parse_html("<>&"«»�", "<>&\"«»�", {}); + check_parse_html("≥<>&"«»�", "≥<>&\"«»�", {}); + check_parse_html("⩔", "⩔", {}); check_parse_html("➡️ ➡️➡️ ➡️", "➡️ ➡️➡️ ➡️", {{td::MessageEntity::Type::Italic, 5, 5}}); check_parse_html("➡️ ➡️➡️ ➡️", "➡️ ➡️➡️ ➡️", From eeb73d4eb22d201655ed30e0e5b76ced3ef2323f Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 9 Jan 2023 17:14:31 +0300 Subject: [PATCH 42/44] Optimize parsing of argument value in HTML tags. --- td/telegram/MessageEntity.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index 0642f106a..db9ca1013 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -2936,11 +2936,7 @@ FormattedText get_markdown_v3(FormattedText text) { } static uint32 decode_html_entity(CSlice text, size_t &pos) { - auto c = static_cast(text[pos]); - if (c != '&') { - return 0; - } - + CHECK(text[pos] == '&'); size_t end_pos = pos + 1; uint32 res = 0; if (text[pos + 1] == '#') { @@ -3016,15 +3012,15 @@ Result> parse_html(string &str) { for (size_t i = 0; i < str_size; i++) { auto c = static_cast(text[i]); if (c == '&') { - auto ch = decode_html_entity(str, i); - if (ch != 0) { + auto code = decode_html_entity(str, i); + if (code != 0) { i--; // i will be incremented in for - utf16_offset += 1 + (ch > 0xffff); - if (ch >= 0xd800 && ch <= 0xdfff) { + utf16_offset += 1 + (code > 0xffff); + if (code >= 0xd800 && code <= 0xdfff) { // half of a surrogate pair need_recheck_utf8 = true; } - result_end = append_utf8_character_unsafe(result_end, ch); + result_end = append_utf8_character_unsafe(result_end, code); CHECK(result_end <= result_begin + i); continue; } @@ -3104,19 +3100,23 @@ Result> parse_html(string &str) { } else { // A string literal char end_character = text[i++]; + char *attribute_end = &str[i]; + const char *attribute_begin = attribute_end; while (text[i] != end_character && text[i] != 0) { if (text[i] == '&') { - auto ch = decode_html_entity(str, i); - if (ch != 0) { - append_utf8_character(attribute_value, ch); + auto code = decode_html_entity(str, i); + if (code != 0) { + attribute_end = reinterpret_cast( + append_utf8_character_unsafe(reinterpret_cast(attribute_end), code)); continue; } } - attribute_value.push_back(text[i++]); + *attribute_end++ = text[i++]; } if (text[i] == end_character) { i++; } + attribute_value.assign(attribute_begin, static_cast(attribute_end - attribute_begin)); } if (text[i] == 0) { return Status::Error(400, PSLICE() << "Unclosed start tag at byte offset " << begin_pos); From 8ceeb39f5d51b49045b001b04f6f660c04f76698 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 9 Jan 2023 17:53:07 +0300 Subject: [PATCH 43/44] Update answers in get_phone_number_info test. --- test/country_info.cpp | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/test/country_info.cpp b/test/country_info.cpp index 3b5820c7f..f48484d78 100644 --- a/test/country_info.cpp +++ b/test/country_info.cpp @@ -11,7 +11,7 @@ static void check_phone_number_info(td::string phone_number_prefix, const td::string &country_code, const td::string &calling_code, const td::string &formatted_phone_number) { - auto result = td::CountryInfoManager::get_phone_number_info_sync(td::string(), std::move(phone_number_prefix)); + auto result = td::CountryInfoManager::get_phone_number_info_sync(td::string(), phone_number_prefix); CHECK(result != nullptr); if (result->country_ == nullptr) { CHECK(country_code.empty()); @@ -19,6 +19,7 @@ static void check_phone_number_info(td::string phone_number_prefix, const td::st CHECK(result->country_->country_code_ == country_code); } CHECK(result->country_calling_code_ == calling_code); + // LOG(ERROR) << phone_number_prefix << ' ' << result->formatted_phone_number_ << ' ' << formatted_phone_number; CHECK(result->formatted_phone_number_ == formatted_phone_number); } @@ -52,7 +53,7 @@ TEST(CountryInfo, phone_number_info) { check_phone_number_info("77654321", "KZ", "7", "765 432 1- --"); check_phone_number_info("3", "", "3", ""); check_phone_number_info("37", "", "37", ""); - check_phone_number_info("372", "EE", "372", "---- ----"); + check_phone_number_info("372", "EE", "372", "---- ---"); check_phone_number_info("42", "", "42", ""); check_phone_number_info("420", "CZ", "420", "--- --- ---"); check_phone_number_info("421", "SK", "421", "--- --- ---"); @@ -61,27 +62,27 @@ TEST(CountryInfo, phone_number_info) { check_phone_number_info("424", "YL", "42", "4"); check_phone_number_info("4241234567890", "YL", "42", "41234567890"); check_phone_number_info("4", "", "4", ""); - check_phone_number_info("49", "DE", "49", "---- -------"); - check_phone_number_info("491", "DE", "49", "1--- -------"); - check_phone_number_info("492", "DE", "49", "2--- -------"); - check_phone_number_info("4915", "DE", "49", "15-- -------"); - check_phone_number_info("4916", "DE", "49", "16- -------"); - check_phone_number_info("4917", "DE", "49", "17- -------"); - check_phone_number_info("4918", "DE", "49", "18-- -------"); - check_phone_number_info("493", "DE", "49", "3--- -------"); - check_phone_number_info("4936", "DE", "49", "36-- -------"); - check_phone_number_info("49360", "DE", "49", "360- -------"); - check_phone_number_info("493601", "DE", "49", "3601 -------"); - check_phone_number_info("4936014", "DE", "49", "3601 4------"); - check_phone_number_info("4936015", "DE", "49", "3601 5------"); - check_phone_number_info("493601419", "DE", "49", "3601 419----"); - check_phone_number_info("4936014198", "DE", "49", "3601 4198--"); - check_phone_number_info("49360141980", "DE", "49", "3601 41980-"); + check_phone_number_info("49", "DE", "49", ""); + check_phone_number_info("491", "DE", "49", "1"); + check_phone_number_info("492", "DE", "49", "2"); + check_phone_number_info("4915", "DE", "49", "15"); + check_phone_number_info("4916", "DE", "49", "16"); + check_phone_number_info("4917", "DE", "49", "17"); + check_phone_number_info("4918", "DE", "49", "18"); + check_phone_number_info("493", "DE", "49", "3"); + check_phone_number_info("4936", "DE", "49", "36"); + check_phone_number_info("49360", "DE", "49", "360"); + check_phone_number_info("493601", "DE", "49", "3601"); + check_phone_number_info("4936014", "DE", "49", "36014"); + check_phone_number_info("4936015", "DE", "49", "36015"); + check_phone_number_info("493601419", "DE", "49", "3601419"); + check_phone_number_info("4936014198", "DE", "49", "36014198"); + check_phone_number_info("49360141980", "DE", "49", "360141980"); check_phone_number_info("841234567890", "VN", "84", "1234567890"); check_phone_number_info("31", "NL", "31", "- -- -- -- --"); check_phone_number_info("318", "NL", "31", "8 -- -- -- --"); check_phone_number_info("319", "NL", "31", "9 -- -- -- --"); check_phone_number_info("3196", "NL", "31", "9 6- -- -- --"); - check_phone_number_info("3197", "NL", "31", "97 ---- -----"); + check_phone_number_info("3197", "NL", "31", "9 7- -- -- --"); check_phone_number_info("3198", "NL", "31", "9 8- -- -- --"); } From ed2644c2fdb9ba389317afcaa019d24c561ca5eb Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 9 Jan 2023 18:19:02 +0300 Subject: [PATCH 44/44] Add tests for anonymous phone numbers. --- test/country_info.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/country_info.cpp b/test/country_info.cpp index f48484d78..4adb0689c 100644 --- a/test/country_info.cpp +++ b/test/country_info.cpp @@ -10,7 +10,8 @@ #include "td/utils/tests.h" static void check_phone_number_info(td::string phone_number_prefix, const td::string &country_code, - const td::string &calling_code, const td::string &formatted_phone_number) { + const td::string &calling_code, const td::string &formatted_phone_number, + bool is_anonymous = false) { auto result = td::CountryInfoManager::get_phone_number_info_sync(td::string(), phone_number_prefix); CHECK(result != nullptr); if (result->country_ == nullptr) { @@ -21,6 +22,7 @@ static void check_phone_number_info(td::string phone_number_prefix, const td::st CHECK(result->country_calling_code_ == calling_code); // LOG(ERROR) << phone_number_prefix << ' ' << result->formatted_phone_number_ << ' ' << formatted_phone_number; CHECK(result->formatted_phone_number_ == formatted_phone_number); + CHECK(result->is_anonymous_ == is_anonymous); } TEST(CountryInfo, phone_number_info) { @@ -85,4 +87,13 @@ TEST(CountryInfo, phone_number_info) { check_phone_number_info("3196", "NL", "31", "9 6- -- -- --"); check_phone_number_info("3197", "NL", "31", "9 7- -- -- --"); check_phone_number_info("3198", "NL", "31", "9 8- -- -- --"); + check_phone_number_info("88", "", "88", ""); + check_phone_number_info("888", "FT", "888", "---- ----", true); + check_phone_number_info("8888", "FT", "888", "8 ---", true); + check_phone_number_info("88888", "FT", "888", "8 8--", true); + check_phone_number_info("888888", "FT", "888", "8 88-", true); + check_phone_number_info("8888888", "FT", "888", "8 888", true); + check_phone_number_info("88888888", "FT", "888", "8 8888", true); + check_phone_number_info("888888888", "FT", "888", "8 88888", true); + check_phone_number_info("8888888888", "FT", "888", "8 888888", true); }