From c50e05db278a75c170edf3ee714bf51c0715de62 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 4 Jul 2022 18:38:34 +0300 Subject: [PATCH 01/55] Add helper for td_api::chatAdministratorRights creation. --- test/link.cpp | 60 +++++++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/test/link.cpp b/test/link.cpp index dd5ebb17f..ace300ff5 100644 --- a/test/link.cpp +++ b/test/link.cpp @@ -95,6 +95,14 @@ static void parse_internal_link(const td::string &url, td::td_api::object_ptr( + can_manage_chat, can_change_info, can_post_messages, can_edit_messages, can_delete_messages, can_invite_users, + can_restrict_members, can_pin_messages, can_promote_members, can_manage_video_chats, is_anonymous); + }; auto target_chat_chosen = [](bool allow_users, bool allow_bots, bool allow_groups, bool allow_channels) { return td::td_api::make_object(allow_users, allow_bots, allow_groups, allow_channels); }; @@ -734,29 +742,27 @@ TEST(Link, parse_internal_link) { parse_internal_link("tg:resolve?domain=username&startgroup&admin=asdas", bot_start_in_group("username", "", nullptr)); parse_internal_link("tg:resolve?domain=username&startgroup&admin=post_messages", bot_start_in_group("username", "", nullptr)); - parse_internal_link( - "tg:resolve?domain=username&startgroup=1&admin=delete_messages+anonymous", - bot_start_in_group("username", "1", - td::td_api::make_object( - true, false, false, false, true, false, false, false, false, false, true))); + parse_internal_link("tg:resolve?domain=username&startgroup=1&admin=delete_messages+anonymous", + bot_start_in_group("username", "1", + chat_administrator_rights(true, false, false, false, true, false, false, false, + false, false, true))); parse_internal_link( "tg:resolve?domain=username&startgroup&admin=manage_chat+change_info+post_messages+edit_messages+delete_messages+" "invite_users+restrict_members+pin_messages+promote_members+manage_video_chats+anonymous", - bot_start_in_group("username", "", - td::td_api::make_object( - true, true, false, false, true, true, true, true, true, true, true))); + bot_start_in_group( + "username", "", + chat_administrator_rights(true, true, false, false, true, true, true, true, true, true, true))); parse_internal_link("tg:resolve?domain=username&startchannel", public_chat("username")); parse_internal_link("tg:resolve?domain=username&startchannel&admin=", public_chat("username")); - parse_internal_link( - "tg:resolve?domain=username&startchannel&admin=post_messages", - bot_add_to_channel("username", td::td_api::make_object( - true, false, true, false, false, false, true, false, false, false, false))); + parse_internal_link("tg:resolve?domain=username&startchannel&admin=post_messages", + bot_add_to_channel("username", chat_administrator_rights(true, false, true, false, false, false, + true, false, false, false, false))); parse_internal_link( "tg:resolve?domain=username&startchannel&admin=manage_chat+change_info+post_messages+edit_messages+delete_" "messages+invite_users+restrict_members+pin_messages+promote_members+manage_video_chats+anonymous", - bot_add_to_channel("username", td::td_api::make_object( - true, true, true, true, true, true, true, false, true, true, false))); + bot_add_to_channel( + "username", chat_administrator_rights(true, true, true, true, true, true, true, false, true, true, false))); parse_internal_link("t.me/username/0/a//s/as?startgroup=", bot_start_in_group("username", "", nullptr)); parse_internal_link("t.me/username/aasdas?test=1&startgroup=#12312", bot_start_in_group("username", "", nullptr)); @@ -771,31 +777,29 @@ TEST(Link, parse_internal_link) { parse_internal_link("t.me/username?startgroup", bot_start_in_group("username", "", nullptr)); parse_internal_link("t.me/username?startgroup&admin=asdas", bot_start_in_group("username", "", nullptr)); parse_internal_link("t.me/username?startgroup&admin=post_messages", bot_start_in_group("username", "", nullptr)); - parse_internal_link( - "t.me/username?startgroup=1&admin=delete_messages+anonymous", - bot_start_in_group("username", "1", - td::td_api::make_object( - true, false, false, false, true, false, false, false, false, false, true))); + parse_internal_link("t.me/username?startgroup=1&admin=delete_messages+anonymous", + bot_start_in_group("username", "1", + chat_administrator_rights(true, false, false, false, true, false, false, false, + false, false, true))); parse_internal_link( "t.me/" "username?startgroup&admin=manage_chat+change_info+post_messages+edit_messages+delete_messages+invite_users+" "restrict_members+pin_messages+promote_members+manage_video_chats+anonymous", - bot_start_in_group("username", "", - td::td_api::make_object( - true, true, false, false, true, true, true, true, true, true, true))); + bot_start_in_group( + "username", "", + chat_administrator_rights(true, true, false, false, true, true, true, true, true, true, true))); parse_internal_link("t.me/username?startchannel", public_chat("username")); parse_internal_link("t.me/username?startchannel&admin=", public_chat("username")); - parse_internal_link( - "t.me/username?startchannel&admin=post_messages", - bot_add_to_channel("username", td::td_api::make_object( - true, false, true, false, false, false, true, false, false, false, false))); + parse_internal_link("t.me/username?startchannel&admin=post_messages", + bot_add_to_channel("username", chat_administrator_rights(true, false, true, false, false, false, + true, false, false, false, false))); parse_internal_link( "t.me/" "username?startchannel&admin=manage_chat+change_info+post_messages+edit_messages+delete_messages+invite_users+" "restrict_members+pin_messages+promote_members+manage_video_chats+anonymous", - bot_add_to_channel("username", td::td_api::make_object( - true, true, true, true, true, true, true, false, true, true, false))); + bot_add_to_channel( + "username", chat_administrator_rights(true, true, true, true, true, true, true, false, true, true, false))); parse_internal_link("tg:resolve?domain=username&game=aasdasd", game("username", "aasdasd")); parse_internal_link("TG://resolve?domain=username&game=", public_chat("username")); From 76080dfb1cf253b40c3aecf21f0f51111de36bee Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 5 Jul 2022 22:10:37 +0300 Subject: [PATCH 02/55] Update example list. --- README.md | 4 ++-- example/README.md | 52 ++++++++++++++++++++++++++++------------------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index db35f0185..a86ec158d 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ Or you could install `TDLib` and then reference it in your CMakeLists.txt like t find_package(Td 1.8.4 REQUIRED) target_link_libraries(YourTarget PRIVATE Td::TdStatic) ``` -See [example/cpp/CMakeLists.txt](https://github.com/tdlib/td/tree/master/example/cpp/CMakeLists.txt). +See [example/cpp/CMakeLists.txt](https://github.com/tdlib/td/blob/master/example/cpp/CMakeLists.txt). ## Using in Java projects @@ -138,7 +138,7 @@ all available `TDLib` [methods](https://core.telegram.org/tdlib/docs/classtd_1_1 `TDLib` JSON interface adheres to semantic versioning and versions with the same major version number are binary and backward compatible, but the underlying `TDLib` API can be different for different minor and even patch versions. If you need to support different `TDLib` versions, then you can use a value of the `version` option to find exact `TDLib` version to use appropriate API methods. -See [example/python/tdjson_example.py](https://github.com/tdlib/td/tree/master/example/python/tdjson_example.py) for an example of such usage. +See [example/python/tdjson_example.py](https://github.com/tdlib/td/blob/master/example/python/tdjson_example.py) for an example of such usage. ## License diff --git a/example/README.md b/example/README.md index 52835e8b1..ce4b0176b 100644 --- a/example/README.md +++ b/example/README.md @@ -46,16 +46,15 @@ Convenient Python wrappers already exist for our JSON interface. If you use Python >= 3.6, take a look at [python-telegram](https://github.com/alexander-akhmetov/python-telegram). The wrapper uses the full power of asyncio, has a good documentation and has several examples. It can be installed through pip or used in a Docker container. -You can also try a fork [python-telegram](https://github.com/iTeam-co/python-telegram) of this library. +You can also try a fork [python-telegram](https://github.com/iTeam-co/pytglib) of this library. -If you want to use TDLib with asyncio and Python >= 3.9, take a look at [aiotdlib](https://github.com/pylakey/aiotdlib). -This wrapper contains automatically generated fully-documented classes for all TDLib API types and functions and provides set of helper methods which makes work with TDLib much simpler. +If you want to use TDLib with asyncio and Python >= 3.9, take a look at [aiotdlib](https://github.com/pylakey/aiotdlib) or [Pytdbot](https://github.com/pytdbot/client). For older Python versions you can use [pytdlib](https://github.com/pytdlib/pytdlib). This wrapper contains generator for TDLib API classes and basic interface for interaction with TDLib. -You can also check out [example/python/tdjson_example.py](https://github.com/tdlib/td/tree/master/example/python/tdjson_example.py) and -[tdlib-python](https://github.com/JunaidBabu/tdlib-python) for some very basic examples of TDLib JSON interface integration with Python. +You can also check out [example/python/tdjson_example.py](https://github.com/tdlib/td/blob/master/example/python/tdjson_example.py), +[tdlib-python](https://github.com/JunaidBabu/tdlib-python), or [Python Wrapper TDLib](https://github.com/alvhix/pywtdlib) for some basic examples of TDLib JSON interface integration with Python. ## Using TDLib in JavaScript projects @@ -63,6 +62,8 @@ You can also check out [example/python/tdjson_example.py](https://github.com/tdl TDLib can be compiled to WebAssembly or asm.js and used in a browser from JavaScript. See [tdweb](https://github.com/tdlib/td/tree/master/example/web) as a convenient wrapper for TDLib in a browser and [telegram-react](https://github.com/evgeny-nadymov/telegram-react) as an example of a TDLib-based Telegram client. +See also [Svelte-tdweb-starter](https://github.com/gennadypolakov/svelte-tdweb-starter) - Svelte wrapper for tdweb, and [Telegram-Photoframe](https://github.com/lukefx/telegram-photoframe) - a web application that displays your prefered group or channel as Photoframe. + TDLib can be used from Node.js through the [JSON](https://github.com/tdlib/td#using-json) interface. Convenient Node.js wrappers already exist for our JSON interface. @@ -70,7 +71,9 @@ For example, take a look at [Airgram](https://github.com/airgram/airgram) – mo at [tdl](https://github.com/Bannerets/tdl), which provides a convenient, fully-asynchronous interface for interaction with TDLib and contains a bunch of examples. You can also see [TdNode](https://github.com/puppy0cam/TdNode), [tglib](https://github.com/nodegin/tglib), [node-tdlib](https://github.com/wfjsw/node-tdlib), [tdlnode](https://github.com/fonbah/tdlnode), -[Paper Plane](https://github.com/BlackSuited/paper-plane), or [node-tlg](https://github.com/dilongfa/node-tlg) for other examples of TDLib JSON interface integration with Node.js. +[Paper Plane](https://github.com/par6n/paper-plane), or [node-tlg](https://github.com/dilongfa/node-tlg) for other examples of TDLib JSON interface integration with Node.js. + +See also the source code of [DIBgram](https://github.com/DIBgram/DIBgram) - an unofficial Telegram web app which looks like Telegram Desktop. TDLib can be used also from NativeScript through the [JSON](https://github.com/tdlib/td#using-json) interface. See [nativescript-tglib](https://github.com/arpit2438735/nativescript-tglib) as an example of a NativeScript library for building Telegram clients. @@ -94,8 +97,6 @@ We provide a generator for JNI bridge methods and Java classes for all TDLib API See [example/java](https://github.com/tdlib/td/tree/master/example/java) for an example of using TDLib from desktop Java along with detailed building and usage instructions. To use TDLib to create Android Java applications, use our [prebuilt library for Android](https://core.telegram.org/tdlib/tdlib.zip). -You can also see [JTDLib](https://github.com/ErnyTech/JTDLib) for another example of Java wrapper for TDLib. - ## Using TDLib in Kotlin projects @@ -103,8 +104,7 @@ TDLib can be used from the Kotlin/JVM programming language through same way as i You can also use [ktd](https://github.com/whyoleg/ktd) library with Kotlin-specific bindings. -See also [KtLib](https://github.com/nekohasekai/KtLib), which provides an automatically generated classes for TDLib API, and -[td-ktx](https://github.com/tdlibx/td-ktx) - Kotlin coroutines wrapper for TDLib. +See also [td-ktx](https://github.com/tdlibx/td-ktx) - Kotlin coroutines wrapper for TDLib. ## Using TDLib in C# projects @@ -117,6 +117,8 @@ See [example/csharp](https://github.com/tdlib/td/tree/master/example/csharp) for If you want to write a cross-platform C# application using .NET Core, see [tdsharp](https://github.com/egramtel/tdsharp). It uses our [JSON](https://github.com/tdlib/td#using-json) interface, provides an asynchronous interface for interaction with TDLib, automatically generated classes for TDLib API and has some examples. +You can also use [TDLibCore](https://github.com/ph09nix/TDLibCore) library. + Also see [Unigram](https://github.com/UnigramDev/Unigram), which is a full-featured client rewritten from scratch in C# using TDLib SDK for Universal Windows Platform in less than 2 months, [egram.tel](https://github.com/egramtel/egram.tel) – a cross-platform Telegram client written in C#, .NET Core, ReactiveUI and Avalonia, or [telewear](https://github.com/telewear/telewear) - a Telegram client for Samsung watches. @@ -127,10 +129,12 @@ Also see [Unigram](https://github.com/UnigramDev/Unigram), which is a full-featu TDLib has a simple and convenient C++11-interface for sending and receiving requests and can be statically linked to your application. See [example/cpp](https://github.com/tdlib/td/tree/master/example/cpp) for an example of TDLib usage from C++. -[td_example.cpp](https://github.com/tdlib/td/tree/master/example/cpp/td_example.cpp) contains an example of authorization, processing new incoming messages, getting a list of chats and sending a text message. +[td_example.cpp](https://github.com/tdlib/td/blob/master/example/cpp/td_example.cpp) contains an example of authorization, processing new incoming messages, getting a list of chats and sending a text message. See also the source code of [Fernschreiber](https://github.com/Wunderfitz/harbour-fernschreiber) and [Depecher](https://github.com/blacksailer/depecher) – Telegram apps for Sailfish OS, -[TELEports](https://gitlab.com/ubports/apps/teleports) – a Qt-client for Ubuntu Touch, or [tdlib-purple](https://github.com/ars3niy/tdlib-purple) - Telegram plugin for Pidgin, all of which are based on TDLib. +[TELEports](https://gitlab.com/ubports/development/apps/teleports) – a Qt-client for Ubuntu Touch, [tdlib-purple](https://github.com/ars3niy/tdlib-purple) - Telegram plugin for Pidgin, +or [MeeGram](https://github.com/qtinsider/meegram2) - a Telegram client for Nokia N9, +[TDLib Native Sciter Extension](https://github.com/RadRussianRus/TDLibNSE) - a Sciter native extension for TDLib's JSON interface, all of which are based on TDLib. ## Using TDLib in Swift projects @@ -141,6 +145,8 @@ See [example/ios](https://github.com/tdlib/td/tree/master/example/ios) for an ex See [TDLibKit](https://github.com/Swiftgram/TDLibKit), [tdlib-swift](https://github.com/modestman/tdlib-swift), or [TDLib-iOS](https://github.com/leoMehlig/TDLib-iOS), which provide convenient TDLib clients with automatically generated and fully-documented classes for all TDLib API methods and objects. +See also the source code of [Moc](https://github.com/ggoraa/moc) - a native and powerful macOS and iPadOS Telegram client, optimized for moderating large communities and personal use. + See [example/swift](https://github.com/tdlib/td/tree/master/example/swift) for an example of a macOS Swift application. @@ -164,9 +170,12 @@ See [tdlib-lazarus](https://github.com/dieletro/tdlib-lazarus) for an example of TDLib can be used from the Dart programming language through the [JSON](https://github.com/tdlib/td#using-json) interface and a Dart Native Extension or Dart FFI. -See [dart_tdlib](https://github.com/periodicaidan/dart_tdlib) or [Dart wrapper for TDLib](https://github.com/tdlib/td/pull/708/commits/237060abd4c205768153180e9f814298d1aa9d49) for an example of a TDLib Dart bindings through FFI. +See [dart_tdlib](https://github.com/periodicaidan/dart_tdlib), [flutter_libtdjson](https://github.com/up9cloud/flutter_libtdjson), [Dart wrapper for TDLib](https://github.com/tdlib/td/pull/708/commits/237060abd4c205768153180e9f814298d1aa9d49), or [tdlib_bindings](https://github.com/lesnitsky/tdlib_bindings) for an example of a TDLib Dart bindings through FFI. -See [project.scarlet](https://github.com/aaugmentum/project.scarlet), [tdlib](https://github.com/i-Naji/tdlib), [tdlib-dart](https://github.com/drewpayment/tdlib-dart), [tdlib-dart](https://github.com/triedcatched/tdlib-dart), or [telegram-service](https://github.com/igorder-dev/telegram-service) for examples of using TDLib from Dart. +See [Telegram Client library](https://github.com/azkadev/telegram_client), [project.scarlet](https://github.com/aaugmentum/project.scarlet), [tdlib](https://github.com/i-Naji/tdlib), +[tdlib-dart](https://github.com/drewpayment/tdlib-dart), [FluGram](https://github.com/triedcatched/tdlib-dart), or [telegram-service](https://github.com/igorder-dev/telegram-service) for examples of using TDLib from Dart. + +See also [f-Telegram](https://github.com/evgfilim1/ftg) - Flutter Telegram client. ## Using TDLib in Rust projects @@ -176,8 +185,7 @@ TDLib can be used from the Rust programming language through the [JSON](https:// See [rust-tdlib](https://github.com/aCLr/rust-tdlib), [tdlib](https://github.com/melix99/tdlib-rs), or [tdlib-rs](https://github.com/agnipau/tdlib-rs), which provide convenient TDLib clients with automatically generated and fully-documented classes for all TDLib API methods and objects. See [rtdlib](https://github.com/fewensa/rtdlib), [tdlib-rs](https://github.com/d653/tdlib-rs), [tdlib-futures](https://github.com/yuri91/tdlib-futures), -[tdlib-sys](https://github.com/nuxeh/tdlib-sys), or -[tdjson-rs](https://github.com/mersinvald/tdjson-rs) for examples of TDLib Rust bindings. +[tdlib-sys](https://github.com/nuxeh/tdlib-sys), [tdjson-rs](https://github.com/mersinvald/tdjson-rs), [rust-tdlib](https://github.com/vhaoran/rust-tdlib), or [tdlib-json-sys](https://github.com/aykxt/tdlib-json-sys) for examples of TDLib Rust bindings. Also see [Telegrand](https://github.com/melix99/telegrand) – a Telegram client optimized for the GNOME desktop. @@ -191,7 +199,7 @@ See [erl-tdlib](https://github.com/lattenwald/erl-tdlib) for an example of TDLib ## Using TDLib in PHP projects -If you use modern PHP >= 7.4, you can use TDLib via a PHP FFI extension. For example, take a look at [ffi-tdlib](https://github.com/aurimasniekis/php-ffi-tdlib) - an FFI-based TDLib wrapper. +If you use modern PHP >= 7.4, you can use TDLib via a PHP FFI extension. For example, take a look at [ffi-tdlib](https://github.com/aurimasniekis/php-ffi-tdlib), or [tdlib-php-ffi](https://github.com/thisismzm/tdlib-php-ffi) - FFI-based TDLib wrappers. See also [tdlib-schema](https://github.com/aurimasniekis/php-tdlib-schema) - a generator for TDLib API classes. @@ -224,7 +232,7 @@ See [d-tdlib-service](https://github.com/Lord-Evil/d-tdlib-service) for an examp TDLib can be used from the Ruby programming language through the [JSON](https://github.com/tdlib/td#using-json) interface. -See [tdlib-ruby](https://github.com/centosadmin/tdlib-ruby) for examples of Ruby bindings and a client for TDLib. +See [tdlib-ruby](https://github.com/southbridgeio/tdlib-ruby) for examples of Ruby bindings and a client for TDLib. ## Using TDLib in Crystal projects @@ -267,21 +275,23 @@ See [telega.el](https://github.com/zevlg/telega.el) for an example of a GNU Emac TDLib can be used from the Elixir programming language. -See [Elixir TDLib](https://gitlab.com/Fnux/elixir-tdlib) or [Elixir TDLib](https://github.com/QuantLayer/elixir-tdlib) for examples of such usage and an Elixir client for TDLib. -These libraries contain automatically generated and fully-documented classes for all TDLib API methods and objects. +See [Elixir TDLib](https://github.com/QuantLayer/elixir-tdlib) for an example of such usage and an Elixir client for TDLib. +The library contains automatically generated and fully-documented classes for all TDLib API methods and objects. ## Using TDLib from 1С:Enterprise TDLib can be used from the 1С programming language. -See [TDLib bindings for 1С:Enterprise](https://github.com/Infactum/telegram-native) for examples of such usage. +See [TDLib bindings for 1С:Enterprise](https://github.com/Infactum/telegram-native) and [e1c.tAgents](https://github.com/fedbka/e1c.tAgents) for examples of such usage. ## Using TDLib in C projects TDLib can be used from the C programming language through the [JSON](https://github.com/tdlib/td#using-json) interface and can be linked statically or dynamically. +See [easy-tg](https://github.com/Trumeet/easy-tg) for an example of such usage. + You can also try to use our [C](https://github.com/tdlib/td/blob/master/td/telegram/td_c_client.h) client, which was used by the private TDLib-based version of [telegram-cli](https://github.com/vysheng/tg). From 81cb929b37c66413f6c06c52a19201c6795cfc5a Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 6 Jul 2022 21:53:50 +0300 Subject: [PATCH 03/55] Avoid warning about set but not used variable. --- benchmark/bench_actor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark/bench_actor.cpp b/benchmark/bench_actor.cpp index f68a8d8e0..b3a04f592 100644 --- a/benchmark/bench_actor.cpp +++ b/benchmark/bench_actor.cpp @@ -260,6 +260,7 @@ class QueryBench final : public td::Benchmark { } else if (type == 4) { int val = 0; send_lambda(client_, [&] { val = n_ * n_; }); + CHECK(val == n_ * n_); } else if (type == 5) { send_closure(client_, &ClientActor::f_promise, td::PromiseCreator::lambda([actor_id = actor_id(this), n = n_](td::Unit) { From f8b49fe42102eb46d77407d67d70b0a14eb57fec Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 7 Jul 2022 20:38:44 +0300 Subject: [PATCH 04/55] Ensure that gen_git_commit_h works even without git. --- gen_git_commit_h.ps1 | 6 +++--- gen_git_commit_h.sh | 12 +++++++++--- tdutils/td/utils/GitInfo.cpp | 11 ++++++++++- tdutils/td/utils/GitInfo.h | 4 +--- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/gen_git_commit_h.ps1 b/gen_git_commit_h.ps1 index 2893f89c2..dadc225cd 100644 --- a/gen_git_commit_h.ps1 +++ b/gen_git_commit_h.ps1 @@ -1,6 +1,6 @@ -$commit = git rev-parse HEAD -git diff-index --quiet HEAD -$dirty = $LASTEXITCODE +$commit = try { git rev-parse HEAD } catch { "unknown" } +try { git diff-index --quiet HEAD } catch {} +$dirty = if ($LASTEXITCODE) { "true" } else { "false" } echo "#pragma once`r`n#define GIT_COMMIT `"$commit`"`r`n#define GIT_DIRTY $dirty" | out-file -encoding ASCII auto/git_info.h.new if (-not (Test-Path .\auto\git_info.h) -or (Compare-Object $(Get-Content .\auto\git_info.h.new) $(Get-Content .\auto\git_info.h))) { mv -Force auto/git_info.h.new auto/git_info.h diff --git a/gen_git_commit_h.sh b/gen_git_commit_h.sh index 15ede4cd7..bc6bc2d35 100755 --- a/gen_git_commit_h.sh +++ b/gen_git_commit_h.sh @@ -1,8 +1,14 @@ #!/bin/sh cd $(dirname $0) -commit=$(git rev-parse HEAD) -git diff-index --quiet HEAD -dirty=$? +commit="$(git rev-parse HEAD 2> /dev/null)" +commit="${commit:-unknown}" +git diff-index --quiet HEAD 2> /dev/null +if [ $? -ne 0 ] +then + dirty="true" +else + dirty="false" +fi printf "#pragma once\n#define GIT_COMMIT \"$commit\"\n#define GIT_DIRTY $dirty\n" > auto/git_info.h.new if cmp -s auto/git_info.h.new auto/git_info.h 2>&1 > /dev/null then diff --git a/tdutils/td/utils/GitInfo.cpp b/tdutils/td/utils/GitInfo.cpp index 9adc64746..f51c6294c 100644 --- a/tdutils/td/utils/GitInfo.cpp +++ b/tdutils/td/utils/GitInfo.cpp @@ -8,11 +8,20 @@ #include "auto/git_info.h" +#if !defined(GIT_COMMIT) +#define GIT_COMMIT "unknown" +#endif + +#if !defined(GIT_DIRTY) +#define GIT_DIRTY false +#endif + namespace td { -CSlice GitInfo::commit() { +const char *GitInfo::commit() { return GIT_COMMIT; } + bool GitInfo::is_dirty() { return GIT_DIRTY; } diff --git a/tdutils/td/utils/GitInfo.h b/tdutils/td/utils/GitInfo.h index c160509ce..194f17698 100644 --- a/tdutils/td/utils/GitInfo.h +++ b/tdutils/td/utils/GitInfo.h @@ -6,13 +6,11 @@ // #pragma once -#include "td/utils/Slice.h" - namespace td { class GitInfo { public: - static CSlice commit(); + static const char *commit(); static bool is_dirty(); }; From ff3235e6d55d4d7c9b86f6d3ca2365348978a1e6 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 7 Jul 2022 22:23:58 +0300 Subject: [PATCH 05/55] Update comment. --- tdutils/generate/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdutils/generate/CMakeLists.txt b/tdutils/generate/CMakeLists.txt index 82f544934..b36a1a4f6 100644 --- a/tdutils/generate/CMakeLists.txt +++ b/tdutils/generate/CMakeLists.txt @@ -3,7 +3,7 @@ if ((CMAKE_MAJOR_VERSION LESS 3) OR (CMAKE_VERSION VERSION_LESS "3.0.2")) endif() # Generates files for MIME type <-> extension conversions -# DEPENDS ON: gperf grep bash/powershell +# DEPENDS ON: gperf grep if (NOT TDUTILS_MIME_TYPE) return() From 435e4e483d0293c6825ac96a2b97587d6af9d51a Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 8 Jul 2022 11:21:31 +0300 Subject: [PATCH 06/55] Clarify verbosiy level range in the documentation. --- td/telegram/Client.h | 2 +- td/telegram/ClientDotNet.cpp | 2 +- td/telegram/td_json_client.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/td/telegram/Client.h b/td/telegram/Client.h index b8c1e814f..01dc8ac02 100644 --- a/td/telegram/Client.h +++ b/td/telegram/Client.h @@ -126,7 +126,7 @@ class ClientManager final { /** * A type of callback function that will be called when a message is added to the internal TDLib log. * - * \param verbosity_level Log verbosity level with which the message was added (-1 - 1024). + * \param verbosity_level Log verbosity level with which the message was added from -1 up to 1024. * If 0, then TDLib will crash as soon as the callback returns. * None of the TDLib methods can be called from the callback. * \param message Null-terminated UTF-8-encoded string with the message added to the log. diff --git a/td/telegram/ClientDotNet.cpp b/td/telegram/ClientDotNet.cpp index 8d4709e18..25b2a8b07 100644 --- a/td/telegram/ClientDotNet.cpp +++ b/td/telegram/ClientDotNet.cpp @@ -25,7 +25,7 @@ using namespace CxCli; /// /// A type of callback function that will be called when a message is added to the internal TDLib log. /// -/// Log verbosity level with which the message was added (-1 - 1024). +/// Log verbosity level with which the message was added from -1 up to 1024. /// If 0, then TDLib will crash as soon as the callback returns. /// None of the TDLib methods can be called from the callback. /// Null-terminated string with the message added to the log. diff --git a/td/telegram/td_json_client.h b/td/telegram/td_json_client.h index e6e0c2ded..a12fa01b6 100644 --- a/td/telegram/td_json_client.h +++ b/td/telegram/td_json_client.h @@ -88,7 +88,7 @@ TDJSON_EXPORT const char *td_execute(const char *request); /** * A type of callback function that will be called when a message is added to the internal TDLib log. * - * \param verbosity_level Log verbosity level with which the message was added (-1 - 1024). + * \param verbosity_level Log verbosity level with which the message was added from -1 up to 1024. * If 0, then TDLib will crash as soon as the callback returns. * None of the TDLib methods can be called from the callback. * \param message Null-terminated UTF-8-encoded string with the message added to the log. From 20fa839dfe1d96b372a524e97bf9b24126be0721 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 8 Jul 2022 11:22:13 +0300 Subject: [PATCH 07/55] Crash on "Lost promise" errors. --- td/telegram/Td.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index d7686486d..be99c16a7 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -4183,6 +4183,9 @@ void Td::send_error_impl(uint64 id, tl_object_ptr error) { if (it != request_set_.end()) { request_set_.erase(it); VLOG(td_requests) << "Sending error for request " << id << ": " << oneline(to_string(error)); + if (error->code_ == 0 && error->message_ == "Lost promise") { + LOG(FATAL) << "Lost promise for query " << id; + } callback_->on_error(id, std::move(error)); } } From 89aaf648fca9dbc2498fbea7dfe165076d39a75d Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 8 Jul 2022 15:01:38 +0300 Subject: [PATCH 08/55] Add Client.setLogMessageHandler to Java example. --- example/java/org/drinkless/tdlib/Client.java | 28 ++++++++++ example/java/org/drinkless/tdlib/Log.java | 30 ---------- .../org/drinkless/tdlib/example/Example.java | 15 ++++- example/java/td_jni.cpp | 56 ++++++++++++++----- td/telegram/ClientDotNet.cpp | 2 +- 5 files changed, 86 insertions(+), 45 deletions(-) diff --git a/example/java/org/drinkless/tdlib/Client.java b/example/java/org/drinkless/tdlib/Client.java index 3ff95f46c..e9cf5e180 100644 --- a/example/java/org/drinkless/tdlib/Client.java +++ b/example/java/org/drinkless/tdlib/Client.java @@ -39,6 +39,21 @@ public final class Client { void onException(Throwable e); } + /** + * Interface for handler of messages that are added to the internal TDLib log. + */ + public interface LogMessageHandler { + /** + * Callback called on messages that are added to the internal TDLib log. + * + * @param verbosityLevel Log verbosity level with which the message was added from -1 up to 1024. + * If 0, then TDLib will crash as soon as the callback returns. + * None of the TDLib methods can be called from the callback. + * @param message The message added to the internal TDLib log. + */ + void onLogMessage(int verbosityLevel, String message); + } + /** * Sends a request to the TDLib. * @@ -105,6 +120,17 @@ public final class Client { return client; } + /** + * Sets the handler for messages that are added to the internal TDLib log. + * None of the TDLib methods can be called from the callback. + * + * @param maxVerbosityLevel The maximum verbosity level of messages for which the callback will be called. + * @param logMessageHandler Handler for messages that are added to the internal TDLib log. Pass null to remove the handler. + */ + public static void setLogMessageHandler(int maxVerbosityLevel, Client.LogMessageHandler logMessageHandler) { + nativeClientSetLogMessageHandler(maxVerbosityLevel, logMessageHandler); + } + private static class ResponseReceiver implements Runnable { public boolean isRun = false; @@ -203,4 +229,6 @@ public final class Client { private static native int nativeClientReceive(int[] clientIds, long[] eventIds, TdApi.Object[] events, double timeout); private static native TdApi.Object nativeClientExecute(TdApi.Function function); + + private static native void nativeClientSetLogMessageHandler(int maxVerbosityLevel, LogMessageHandler logMessageHandler); } diff --git a/example/java/org/drinkless/tdlib/Log.java b/example/java/org/drinkless/tdlib/Log.java index 1a7f637b1..a5de7c323 100644 --- a/example/java/org/drinkless/tdlib/Log.java +++ b/example/java/org/drinkless/tdlib/Log.java @@ -49,34 +49,4 @@ public final class Log { */ @Deprecated public static native void setMaxFileSize(long maxFileSize); - - /** - * This function is called from the JNI when a fatal error happens to provide a better error message. - * The function does not return. - * - * @param errorMessage Error message. - */ - private static void onFatalError(String errorMessage) { - class ThrowError implements Runnable { - private ThrowError(String errorMessage) { - this.errorMessage = errorMessage; - } - - @Override - public void run() { - throw new RuntimeException("TDLib fatal error: " + errorMessage); - } - - private final String errorMessage; - } - - new Thread(new ThrowError(errorMessage), "TDLib fatal error thread").start(); - while (true) { - try { - Thread.sleep(1000); // milliseconds - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - } - } - } } diff --git a/example/java/org/drinkless/tdlib/example/Example.java b/example/java/org/drinkless/tdlib/example/Example.java index f657c3c8b..8a6ea24a8 100644 --- a/example/java/org/drinkless/tdlib/example/Example.java +++ b/example/java/org/drinkless/tdlib/example/Example.java @@ -298,7 +298,10 @@ public final class Example { } public static void main(String[] args) throws InterruptedException { - // disable TDLib log + // set log message handler to handle only fatal errors (0) and plain log messages (-1) + Client.setLogMessageHandler(0, new LogMessageHandler()); + + // disable TDLib log and redirect fatal errors and plain log messages to a file Client.execute(new TdApi.SetLogVerbosityLevel(0)); if (Client.execute(new TdApi.SetLogStream(new TdApi.LogStreamFile("tdlib.log", 1 << 27, false))) instanceof TdApi.Error) { throw new IOError(new IOException("Write access to the current directory is required")); @@ -598,4 +601,14 @@ public final class Example { } } } + + private static class LogMessageHandler implements Client.LogMessageHandler { + @Override + public void onLogMessage(int verbosityLevel, String message) { + if (verbosityLevel == 0) { + // a fatal error, the app will crash right after the function returns + } + System.err.println(message); + } + } } diff --git a/example/java/td_jni.cpp b/example/java/td_jni.cpp index 2621e534c..cbb03c3b9 100644 --- a/example/java/td_jni.cpp +++ b/example/java/td_jni.cpp @@ -99,23 +99,52 @@ static jstring Function_toString(JNIEnv *env, jobject object) { static constexpr jint JAVA_VERSION = JNI_VERSION_1_6; static JavaVM *java_vm; -static jclass log_class; +static jobject log_message_handler; -static void on_log_message(int verbosity_level, const char *error_message) { - if (verbosity_level != 0) { - return; - } +static void on_log_message(int verbosity_level, const char *log_message) { auto env = td::jni::get_jni_env(java_vm, JAVA_VERSION); if (env == nullptr) { return; } - jmethodID on_fatal_error_method = env->GetStaticMethodID(log_class, "onFatalError", "(Ljava/lang/String;)V"); - if (on_fatal_error_method) { - jstring error_str = td::jni::to_jstring(env.get(), error_message); - env->CallStaticVoidMethod(log_class, on_fatal_error_method, error_str); - if (error_str) { - env->DeleteLocalRef(error_str); + + jobject handler = env->NewLocalRef(log_message_handler); + if (!handler) { + return; + } + + jclass handler_class = env->GetObjectClass(handler); + if (handler_class) { + jmethodID on_log_message_method = env->GetMethodID(handler_class, "onLogMessage", "(ILjava/lang/String;)V"); + if (on_log_message_method) { + jstring log_message_str = td::jni::to_jstring(env.get(), log_message); + if (log_message_str) { + env->CallVoidMethod(handler, on_log_message_method, static_cast(verbosity_level), log_message_str); + env->DeleteLocalRef((jobject)log_message_str); + } } + env->DeleteLocalRef((jobject)handler_class); + } + + env->DeleteLocalRef(handler); +} + +static void Client_nativeClientSetLogMessageHandler(JNIEnv *env, jclass clazz, jint max_verbosity_level, + jobject new_log_message_handler) { + if (log_message_handler) { + td::ClientManager::set_log_message_callback(0, nullptr); + jobject old_log_message_handler = log_message_handler; + log_message_handler = jobject(); + env->DeleteGlobalRef(old_log_message_handler); + } + + if (new_log_message_handler) { + log_message_handler = env->NewGlobalRef(new_log_message_handler); + if (!log_message_handler) { + // out of memory + return; + } + + td::ClientManager::set_log_message_callback(static_cast(max_verbosity_level), on_log_message); } } @@ -133,7 +162,7 @@ static jint register_native(JavaVM *vm) { }; auto client_class = td::jni::get_jclass(env, PACKAGE_NAME "/Client"); - log_class = td::jni::get_jclass(env, PACKAGE_NAME "/Log"); + auto log_class = td::jni::get_jclass(env, PACKAGE_NAME "/Log"); auto object_class = td::jni::get_jclass(env, PACKAGE_NAME "/TdApi$Object"); auto function_class = td::jni::get_jclass(env, PACKAGE_NAME "/TdApi$Function"); @@ -143,6 +172,8 @@ static jint register_native(JavaVM *vm) { register_method(client_class, "nativeClientSend", "(IJ" TD_FUNCTION ")V", Client_nativeClientSend); register_method(client_class, "nativeClientReceive", "([I[J[" TD_OBJECT "D)I", Client_nativeClientReceive); register_method(client_class, "nativeClientExecute", "(" TD_FUNCTION ")" TD_OBJECT, Client_nativeClientExecute); + register_method(client_class, "nativeClientSetLogMessageHandler", "(IL" PACKAGE_NAME "/Client$LogMessageHandler;)V", + Client_nativeClientSetLogMessageHandler); register_method(log_class, "setVerbosityLevel", "(I)V", Log_setVerbosityLevel); register_method(log_class, "setFilePath", "(Ljava/lang/String;)Z", Log_setFilePath); @@ -157,7 +188,6 @@ static jint register_native(JavaVM *vm) { td::jni::init_vars(env, PACKAGE_NAME); td::td_api::Object::init_jni_vars(env, PACKAGE_NAME); td::td_api::Function::init_jni_vars(env, PACKAGE_NAME); - td::ClientManager::set_log_message_callback(0, on_log_message); return JAVA_VERSION; } diff --git a/td/telegram/ClientDotNet.cpp b/td/telegram/ClientDotNet.cpp index 25b2a8b07..44195978e 100644 --- a/td/telegram/ClientDotNet.cpp +++ b/td/telegram/ClientDotNet.cpp @@ -28,7 +28,7 @@ using namespace CxCli; /// Log verbosity level with which the message was added from -1 up to 1024. /// If 0, then TDLib will crash as soon as the callback returns. /// None of the TDLib methods can be called from the callback. -/// Null-terminated string with the message added to the log. +/// The message added to the log. public delegate void LogMessageCallback(int verbosityLevel, String^ message); #endif From d94a750685a7ab100a97d3b36459f98e6c51973b Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 8 Jul 2022 15:08:50 +0300 Subject: [PATCH 09/55] Remove deprecated for more than 3 years Log.java from example. --- example/java/CMakeLists.txt | 2 +- example/java/org/drinkless/tdlib/Log.java | 52 ----------------------- example/java/td_jni.cpp | 18 -------- 3 files changed, 1 insertion(+), 71 deletions(-) delete mode 100644 example/java/org/drinkless/tdlib/Log.java diff --git a/example/java/CMakeLists.txt b/example/java/CMakeLists.txt index 2c685a774..576eee2cf 100644 --- a/example/java/CMakeLists.txt +++ b/example/java/CMakeLists.txt @@ -59,7 +59,7 @@ set(JAVA_SOURCE_PATH "${TD_API_JAVA_PATH}/${TD_API_JAVA_PACKAGE}") get_filename_component(JAVA_OUTPUT_DIRECTORY ${CMAKE_INSTALL_PREFIX}/bin REALPATH BASE_DIR "${CMAKE_CURRENT_BINARY_DIR}") file(MAKE_DIRECTORY ${JAVA_OUTPUT_DIRECTORY}) add_custom_target(build_java - COMMAND ${Java_JAVAC_EXECUTABLE} -encoding UTF-8 -d ${JAVA_OUTPUT_DIRECTORY} ${JAVA_SOURCE_PATH}/example/Example.java ${JAVA_SOURCE_PATH}/Client.java ${JAVA_SOURCE_PATH}/Log.java ${JAVA_SOURCE_PATH}/TdApi.java + COMMAND ${Java_JAVAC_EXECUTABLE} -encoding UTF-8 -d ${JAVA_OUTPUT_DIRECTORY} ${JAVA_SOURCE_PATH}/example/Example.java ${JAVA_SOURCE_PATH}/Client.java ${JAVA_SOURCE_PATH}/TdApi.java COMMENT "Building Java code" DEPENDS td_generate_java_api ) diff --git a/example/java/org/drinkless/tdlib/Log.java b/example/java/org/drinkless/tdlib/Log.java deleted file mode 100644 index a5de7c323..000000000 --- a/example/java/org/drinkless/tdlib/Log.java +++ /dev/null @@ -1,52 +0,0 @@ -// -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 -// -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -// -package org.drinkless.tdlib; - -/** - * Class used for managing internal TDLib logging. - * Use TdApi.*Log* methods instead. - */ -public final class Log { - /** - * Changes TDLib log verbosity. - * - * @deprecated As of TDLib 1.4.0 in favor of {@link TdApi.SetLogVerbosityLevel}, to be removed in the future. - * @param verbosityLevel New value of log verbosity level. Must be non-negative. - * Value 0 corresponds to fatal errors, - * value 1 corresponds to java.util.logging.Level.SEVERE, - * value 2 corresponds to java.util.logging.Level.WARNING, - * value 3 corresponds to java.util.logging.Level.INFO, - * value 4 corresponds to java.util.logging.Level.FINE, - * value 5 corresponds to java.util.logging.Level.FINER, - * value greater than 5 can be used to enable even more logging. - * Default value of the log verbosity level is 5. - */ - @Deprecated - public static native void setVerbosityLevel(int verbosityLevel); - - /** - * Sets file path for writing TDLib internal log. By default TDLib writes logs to the System.err. - * Use this method to write the log to a file instead. - * - * @deprecated As of TDLib 1.4.0 in favor of {@link TdApi.SetLogStream}, to be removed in the future. - * @param filePath Path to a file for writing TDLib internal log. Use an empty path to - * switch back to logging to the System.err. - * @return whether opening the log file succeeded. - */ - @Deprecated - public static native boolean setFilePath(String filePath); - - /** - * Changes the maximum size of TDLib log file. - * - * @deprecated As of TDLib 1.4.0 in favor of {@link TdApi.SetLogStream}, to be removed in the future. - * @param maxFileSize The maximum size of the file to where the internal TDLib log is written - * before the file will be auto-rotated. Must be positive. Defaults to 10 MB. - */ - @Deprecated - public static native void setMaxFileSize(long maxFileSize); -} diff --git a/example/java/td_jni.cpp b/example/java/td_jni.cpp index cbb03c3b9..15c60a954 100644 --- a/example/java/td_jni.cpp +++ b/example/java/td_jni.cpp @@ -5,7 +5,6 @@ // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // #include -#include #include #include @@ -77,18 +76,6 @@ static jobject Client_nativeClientExecute(JNIEnv *env, jclass clazz, jobject fun return result; } -static void Log_setVerbosityLevel(JNIEnv *env, jclass clazz, jint new_log_verbosity_level) { - td::Log::set_verbosity_level(static_cast(new_log_verbosity_level)); -} - -static jboolean Log_setFilePath(JNIEnv *env, jclass clazz, jstring file_path) { - return td::Log::set_file_path(td::jni::from_jstring(env, file_path)) ? JNI_TRUE : JNI_FALSE; -} - -static void Log_setMaxFileSize(JNIEnv *env, jclass clazz, jlong max_file_size) { - td::Log::set_max_file_size(max_file_size); -} - static jstring Object_toString(JNIEnv *env, jobject object) { return td::jni::to_jstring(env, to_string(td::td_api::Object::fetch(env, object))); } @@ -162,7 +149,6 @@ static jint register_native(JavaVM *vm) { }; auto client_class = td::jni::get_jclass(env, PACKAGE_NAME "/Client"); - auto log_class = td::jni::get_jclass(env, PACKAGE_NAME "/Log"); auto object_class = td::jni::get_jclass(env, PACKAGE_NAME "/TdApi$Object"); auto function_class = td::jni::get_jclass(env, PACKAGE_NAME "/TdApi$Function"); @@ -175,10 +161,6 @@ static jint register_native(JavaVM *vm) { register_method(client_class, "nativeClientSetLogMessageHandler", "(IL" PACKAGE_NAME "/Client$LogMessageHandler;)V", Client_nativeClientSetLogMessageHandler); - register_method(log_class, "setVerbosityLevel", "(I)V", Log_setVerbosityLevel); - register_method(log_class, "setFilePath", "(Ljava/lang/String;)Z", Log_setFilePath); - register_method(log_class, "setMaxFileSize", "(J)V", Log_setMaxFileSize); - register_method(object_class, "toString", "()Ljava/lang/String;", Object_toString); register_method(function_class, "toString", "()Ljava/lang/String;", Function_toString); From 9a58bc03ab5ee7cff5d2b9d11cd7bfbda4c4fe37 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 8 Jul 2022 16:12:31 +0300 Subject: [PATCH 10/55] Fix Java example formatting. --- example/java/org/drinkless/tdlib/example/Example.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/example/java/org/drinkless/tdlib/example/Example.java b/example/java/org/drinkless/tdlib/example/Example.java index 8a6ea24a8..1f587a1c9 100644 --- a/example/java/org/drinkless/tdlib/example/Example.java +++ b/example/java/org/drinkless/tdlib/example/Example.java @@ -9,9 +9,9 @@ package org.drinkless.tdlib.example; import org.drinkless.tdlib.Client; import org.drinkless.tdlib.TdApi; +import java.io.BufferedReader; import java.io.IOError; import java.io.IOException; -import java.io.BufferedReader; import java.io.InputStreamReader; import java.util.NavigableSet; import java.util.TreeSet; @@ -380,7 +380,7 @@ public final class Example { TdApi.UpdateUser updateUser = (TdApi.UpdateUser) object; users.put(updateUser.user.id, updateUser.user); break; - case TdApi.UpdateUserStatus.CONSTRUCTOR: { + case TdApi.UpdateUserStatus.CONSTRUCTOR: { TdApi.UpdateUserStatus updateUserStatus = (TdApi.UpdateUserStatus) object; TdApi.User user = users.get(updateUserStatus.userId); synchronized (user) { @@ -441,7 +441,7 @@ public final class Example { case TdApi.UpdateChatPosition.CONSTRUCTOR: { TdApi.UpdateChatPosition updateChat = (TdApi.UpdateChatPosition) object; if (updateChat.position.list.getConstructor() != TdApi.ChatListMain.CONSTRUCTOR) { - break; + break; } TdApi.Chat chat = chats.get(updateChat.chatId); @@ -455,7 +455,7 @@ public final class Example { TdApi.ChatPosition[] new_positions = new TdApi.ChatPosition[chat.positions.length + (updateChat.position.order == 0 ? 0 : 1) - (i < chat.positions.length ? 1 : 0)]; int pos = 0; if (updateChat.position.order != 0) { - new_positions[pos++] = updateChat.position; + new_positions[pos++] = updateChat.position; } for (int j = 0; j < chat.positions.length; j++) { if (j != i) { @@ -606,7 +606,7 @@ public final class Example { @Override public void onLogMessage(int verbosityLevel, String message) { if (verbosityLevel == 0) { - // a fatal error, the app will crash right after the function returns + // a fatal error, the app will crash right after the function returns } System.err.println(message); } From 1dec0e203ca8110ea53f57830eb5d890a648d0e0 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 8 Jul 2022 18:44:09 +0300 Subject: [PATCH 11/55] Add Java example of fatal error handler. --- .../org/drinkless/tdlib/example/Example.java | 74 ++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/example/java/org/drinkless/tdlib/example/Example.java b/example/java/org/drinkless/tdlib/example/Example.java index 1f587a1c9..2923ecea7 100644 --- a/example/java/org/drinkless/tdlib/example/Example.java +++ b/example/java/org/drinkless/tdlib/example/Example.java @@ -17,6 +17,7 @@ import java.util.NavigableSet; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -606,9 +607,80 @@ public final class Example { @Override public void onLogMessage(int verbosityLevel, String message) { if (verbosityLevel == 0) { - // a fatal error, the app will crash right after the function returns + onFatalError(message); + return; } System.err.println(message); } } + + private static void onFatalError(String errorMessage) { + final class ThrowError implements Runnable { + private final String errorMessage; + private final AtomicLong errorThrowTime; + + private ThrowError(String errorMessage, AtomicLong errorThrowTime) { + this.errorMessage = errorMessage; + this.errorThrowTime = errorThrowTime; + } + + @Override + public void run() { + if (isDatabaseBrokenError(errorMessage) || isDiskFullError(errorMessage) || isDiskError(errorMessage)) { + processExternalError(); + return; + } + + errorThrowTime.set(System.currentTimeMillis()); + throw new ClientError("TDLib fatal error: " + errorMessage); + } + + private void processExternalError() { + errorThrowTime.set(System.currentTimeMillis()); + throw new ExternalClientError("Fatal error: " + errorMessage); + } + + private static final class ClientError extends Error { + private ClientError(String message) { + super(message); + } + } + + private static final class ExternalClientError extends Error { + public ExternalClientError(String message) { + super(message); + } + } + + private static boolean isDatabaseBrokenError(String message) { + return message.contains("Wrong key or database is corrupted") || + message.contains("SQL logic error or missing database") || + message.contains("database disk image is malformed") || + message.contains("file is encrypted or is not a database") || + message.contains("unsupported file format") || + message.contains("Database was corrupted and deleted during execution and can't be recreated"); + } + + private static boolean isDiskFullError(String message) { + return message.contains("PosixError : No space left on device") || + message.contains("database or disk is full"); + } + + private static boolean isDiskError(String message) { + return message.contains("I/O error") || message.contains("Structure needs cleaning"); + } + } + + final AtomicLong errorThrowTime = new AtomicLong(Long.MAX_VALUE); + new Thread(new ThrowError(errorMessage, errorThrowTime), "TDLib fatal error thread").start(); + + // wait at least 10 seconds after the error is thrown + while (errorThrowTime.get() >= System.currentTimeMillis() - 10000) { + try { + Thread.sleep(1000 /* milliseconds */); + } catch (InterruptedException ignore) { + Thread.currentThread().interrupt(); + } + } + } } From d37ad61f86d0e3fd8df6dd36776b74b2496bf4e5 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 8 Jul 2022 20:56:43 +0300 Subject: [PATCH 12/55] Generate GIT_COMMIT and GIT_DIRTY from CMake. --- CMake/GetGitInfo.cmake | 29 +++++++++++++++++++++++++++++ CMakeLists.txt | 21 +++++---------------- tdutils/CMakeLists.txt | 10 +++++++++- tdutils/td/utils/GitInfo.cpp | 2 -- 4 files changed, 43 insertions(+), 19 deletions(-) create mode 100644 CMake/GetGitInfo.cmake diff --git a/CMake/GetGitInfo.cmake b/CMake/GetGitInfo.cmake new file mode 100644 index 000000000..bba1a0b5c --- /dev/null +++ b/CMake/GetGitInfo.cmake @@ -0,0 +1,29 @@ +# sets GIT_COMMIT and GIT_DIRTY CMake variables +function(get_git_info) + find_program(GIT_EXECUTABLE git) + + unset(TD_GIT_COMMIT PARENT_SCOPE) + unset(TD_GIT_DIRTY PARENT_SCOPE) + + execute_process( + COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE GIT_COMMIT + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + + if (NOT GIT_COMMIT) + return() + endif() + + set(TD_GIT_COMMIT "${GIT_COMMIT}" PARENT_SCOPE) + + execute_process( + COMMAND "${GIT_EXECUTABLE}" diff-index --quiet HEAD + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + RESULT_VARIABLE GIT_DIRTY_RESULT + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + + if (GIT_DIRTY_RESULT) + set(TD_GIT_DIRTY "true" PARENT_SCOPE) + endif() +endfunction() diff --git a/CMakeLists.txt b/CMakeLists.txt index eaee9977b..2a833689a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -152,6 +152,11 @@ if (CLANG OR GCC) endif() endif() +if (NOT DEFINED TD_GIT_COMMIT) + include(GetGitInfo) + get_git_info() +endif() + add_subdirectory(tdtl) add_subdirectory(tdutils) @@ -744,22 +749,6 @@ set(MEMPROF_STAT_SOURCE memprof/memprof_stat.h ) -#RULES - -file(MAKE_DIRECTORY auto) - -if (CMAKE_HOST_WIN32) - set(GIT_COMMIT_CMD powershell -ExecutionPolicy ByPass ./gen_git_commit_h.ps1) -else() - set(GIT_COMMIT_CMD ./gen_git_commit_h.sh) -endif() - -add_custom_target(git_commit ALL - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - COMMAND ${GIT_COMMIT_CMD} - COMMENT "Generate git_commit.h" -) - #LIBRARIES # memprof - simple library for memory usage profiling diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index c0d26b9ec..2aed70b52 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -100,6 +100,7 @@ set(TDUTILS_SOURCE td/utils/find_boundary.cpp td/utils/FlatHashTable.cpp td/utils/FloodControlGlobal.cpp + td/utils/GitInfo.cpp td/utils/Gzip.cpp td/utils/GzipByteFlow.cpp td/utils/Hints.cpp @@ -217,6 +218,7 @@ set(TDUTILS_SOURCE td/utils/FloodControlGlobal.h td/utils/FloodControlStrict.h td/utils/format.h + td/utils/GitInfo.h td/utils/Gzip.h td/utils/GzipByteFlow.h td/utils/Hash.h @@ -337,10 +339,16 @@ set(TDUTILS_TEST_SOURCE PARENT_SCOPE ) -#RULES #LIBRARIES add_library(tdutils STATIC ${TDUTILS_SOURCE}) +if (DEFINED TD_GIT_COMMIT) + set_property(SOURCE td/utils/GitInfo.cpp APPEND PROPERTY COMPILE_DEFINITIONS "GIT_COMMIT=\"${TD_GIT_COMMIT}\"") +endif() +if (DEFINED TD_GIT_DIRTY) + set_property(SOURCE td/utils/GitInfo.cpp APPEND PROPERTY COMPILE_DEFINITIONS "GIT_DIRTY=true") +endif() + if (NOT CMAKE_CROSSCOMPILING AND TDUTILS_MIME_TYPE) add_dependencies(tdutils tdmime_auto) endif() diff --git a/tdutils/td/utils/GitInfo.cpp b/tdutils/td/utils/GitInfo.cpp index f51c6294c..219778f63 100644 --- a/tdutils/td/utils/GitInfo.cpp +++ b/tdutils/td/utils/GitInfo.cpp @@ -6,8 +6,6 @@ // #include "td/utils/GitInfo.h" -#include "auto/git_info.h" - #if !defined(GIT_COMMIT) #define GIT_COMMIT "unknown" #endif From bb5ebc09effefe78228d9320bf797ca535092856 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 9 Jul 2022 22:27:41 +0300 Subject: [PATCH 13/55] Disable git commit hash calculation, because the current approach must be completely replaced. --- CMakeLists.txt | 5 ----- tdutils/CMakeLists.txt | 3 +++ 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a833689a..77f32011d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -152,11 +152,6 @@ if (CLANG OR GCC) endif() endif() -if (NOT DEFINED TD_GIT_COMMIT) - include(GetGitInfo) - get_git_info() -endif() - add_subdirectory(tdtl) add_subdirectory(tdutils) diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index 2aed70b52..5b96fd555 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -343,6 +343,9 @@ set(TDUTILS_TEST_SOURCE add_library(tdutils STATIC ${TDUTILS_SOURCE}) if (DEFINED TD_GIT_COMMIT) + # this forces the whole tdutils to be recompiled after each commit with Makefile generator as explained at + # https://cmake.org/pipermail/cmake-developers/2012-October/005411.html, + # so this can't be improved for production-ready state set_property(SOURCE td/utils/GitInfo.cpp APPEND PROPERTY COMPILE_DEFINITIONS "GIT_COMMIT=\"${TD_GIT_COMMIT}\"") endif() if (DEFINED TD_GIT_DIRTY) From f083da4c1efb0d897a89f079bfcc25401d9da513 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 9 Jul 2022 23:40:22 +0300 Subject: [PATCH 14/55] Ignore CMAKE_INSTALL_PREFIX in pkgconfig, if CMAKE_INSTALL_*DIR is absolute. --- CMake/GeneratePkgConfig.cmake | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/CMake/GeneratePkgConfig.cmake b/CMake/GeneratePkgConfig.cmake index b9154de13..afbe06ac6 100644 --- a/CMake/GeneratePkgConfig.cmake +++ b/CMake/GeneratePkgConfig.cmake @@ -64,6 +64,18 @@ function(generate_pkgconfig TARGET DESCRIPTION) set(LIBRARIES "Libs.private:${LIBRARIES}\n") endif() + if (IS_ABSOLUTE "${CMAKE_INSTALL_INCLUDEDIR}") + set(PKGCONFIG_INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}") + else() + set(PKGCONFIG_INCLUDEDIR "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}") + endif() + + if (IS_ABSOLUTE "${CMAKE_INSTALL_LIBDIR}") + set(PKGCONFIG_LIBDIR "${CMAKE_INSTALL_LIBDIR}") + else() + set(PKGCONFIG_LIBDIR "\${prefix}/${CMAKE_INSTALL_LIBDIR}") + endif() + file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/pkgconfig") file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/pkgconfig/${TARGET}.pc" CONTENT "prefix=${PREFIX} @@ -72,8 +84,8 @@ Name: ${TARGET} Description: ${DESCRIPTION} Version: ${PROJECT_VERSION} -CFlags: -I\"\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}\" -Libs: -L\"\${prefix}/${CMAKE_INSTALL_LIBDIR}\" -l${TARGET} +CFlags: -I\"${PKGCONFIG_INCLUDEDIR}\" +Libs: -L\"${PKGCONFIG_LIBDIR}\" -l${TARGET} ${REQUIRES}${LIBRARIES}") install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pkgconfig/${TARGET}.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") From 22ce9df3522dca44bb7f21aab499a72d78ae7eaf Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 10 Jul 2022 17:54:43 +0300 Subject: [PATCH 15/55] Use scripts from https://github.com/rpavlik/cmake-modules to find git revision description. --- CMake/GetGitInfo.cmake | 29 ---- CMake/GetGitRevisionDescription.cmake | 167 +++++++++++++++++++++++ CMake/GetGitRevisionDescription.cmake.in | 43 ++++++ CMakeLists.txt | 8 ++ 4 files changed, 218 insertions(+), 29 deletions(-) delete mode 100644 CMake/GetGitInfo.cmake create mode 100644 CMake/GetGitRevisionDescription.cmake create mode 100644 CMake/GetGitRevisionDescription.cmake.in diff --git a/CMake/GetGitInfo.cmake b/CMake/GetGitInfo.cmake deleted file mode 100644 index bba1a0b5c..000000000 --- a/CMake/GetGitInfo.cmake +++ /dev/null @@ -1,29 +0,0 @@ -# sets GIT_COMMIT and GIT_DIRTY CMake variables -function(get_git_info) - find_program(GIT_EXECUTABLE git) - - unset(TD_GIT_COMMIT PARENT_SCOPE) - unset(TD_GIT_DIRTY PARENT_SCOPE) - - execute_process( - COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - OUTPUT_VARIABLE GIT_COMMIT - ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) - - if (NOT GIT_COMMIT) - return() - endif() - - set(TD_GIT_COMMIT "${GIT_COMMIT}" PARENT_SCOPE) - - execute_process( - COMMAND "${GIT_EXECUTABLE}" diff-index --quiet HEAD - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - RESULT_VARIABLE GIT_DIRTY_RESULT - ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) - - if (GIT_DIRTY_RESULT) - set(TD_GIT_DIRTY "true" PARENT_SCOPE) - endif() -endfunction() diff --git a/CMake/GetGitRevisionDescription.cmake b/CMake/GetGitRevisionDescription.cmake new file mode 100644 index 000000000..a9555fc3c --- /dev/null +++ b/CMake/GetGitRevisionDescription.cmake @@ -0,0 +1,167 @@ +# - Returns a version string from Git +# +# These functions force a re-configure on each git commit so that you can +# trust the values of the variables in your build system. +# +# get_git_head_revision( [ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR]) +# +# Returns the results of git describe --exact-match on the source tree, +# and adjusting the output so that it tests false if there was no exact +# matching tag. +# +# git_local_changes() +# +# Returns either "CLEAN" or "DIRTY" with respect to uncommitted changes. +# Uses the return code of "git diff-index --quiet HEAD --". +# Does not regard untracked files. +# +# Requires CMake 2.6 or newer (uses the 'function' command) +# +# Original Author: +# 2009-2020 Ryan Pavlik +# http://academic.cleardefinition.com +# +# Copyright 2009-2013, Iowa State University. +# Copyright 2013-2020, Ryan Pavlik +# Copyright 2013-2020, Contributors +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) + +if (__get_git_revision_description) + return() +endif() +set(__get_git_revision_description YES) + +# We must run the following at "include" time, not at function call time, +# to find the path to this module rather than the path to a calling list file +get_filename_component(_gitdescmoddir ${CMAKE_CURRENT_LIST_FILE} PATH) + +# Function _git_find_closest_git_dir finds the next closest .git directory +# that is part of any directory in the path defined by _start_dir. +# The result is returned in the parent scope variable whose name is passed +# as variable _git_dir_var. If no .git directory can be found, the +# function returns an empty string via _git_dir_var. +# +# Example: Given a path C:/bla/foo/bar and assuming C:/bla/.git exists and +# neither foo nor bar contain a file/directory .git. This wil return +# C:/bla/.git +# +function(_git_find_closest_git_dir _start_dir _git_dir_var) + set(cur_dir "${_start_dir}") + set(git_dir "${_start_dir}/.git") + while (NOT EXISTS "${git_dir}") + # .git dir not found, search parent directories + set(git_previous_parent "${cur_dir}") + get_filename_component(cur_dir "${cur_dir}" DIRECTORY) + if (cur_dir STREQUAL git_previous_parent) + # We have reached the root directory, we are not in git + set(${_git_dir_var} "" PARENT_SCOPE) + return() + endif() + set(git_dir "${cur_dir}/.git") + endwhile() + set(${_git_dir_var} "${git_dir}" PARENT_SCOPE) +endfunction() + +function(get_git_head_revision _refspecvar _hashvar) + _git_find_closest_git_dir("${CMAKE_CURRENT_SOURCE_DIR}" GIT_DIR) + + if ("${ARGN}" STREQUAL "ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR") + set(ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR TRUE) + else() + set(ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR FALSE) + endif() + if (NOT "${GIT_DIR}" STREQUAL "") + file(RELATIVE_PATH _relative_to_source_dir "${CMAKE_SOURCE_DIR}" "${GIT_DIR}") + if ("${_relative_to_source_dir}" MATCHES "[.][.]" AND NOT ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR) + # We've gone above the CMake root dir. + set(GIT_DIR "") + endif() + endif() + if ("${GIT_DIR}" STREQUAL "") + set(${_refspecvar} "GITDIR-NOTFOUND" PARENT_SCOPE) + set(${_hashvar} "GITDIR-NOTFOUND" PARENT_SCOPE) + return() + endif() + + # Check if the current source dir is a git submodule or a worktree. + # In both cases .git is a file instead of a directory. + # + if (NOT IS_DIRECTORY ${GIT_DIR}) + # The following git command will return a non empty string that + # points to the super project working tree if the current + # source dir is inside a git submodule. + # Otherwise the command will return an empty string. + # + execute_process( + COMMAND "${GIT_EXECUTABLE}" rev-parse --show-superproject-working-tree + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE out + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + if (NOT "${out}" STREQUAL "") + # If out is empty, GIT_DIR/CMAKE_CURRENT_SOURCE_DIR is in a submodule + file(READ ${GIT_DIR} submodule) + string(REGEX REPLACE "gitdir: (.*)$" "\\1" GIT_DIR_RELATIVE ${submodule}) + string(STRIP ${GIT_DIR_RELATIVE} GIT_DIR_RELATIVE) + get_filename_component(SUBMODULE_DIR ${GIT_DIR} PATH) + get_filename_component(GIT_DIR ${SUBMODULE_DIR}/${GIT_DIR_RELATIVE} ABSOLUTE) + set(HEAD_SOURCE_FILE "${GIT_DIR}/HEAD") + else() + # GIT_DIR/CMAKE_CURRENT_SOURCE_DIR is in a worktree + file(READ ${GIT_DIR} worktree_ref) + # The .git directory contains a path to the worktree information directory + # inside the parent git repo of the worktree. + string(REGEX REPLACE "gitdir: (.*)$" "\\1" git_worktree_dir ${worktree_ref}) + string(STRIP ${git_worktree_dir} git_worktree_dir) + _git_find_closest_git_dir("${git_worktree_dir}" GIT_DIR) + set(HEAD_SOURCE_FILE "${git_worktree_dir}/HEAD") + endif() + else() + set(HEAD_SOURCE_FILE "${GIT_DIR}/HEAD") + endif() + set(GIT_DATA "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/git-data") + if (NOT EXISTS "${GIT_DATA}") + file(MAKE_DIRECTORY "${GIT_DATA}") + endif() + + if (NOT EXISTS "${HEAD_SOURCE_FILE}") + return() + endif() + set(HEAD_FILE "${GIT_DATA}/HEAD") + configure_file("${HEAD_SOURCE_FILE}" "${HEAD_FILE}" COPYONLY) + + configure_file("${_gitdescmoddir}/GetGitRevisionDescription.cmake.in" "${GIT_DATA}/grabRef.cmake" @ONLY) + include("${GIT_DATA}/grabRef.cmake") + + set(${_refspecvar} "${HEAD_REF}" PARENT_SCOPE) + set(${_hashvar} "${HEAD_HASH}" PARENT_SCOPE) +endfunction() + +function(git_local_changes _var) + if (NOT GIT_FOUND) + find_package(Git QUIET) + endif() + get_git_head_revision(refspec hash) + if (NOT GIT_FOUND) + set(${_var} "GIT-NOTFOUND" PARENT_SCOPE) + return() + endif() + if (NOT hash) + set(${_var} "HEAD-HASH-NOTFOUND" PARENT_SCOPE) + return() + endif() + + execute_process( + COMMAND "${GIT_EXECUTABLE}" diff-index --quiet HEAD -- + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + RESULT_VARIABLE res + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + + if (res EQUAL 0) + set(${_var} "CLEAN" PARENT_SCOPE) + else() + set(${_var} "DIRTY" PARENT_SCOPE) + endif() +endfunction() diff --git a/CMake/GetGitRevisionDescription.cmake.in b/CMake/GetGitRevisionDescription.cmake.in new file mode 100644 index 000000000..e35156f25 --- /dev/null +++ b/CMake/GetGitRevisionDescription.cmake.in @@ -0,0 +1,43 @@ +# +# Internal file for GetGitRevisionDescription.cmake +# +# Requires CMake 2.6 or newer (uses the 'function' command) +# +# Original Author: +# 2009-2010 Ryan Pavlik +# http://academic.cleardefinition.com +# Iowa State University HCI Graduate Program/VRAC +# +# Copyright 2009-2012, Iowa State University +# Copyright 2011-2015, Contributors +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) +# SPDX-License-Identifier: BSL-1.0 + +set(HEAD_HASH) + +file(READ "@HEAD_FILE@" HEAD_CONTENTS LIMIT 1024) + +string(STRIP "${HEAD_CONTENTS}" HEAD_CONTENTS) +if(HEAD_CONTENTS MATCHES "ref") + # named branch + string(REPLACE "ref: " "" HEAD_REF "${HEAD_CONTENTS}") + if(EXISTS "@GIT_DIR@/${HEAD_REF}") + configure_file("@GIT_DIR@/${HEAD_REF}" "@GIT_DATA@/head-ref" COPYONLY) + else() + configure_file("@GIT_DIR@/packed-refs" "@GIT_DATA@/packed-refs" COPYONLY) + file(READ "@GIT_DATA@/packed-refs" PACKED_REFS) + if(${PACKED_REFS} MATCHES "([0-9a-z]*) ${HEAD_REF}") + set(HEAD_HASH "${CMAKE_MATCH_1}") + endif() + endif() +else() + # detached HEAD + configure_file("@GIT_DIR@/HEAD" "@GIT_DATA@/head-ref" COPYONLY) +endif() + +if(NOT HEAD_HASH) + file(READ "@GIT_DATA@/head-ref" HEAD_HASH LIMIT 1024) + string(STRIP "${HEAD_HASH}" HEAD_HASH) +endif() \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 77f32011d..17f2b6d5a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -152,6 +152,14 @@ if (CLANG OR GCC) endif() endif() +include(GetGitRevisionDescription) +get_git_head_revision(TD_GIT_REFSPEC TD_GIT_COMMIT) +git_local_changes(TD_GIT_DIRTY) +message(STATUS "Git state: ${TD_GIT_COMMIT} ${TD_GIT_DIRTY}") +if (NOT TD_GIT_DIRTY STREQUAL "DIRTY") + unset(TD_GIT_DIRTY) +endif() + add_subdirectory(tdtl) add_subdirectory(tdutils) From be18035b5d8218d61570286690309fdce2f42c83 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 10 Jul 2022 18:20:23 +0300 Subject: [PATCH 16/55] Remove non-working git_local_changes, which isn't updated on each rebuild, only when hash changes. --- CMake/GetGitRevisionDescription.cmake | 27 --------------------------- CMakeLists.txt | 6 +----- 2 files changed, 1 insertion(+), 32 deletions(-) diff --git a/CMake/GetGitRevisionDescription.cmake b/CMake/GetGitRevisionDescription.cmake index a9555fc3c..3cb24ecda 100644 --- a/CMake/GetGitRevisionDescription.cmake +++ b/CMake/GetGitRevisionDescription.cmake @@ -138,30 +138,3 @@ function(get_git_head_revision _refspecvar _hashvar) set(${_refspecvar} "${HEAD_REF}" PARENT_SCOPE) set(${_hashvar} "${HEAD_HASH}" PARENT_SCOPE) endfunction() - -function(git_local_changes _var) - if (NOT GIT_FOUND) - find_package(Git QUIET) - endif() - get_git_head_revision(refspec hash) - if (NOT GIT_FOUND) - set(${_var} "GIT-NOTFOUND" PARENT_SCOPE) - return() - endif() - if (NOT hash) - set(${_var} "HEAD-HASH-NOTFOUND" PARENT_SCOPE) - return() - endif() - - execute_process( - COMMAND "${GIT_EXECUTABLE}" diff-index --quiet HEAD -- - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - RESULT_VARIABLE res - ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) - - if (res EQUAL 0) - set(${_var} "CLEAN" PARENT_SCOPE) - else() - set(${_var} "DIRTY" PARENT_SCOPE) - endif() -endfunction() diff --git a/CMakeLists.txt b/CMakeLists.txt index 17f2b6d5a..e2803b6b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -154,11 +154,7 @@ endif() include(GetGitRevisionDescription) get_git_head_revision(TD_GIT_REFSPEC TD_GIT_COMMIT) -git_local_changes(TD_GIT_DIRTY) -message(STATUS "Git state: ${TD_GIT_COMMIT} ${TD_GIT_DIRTY}") -if (NOT TD_GIT_DIRTY STREQUAL "DIRTY") - unset(TD_GIT_DIRTY) -endif() +message(STATUS "Git state: ${TD_GIT_COMMIT}") add_subdirectory(tdtl) From 954444e692dadacf7c8f200243b50f5fc3bf5c7c Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 10 Jul 2022 19:07:15 +0300 Subject: [PATCH 17/55] Don't allow to look above CMAKE_CURRENT_SOURCE_DIR to support gitrepo. Fix a regexp. --- CMake/GetGitRevisionDescription.cmake | 37 ++++++++------------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/CMake/GetGitRevisionDescription.cmake b/CMake/GetGitRevisionDescription.cmake index 3cb24ecda..32be4fa85 100644 --- a/CMake/GetGitRevisionDescription.cmake +++ b/CMake/GetGitRevisionDescription.cmake @@ -3,17 +3,7 @@ # These functions force a re-configure on each git commit so that you can # trust the values of the variables in your build system. # -# get_git_head_revision( [ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR]) -# -# Returns the results of git describe --exact-match on the source tree, -# and adjusting the output so that it tests false if there was no exact -# matching tag. -# -# git_local_changes() -# -# Returns either "CLEAN" or "DIRTY" with respect to uncommitted changes. -# Uses the return code of "git diff-index --quiet HEAD --". -# Does not regard untracked files. +# get_git_head_revision( ) # # Requires CMake 2.6 or newer (uses the 'function' command) # @@ -68,17 +58,10 @@ endfunction() function(get_git_head_revision _refspecvar _hashvar) _git_find_closest_git_dir("${CMAKE_CURRENT_SOURCE_DIR}" GIT_DIR) - if ("${ARGN}" STREQUAL "ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR") - set(ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR TRUE) - else() - set(ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR FALSE) - endif() - if (NOT "${GIT_DIR}" STREQUAL "") - file(RELATIVE_PATH _relative_to_source_dir "${CMAKE_SOURCE_DIR}" "${GIT_DIR}") - if ("${_relative_to_source_dir}" MATCHES "[.][.]" AND NOT ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR) - # We've gone above the CMake root dir. - set(GIT_DIR "") - endif() + file(RELATIVE_PATH _relative_to_source_dir "${CMAKE_CURRENT_SOURCE_DIR}" "${GIT_DIR}") + if ("${_relative_to_source_dir}" MATCHES "^[.][.]") + # We've gone above the CMake root dir. + set(GIT_DIR "") endif() if ("${GIT_DIR}" STREQUAL "") set(${_refspecvar} "GITDIR-NOTFOUND" PARENT_SCOPE) @@ -101,7 +84,7 @@ function(get_git_head_revision _refspecvar _hashvar) OUTPUT_VARIABLE out ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) if (NOT "${out}" STREQUAL "") - # If out is empty, GIT_DIR/CMAKE_CURRENT_SOURCE_DIR is in a submodule + # If out is non-empty, GIT_DIR/CMAKE_CURRENT_SOURCE_DIR is in a submodule file(READ ${GIT_DIR} submodule) string(REGEX REPLACE "gitdir: (.*)$" "\\1" GIT_DIR_RELATIVE ${submodule}) string(STRIP ${GIT_DIR_RELATIVE} GIT_DIR_RELATIVE) @@ -121,14 +104,14 @@ function(get_git_head_revision _refspecvar _hashvar) else() set(HEAD_SOURCE_FILE "${GIT_DIR}/HEAD") endif() + if (NOT EXISTS "${HEAD_SOURCE_FILE}") + return() + endif() + set(GIT_DATA "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/git-data") if (NOT EXISTS "${GIT_DATA}") file(MAKE_DIRECTORY "${GIT_DATA}") endif() - - if (NOT EXISTS "${HEAD_SOURCE_FILE}") - return() - endif() set(HEAD_FILE "${GIT_DATA}/HEAD") configure_file("${HEAD_SOURCE_FILE}" "${HEAD_FILE}" COPYONLY) From 6eb03bdc8b3a93462748907b975721ce98b10394 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 11 Jul 2022 12:20:31 +0300 Subject: [PATCH 18/55] Generate GitCommitHash.cpp during building. --- CMakeLists.txt | 9 +++++++-- .../telegram/GitCommitHash.cpp.in | 18 +++--------------- .../GitInfo.h => td/telegram/GitCommitHash.h | 6 +----- tdutils/CMakeLists.txt | 12 ------------ 4 files changed, 11 insertions(+), 34 deletions(-) rename tdutils/td/utils/GitInfo.cpp => td/telegram/GitCommitHash.cpp.in (54%) rename tdutils/td/utils/GitInfo.h => td/telegram/GitCommitHash.h (78%) diff --git a/CMakeLists.txt b/CMakeLists.txt index e2803b6b4..a10015f59 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -153,8 +153,10 @@ if (CLANG OR GCC) endif() include(GetGitRevisionDescription) -get_git_head_revision(TD_GIT_REFSPEC TD_GIT_COMMIT) -message(STATUS "Git state: ${TD_GIT_COMMIT}") +get_git_head_revision(TD_GIT_REFSPEC TD_GIT_COMMIT_HASH) +message(STATUS "Git state: ${TD_GIT_COMMIT_HASH}") + +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/td/telegram/GitCommitHash.cpp.in" "${CMAKE_CURRENT_BINARY_DIR}/td/telegram/GitCommitHash.cpp" @ONLY) add_subdirectory(tdtl) @@ -566,6 +568,7 @@ set(TDLIB_SOURCE td/telegram/FullMessageId.h td/telegram/Game.h td/telegram/GameManager.h + td/telegram/GitCommitHash.h td/telegram/Global.h td/telegram/GroupCallId.h td/telegram/GroupCallManager.h @@ -736,6 +739,8 @@ set(TDLIB_SOURCE td/telegram/VoiceNotesManager.hpp ${TL_TD_SCHEME_SOURCE} + + ${CMAKE_CURRENT_BINARY_DIR}/td/telegram/GitCommitHash.cpp ) set(MEMPROF_SOURCE diff --git a/tdutils/td/utils/GitInfo.cpp b/td/telegram/GitCommitHash.cpp.in similarity index 54% rename from tdutils/td/utils/GitInfo.cpp rename to td/telegram/GitCommitHash.cpp.in index 219778f63..89706f132 100644 --- a/tdutils/td/utils/GitInfo.cpp +++ b/td/telegram/GitCommitHash.cpp.in @@ -4,24 +4,12 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // -#include "td/utils/GitInfo.h" - -#if !defined(GIT_COMMIT) -#define GIT_COMMIT "unknown" -#endif - -#if !defined(GIT_DIRTY) -#define GIT_DIRTY false -#endif +#include "td/telegram/GitCommitHash.h" namespace td { -const char *GitInfo::commit() { - return GIT_COMMIT; -} - -bool GitInfo::is_dirty() { - return GIT_DIRTY; +const char *get_git_commit_hash() { + return "@TD_GIT_COMMIT_HASH@"; } } // namespace td diff --git a/tdutils/td/utils/GitInfo.h b/td/telegram/GitCommitHash.h similarity index 78% rename from tdutils/td/utils/GitInfo.h rename to td/telegram/GitCommitHash.h index 194f17698..f4963d2e8 100644 --- a/tdutils/td/utils/GitInfo.h +++ b/td/telegram/GitCommitHash.h @@ -8,10 +8,6 @@ namespace td { -class GitInfo { - public: - static const char *commit(); - static bool is_dirty(); -}; +const char *get_git_commit_hash(); } // namespace td diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index 5b96fd555..6603c6846 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -100,7 +100,6 @@ set(TDUTILS_SOURCE td/utils/find_boundary.cpp td/utils/FlatHashTable.cpp td/utils/FloodControlGlobal.cpp - td/utils/GitInfo.cpp td/utils/Gzip.cpp td/utils/GzipByteFlow.cpp td/utils/Hints.cpp @@ -218,7 +217,6 @@ set(TDUTILS_SOURCE td/utils/FloodControlGlobal.h td/utils/FloodControlStrict.h td/utils/format.h - td/utils/GitInfo.h td/utils/Gzip.h td/utils/GzipByteFlow.h td/utils/Hash.h @@ -342,16 +340,6 @@ set(TDUTILS_TEST_SOURCE #LIBRARIES add_library(tdutils STATIC ${TDUTILS_SOURCE}) -if (DEFINED TD_GIT_COMMIT) - # this forces the whole tdutils to be recompiled after each commit with Makefile generator as explained at - # https://cmake.org/pipermail/cmake-developers/2012-October/005411.html, - # so this can't be improved for production-ready state - set_property(SOURCE td/utils/GitInfo.cpp APPEND PROPERTY COMPILE_DEFINITIONS "GIT_COMMIT=\"${TD_GIT_COMMIT}\"") -endif() -if (DEFINED TD_GIT_DIRTY) - set_property(SOURCE td/utils/GitInfo.cpp APPEND PROPERTY COMPILE_DEFINITIONS "GIT_DIRTY=true") -endif() - if (NOT CMAKE_CROSSCOMPILING AND TDUTILS_MIME_TYPE) add_dependencies(tdutils tdmime_auto) endif() From 4300c64a3061ee27c80f404fca7d067a09abffc9 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 11 Jul 2022 12:47:23 +0300 Subject: [PATCH 19/55] Add Td::get_version_option_value_object(). --- td/telegram/OptionManager.cpp | 5 ++--- td/telegram/Td.cpp | 10 ++++++---- td/telegram/Td.h | 6 ++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/td/telegram/OptionManager.cpp b/td/telegram/OptionManager.cpp index 0f92b6180..ba8248627 100644 --- a/td/telegram/OptionManager.cpp +++ b/td/telegram/OptionManager.cpp @@ -426,7 +426,7 @@ void OptionManager::get_option(const string &name, Promise(Td::TDLIB_VERSION)); + return promise.set_value(Td::get_version_option_value_object()); } break; } @@ -760,8 +760,7 @@ td_api::object_ptr OptionManager::get_option_value_object(S } void OptionManager::get_current_state(vector> &updates) const { - updates.push_back(td_api::make_object( - "version", td_api::make_object(Td::TDLIB_VERSION))); + updates.push_back(td_api::make_object("version", Td::get_version_option_value_object())); updates.push_back(td_api::make_object( "online", td_api::make_object(td_->is_online()))); diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index be99c16a7..528be8bb5 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -3057,8 +3057,7 @@ void Td::run_request(uint64 id, tl_object_ptr function) { return send_result(id, get_fake_authorization_state_object()); case td_api::getCurrentState::ID: { vector> updates; - updates.push_back(td_api::make_object( - "version", td_api::make_object(TDLIB_VERSION))); + updates.push_back(td_api::make_object("version", get_version_option_value_object())); updates.push_back(td_api::make_object(get_fake_authorization_state_object())); // send response synchronously to prevent "Request aborted" return send_result(id, td_api::make_object(std::move(updates))); @@ -3255,6 +3254,10 @@ void Td::on_connection_state_changed(ConnectionState new_state) { send_closure(actor_id(this), &Td::send_update, get_update_connection_state_object(connection_state_)); } +td_api::object_ptr Td::get_version_option_value_object() { + return td_api::make_object(TDLIB_VERSION); +} + void Td::start_up() { always_wait_for_mailbox(); @@ -3275,8 +3278,7 @@ void Td::start_up() { alarm_timeout_.set_callback_data(static_cast(this)); CHECK(state_ == State::WaitParameters); - send_update(td_api::make_object("version", - td_api::make_object(TDLIB_VERSION))); + send_update(td_api::make_object("version", get_version_option_value_object())); send_update(td_api::make_object( td_api::make_object())); } diff --git a/td/telegram/Td.h b/td/telegram/Td.h index d1fa16b14..db6ad82ad 100644 --- a/td/telegram/Td.h +++ b/td/telegram/Td.h @@ -102,8 +102,6 @@ class Td final : public Actor { Td &operator=(Td &&) = delete; ~Td() final; - static constexpr const char *TDLIB_VERSION = "1.8.4"; - struct Options { std::shared_ptr net_query_stats; }; @@ -248,6 +246,8 @@ class Td final : public Actor { static td_api::object_ptr static_request(td_api::object_ptr function); + static td_api::object_ptr get_version_option_value_object(); + private: static constexpr int64 ONLINE_ALARM_ID = 0; static constexpr int64 PING_SERVER_ALARM_ID = -1; @@ -255,6 +255,8 @@ class Td final : public Actor { static constexpr int64 TERMS_OF_SERVICE_ALARM_ID = -2; static constexpr int64 PROMO_DATA_ALARM_ID = -3; + static constexpr const char *TDLIB_VERSION = "1.8.4"; + void on_connection_state_changed(ConnectionState new_state); void run_request(uint64 id, tl_object_ptr function); From 61f7da7215c137ff4f8854255bd8991ef7017dc3 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 11 Jul 2022 12:57:42 +0300 Subject: [PATCH 20/55] Add option "commit_hash". --- td/telegram/OptionManager.cpp | 4 ++++ td/telegram/Td.cpp | 7 +++++++ td/telegram/Td.h | 2 ++ 3 files changed, 13 insertions(+) diff --git a/td/telegram/OptionManager.cpp b/td/telegram/OptionManager.cpp index ba8248627..757ba7577 100644 --- a/td/telegram/OptionManager.cpp +++ b/td/telegram/OptionManager.cpp @@ -398,6 +398,9 @@ void OptionManager::get_option(const string &name, Promiseconfig_manager_, &ConfigManager::get_content_settings, wrap_promise()); } + if (name == "commit_hash") { + return promise.set_value(Td::get_commit_hash_option_value_object()); + } break; case 'd': if (!is_bot && name == "disable_contact_registered_notifications") { @@ -761,6 +764,7 @@ td_api::object_ptr OptionManager::get_option_value_object(S void OptionManager::get_current_state(vector> &updates) const { updates.push_back(td_api::make_object("version", Td::get_version_option_value_object())); + updates.push_back(td_api::make_object("commit_hash", Td::get_commit_hash_option_value_object())); updates.push_back(td_api::make_object( "online", td_api::make_object(td_->is_online()))); diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index 528be8bb5..104ffc0dd 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -52,6 +52,7 @@ #include "td/telegram/FolderId.h" #include "td/telegram/FullMessageId.h" #include "td/telegram/GameManager.h" +#include "td/telegram/GitCommitHash.h" #include "td/telegram/Global.h" #include "td/telegram/GroupCallId.h" #include "td/telegram/GroupCallManager.h" @@ -3058,6 +3059,7 @@ void Td::run_request(uint64 id, tl_object_ptr function) { case td_api::getCurrentState::ID: { vector> updates; updates.push_back(td_api::make_object("version", get_version_option_value_object())); + updates.push_back(td_api::make_object("commit_hash", get_commit_hash_option_value_object())); updates.push_back(td_api::make_object(get_fake_authorization_state_object())); // send response synchronously to prevent "Request aborted" return send_result(id, td_api::make_object(std::move(updates))); @@ -3258,6 +3260,10 @@ td_api::object_ptr Td::get_version_option_value_object() { return td_api::make_object(TDLIB_VERSION); } +td_api::object_ptr Td::get_commit_hash_option_value_object() { + return td_api::make_object(get_git_commit_hash()); +} + void Td::start_up() { always_wait_for_mailbox(); @@ -3279,6 +3285,7 @@ void Td::start_up() { CHECK(state_ == State::WaitParameters); send_update(td_api::make_object("version", get_version_option_value_object())); + send_update(td_api::make_object("commit_hash", get_commit_hash_option_value_object())); send_update(td_api::make_object( td_api::make_object())); } diff --git a/td/telegram/Td.h b/td/telegram/Td.h index db6ad82ad..bee0d6bdd 100644 --- a/td/telegram/Td.h +++ b/td/telegram/Td.h @@ -248,6 +248,8 @@ class Td final : public Actor { static td_api::object_ptr get_version_option_value_object(); + static td_api::object_ptr get_commit_hash_option_value_object(); + private: static constexpr int64 ONLINE_ALARM_ID = 0; static constexpr int64 PING_SERVER_ALARM_ID = -1; From 17a548292b2967bd694e6665ed012c1825d1e892 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 11 Jul 2022 13:33:56 +0300 Subject: [PATCH 21/55] Allow to get synchronously options "version" and "commit_hash". --- td/generate/scheme/td_api.tl | 2 +- td/telegram/OptionManager.cpp | 25 +++++++++++++++++++------ td/telegram/OptionManager.h | 4 ++++ td/telegram/Td.cpp | 18 ++++++++++++++---- td/telegram/Td.h | 3 ++- td/telegram/cli.cpp | 2 ++ 6 files changed, 42 insertions(+), 12 deletions(-) diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index aa0d1fd57..0096fd613 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -6196,7 +6196,7 @@ setUserPrivacySettingRules setting:UserPrivacySetting rules:userPrivacySettingRu getUserPrivacySettingRules setting:UserPrivacySetting = UserPrivacySettingRules; -//@description Returns the value of an option by its name. (Check the list of available options on https://core.telegram.org/tdlib/options.) Can be called before authorization +//@description Returns the value of an option by its name. (Check the list of available options on https://core.telegram.org/tdlib/options.) Can be called before authorization. Can be called synchronously for options "version" and "commit_hash" //@name The name of the option getOption name:string = OptionValue; diff --git a/td/telegram/OptionManager.cpp b/td/telegram/OptionManager.cpp index 757ba7577..4452f25b9 100644 --- a/td/telegram/OptionManager.cpp +++ b/td/telegram/OptionManager.cpp @@ -210,6 +210,10 @@ bool OptionManager::is_internal_option(Slice name) { } } +bool OptionManager::is_synchronous_option(Slice name) { + return name == "version" || name == "commit_hash"; +} + void OptionManager::on_option_updated(const string &name) { if (G()->close_flag()) { return; @@ -398,9 +402,6 @@ void OptionManager::get_option(const string &name, Promiseconfig_manager_, &ConfigManager::get_content_settings, wrap_promise()); } - if (name == "commit_hash") { - return promise.set_value(Td::get_commit_hash_option_value_object()); - } break; case 'd': if (!is_bot && name == "disable_contact_registered_notifications") { @@ -427,13 +428,24 @@ void OptionManager::get_option(const string &name, Promise OptionManager::get_option_synchronously(const string &name) { + switch (name[0]) { + case 'c': + if (name == "commit_hash") { + return Td::get_commit_hash_option_value_object(); + } + break; case 'v': if (name == "version") { - return promise.set_value(Td::get_version_option_value_object()); + return Td::get_version_option_value_object(); } break; } - wrap_promise().set_value(Unit()); + UNREACHABLE(); } void OptionManager::set_option(const string &name, td_api::object_ptr &&value, @@ -764,7 +776,8 @@ td_api::object_ptr OptionManager::get_option_value_object(S void OptionManager::get_current_state(vector> &updates) const { updates.push_back(td_api::make_object("version", Td::get_version_option_value_object())); - updates.push_back(td_api::make_object("commit_hash", Td::get_commit_hash_option_value_object())); + updates.push_back( + td_api::make_object("commit_hash", Td::get_commit_hash_option_value_object())); updates.push_back(td_api::make_object( "online", td_api::make_object(td_->is_online()))); diff --git a/td/telegram/OptionManager.h b/td/telegram/OptionManager.h index 2bfd1d341..57a439117 100644 --- a/td/telegram/OptionManager.h +++ b/td/telegram/OptionManager.h @@ -38,6 +38,10 @@ class OptionManager final : public Actor { static void clear_options(); + static bool is_synchronous_option(Slice name); + + static td_api::object_ptr get_option_synchronously(const string &name); + void get_current_state(vector> &updates) const; private: diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index 104ffc0dd..8d17923fc 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -2894,8 +2894,8 @@ bool Td::is_authentication_request(int32 id) { } } -bool Td::is_synchronous_request(int32 id) { - switch (id) { +bool Td::is_synchronous_request(const td_api::Function *function) { + switch (function->get_id()) { case td_api::getTextEntities::ID: case td_api::parseTextEntities::ID: case td_api::parseMarkdown::ID: @@ -2920,6 +2920,8 @@ bool Td::is_synchronous_request(int32 id) { case td_api::addLogMessage::ID: case td_api::testReturnError::ID: return true; + case td_api::getOption::ID: + return OptionManager::is_synchronous_option(static_cast(function)->name_); default: return false; } @@ -3032,7 +3034,7 @@ void Td::request(uint64 id, tl_object_ptr function) { } VLOG(td_requests) << "Receive request " << id << ": " << to_string(function); - if (is_synchronous_request(function->get_id())) { + if (is_synchronous_request(function.get())) { // send response synchronously return send_result(id, static_request(std::move(function))); } @@ -3059,7 +3061,8 @@ void Td::run_request(uint64 id, tl_object_ptr function) { case td_api::getCurrentState::ID: { vector> updates; updates.push_back(td_api::make_object("version", get_version_option_value_object())); - updates.push_back(td_api::make_object("commit_hash", get_commit_hash_option_value_object())); + updates.push_back( + td_api::make_object("commit_hash", get_commit_hash_option_value_object())); updates.push_back(td_api::make_object(get_fake_authorization_state_object())); // send response synchronously to prevent "Request aborted" return send_result(id, td_api::make_object(std::move(updates))); @@ -8208,6 +8211,13 @@ td_api::object_ptr Td::do_static_request(td_api::parseMarkdown & return get_formatted_text_object(parsed_text, false, std::numeric_limits::max()); } +td_api::object_ptr Td::do_static_request(const td_api::getOption &request) { + if (!is_synchronous_request(&request)) { + return make_error(400, "The option can't be get synchronously"); + } + return OptionManager::get_option_synchronously(request.name_); +} + td_api::object_ptr Td::do_static_request(td_api::getMarkdownText &request) { if (request.text_ == nullptr) { return make_error(400, "Text must be non-empty"); diff --git a/td/telegram/Td.h b/td/telegram/Td.h index bee0d6bdd..09473aa74 100644 --- a/td/telegram/Td.h +++ b/td/telegram/Td.h @@ -384,7 +384,7 @@ class Td final : public Actor { static bool is_authentication_request(int32 id); - static bool is_synchronous_request(int32 id); + static bool is_synchronous_request(const td_api::Function *function); static bool is_preinitialization_request(int32 id); @@ -1412,6 +1412,7 @@ class Td final : public Actor { static td_api::object_ptr do_static_request(const T &request) { return td_api::make_object(400, "The method can't be executed synchronously"); } + static td_api::object_ptr do_static_request(const td_api::getOption &request); static td_api::object_ptr do_static_request(const td_api::getTextEntities &request); static td_api::object_ptr do_static_request(td_api::parseTextEntities &request); static td_api::object_ptr do_static_request(td_api::parseMarkdown &request); diff --git a/td/telegram/cli.cpp b/td/telegram/cli.cpp index 8a9bf6323..1719b55c5 100644 --- a/td/telegram/cli.cpp +++ b/td/telegram/cli.cpp @@ -2400,6 +2400,8 @@ class CliClient final : public Actor { td_api::make_object(op == "on"))); } else if (op == "go") { send_request(td_api::make_object(args)); + } else if (op == "gos") { + execute(td_api::make_object(args)); } else if (op == "sob") { string name; bool value; From ee7011538a86cbab11d986b35e62891d3c3e1dee Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 11 Jul 2022 14:01:56 +0300 Subject: [PATCH 22/55] Use OptionManager::get_option_synchronously internally. --- td/telegram/OptionManager.cpp | 10 +++++----- td/telegram/Td.cpp | 25 +++++++++---------------- td/telegram/Td.h | 6 ------ 3 files changed, 14 insertions(+), 27 deletions(-) diff --git a/td/telegram/OptionManager.cpp b/td/telegram/OptionManager.cpp index 4452f25b9..2ad71e848 100644 --- a/td/telegram/OptionManager.cpp +++ b/td/telegram/OptionManager.cpp @@ -13,6 +13,7 @@ #include "td/telegram/ConfigShared.h" #include "td/telegram/ContactsManager.h" #include "td/telegram/DialogId.h" +#include "td/telegram/GitCommitHash.h" #include "td/telegram/Global.h" #include "td/telegram/JsonValue.h" #include "td/telegram/LanguagePackManager.h" @@ -436,12 +437,12 @@ td_api::object_ptr OptionManager::get_option_synchronously( switch (name[0]) { case 'c': if (name == "commit_hash") { - return Td::get_commit_hash_option_value_object(); + return td_api::make_object(get_git_commit_hash()); } break; case 'v': if (name == "version") { - return Td::get_version_option_value_object(); + return td_api::make_object("1.8.4"); } break; } @@ -775,9 +776,8 @@ td_api::object_ptr OptionManager::get_option_value_object(S } void OptionManager::get_current_state(vector> &updates) const { - updates.push_back(td_api::make_object("version", Td::get_version_option_value_object())); - updates.push_back( - td_api::make_object("commit_hash", Td::get_commit_hash_option_value_object())); + updates.push_back(td_api::make_object("version", get_option_synchronously("version"))); + updates.push_back(td_api::make_object("commit_hash", get_option_synchronously("commit_hash"))); updates.push_back(td_api::make_object( "online", td_api::make_object(td_->is_online()))); diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index 8d17923fc..ab90a955c 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -52,7 +52,6 @@ #include "td/telegram/FolderId.h" #include "td/telegram/FullMessageId.h" #include "td/telegram/GameManager.h" -#include "td/telegram/GitCommitHash.h" #include "td/telegram/Global.h" #include "td/telegram/GroupCallId.h" #include "td/telegram/GroupCallManager.h" @@ -3060,9 +3059,10 @@ void Td::run_request(uint64 id, tl_object_ptr function) { return send_result(id, get_fake_authorization_state_object()); case td_api::getCurrentState::ID: { vector> updates; - updates.push_back(td_api::make_object("version", get_version_option_value_object())); updates.push_back( - td_api::make_object("commit_hash", get_commit_hash_option_value_object())); + td_api::make_object("version", OptionManager::get_option_synchronously("version"))); + updates.push_back(td_api::make_object( + "commit_hash", OptionManager::get_option_synchronously("commit_hash"))); updates.push_back(td_api::make_object(get_fake_authorization_state_object())); // send response synchronously to prevent "Request aborted" return send_result(id, td_api::make_object(std::move(updates))); @@ -3259,14 +3259,6 @@ void Td::on_connection_state_changed(ConnectionState new_state) { send_closure(actor_id(this), &Td::send_update, get_update_connection_state_object(connection_state_)); } -td_api::object_ptr Td::get_version_option_value_object() { - return td_api::make_object(TDLIB_VERSION); -} - -td_api::object_ptr Td::get_commit_hash_option_value_object() { - return td_api::make_object(get_git_commit_hash()); -} - void Td::start_up() { always_wait_for_mailbox(); @@ -3287,8 +3279,9 @@ void Td::start_up() { alarm_timeout_.set_callback_data(static_cast(this)); CHECK(state_ == State::WaitParameters); - send_update(td_api::make_object("version", get_version_option_value_object())); - send_update(td_api::make_object("commit_hash", get_commit_hash_option_value_object())); + send_update(td_api::make_object("version", OptionManager::get_option_synchronously("version"))); + send_update( + td_api::make_object("commit_hash", OptionManager::get_option_synchronously("commit_hash"))); send_update(td_api::make_object( td_api::make_object())); } @@ -4332,7 +4325,9 @@ Status Td::set_parameters(td_api::object_ptr parameters } if (options_.api_id != 21724) { options_.application_version += ", TDLib "; - options_.application_version += TDLIB_VERSION; + auto version = OptionManager::get_option_synchronously("version"); + CHECK(version->get_id() == td_api::optionValueString::ID); + options_.application_version += static_cast(version.get())->value_; } options_.language_pack = string(); options_.language_code = string(); @@ -8442,6 +8437,4 @@ void Td::on_request(uint64 id, td_api::testCallVectorStringObject &request) { #undef CREATE_REQUEST_PROMISE #undef CREATE_OK_REQUEST_PROMISE -constexpr const char *Td::TDLIB_VERSION; - } // namespace td diff --git a/td/telegram/Td.h b/td/telegram/Td.h index 09473aa74..c25b3ff5e 100644 --- a/td/telegram/Td.h +++ b/td/telegram/Td.h @@ -246,10 +246,6 @@ class Td final : public Actor { static td_api::object_ptr static_request(td_api::object_ptr function); - static td_api::object_ptr get_version_option_value_object(); - - static td_api::object_ptr get_commit_hash_option_value_object(); - private: static constexpr int64 ONLINE_ALARM_ID = 0; static constexpr int64 PING_SERVER_ALARM_ID = -1; @@ -257,8 +253,6 @@ class Td final : public Actor { static constexpr int64 TERMS_OF_SERVICE_ALARM_ID = -2; static constexpr int64 PROMO_DATA_ALARM_ID = -3; - static constexpr const char *TDLIB_VERSION = "1.8.4"; - void on_connection_state_changed(ConnectionState new_state); void run_request(uint64 id, tl_object_ptr function); From e44418f2b3a1533b71a20b80e201802efe55a6d9 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 11 Jul 2022 15:36:29 +0300 Subject: [PATCH 23/55] Add and use everywhere OptionsManager::get_synchronous_options(). --- td/telegram/OptionManager.cpp | 21 +++++++++++++++++---- td/telegram/OptionManager.h | 6 +++++- td/telegram/Td.cpp | 27 +++++++++++++-------------- td/telegram/Td.h | 2 ++ 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/td/telegram/OptionManager.cpp b/td/telegram/OptionManager.cpp index 2ad71e848..d892da653 100644 --- a/td/telegram/OptionManager.cpp +++ b/td/telegram/OptionManager.cpp @@ -29,6 +29,7 @@ #include "td/telegram/telegram_api.h" #include "td/telegram/TopDialogManager.h" +#include "td/utils/algorithm.h" #include "td/utils/buffer.h" #include "td/utils/logging.h" #include "td/utils/misc.h" @@ -211,8 +212,13 @@ bool OptionManager::is_internal_option(Slice name) { } } +const vector &OptionManager::get_synchronous_options() { + static const vector options{"version", "commit_hash"}; + return options; +} + bool OptionManager::is_synchronous_option(Slice name) { - return name == "version" || name == "commit_hash"; + return td::contains(get_synchronous_options(), name); } void OptionManager::on_option_updated(const string &name) { @@ -433,7 +439,8 @@ void OptionManager::get_option(const string &name, Promise OptionManager::get_option_synchronously(const string &name) { +td_api::object_ptr OptionManager::get_option_synchronously(Slice name) { + CHECK(!name.empty()); switch (name[0]) { case 'c': if (name == "commit_hash") { @@ -775,9 +782,15 @@ td_api::object_ptr OptionManager::get_option_value_object(S return td_api::make_object(value.str()); } +void OptionManager::get_common_state(vector> &updates) { + for (auto option_name : get_synchronous_options()) { + updates.push_back( + td_api::make_object(option_name.str(), get_option_synchronously(option_name))); + } +} + void OptionManager::get_current_state(vector> &updates) const { - updates.push_back(td_api::make_object("version", get_option_synchronously("version"))); - updates.push_back(td_api::make_object("commit_hash", get_option_synchronously("commit_hash"))); + get_common_state(updates); updates.push_back(td_api::make_object( "online", td_api::make_object(td_->is_online()))); diff --git a/td/telegram/OptionManager.h b/td/telegram/OptionManager.h index 57a439117..5b21739d0 100644 --- a/td/telegram/OptionManager.h +++ b/td/telegram/OptionManager.h @@ -40,7 +40,9 @@ class OptionManager final : public Actor { static bool is_synchronous_option(Slice name); - static td_api::object_ptr get_option_synchronously(const string &name); + static td_api::object_ptr get_option_synchronously(Slice name); + + static void get_common_state(vector> &updates); void get_current_state(vector> &updates) const; @@ -49,6 +51,8 @@ class OptionManager final : public Actor { static bool is_internal_option(Slice name); + static const vector &get_synchronous_options(); + static td_api::object_ptr get_unix_time_option_value_object(); static td_api::object_ptr get_option_value_object(Slice value); diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index ab90a955c..c6216e1bf 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -3010,6 +3010,14 @@ td_api::object_ptr Td::get_fake_authorization_state_ } } +vector> Td::get_fake_current_state() const { + CHECK(state_ != State::Run); + vector> updates; + OptionManager::get_common_state(updates); + updates.push_back(td_api::make_object(get_fake_authorization_state_object())); + return updates; +} + DbKey Td::as_db_key(string key) { // Database will still be effectively not encrypted, but // 1. SQLite database will be protected from corruption, because that's how sqlcipher works @@ -3057,16 +3065,9 @@ void Td::run_request(uint64 id, tl_object_ptr function) { case td_api::getAuthorizationState::ID: // send response synchronously to prevent "Request aborted" return send_result(id, get_fake_authorization_state_object()); - case td_api::getCurrentState::ID: { - vector> updates; - updates.push_back( - td_api::make_object("version", OptionManager::get_option_synchronously("version"))); - updates.push_back(td_api::make_object( - "commit_hash", OptionManager::get_option_synchronously("commit_hash"))); - updates.push_back(td_api::make_object(get_fake_authorization_state_object())); + case td_api::getCurrentState::ID: // send response synchronously to prevent "Request aborted" - return send_result(id, td_api::make_object(std::move(updates))); - } + return send_result(id, td_api::make_object(get_fake_current_state())); case td_api::close::ID: // need to send response before actual closing send_closure(actor_id(this), &Td::send_result, id, td_api::make_object()); @@ -3279,11 +3280,9 @@ void Td::start_up() { alarm_timeout_.set_callback_data(static_cast(this)); CHECK(state_ == State::WaitParameters); - send_update(td_api::make_object("version", OptionManager::get_option_synchronously("version"))); - send_update( - td_api::make_object("commit_hash", OptionManager::get_option_synchronously("commit_hash"))); - send_update(td_api::make_object( - td_api::make_object())); + for (auto &update : get_fake_current_state()) { + send_update(std::move(update)); + } } void Td::tear_down() { diff --git a/td/telegram/Td.h b/td/telegram/Td.h index c25b3ff5e..e67af1ae9 100644 --- a/td/telegram/Td.h +++ b/td/telegram/Td.h @@ -328,6 +328,8 @@ class Td final : public Actor { td_api::object_ptr get_fake_authorization_state_object() const; + vector> get_fake_current_state() const; + static void on_alarm_timeout_callback(void *td_ptr, int64 alarm_id); void on_alarm_timeout(int64 alarm_id); From ef9df9f99b1a912573b03c18f33af085000c497a Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 12 Jul 2022 20:20:13 +0300 Subject: [PATCH 24/55] Add Ubuntu 22 to the list of Linux distros. --- build.html | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/build.html b/build.html index 5ca4a22ac..88739752d 100644 --- a/build.html +++ b/build.html @@ -246,6 +246,7 @@ +

@@ -638,7 +639,7 @@ function onOptionsChanged() { var use_vcpkg = os_windows; var use_lto = false; - if (!use_msvc && language !== 'Java' && language !== 'Kotlin' && (os_mac || (os_linux && (linux_distro === 'Ubuntu 18' || linux_distro === 'Ubuntu 20' || linux_distro === 'Other')))) { + if (!use_msvc && language !== 'Java' && language !== 'Kotlin' && (os_mac || (os_linux && (linux_distro === 'Ubuntu 18' || linux_distro === 'Ubuntu 20' || linux_distro === 'Ubuntu 22' || linux_distro === 'Other')))) { document.getElementById('buildLtoDiv').style.display = 'block'; use_lto = document.getElementById('buildLtoCheckbox').checked; } else { @@ -797,6 +798,8 @@ function onOptionsChanged() { return '-6.0'; case 'Ubuntu 20': return '-10'; + case 'Ubuntu 22': + return '-14'; default: return ''; // use default version } @@ -849,6 +852,7 @@ function onOptionsChanged() { case 'Ubuntu 16': case 'Ubuntu 18': case 'Ubuntu 20': + case 'Ubuntu 22': if (linux_distro.includes('Debian') && !use_root) { commands.push('su -'); } @@ -873,7 +877,7 @@ function onOptionsChanged() { } if (use_clang) { packages += ' clang' + getClangVersionSuffix() + ' libc++-dev'; - if (linux_distro === 'Debian 10+' || linux_distro === 'Ubuntu 18' || linux_distro === 'Ubuntu 20') { + if (linux_distro === 'Debian 10+' || linux_distro === 'Ubuntu 18' || linux_distro === 'Ubuntu 20' || linux_distro === 'Ubuntu 22') { packages += ' libc++abi-dev'; } } else { From c01dd0d4616dc0f4d45204ed7f8c64e27dbcea29 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 12 Jul 2022 22:50:28 +0300 Subject: [PATCH 25/55] Increase MessageUnsupported version. --- td/telegram/MessageContent.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/td/telegram/MessageContent.cpp b/td/telegram/MessageContent.cpp index 9ce184579..9f4117c73 100644 --- a/td/telegram/MessageContent.cpp +++ b/td/telegram/MessageContent.cpp @@ -453,7 +453,7 @@ class MessageChatSetTtl final : public MessageContent { class MessageUnsupported final : public MessageContent { public: - static constexpr int32 CURRENT_VERSION = 11; + static constexpr int32 CURRENT_VERSION = 12; int32 version = CURRENT_VERSION; MessageUnsupported() = default; From 98bfbc90001d01ae5748f536e0f27ab34bbef6fd Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 12 Jul 2022 23:04:01 +0300 Subject: [PATCH 26/55] Use random_id == 1 in ChangeImportedContacts. --- td/telegram/ContactsManager.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/td/telegram/ContactsManager.cpp b/td/telegram/ContactsManager.cpp index 9b217baee..8645a7f17 100644 --- a/td/telegram/ContactsManager.cpp +++ b/td/telegram/ContactsManager.cpp @@ -5413,7 +5413,7 @@ std::pair, vector> ContactsManager::import_contacts(const do { random_id = Random::secure_int64(); - } while (random_id == 0 || imported_contacts_.count(random_id) > 0); + } while (random_id == 0 || random_id == 1 || imported_contacts_.count(random_id) > 0); imported_contacts_[random_id]; // reserve place for result do_import_contacts(contacts, random_id, std::move(promise)); @@ -5758,7 +5758,7 @@ void ContactsManager::on_clear_imported_contacts(vector &&contacts, vec imported_contacts_unique_id_ = std::move(contacts_unique_id); imported_contacts_pos_ = std::move(to_add.first); - do_import_contacts(std::move(to_add.second), 0, std::move(promise)); + do_import_contacts(std::move(to_add.second), 1, std::move(promise)); } void ContactsManager::clear_imported_contacts(Promise &&promise) { @@ -8249,7 +8249,7 @@ void ContactsManager::on_import_contacts_finished(int64 random_id, vector unimported_contact_invites) { LOG(INFO) << "Contacts import with random_id " << random_id << " has finished: " << format::as_array(imported_contact_user_ids); - if (random_id == 0) { + if (random_id == 1) { // import from change_imported_contacts all_imported_contacts_ = std::move(next_all_imported_contacts_); next_all_imported_contacts_.clear(); From ac4e76edf14510e4310b134a6f759497c5ee19ee Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 13 Jul 2022 12:01:41 +0300 Subject: [PATCH 27/55] Fix SslStream support of OpenSSL 3.0. --- tdnet/td/net/SslStream.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tdnet/td/net/SslStream.cpp b/tdnet/td/net/SslStream.cpp index 8cdf88b9c..2c88632f3 100644 --- a/tdnet/td/net/SslStream.cpp +++ b/tdnet/td/net/SslStream.cpp @@ -98,6 +98,11 @@ long strm_ctrl(BIO *b, int cmd, long num, void *ptr) { case BIO_CTRL_PUSH: case BIO_CTRL_POP: return 0; +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + case BIO_CTRL_GET_KTLS_SEND: + case BIO_CTRL_GET_KTLS_RECV: + return 0; +#endif default: LOG(FATAL) << b << " " << cmd << " " << num << " " << ptr; } From 7618c9aa64d3a108ac5cf79db9bf8aa26b1c00e3 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 13 Jul 2022 23:02:46 +0300 Subject: [PATCH 28/55] Add get_sticker_type function. --- td/telegram/StickerFormat.cpp | 17 +++++++++++++++++ td/telegram/StickerFormat.h | 2 ++ td/telegram/StickersManager.cpp | 14 +------------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/td/telegram/StickerFormat.cpp b/td/telegram/StickerFormat.cpp index ddf05a5fa..88e8f573f 100644 --- a/td/telegram/StickerFormat.cpp +++ b/td/telegram/StickerFormat.cpp @@ -10,6 +10,23 @@ namespace td { +StickerFormat get_sticker_format(const td_api::object_ptr &type) { + CHECK(type != nullptr); + switch (type->get_id()) { + case td_api::stickerTypeStatic::ID: + return StickerFormat::Webp; + case td_api::stickerTypeAnimated::ID: + return StickerFormat::Tgs; + case td_api::stickerTypeVideo::ID: + return StickerFormat::Webm; + case td_api::stickerTypeMask::ID: + return StickerFormat::Webp; + default: + UNREACHABLE(); + return StickerFormat::Unknown; + } +} + StickerFormat get_sticker_format_by_mime_type(Slice mime_type) { if (mime_type == "application/x-tgsticker") { return StickerFormat::Tgs; diff --git a/td/telegram/StickerFormat.h b/td/telegram/StickerFormat.h index 6e1d537ee..b367a6cab 100644 --- a/td/telegram/StickerFormat.h +++ b/td/telegram/StickerFormat.h @@ -18,6 +18,8 @@ namespace td { // update store_sticker/store_sticker_set when this type changes enum class StickerFormat : int32 { Unknown, Webp, Tgs, Webm }; +StickerFormat get_sticker_format(const td_api::object_ptr &type); + StickerFormat get_sticker_format_by_mime_type(Slice mime_type); StickerFormat get_sticker_format_by_extension(Slice extension); diff --git a/td/telegram/StickersManager.cpp b/td/telegram/StickersManager.cpp index 4f70f09b6..804aa51bb 100644 --- a/td/telegram/StickersManager.cpp +++ b/td/telegram/StickersManager.cpp @@ -5794,19 +5794,7 @@ Result> StickersManager::prepare_i return Status::Error(400, "Sticker type must be non-empty"); } - switch (sticker->type_->get_id()) { - case td_api::stickerTypeStatic::ID: - return prepare_input_file(sticker->sticker_, StickerFormat::Webp, false); - case td_api::stickerTypeAnimated::ID: - return prepare_input_file(sticker->sticker_, StickerFormat::Tgs, false); - case td_api::stickerTypeVideo::ID: - return prepare_input_file(sticker->sticker_, StickerFormat::Webm, false); - case td_api::stickerTypeMask::ID: - return prepare_input_file(sticker->sticker_, StickerFormat::Webp, false); - default: - UNREACHABLE(); - return {}; - } + return prepare_input_file(sticker->sticker_, get_sticker_format(sticker->type_), false); } Result> StickersManager::prepare_input_file( From b8bd39dc6f53429cd7ec2af2ce5ba39e18b039df Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 14 Jul 2022 00:13:52 +0300 Subject: [PATCH 29/55] Fix PaymentSuccessful comparison. --- td/telegram/MessageContent.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/td/telegram/MessageContent.cpp b/td/telegram/MessageContent.cpp index 9f4117c73..83e6c3680 100644 --- a/td/telegram/MessageContent.cpp +++ b/td/telegram/MessageContent.cpp @@ -3464,8 +3464,8 @@ void merge_message_contents(Td *td, const MessageContent *old_content, MessageCo old_->telegram_payment_charge_id != new_->telegram_payment_charge_id || old_->provider_payment_charge_id != new_->provider_payment_charge_id || ((old_->order_info != nullptr || new_->order_info != nullptr) && - (old_->order_info == nullptr || new_->order_info == nullptr || *old_->order_info != *new_->order_info || - old_->is_recurring != new_->is_recurring || old_->is_first_recurring != new_->is_first_recurring))) { + (old_->order_info == nullptr || new_->order_info == nullptr || *old_->order_info != *new_->order_info)) || + old_->is_recurring != new_->is_recurring || old_->is_first_recurring != new_->is_first_recurring) { need_update = true; } break; From 7ddc3099f646af6d0f57a5bdb7251853638ee4ee Mon Sep 17 00:00:00 2001 From: Arseny Smirnov Date: Thu, 14 Jul 2022 16:15:17 +0400 Subject: [PATCH 30/55] remove td::this_thread::yield function. usleep_for(1) used instead --- benchmark/bench_queue.cpp | 2 +- td/telegram/net/NetQueryDispatcher.cpp | 2 +- tdutils/td/utils/AtomicRead.h | 12 ++++++++- tdutils/td/utils/MpmcQueue.h | 5 ++-- tdutils/td/utils/MpmcWaiter.h | 16 ++++++++---- tdutils/td/utils/SpinLock.h | 3 ++- .../td/utils/port/detail/ThreadPthread.cpp | 3 --- tdutils/td/utils/port/detail/ThreadPthread.h | 1 - tdutils/td/utils/port/detail/ThreadStl.h | 4 ++- tdutils/td/utils/queue.h | 4 +-- tdutils/td/utils/tests.h | 26 +++++++++++++++++-- 11 files changed, 58 insertions(+), 20 deletions(-) diff --git a/benchmark/bench_queue.cpp b/benchmark/bench_queue.cpp index 0d4911729..6f7cf20bc 100644 --- a/benchmark/bench_queue.cpp +++ b/benchmark/bench_queue.cpp @@ -51,7 +51,7 @@ class Backoff { if (cnt < 50) { return true; } else { - td::this_thread::yield(); + td::usleep_for(1); return cnt < 500; } } diff --git a/td/telegram/net/NetQueryDispatcher.cpp b/td/telegram/net/NetQueryDispatcher.cpp index ccbe5a642..e477379b5 100644 --- a/td/telegram/net/NetQueryDispatcher.cpp +++ b/td/telegram/net/NetQueryDispatcher.cpp @@ -194,7 +194,7 @@ Status NetQueryDispatcher::wait_dc_init(DcId dc_id, bool force) { return Status::Error("Closing"); } #if !TD_THREAD_UNSUPPORTED - td::this_thread::yield(); + td::usleep_for(1); #endif } } diff --git a/tdutils/td/utils/AtomicRead.h b/tdutils/td/utils/AtomicRead.h index a51a91549..58d96e436 100644 --- a/tdutils/td/utils/AtomicRead.h +++ b/tdutils/td/utils/AtomicRead.h @@ -7,6 +7,7 @@ #pragma once #include "td/utils/common.h" +#include "td/utils/port/sleep.h" #include "td/utils/port/thread.h" #include "td/utils/type_traits.h" @@ -20,6 +21,15 @@ template class AtomicRead { public: void read(T &dest) const { + int it = 0; + const int wait_each_it = 4; + auto wait = [&]() { + it++; + if (it % wait_each_it == 0) { + usleep_for(1); + } + }; + while (true) { static_assert(TD_IS_TRIVIALLY_COPYABLE(T), "T must be trivially copyable"); auto version_before = version.load(); @@ -30,7 +40,7 @@ class AtomicRead { break; } } - td::this_thread::yield(); + wait(); } } diff --git a/tdutils/td/utils/MpmcQueue.h b/tdutils/td/utils/MpmcQueue.h index 526bc00be..9d5708586 100644 --- a/tdutils/td/utils/MpmcQueue.h +++ b/tdutils/td/utils/MpmcQueue.h @@ -14,6 +14,7 @@ #include "td/utils/format.h" #include "td/utils/HazardPointers.h" #include "td/utils/logging.h" +#include "td/utils/port/sleep.h" #include "td/utils/port/thread.h" #include "td/utils/ScopeGuard.h" @@ -305,7 +306,7 @@ class MpmcQueueOld { if (try_pop(value, thread_id)) { return value; } - td::this_thread::yield(); + td::usleep_for(1); } } @@ -429,7 +430,7 @@ class MpmcQueue { if (try_pop(value, thread_id)) { return value; } - td::this_thread::yield(); + td::usleep_for(1); } } diff --git a/tdutils/td/utils/MpmcWaiter.h b/tdutils/td/utils/MpmcWaiter.h index b6e0bee0c..7075c3f7c 100644 --- a/tdutils/td/utils/MpmcWaiter.h +++ b/tdutils/td/utils/MpmcWaiter.h @@ -8,6 +8,7 @@ #include "td/utils/common.h" #include "td/utils/logging.h" +#include "td/utils/port/sleep.h" #include "td/utils/port/thread.h" #include @@ -29,16 +30,17 @@ class MpmcEagerWaiter { slot.yields = 0; slot.worker_id = worker_id; } + void wait(Slot &slot) { if (slot.yields < RoundsTillSleepy) { - td::this_thread::yield(); + yield(); slot.yields++; } else if (slot.yields == RoundsTillSleepy) { auto state = state_.load(std::memory_order_relaxed); if (!State::has_worker(state)) { auto new_state = State::with_worker(state, slot.worker_id); if (state_.compare_exchange_strong(state, new_state, std::memory_order_acq_rel)) { - td::this_thread::yield(); + yield(); slot.yields++; return; } @@ -47,12 +49,12 @@ class MpmcEagerWaiter { return; } } - td::this_thread::yield(); + yield(); slot.yields = 0; } else if (slot.yields < RoundsTillAsleep) { auto state = state_.load(std::memory_order_acquire); if (State::still_sleepy(state, slot.worker_id)) { - td::this_thread::yield(); + yield(); slot.yields++; return; } @@ -121,6 +123,10 @@ class MpmcEagerWaiter { condition_variable_.notify_all(); } } + static void yield() { + // whatever, this is better than sched_yield + usleep_for(1); + } }; class MpmcSleepyWaiter { @@ -208,7 +214,7 @@ class MpmcSleepyWaiter { } if (slot.state_ == Slot::Search) { if (slot.yield_cnt++ < 10 && false) { - td::this_thread::yield(); + // TODO some sleep backoff is possible return; } diff --git a/tdutils/td/utils/SpinLock.h b/tdutils/td/utils/SpinLock.h index 33230b8db..abaee7dff 100644 --- a/tdutils/td/utils/SpinLock.h +++ b/tdutils/td/utils/SpinLock.h @@ -6,6 +6,7 @@ // #pragma once +#include "td/utils/port/sleep.h" #include "td/utils/port/thread.h" #include @@ -30,7 +31,7 @@ class SpinLock { //TODO pause return true; } else { - td::this_thread::yield(); + usleep_for(1); return true; } } diff --git a/tdutils/td/utils/port/detail/ThreadPthread.cpp b/tdutils/td/utils/port/detail/ThreadPthread.cpp index a342c3ad1..30234d0aa 100644 --- a/tdutils/td/utils/port/detail/ThreadPthread.cpp +++ b/tdutils/td/utils/port/detail/ThreadPthread.cpp @@ -88,9 +88,6 @@ int ThreadPthread::do_pthread_create(pthread_t *thread, const pthread_attr_t *at } namespace this_thread_pthread { -void yield() { - sched_yield(); -} ThreadPthread::id get_id() { return pthread_self(); } diff --git a/tdutils/td/utils/port/detail/ThreadPthread.h b/tdutils/td/utils/port/detail/ThreadPthread.h index 5342c76c0..c50546c14 100644 --- a/tdutils/td/utils/port/detail/ThreadPthread.h +++ b/tdutils/td/utils/port/detail/ThreadPthread.h @@ -83,7 +83,6 @@ class ThreadPthread { }; namespace this_thread_pthread { -void yield(); ThreadPthread::id get_id(); } // namespace this_thread_pthread } // namespace detail diff --git a/tdutils/td/utils/port/detail/ThreadStl.h b/tdutils/td/utils/port/detail/ThreadStl.h index 5c1a61205..0d9f8f2ff 100644 --- a/tdutils/td/utils/port/detail/ThreadStl.h +++ b/tdutils/td/utils/port/detail/ThreadStl.h @@ -70,7 +70,9 @@ class ThreadStl { return std::forward(v); } }; -namespace this_thread_stl = std::this_thread; +namespace this_thread_stl { +using std::this_thread::get_id; +} // namespace this_thread_stl } // namespace detail } // namespace td diff --git a/tdutils/td/utils/queue.h b/tdutils/td/utils/queue.h index 6157cc808..6e5b67023 100644 --- a/tdutils/td/utils/queue.h +++ b/tdutils/td/utils/queue.h @@ -32,7 +32,7 @@ class Backoff { if (cnt < 1) { // 50 return true; } else { - td::this_thread::yield(); + td::usleep_for(1); return cnt < 3; // 500 } } @@ -47,7 +47,7 @@ class InfBackoff { if (cnt < 50) { return true; } else { - td::this_thread::yield(); + td::usleep_for(1); return true; } } diff --git a/tdutils/td/utils/tests.h b/tdutils/td/utils/tests.h index 183cfe640..2d1b92877 100644 --- a/tdutils/td/utils/tests.h +++ b/tdutils/td/utils/tests.h @@ -10,12 +10,14 @@ #include "td/utils/Context.h" #include "td/utils/format.h" #include "td/utils/logging.h" +#include "td/utils/port/sleep.h" #include "td/utils/port/thread.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" #include #include +#include #include namespace td { @@ -126,12 +128,12 @@ class RegisterTest { } }; -class Stage { +class StageWait { public: void wait(uint64 need) { value_.fetch_add(1, std::memory_order_release); while (value_.load(std::memory_order_acquire) < need) { - td::this_thread::yield(); + usleep_for(1); } }; @@ -139,6 +141,26 @@ class Stage { std::atomic value_{0}; }; +class StageMutex { + public: + void wait(uint64 need) { + std::unique_lock lock{mutex_}; + value_++; + if (value_ == need) { + cond_.notify_all(); + return; + } + cond_.wait(lock, [&] { return value_ >= need; }); + }; + + private: + std::mutex mutex_; + std::condition_variable cond_; + uint64 value_{0}; +}; + +using Stage = StageMutex; + string rand_string(int from, int to, size_t len); vector rand_split(Slice str); From f4c97b25ec5ba0f652441c7894283f076a2b925c Mon Sep 17 00:00:00 2001 From: Arseny Smirnov Date: Thu, 14 Jul 2022 16:27:06 +0400 Subject: [PATCH 31/55] add Mutex wrapper around std::mutex and use it instead of SpinLock --- tdutils/CMakeLists.txt | 1 + tdutils/td/utils/MpscPollableQueue.h | 4 ++-- tdutils/td/utils/port/Mutex.h | 20 ++++++++++++++++++++ tdutils/td/utils/port/ServerSocketFd.cpp | 4 ++-- tdutils/td/utils/port/SocketFd.cpp | 4 ++-- tdutils/td/utils/port/UdpSocketFd.cpp | 4 ++-- tdutils/td/utils/port/detail/PollableFd.h | 4 ++-- tdutils/test/ConcurrentHashMap.cpp | 7 ++++--- 8 files changed, 35 insertions(+), 13 deletions(-) create mode 100644 tdutils/td/utils/port/Mutex.h diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index 6603c6846..c56438cb3 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -138,6 +138,7 @@ set(TDUTILS_SOURCE td/utils/port/IPAddress.h td/utils/port/IoSlice.h td/utils/port/MemoryMapping.h + td/utils/port/Mutex.h td/utils/port/path.h td/utils/port/platform.h td/utils/port/Poll.h diff --git a/tdutils/td/utils/MpscPollableQueue.h b/tdutils/td/utils/MpscPollableQueue.h index e4c7ba2b2..4daf4a0b9 100644 --- a/tdutils/td/utils/MpscPollableQueue.h +++ b/tdutils/td/utils/MpscPollableQueue.h @@ -12,7 +12,7 @@ #if !TD_EVENTFD_UNSUPPORTED -#include "td/utils/SpinLock.h" +#include "td/utils/port/Mutex.h" #include @@ -91,7 +91,7 @@ class MpscPollableQueue { } private: - SpinLock lock_; + Mutex lock_; bool wait_event_fd_{false}; EventFd event_fd_; std::vector writer_vector_; diff --git a/tdutils/td/utils/port/Mutex.h b/tdutils/td/utils/port/Mutex.h new file mode 100644 index 000000000..44710dd1d --- /dev/null +++ b/tdutils/td/utils/port/Mutex.h @@ -0,0 +1,20 @@ +#include + +namespace td { +class Mutex { + public: + struct Guard { + std::unique_lock guard; + void reset() { + guard.unlock(); + } + }; + + Guard lock() { + return {std::unique_lock(mutex_)}; + } + + private: + std::mutex mutex_; +}; +} // namespace td diff --git a/tdutils/td/utils/port/ServerSocketFd.cpp b/tdutils/td/utils/port/ServerSocketFd.cpp index 6e4158c1c..139252b40 100644 --- a/tdutils/td/utils/port/ServerSocketFd.cpp +++ b/tdutils/td/utils/port/ServerSocketFd.cpp @@ -29,7 +29,7 @@ #if TD_PORT_WINDOWS #include "td/utils/port/detail/Iocp.h" -#include "td/utils/SpinLock.h" +#include "td/utils/port/Mutex.h" #include "td/utils/VectorQueue.h" #endif @@ -87,7 +87,7 @@ class ServerSocketFdImpl final : private Iocp::Callback { private: PollableFdInfo info_; - SpinLock lock_; + Mutex lock_; VectorQueue accepted_; VectorQueue pending_errors_; static constexpr size_t MAX_ADDR_SIZE = sizeof(sockaddr_in6) + 16; diff --git a/tdutils/td/utils/port/SocketFd.cpp b/tdutils/td/utils/port/SocketFd.cpp index 9076c4e4a..90d516db3 100644 --- a/tdutils/td/utils/port/SocketFd.cpp +++ b/tdutils/td/utils/port/SocketFd.cpp @@ -17,7 +17,7 @@ #if TD_PORT_WINDOWS #include "td/utils/buffer.h" #include "td/utils/port/detail/Iocp.h" -#include "td/utils/SpinLock.h" +#include "td/utils/port/Mutex.h" #include "td/utils/VectorQueue.h" #include @@ -164,7 +164,7 @@ class SocketFdImpl final : private Iocp::Callback { private: PollableFdInfo info_; - SpinLock lock_; + Mutex lock_; std::atomic refcnt_{1}; bool close_flag_{false}; diff --git a/tdutils/td/utils/port/UdpSocketFd.cpp b/tdutils/td/utils/port/UdpSocketFd.cpp index d33d24340..eee487d44 100644 --- a/tdutils/td/utils/port/UdpSocketFd.cpp +++ b/tdutils/td/utils/port/UdpSocketFd.cpp @@ -18,7 +18,7 @@ #if TD_PORT_WINDOWS #include "td/utils/port/detail/Iocp.h" -#include "td/utils/SpinLock.h" +#include "td/utils/port/Mutex.h" #endif #if TD_PORT_POSIX @@ -154,7 +154,7 @@ class UdpSocketFdImpl final : private Iocp::Callback { private: PollableFdInfo info_; - SpinLock lock_; + Mutex lock_; std::atomic refcnt_{1}; bool is_connected_{false}; diff --git a/tdutils/td/utils/port/detail/PollableFd.h b/tdutils/td/utils/port/detail/PollableFd.h index ba8c8891b..01fb5857e 100644 --- a/tdutils/td/utils/port/detail/PollableFd.h +++ b/tdutils/td/utils/port/detail/PollableFd.h @@ -12,8 +12,8 @@ #include "td/utils/logging.h" #include "td/utils/Observer.h" #include "td/utils/port/detail/NativeFd.h" +#include "td/utils/port/Mutex.h" #include "td/utils/port/PollFlags.h" -#include "td/utils/SpinLock.h" #include #include @@ -140,7 +140,7 @@ class PollableFdInfo final : private ListNode { std::atomic_flag lock_ = ATOMIC_FLAG_INIT; PollFlagsSet flags_; #if TD_PORT_WINDOWS - SpinLock observer_lock_; + Mutex observer_lock_; #endif ObserverBase *observer_{nullptr}; diff --git a/tdutils/test/ConcurrentHashMap.cpp b/tdutils/test/ConcurrentHashMap.cpp index 3e2219f1a..de8ef9a71 100644 --- a/tdutils/test/ConcurrentHashMap.cpp +++ b/tdutils/test/ConcurrentHashMap.cpp @@ -7,6 +7,7 @@ #include "td/utils/benchmark.h" #include "td/utils/ConcurrentHashTable.h" #include "td/utils/misc.h" +#include "td/utils/port/Mutex.h" #include "td/utils/port/thread.h" #include "td/utils/SpinLock.h" #include "td/utils/tests.h" @@ -72,11 +73,11 @@ class ConcurrentHashMapMutex { return "ConcurrentHashMapMutex"; } void insert(KeyT key, ValueT value) { - std::unique_lock lock(mutex_); + auto guard = mutex_.lock(); hash_map_.emplace(key, value); } ValueT find(KeyT key, ValueT default_value) { - std::unique_lock lock(mutex_); + auto guard = mutex_.lock(); auto it = hash_map_.find(key); if (it == hash_map_.end()) { return default_value; @@ -85,7 +86,7 @@ class ConcurrentHashMapMutex { } private: - std::mutex mutex_; + Mutex mutex_; #if TD_HAVE_ABSL absl::flat_hash_map hash_map_; #else From bf80c43c05732905df4d27dee31536759b6f1e06 Mon Sep 17 00:00:00 2001 From: Arseny Smirnov Date: Thu, 14 Jul 2022 18:36:46 +0400 Subject: [PATCH 32/55] fix CE --- td/telegram/net/NetQueryDispatcher.cpp | 1 + tdutils/td/utils/port/Mutex.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/td/telegram/net/NetQueryDispatcher.cpp b/td/telegram/net/NetQueryDispatcher.cpp index e477379b5..eea49a34b 100644 --- a/td/telegram/net/NetQueryDispatcher.cpp +++ b/td/telegram/net/NetQueryDispatcher.cpp @@ -24,6 +24,7 @@ #include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/misc.h" +#include "td/utils/port/sleep.h" #include "td/utils/port/thread.h" #include "td/utils/Slice.h" #include "td/utils/SliceBuilder.h" diff --git a/tdutils/td/utils/port/Mutex.h b/tdutils/td/utils/port/Mutex.h index 44710dd1d..2cced68d6 100644 --- a/tdutils/td/utils/port/Mutex.h +++ b/tdutils/td/utils/port/Mutex.h @@ -1,3 +1,5 @@ +#pragma once + #include namespace td { From abc9ed2cd90129d41e69d413d2c48ba3f09a090e Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 14 Jul 2022 18:38:48 +0300 Subject: [PATCH 33/55] Minor fixes. --- CMake/GetGitRevisionDescription.cmake | 6 ++-- CMake/GetGitRevisionDescription.cmake.in | 36 ++++++++++++------------ example/README.md | 2 +- td/generate/scheme/td_api.tl | 4 +-- td/mtproto/AuthData.cpp | 2 +- td/telegram/net/NetQueryDispatcher.cpp | 5 ++-- tdactor/td/actor/Timeout.h | 2 ++ tdactor/td/actor/impl/Scheduler-decl.h | 1 + tdactor/td/actor/impl/Scheduler.cpp | 1 + tdutils/generate/CMakeLists.txt | 2 +- tdutils/td/utils/AtomicRead.h | 11 ++++---- tdutils/td/utils/MpmcQueue.h | 5 ++-- tdutils/td/utils/MpmcWaiter.h | 1 - tdutils/td/utils/SpinLock.h | 1 - tdutils/td/utils/port/Mutex.h | 8 ++++++ tdutils/td/utils/queue.h | 6 ++-- tdutils/td/utils/tests.h | 2 +- tdutils/test/ConcurrentHashMap.cpp | 1 - 18 files changed, 51 insertions(+), 45 deletions(-) diff --git a/CMake/GetGitRevisionDescription.cmake b/CMake/GetGitRevisionDescription.cmake index 32be4fa85..73fe1cc14 100644 --- a/CMake/GetGitRevisionDescription.cmake +++ b/CMake/GetGitRevisionDescription.cmake @@ -59,11 +59,11 @@ function(get_git_head_revision _refspecvar _hashvar) _git_find_closest_git_dir("${CMAKE_CURRENT_SOURCE_DIR}" GIT_DIR) file(RELATIVE_PATH _relative_to_source_dir "${CMAKE_CURRENT_SOURCE_DIR}" "${GIT_DIR}") - if ("${_relative_to_source_dir}" MATCHES "^[.][.]") + if (_relative_to_source_dir MATCHES "^[.][.]") # We've gone above the CMake root dir. set(GIT_DIR "") endif() - if ("${GIT_DIR}" STREQUAL "") + if (GIT_DIR STREQUAL "") set(${_refspecvar} "GITDIR-NOTFOUND" PARENT_SCOPE) set(${_hashvar} "GITDIR-NOTFOUND" PARENT_SCOPE) return() @@ -83,7 +83,7 @@ function(get_git_head_revision _refspecvar _hashvar) WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE out ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) - if (NOT "${out}" STREQUAL "") + if (NOT out STREQUAL "") # If out is non-empty, GIT_DIR/CMAKE_CURRENT_SOURCE_DIR is in a submodule file(READ ${GIT_DIR} submodule) string(REGEX REPLACE "gitdir: (.*)$" "\\1" GIT_DIR_RELATIVE ${submodule}) diff --git a/CMake/GetGitRevisionDescription.cmake.in b/CMake/GetGitRevisionDescription.cmake.in index e35156f25..bfc1e54c1 100644 --- a/CMake/GetGitRevisionDescription.cmake.in +++ b/CMake/GetGitRevisionDescription.cmake.in @@ -20,24 +20,24 @@ set(HEAD_HASH) file(READ "@HEAD_FILE@" HEAD_CONTENTS LIMIT 1024) string(STRIP "${HEAD_CONTENTS}" HEAD_CONTENTS) -if(HEAD_CONTENTS MATCHES "ref") - # named branch - string(REPLACE "ref: " "" HEAD_REF "${HEAD_CONTENTS}") - if(EXISTS "@GIT_DIR@/${HEAD_REF}") - configure_file("@GIT_DIR@/${HEAD_REF}" "@GIT_DATA@/head-ref" COPYONLY) - else() - configure_file("@GIT_DIR@/packed-refs" "@GIT_DATA@/packed-refs" COPYONLY) - file(READ "@GIT_DATA@/packed-refs" PACKED_REFS) - if(${PACKED_REFS} MATCHES "([0-9a-z]*) ${HEAD_REF}") - set(HEAD_HASH "${CMAKE_MATCH_1}") - endif() - endif() +if (HEAD_CONTENTS MATCHES "ref") + # named branch + string(REPLACE "ref: " "" HEAD_REF "${HEAD_CONTENTS}") + if (EXISTS "@GIT_DIR@/${HEAD_REF}") + configure_file("@GIT_DIR@/${HEAD_REF}" "@GIT_DATA@/head-ref" COPYONLY) + else() + configure_file("@GIT_DIR@/packed-refs" "@GIT_DATA@/packed-refs" COPYONLY) + file(READ "@GIT_DATA@/packed-refs" PACKED_REFS) + if (PACKED_REFS MATCHES "([0-9a-z]*) ${HEAD_REF}") + set(HEAD_HASH "${CMAKE_MATCH_1}") + endif() + endif() else() - # detached HEAD - configure_file("@GIT_DIR@/HEAD" "@GIT_DATA@/head-ref" COPYONLY) + # detached HEAD + configure_file("@GIT_DIR@/HEAD" "@GIT_DATA@/head-ref" COPYONLY) endif() -if(NOT HEAD_HASH) - file(READ "@GIT_DATA@/head-ref" HEAD_HASH LIMIT 1024) - string(STRIP "${HEAD_HASH}" HEAD_HASH) -endif() \ No newline at end of file +if (NOT HEAD_HASH) + file(READ "@GIT_DATA@/head-ref" HEAD_HASH LIMIT 1024) + string(STRIP "${HEAD_HASH}" HEAD_HASH) +endif() diff --git a/example/README.md b/example/README.md index ce4b0176b..e3cf1b7ac 100644 --- a/example/README.md +++ b/example/README.md @@ -73,7 +73,7 @@ at [tdl](https://github.com/Bannerets/tdl), which provides a convenient, fully-a You can also see [TdNode](https://github.com/puppy0cam/TdNode), [tglib](https://github.com/nodegin/tglib), [node-tdlib](https://github.com/wfjsw/node-tdlib), [tdlnode](https://github.com/fonbah/tdlnode), [Paper Plane](https://github.com/par6n/paper-plane), or [node-tlg](https://github.com/dilongfa/node-tlg) for other examples of TDLib JSON interface integration with Node.js. -See also the source code of [DIBgram](https://github.com/DIBgram/DIBgram) - an unofficial Telegram web app which looks like Telegram Desktop. +See also the source code of [DIBgram](https://github.com/DIBgram/DIBgram) - an unofficial Telegram web application which looks like Telegram Desktop. TDLib can be used also from NativeScript through the [JSON](https://github.com/tdlib/td#using-json) interface. See [nativescript-tglib](https://github.com/arpit2438735/nativescript-tglib) as an example of a NativeScript library for building Telegram clients. diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index 0096fd613..7add74a29 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -4793,7 +4793,7 @@ searchMessages chat_list:ChatList query:string offset_date:int32 offset_chat_id: //@filter Additional filter for messages to search; pass null to search for all messages searchSecretMessages chat_id:int53 query:string offset:string limit:int32 filter:SearchMessagesFilter = FoundMessages; -//@description Searches for call messages. Returns the results in reverse chronological order (i. e., in order of decreasing message_id). For optimal performance, the number of returned messages is chosen by TDLib +//@description Searches for call messages. Returns the results in reverse chronological order (i.e., in order of decreasing message_id). For optimal performance, the number of returned messages is chosen by TDLib //@from_message_id Identifier of the message from which to search; use 0 to get results from the last message //@limit The maximum number of messages to be returned; up to 100. For optimal performance, the number of returned messages is chosen by TDLib and can be smaller than the specified limit //@only_missed Pass true to search only for messages with missed/declined calls @@ -6081,7 +6081,7 @@ getSupergroupMembers supergroup_id:int53 filter:SupergroupMembersFilter offset:i closeSecretChat secret_chat_id:int32 = Ok; -//@description Returns a list of service actions taken by chat members and administrators in the last 48 hours. Available only for supergroups and channels. Requires administrator rights. Returns results in reverse chronological order (i. e., in order of decreasing event_id) +//@description Returns a list of service actions taken by chat members and administrators in the last 48 hours. Available only for supergroups and channels. Requires administrator rights. Returns results in reverse chronological order (i.e., in order of decreasing event_id) //@chat_id Chat identifier @query Search query by which to filter events @from_event_id Identifier of an event from which to return results. Use 0 to get results from the latest events @limit The maximum number of events to return; up to 100 //@filters The types of events to return; pass null to get chat events of all types @user_ids User identifiers by which to filter events. By default, events relating to all users will be returned getChatEventLog chat_id:int53 query:string from_event_id:int64 limit:int32 filters:chatEventLogFilters user_ids:vector = ChatEvents; diff --git a/td/mtproto/AuthData.cpp b/td/mtproto/AuthData.cpp index d37f7a9c1..c87ba6ee8 100644 --- a/td/mtproto/AuthData.cpp +++ b/td/mtproto/AuthData.cpp @@ -21,7 +21,7 @@ Status check_message_id_duplicates(int64 *saved_message_ids, size_t max_size, si // In addition, the identifiers (msg_id) of the last N messages received from the other side must be stored, and if // a message comes in with msg_id lower than all or equal to any of the stored values, that message is to be // ignored. Otherwise, the new message msg_id is added to the set, and, if the number of stored msg_id values is - // greater than N, the oldest (i. e. the lowest) is forgotten. + // greater than N, the oldest (i.e. the lowest) is forgotten. if (end_pos == 2 * max_size) { std::copy_n(&saved_message_ids[max_size], max_size, &saved_message_ids[0]); end_pos = max_size; diff --git a/td/telegram/net/NetQueryDispatcher.cpp b/td/telegram/net/NetQueryDispatcher.cpp index eea49a34b..e94845e72 100644 --- a/td/telegram/net/NetQueryDispatcher.cpp +++ b/td/telegram/net/NetQueryDispatcher.cpp @@ -25,7 +25,6 @@ #include "td/utils/logging.h" #include "td/utils/misc.h" #include "td/utils/port/sleep.h" -#include "td/utils/port/thread.h" #include "td/utils/Slice.h" #include "td/utils/SliceBuilder.h" @@ -195,7 +194,7 @@ Status NetQueryDispatcher::wait_dc_init(DcId dc_id, bool force) { return Status::Error("Closing"); } #if !TD_THREAD_UNSUPPORTED - td::usleep_for(1); + usleep_for(1); #endif } } @@ -332,7 +331,7 @@ void NetQueryDispatcher::set_main_dc_id(int32 new_main_dc_id) { return; } - // Very rare event. Mutex is ok. + // Very rare event; mutex is ok. std::lock_guard guard(main_dc_id_mutex_); if (new_main_dc_id == main_dc_id_) { return; diff --git a/tdactor/td/actor/Timeout.h b/tdactor/td/actor/Timeout.h index 5c2293b99..cde72657b 100644 --- a/tdactor/td/actor/Timeout.h +++ b/tdactor/td/actor/Timeout.h @@ -8,6 +8,8 @@ #include "td/actor/actor.h" +#include "td/utils/common.h" + namespace td { class Timeout final : public Actor { diff --git a/tdactor/td/actor/impl/Scheduler-decl.h b/tdactor/td/actor/impl/Scheduler-decl.h index 396995e6c..72893e4e8 100644 --- a/tdactor/td/actor/impl/Scheduler-decl.h +++ b/tdactor/td/actor/impl/Scheduler-decl.h @@ -11,6 +11,7 @@ #include "td/actor/impl/EventFull-decl.h" #include "td/utils/Closure.h" +#include "td/utils/common.h" #include "td/utils/FlatHashMap.h" #include "td/utils/Heap.h" #include "td/utils/List.h" diff --git a/tdactor/td/actor/impl/Scheduler.cpp b/tdactor/td/actor/impl/Scheduler.cpp index 9ad7cb72c..b29b5b34d 100644 --- a/tdactor/td/actor/impl/Scheduler.cpp +++ b/tdactor/td/actor/impl/Scheduler.cpp @@ -21,6 +21,7 @@ #include "td/utils/MpscPollableQueue.h" #include "td/utils/ObjectPool.h" #include "td/utils/port/thread_local.h" +#include "td/utils/Promise.h" #include "td/utils/ScopeGuard.h" #include "td/utils/Time.h" diff --git a/tdutils/generate/CMakeLists.txt b/tdutils/generate/CMakeLists.txt index b36a1a4f6..69b42d914 100644 --- a/tdutils/generate/CMakeLists.txt +++ b/tdutils/generate/CMakeLists.txt @@ -25,7 +25,7 @@ add_custom_target(tdmime_auto DEPENDS ${TDMIME_SOURCE}) if (NOT CMAKE_CROSSCOMPILING) find_program(GPERF_EXECUTABLE gperf) if (NOT GPERF_EXECUTABLE) - message(FATAL_ERROR "Could NOT find gperf. Add path to gperf executable to PATH environment variable or specify it manually using GPERF_EXECUTABLE option, i. e. 'cmake -DGPERF_EXECUTABLE:FILEPATH=\"\"'.") + message(FATAL_ERROR "Could NOT find gperf. Add path to gperf executable to PATH environment variable or specify it manually using GPERF_EXECUTABLE option, i.e. 'cmake -DGPERF_EXECUTABLE:FILEPATH=\"\"'.") endif() set(GPERF_FILES diff --git a/tdutils/td/utils/AtomicRead.h b/tdutils/td/utils/AtomicRead.h index 58d96e436..d30e960a8 100644 --- a/tdutils/td/utils/AtomicRead.h +++ b/tdutils/td/utils/AtomicRead.h @@ -8,7 +8,6 @@ #include "td/utils/common.h" #include "td/utils/port/sleep.h" -#include "td/utils/port/thread.h" #include "td/utils/type_traits.h" #include @@ -21,11 +20,11 @@ template class AtomicRead { public: void read(T &dest) const { - int it = 0; - const int wait_each_it = 4; - auto wait = [&]() { - it++; - if (it % wait_each_it == 0) { + uint32 counter = 0; + auto wait = [&] { + counter++; + const int wait_each_count = 4; + if (counter % wait_each_count == 0) { usleep_for(1); } }; diff --git a/tdutils/td/utils/MpmcQueue.h b/tdutils/td/utils/MpmcQueue.h index 9d5708586..f1f12a37f 100644 --- a/tdutils/td/utils/MpmcQueue.h +++ b/tdutils/td/utils/MpmcQueue.h @@ -15,7 +15,6 @@ #include "td/utils/HazardPointers.h" #include "td/utils/logging.h" #include "td/utils/port/sleep.h" -#include "td/utils/port/thread.h" #include "td/utils/ScopeGuard.h" #include @@ -306,7 +305,7 @@ class MpmcQueueOld { if (try_pop(value, thread_id)) { return value; } - td::usleep_for(1); + usleep_for(1); } } @@ -430,7 +429,7 @@ class MpmcQueue { if (try_pop(value, thread_id)) { return value; } - td::usleep_for(1); + usleep_for(1); } } diff --git a/tdutils/td/utils/MpmcWaiter.h b/tdutils/td/utils/MpmcWaiter.h index 7075c3f7c..7ece498ff 100644 --- a/tdutils/td/utils/MpmcWaiter.h +++ b/tdutils/td/utils/MpmcWaiter.h @@ -9,7 +9,6 @@ #include "td/utils/common.h" #include "td/utils/logging.h" #include "td/utils/port/sleep.h" -#include "td/utils/port/thread.h" #include #include diff --git a/tdutils/td/utils/SpinLock.h b/tdutils/td/utils/SpinLock.h index abaee7dff..3413f6c5b 100644 --- a/tdutils/td/utils/SpinLock.h +++ b/tdutils/td/utils/SpinLock.h @@ -7,7 +7,6 @@ #pragma once #include "td/utils/port/sleep.h" -#include "td/utils/port/thread.h" #include #include diff --git a/tdutils/td/utils/port/Mutex.h b/tdutils/td/utils/port/Mutex.h index 2cced68d6..40d9f482d 100644 --- a/tdutils/td/utils/port/Mutex.h +++ b/tdutils/td/utils/port/Mutex.h @@ -1,8 +1,15 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// #pragma once #include namespace td { + class Mutex { public: struct Guard { @@ -19,4 +26,5 @@ class Mutex { private: std::mutex mutex_; }; + } // namespace td diff --git a/tdutils/td/utils/queue.h b/tdutils/td/utils/queue.h index 6e5b67023..e6d69aae3 100644 --- a/tdutils/td/utils/queue.h +++ b/tdutils/td/utils/queue.h @@ -7,7 +7,7 @@ #pragma once #include "td/utils/port/EventFd.h" -#include "td/utils/port/thread.h" +#include "td/utils/port/sleep.h" #if !TD_THREAD_UNSUPPORTED && !TD_EVENTFD_UNSUPPORTED @@ -32,7 +32,7 @@ class Backoff { if (cnt < 1) { // 50 return true; } else { - td::usleep_for(1); + usleep_for(1); return cnt < 3; // 500 } } @@ -47,7 +47,7 @@ class InfBackoff { if (cnt < 50) { return true; } else { - td::usleep_for(1); + usleep_for(1); return true; } } diff --git a/tdutils/td/utils/tests.h b/tdutils/td/utils/tests.h index 2d1b92877..69b920626 100644 --- a/tdutils/td/utils/tests.h +++ b/tdutils/td/utils/tests.h @@ -11,11 +11,11 @@ #include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/port/sleep.h" -#include "td/utils/port/thread.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" #include +#include #include #include #include diff --git a/tdutils/test/ConcurrentHashMap.cpp b/tdutils/test/ConcurrentHashMap.cpp index de8ef9a71..afdb8aea8 100644 --- a/tdutils/test/ConcurrentHashMap.cpp +++ b/tdutils/test/ConcurrentHashMap.cpp @@ -13,7 +13,6 @@ #include "td/utils/tests.h" #include -#include #if !TD_THREAD_UNSUPPORTED From 0bdd15fe9fd8507211bd4b0fdb612c2f2878d110 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 15 Jul 2022 00:06:41 +0300 Subject: [PATCH 34/55] Fix warning. --- tdutils/test/HashSet.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tdutils/test/HashSet.cpp b/tdutils/test/HashSet.cpp index 83831bd6f..507960c4b 100644 --- a/tdutils/test/HashSet.cpp +++ b/tdutils/test/HashSet.cpp @@ -265,8 +265,7 @@ TEST(FlatHashMap, stress_test) { td::vector steps; auto add_step = [&](td::Slice step_name, td::uint32 weight, auto f) { - auto g = [&, step_name, f = std::move(f)] { - //LOG(ERROR) << step_name; + auto g = [&, f = std::move(f)] { //ASSERT_EQ(ref.size(), tbl.size()); f(); ASSERT_EQ(ref.size(), tbl.size()); From 327babb85114cb09a75208451b019fb6a1136e4b Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 15 Jul 2022 13:58:04 +0300 Subject: [PATCH 35/55] Fix Java example for old Java versions. --- example/java/org/drinkless/tdlib/example/Example.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/java/org/drinkless/tdlib/example/Example.java b/example/java/org/drinkless/tdlib/example/Example.java index 2923ecea7..eb71f4e4b 100644 --- a/example/java/org/drinkless/tdlib/example/Example.java +++ b/example/java/org/drinkless/tdlib/example/Example.java @@ -615,7 +615,7 @@ public final class Example { } private static void onFatalError(String errorMessage) { - final class ThrowError implements Runnable { + final static class ThrowError implements Runnable { private final String errorMessage; private final AtomicLong errorThrowTime; From bedec2c9bae8aa080ad55a92dc191ca1d7554431 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 16 Jul 2022 15:45:30 +0300 Subject: [PATCH 36/55] Add StickersManager::get_sticker_set_thumbnail_object. --- td/telegram/StickersManager.cpp | 15 +++++++++------ td/telegram/StickersManager.h | 2 ++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/td/telegram/StickersManager.cpp b/td/telegram/StickersManager.cpp index 804aa51bb..278b280b6 100644 --- a/td/telegram/StickersManager.cpp +++ b/td/telegram/StickersManager.cpp @@ -1998,6 +1998,13 @@ double StickersManager::get_sticker_set_minithumbnail_zoom(const StickerSet *sti return 1.0; } +td_api::object_ptr StickersManager::get_sticker_set_thumbnail_object( + const StickerSet *sticker_set) const { + CHECK(sticker_set != nullptr); + auto thumbnail_format = get_sticker_set_thumbnail_format(sticker_set->sticker_format); + return get_thumbnail_object(td_->file_manager_.get(), sticker_set->thumbnail, thumbnail_format); +} + tl_object_ptr StickersManager::get_sticker_set_object(StickerSetId sticker_set_id) const { const StickerSet *sticker_set = get_sticker_set(sticker_set_id); CHECK(sticker_set != nullptr); @@ -2016,10 +2023,8 @@ tl_object_ptr StickersManager::get_sticker_set_object(Sticke } emojis.push_back(make_tl_object(std::move(sticker_emojis))); } - auto thumbnail_format = get_sticker_set_thumbnail_format(sticker_set->sticker_format); - auto thumbnail = get_thumbnail_object(td_->file_manager_.get(), sticker_set->thumbnail, thumbnail_format); return make_tl_object( - sticker_set->id.get(), sticker_set->title, sticker_set->short_name, std::move(thumbnail), + sticker_set->id.get(), sticker_set->title, sticker_set->short_name, get_sticker_set_thumbnail_object(sticker_set), get_sticker_minithumbnail(sticker_set->minithumbnail, sticker_set->id, -2, get_sticker_set_minithumbnail_zoom(sticker_set)), sticker_set->is_installed && !sticker_set->is_archived, sticker_set->is_archived, sticker_set->is_official, @@ -2091,11 +2096,9 @@ tl_object_ptr StickersManager::get_sticker_set_info_obje } } - auto thumbnail_format = get_sticker_set_thumbnail_format(sticker_set->sticker_format); - auto thumbnail = get_thumbnail_object(td_->file_manager_.get(), sticker_set->thumbnail, thumbnail_format); auto actual_count = narrow_cast(sticker_set->sticker_ids.size()); return make_tl_object( - sticker_set->id.get(), sticker_set->title, sticker_set->short_name, std::move(thumbnail), + sticker_set->id.get(), sticker_set->title, sticker_set->short_name, get_sticker_set_thumbnail_object(sticker_set), get_sticker_minithumbnail(sticker_set->minithumbnail, sticker_set->id, -3, get_sticker_set_minithumbnail_zoom(sticker_set)), sticker_set->is_installed && !sticker_set->is_archived, sticker_set->is_archived, sticker_set->is_official, diff --git a/td/telegram/StickersManager.h b/td/telegram/StickersManager.h index fb0876b46..1b2058fa6 100644 --- a/td/telegram/StickersManager.h +++ b/td/telegram/StickersManager.h @@ -498,6 +498,8 @@ class StickersManager final : public Actor { static tl_object_ptr get_mask_point_object(int32 point); + td_api::object_ptr get_sticker_set_thumbnail_object(const StickerSet *sticker_set) const; + tl_object_ptr get_sticker_set_info_object(StickerSetId sticker_set_id, size_t covers_limit, bool prefer_premium) const; From 02396be7ebc113bede1d5af1933a1f5221f04988 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 16 Jul 2022 16:10:06 +0300 Subject: [PATCH 37/55] Allow internal links in bio of non-premium users. --- td/telegram/ContactsManager.cpp | 14 +++++++++++++- td/telegram/LinkManager.cpp | 5 +++++ td/telegram/LinkManager.h | 3 +++ td/telegram/MessageEntity.cpp | 18 ++++++++---------- td/telegram/MessageEntity.h | 3 +-- 5 files changed, 30 insertions(+), 13 deletions(-) diff --git a/td/telegram/ContactsManager.cpp b/td/telegram/ContactsManager.cpp index 8645a7f17..18e37079a 100644 --- a/td/telegram/ContactsManager.cpp +++ b/td/telegram/ContactsManager.cpp @@ -16692,7 +16692,19 @@ tl_object_ptr ContactsManager::get_user_full_info_object(U } else { FormattedText bio; bio.text = user_full->about; - bio.entities = find_entities(bio.text, true, true, !is_user_premium(user_id)); + bio.entities = find_entities(bio.text, true, true); + if (!is_user_premium(user_id)) { + td::remove_if(bio.entities, [&](const MessageEntity &entity) { + if (entity.type == MessageEntity::Type::EmailAddress) { + return true; + } + if (entity.type == MessageEntity::Type::Url && + !LinkManager::is_internal_link(utf8_utf16_substr(Slice(bio.text), entity.offset, entity.length))) { + return true; + } + return false; + }); + } bio_object = get_formatted_text_object(bio, true, 0); } return make_tl_object( diff --git a/td/telegram/LinkManager.cpp b/td/telegram/LinkManager.cpp index 4ec740ea5..eff592aee 100644 --- a/td/telegram/LinkManager.cpp +++ b/td/telegram/LinkManager.cpp @@ -924,6 +924,11 @@ LinkManager::LinkInfo LinkManager::get_link_info(Slice link) { return result; } +bool LinkManager::is_internal_link(Slice link) { + auto info = get_link_info(link); + return info.is_internal_; +} + unique_ptr LinkManager::parse_internal_link(Slice link, bool is_trusted) { auto info = get_link_info(link); if (!info.is_internal_) { diff --git a/td/telegram/LinkManager.h b/td/telegram/LinkManager.h index 7ba71ec0b..6c4be78c5 100644 --- a/td/telegram/LinkManager.h +++ b/td/telegram/LinkManager.h @@ -52,6 +52,9 @@ class LinkManager final : public Actor { // same as check_link, but returns an empty string instead of an error static string get_checked_link(Slice link, bool http_only = false, bool https_only = false); + // returns whether a link is an internal link, supported or not + static bool is_internal_link(Slice link); + // checks whether the link is a supported tg or t.me link and parses it static unique_ptr parse_internal_link(Slice link, bool is_trusted = false); diff --git a/td/telegram/MessageEntity.cpp b/td/telegram/MessageEntity.cpp index d4c205c9c..c3f947590 100644 --- a/td/telegram/MessageEntity.cpp +++ b/td/telegram/MessageEntity.cpp @@ -1636,7 +1636,7 @@ static void fix_entity_offsets(Slice text, vector &entities) { } } -vector find_entities(Slice text, bool skip_bot_commands, bool skip_media_timestamps, bool skip_urls) { +vector find_entities(Slice text, bool skip_bot_commands, bool skip_media_timestamps) { vector entities; auto add_entities = [&entities, &text](MessageEntity::Type type, vector (*find_entities_f)(Slice)) mutable { @@ -1655,15 +1655,13 @@ vector find_entities(Slice text, bool skip_bot_commands, bool ski add_entities(MessageEntity::Type::Cashtag, find_cashtags); // TODO find_phone_numbers add_entities(MessageEntity::Type::BankCardNumber, find_bank_card_numbers); - if (!skip_urls) { - add_entities(MessageEntity::Type::Url, find_tg_urls); - auto urls = find_urls(text); - for (auto &url : urls) { - auto type = url.second ? MessageEntity::Type::EmailAddress : MessageEntity::Type::Url; - auto offset = narrow_cast(url.first.begin() - text.begin()); - auto length = narrow_cast(url.first.size()); - entities.emplace_back(type, offset, length); - } + add_entities(MessageEntity::Type::Url, find_tg_urls); + auto urls = find_urls(text); + for (auto &url : urls) { + auto type = url.second ? MessageEntity::Type::EmailAddress : MessageEntity::Type::Url; + auto offset = narrow_cast(url.first.begin() - text.begin()); + auto length = narrow_cast(url.first.size()); + entities.emplace_back(type, offset, length); } if (!skip_media_timestamps) { auto media_timestamps = find_media_timestamps(text); diff --git a/td/telegram/MessageEntity.h b/td/telegram/MessageEntity.h index 88600c26a..4a0ec6e29 100644 --- a/td/telegram/MessageEntity.h +++ b/td/telegram/MessageEntity.h @@ -143,8 +143,7 @@ vector> get_text_entities_object(const vector< td_api::object_ptr get_formatted_text_object(const FormattedText &text, bool skip_bot_commands, int32 max_media_timestamp); -vector find_entities(Slice text, bool skip_bot_commands, bool skip_media_timestamps, - bool skip_urls = false); +vector find_entities(Slice text, bool skip_bot_commands, bool skip_media_timestamps); vector find_mentions(Slice str); vector find_bot_commands(Slice str); From 72bc28563758be854c27eee6b85d620c3240019c Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 16 Jul 2022 17:25:28 +0300 Subject: [PATCH 38/55] Delete from database information about unlinked files. --- td/telegram/files/FileManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index 48703c241..eb8ac4700 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -1099,7 +1099,6 @@ FileId FileManager::register_empty(FileType type) { } void FileManager::on_file_unlink(const FullLocalFileLocation &location) { - // TODO: remove file from the database too auto it = local_location_to_file_id_.find(location); if (it == local_location_to_file_id_.end()) { return; @@ -1107,6 +1106,7 @@ void FileManager::on_file_unlink(const FullLocalFileLocation &location) { auto file_id = it->second; auto file_node = get_sync_file_node(file_id); CHECK(file_node); + clear_from_pmc(file_node); send_closure(G()->download_manager(), &DownloadManager::remove_file_if_finished, file_node->main_file_id_); file_node->drop_local_location(); try_flush_node_info(file_node, "on_file_unlink"); From 04a7583bd0fb0d3796ca61716dcc6be3cb27a252 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 17 Jul 2022 10:41:17 +0300 Subject: [PATCH 39/55] Remove unused Event::clone method. --- tdactor/td/actor/impl/Event.h | 19 ------------------- tdutils/td/utils/Closure.h | 19 ------------------- 2 files changed, 38 deletions(-) diff --git a/tdactor/td/actor/impl/Event.h b/tdactor/td/actor/impl/Event.h index 32270e09a..a0e4d39fb 100644 --- a/tdactor/td/actor/impl/Event.h +++ b/tdactor/td/actor/impl/Event.h @@ -49,7 +49,6 @@ class CustomEvent { virtual ~CustomEvent() = default; virtual void run(Actor *actor) = 0; - virtual CustomEvent *clone() const = 0; virtual void start_migrate(int32 sched_id) { } virtual void finish_migrate() { @@ -62,9 +61,6 @@ class ClosureEvent final : public CustomEvent { void run(Actor *actor) final { closure_.run(static_cast(actor)); } - CustomEvent *clone() const final { - return new ClosureEvent(closure_.clone()); - } template explicit ClosureEvent(ArgsT &&...args) : closure_(std::forward(args)...) { } @@ -93,10 +89,6 @@ class LambdaEvent final : public CustomEvent { void run(Actor *actor) final { f_(); } - CustomEvent *clone() const final { - LOG(FATAL) << "Not supported"; - return nullptr; - } template , LambdaEvent>::value, int> = 0> explicit LambdaEvent(FromLambdaT &&lambda) : f_(std::forward(lambda)) { } @@ -181,17 +173,6 @@ class Event { destroy(); } - Event clone() const { - Event res; - res.type = type; - if (type == Type::Custom) { - res.data.custom_event = data.custom_event->clone(); - } else { - res.data = data; - } - return res; - } - bool empty() const { return type == Type::NoType; } diff --git a/tdutils/td/utils/Closure.h b/tdutils/td/utils/Closure.h index 999c9de1a..6c895c68e 100644 --- a/tdutils/td/utils/Closure.h +++ b/tdutils/td/utils/Closure.h @@ -94,10 +94,6 @@ class DelayedClosure { using ActorType = ActorT; using Delayed = DelayedClosure; - DelayedClosure clone() const { - return do_clone(*this); - } - explicit DelayedClosure(ImmediateClosure &&other) : args(std::move(other.args)) { } @@ -128,21 +124,6 @@ class DelayedClosure { std::abort(); } - template - std::enable_if_t::value...>::value, - DelayedClosure> - do_clone(const DelayedClosure &value) const { - LOG(FATAL) << "Trying to clone DelayedClosure that contains noncopyable elements"; - std::abort(); - } - - template - std::enable_if_t::value...>::value, - DelayedClosure> - do_clone(const DelayedClosure &value) const { - return DelayedClosure(value); - } - public: auto run(ActorT *actor) -> decltype(mem_call_tuple(actor, std::move(args))) { return mem_call_tuple(actor, std::move(args)); From 2cc8f34a7d3ff03daf7082ca9c903149596ead28 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 17 Jul 2022 11:41:31 +0300 Subject: [PATCH 40/55] Remove unused to_delayed_closure functions. --- tdutils/td/utils/Closure.h | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tdutils/td/utils/Closure.h b/tdutils/td/utils/Closure.h index 6c895c68e..9f97c1538 100644 --- a/tdutils/td/utils/Closure.h +++ b/tdutils/td/utils/Closure.h @@ -55,7 +55,6 @@ // // // create_immediate_closure(&ActorT::func, arg1, arg2, ..., argn).run(actor) -// to_delayed_closure(std::move(immediate)).run(actor) namespace td { template @@ -92,7 +91,6 @@ template class DelayedClosure { public: using ActorType = ActorT; - using Delayed = DelayedClosure; explicit DelayedClosure(ImmediateClosure &&other) : args(std::move(other.args)) { } @@ -130,16 +128,6 @@ class DelayedClosure { } }; -template -typename ImmediateClosure::Delayed to_delayed_closure(ImmediateClosure &&other) { - return typename ImmediateClosure::Delayed(std::move(other)); -} - -template -DelayedClosure to_delayed_closure(DelayedClosure &&other) { - return std::move(other); -} - template auto create_delayed_closure(ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&...args) { return DelayedClosure(func, From 6eef50f769915b88a26c194c276100e54d547556 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 17 Jul 2022 12:11:04 +0300 Subject: [PATCH 41/55] Remove unused DelayedClosure private constructors. --- tdactor/td/actor/impl/Event.h | 1 - tdutils/td/utils/Closure.h | 20 +------------------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/tdactor/td/actor/impl/Event.h b/tdactor/td/actor/impl/Event.h index a0e4d39fb..2796c701e 100644 --- a/tdactor/td/actor/impl/Event.h +++ b/tdactor/td/actor/impl/Event.h @@ -8,7 +8,6 @@ #include "td/utils/Closure.h" #include "td/utils/common.h" -#include "td/utils/logging.h" #include "td/utils/StringBuilder.h" #include diff --git a/tdutils/td/utils/Closure.h b/tdutils/td/utils/Closure.h index 9f97c1538..345f1b1f8 100644 --- a/tdutils/td/utils/Closure.h +++ b/tdutils/td/utils/Closure.h @@ -8,9 +8,7 @@ #include "td/utils/common.h" #include "td/utils/invoke.h" -#include "td/utils/logging.h" -#include #include #include #include @@ -104,23 +102,7 @@ class DelayedClosure { } private: - using ArgsStorageT = std::tuple::type...>; - - ArgsStorageT args; - - template - explicit DelayedClosure(const DelayedClosure &other, - std::enable_if_t::value...>::value, int> = 0) - : args(other.args) { - } - - template - explicit DelayedClosure( - const DelayedClosure &other, - std::enable_if_t::value...>::value, int> = 0) { - LOG(FATAL) << "Deleted constructor"; - std::abort(); - } + std::tuple::type...> args; public: auto run(ActorT *actor) -> decltype(mem_call_tuple(actor, std::move(args))) { From 75902bb8a7296597ea8cefd0b4ebc840e166ba3d Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 17 Jul 2022 13:31:43 +0300 Subject: [PATCH 42/55] Improve logging. --- td/telegram/MessagesManager.cpp | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 5aa255414..a0720032c 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -14799,9 +14799,9 @@ void MessagesManager::set_dialog_last_new_message_id(Dialog *d, MessageId last_n auto last_new_message = get_message(d, last_new_message_id); if (last_new_message != nullptr) { - add_message_to_database(d, last_new_message, "set_dialog_last_new_message_id"); - set_dialog_first_database_message_id(d, last_new_message_id, "set_dialog_last_new_message_id"); - set_dialog_last_database_message_id(d, last_new_message_id, "set_dialog_last_new_message_id"); + add_message_to_database(d, last_new_message, source); + set_dialog_first_database_message_id(d, last_new_message_id, source); + set_dialog_last_database_message_id(d, last_new_message_id, source); try_restore_dialog_reply_markup(d, last_new_message); } } @@ -36198,6 +36198,7 @@ MessagesManager::Dialog *MessagesManager::add_new_dialog(unique_ptr &&d, const char *source) { auto dialog_id = d->dialog_id; LOG_CHECK(is_inited_) << dialog_id << ' ' << is_loaded_from_database << ' ' << source; + LOG_CHECK(!have_dialog(dialog_id)) << dialog_id << ' ' << is_loaded_from_database << ' ' << source; switch (dialog_id.get_type()) { case DialogType::User: if (dialog_id == get_my_dialog_id() && d->last_read_inbox_message_id == MessageId::max() && @@ -36340,6 +36341,11 @@ MessagesManager::Dialog *MessagesManager::add_new_dialog(unique_ptr &&d, being_added_new_dialog_id_ = DialogId(); + LOG_CHECK(dialog->messages == nullptr) << dialog->messages->message_id << ' ' << dialog->last_message_id << ' ' + << dialog->last_database_message_id << ' ' + << dialog->debug_set_dialog_last_database_message_id << ' ' + << dialog->messages->debug_source; + fix_new_dialog(dialog, std::move(last_database_message), last_database_message_id, order, last_clear_history_date, last_clear_history_message_id, default_join_group_call_as_dialog_id, default_send_message_as_dialog_id, need_drop_default_send_message_as_dialog_id, is_loaded_from_database); @@ -36699,13 +36705,16 @@ void MessagesManager::fix_new_dialog(Dialog *d, unique_ptr &&last_datab if (d->messages != nullptr) { LOG_CHECK(d->messages->message_id == last_message_id) - << d->messages->message_id << ' ' << last_message_id << ' ' << d->debug_set_dialog_last_database_message_id - << ' ' << d->messages->debug_source; + << d->messages->message_id << ' ' << last_message_id << ' ' << d->last_message_id << ' ' + << d->last_database_message_id << ' ' << d->debug_set_dialog_last_database_message_id << ' ' + << d->messages->debug_source; LOG_CHECK(d->messages->left == nullptr) - << d->messages->left->message_id << ' ' << d->messages->message_id << ' ' << last_message_id << ' ' + << d->messages->left->message_id << ' ' << d->messages->message_id << ' ' << d->messages->left->message_id + << ' ' << last_message_id << ' ' << d->last_message_id << ' ' << d->last_database_message_id << ' ' << d->debug_set_dialog_last_database_message_id << ' ' << d->messages->debug_source; LOG_CHECK(d->messages->right == nullptr) - << d->messages->right->message_id << ' ' << d->messages->message_id << ' ' << last_message_id << ' ' + << d->messages->right->message_id << ' ' << d->messages->message_id << ' ' << d->messages->right->message_id + << ' ' << last_message_id << ' ' << d->last_message_id << ' ' << d->last_database_message_id << ' ' << d->debug_set_dialog_last_database_message_id << ' ' << d->messages->debug_source; } From c7fe53ecaedd8676f55976976fb6370e12910b32 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 17 Jul 2022 14:02:33 +0300 Subject: [PATCH 43/55] Add additional checks in DownloadManager. --- td/telegram/DownloadManager.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/td/telegram/DownloadManager.cpp b/td/telegram/DownloadManager.cpp index 9912610e0..202a0272a 100644 --- a/td/telegram/DownloadManager.cpp +++ b/td/telegram/DownloadManager.cpp @@ -352,7 +352,7 @@ class DownloadManagerImpl final : public DownloadManager { FileId file_id; FileId internal_file_id; FileSourceId file_source_id; - int8 priority; + int8 priority{}; bool is_paused{}; bool is_counted{}; mutable bool is_registered{}; @@ -567,8 +567,11 @@ class DownloadManagerImpl final : public DownloadManager { << file_info->size << '/' << file_info->expected_size << " with downloaded_size = " << file_info->downloaded_size << " and is_paused = " << file_info->is_paused; - auto it = files_.emplace(download_id, std::move(file_info)).first; + auto res = files_.emplace(download_id, std::move(file_info)); + auto it = res.first; bool was_completed = is_completed(*it->second); + LOG_CHECK(!it->second->is_registered) + << res.second << ' ' << download_id << ' ' << max_download_id_ << ' ' << it->second->need_save_to_database; register_file_info(*it->second); // must be called before start_file, which can call update_file_download_state if (is_completed(*it->second)) { bool is_inserted = completed_download_ids_.insert(it->second->download_id).second; From 0147c97f9c6035819e4df6f28bc18dcd8e00bcf0 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 17 Jul 2022 14:17:22 +0300 Subject: [PATCH 44/55] Ensure that the same notification doesn't added and removed simultaneously. --- td/telegram/NotificationManager.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/td/telegram/NotificationManager.cpp b/td/telegram/NotificationManager.cpp index 85cc37766..d9de73483 100644 --- a/td/telegram/NotificationManager.cpp +++ b/td/telegram/NotificationManager.cpp @@ -980,6 +980,17 @@ void NotificationManager::add_update_notification_group(td_api::object_ptrnotification_settings_chat_id_ == 0) { update->notification_settings_chat_id_ = update->chat_id_; } + if (!update->added_notifications_.empty() && !update->removed_notification_ids_.empty()) { + // just in case + td::remove_if(update->added_notifications_, [&](const td_api::object_ptr ¬ification) { + CHECK(notification != nullptr); + if (td::contains(update->removed_notification_ids_, notification->id_)) { + LOG(ERROR) << "Have the same notification as added and removed"; + return true; + } + return false; + }); + } add_update(group_id, std::move(update)); } From 6898d07793929f098e12d3c7ccbb632d1fbe9df6 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 18 Jul 2022 19:21:47 +0300 Subject: [PATCH 45/55] Make td_api::saveApplicationLogEvent persistent. --- td/generate/scheme/telegram_api.tl | 1 + td/telegram/Application.cpp | 61 +++++++++++++++++++++++++++--- td/telegram/Application.h | 4 ++ td/telegram/Td.cpp | 4 ++ td/telegram/TdDb.cpp | 3 ++ td/telegram/TdDb.h | 1 + td/telegram/logevent/LogEvent.h | 1 + 7 files changed, 70 insertions(+), 5 deletions(-) diff --git a/td/generate/scheme/telegram_api.tl b/td/generate/scheme/telegram_api.tl index 61e68c5f9..5d87305f6 100644 --- a/td/generate/scheme/telegram_api.tl +++ b/td/generate/scheme/telegram_api.tl @@ -26,6 +26,7 @@ inputStickerSetThumbLegacy#dbaeae9 stickerset:InputStickerSet volume_id:long loc test.useError = Error; test.useConfigSimple = help.ConfigSimple; +test.parseInputAppEvent = InputAppEvent; ---types--- diff --git a/td/telegram/Application.cpp b/td/telegram/Application.cpp index 8f4e559d4..f733bf7f1 100644 --- a/td/telegram/Application.cpp +++ b/td/telegram/Application.cpp @@ -7,11 +7,18 @@ #include "td/telegram/Application.h" #include "td/telegram/Global.h" +#include "td/telegram/logevent/LogEvent.h" +#include "td/telegram/logevent/LogEventHelper.h" #include "td/telegram/Td.h" +#include "td/telegram/TdDb.h" + +#include "td/db/binlog/BinlogEvent.h" +#include "td/db/binlog/BinlogHelper.h" #include "td/utils/buffer.h" #include "td/utils/logging.h" #include "td/utils/Status.h" +#include "td/utils/tl_parsers.h" namespace td { @@ -48,11 +55,9 @@ class SaveAppLogQuery final : public Td::ResultHandler { explicit SaveAppLogQuery(Promise &&promise) : promise_(std::move(promise)) { } - void send(const string &type, DialogId dialog_id, tl_object_ptr &&data) { - CHECK(data != nullptr); + void send(telegram_api::object_ptr &&input_app_event) { vector> input_app_events; - input_app_events.push_back( - make_tl_object(G()->server_time_cached(), type, dialog_id.get(), std::move(data))); + input_app_events.push_back(std::move(input_app_event)); send_query(G()->net_query_creator().create_unauth(telegram_api::help_saveAppLog(std::move(input_app_events)))); } @@ -76,9 +81,55 @@ void get_invite_text(Td *td, Promise &&promise) { td->create_handler(std::move(promise))->send(); } +class SaveAppLogLogEvent { + public: + const telegram_api::inputAppEvent *input_app_event_in_ = nullptr; + telegram_api::object_ptr input_app_event_out_; + + template + void store(StorerT &storer) const { + input_app_event_in_->store(storer); + } + + template + void parse(ParserT &parser) { + auto buffer = parser.fetch_string_raw(parser.get_left_len()); + TlBufferParser buffer_parser{&buffer}; + input_app_event_out_ = telegram_api::make_object(buffer_parser); + } +}; + +static void save_app_log_impl(Td *td, telegram_api::object_ptr input_app_event, + uint64 log_event_id, Promise &&promise) { + if (log_event_id == 0) { + SaveAppLogLogEvent log_event; + log_event.input_app_event_in_ = input_app_event.get(); + log_event_id = + binlog_add(G()->td_db()->get_binlog(), LogEvent::HandlerType::SaveAppLog, get_log_event_storer(log_event)); + } + + td->create_handler(get_erase_log_event_promise(log_event_id, std::move(promise))) + ->send(std::move(input_app_event)); +} + void save_app_log(Td *td, const string &type, DialogId dialog_id, tl_object_ptr &&data, Promise &&promise) { - td->create_handler(std::move(promise))->send(type, dialog_id, std::move(data)); + CHECK(data != nullptr); + auto input_app_event = telegram_api::make_object(G()->server_time_cached(), type, + dialog_id.get(), std::move(data)); + save_app_log_impl(td, std::move(input_app_event), 0, std::move(promise)); +} + +void on_save_app_log_binlog_event(Td *td, BinlogEvent &&event) { + if (G()->close_flag()) { + return; + } + CHECK(event.id_ != 0); + CHECK(event.type_ == LogEvent::HandlerType::SaveAppLog); + SaveAppLogLogEvent log_event; + log_event_parse(log_event, event.data_).ensure(); + + save_app_log_impl(td, std::move(log_event.input_app_event_out_), event.id_, Promise()); } } // namespace td diff --git a/td/telegram/Application.h b/td/telegram/Application.h index 4ca4ca7ac..e6d792689 100644 --- a/td/telegram/Application.h +++ b/td/telegram/Application.h @@ -14,6 +14,8 @@ namespace td { +struct BinlogEvent; + class Td; void get_invite_text(Td *td, Promise &&promise); @@ -21,4 +23,6 @@ void get_invite_text(Td *td, Promise &&promise); void save_app_log(Td *td, const string &type, DialogId dialog_id, tl_object_ptr &&data, Promise &&promise); +void on_save_app_log_binlog_event(Td *td, BinlogEvent &&event); + } // namespace td diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index c6216e1bf..a1e06e94b 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -3801,6 +3801,10 @@ void Td::init(Result r_opened_database) { web_pages_manager_->on_binlog_web_page_event(std::move(event)); } + for (auto &event : events.save_app_log_events) { + on_save_app_log_binlog_event(this, std::move(event)); + } + if (is_online_) { on_online_updated(true, true); } diff --git a/td/telegram/TdDb.cpp b/td/telegram/TdDb.cpp index 948526d22..96eb70ab5 100644 --- a/td/telegram/TdDb.cpp +++ b/td/telegram/TdDb.cpp @@ -117,6 +117,9 @@ Status init_binlog(Binlog &binlog, string path, BinlogKeyValue &binlog_p case LogEvent::HandlerType::EditMessagePushNotification: events.to_notification_manager.push_back(event.clone()); break; + case LogEvent::HandlerType::SaveAppLog: + events.save_app_log_events.push_back(event.clone()); + break; case LogEvent::HandlerType::BinlogPmcMagic: binlog_pmc.external_init_handle(event); break; diff --git a/td/telegram/TdDb.h b/td/telegram/TdDb.h index 14beacfa3..f38fb7c42 100644 --- a/td/telegram/TdDb.h +++ b/td/telegram/TdDb.h @@ -63,6 +63,7 @@ class TdDb { vector channel_events; vector secret_chat_events; vector web_page_events; + vector save_app_log_events; vector to_poll_manager; vector to_messages_manager; vector to_notification_manager; diff --git a/td/telegram/logevent/LogEvent.h b/td/telegram/logevent/LogEvent.h index d0f0a80d6..2a4fee9b0 100644 --- a/td/telegram/logevent/LogEvent.h +++ b/td/telegram/logevent/LogEvent.h @@ -104,6 +104,7 @@ class LogEvent { GetChannelDifference = 0x140, AddMessagePushNotification = 0x200, EditMessagePushNotification = 0x201, + SaveAppLog = 0x300, ConfigPmcMagic = 0x1f18, BinlogPmcMagic = 0x4327 }; From 87a511139935de357d8c5d0ad8ba95f0e8b1637d Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 19 Jul 2022 12:40:34 +0300 Subject: [PATCH 46/55] Fix compilation error. --- td/telegram/Application.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/td/telegram/Application.cpp b/td/telegram/Application.cpp index f733bf7f1..ca0ac61a2 100644 --- a/td/telegram/Application.cpp +++ b/td/telegram/Application.cpp @@ -93,7 +93,7 @@ class SaveAppLogLogEvent { template void parse(ParserT &parser) { - auto buffer = parser.fetch_string_raw(parser.get_left_len()); + auto buffer = parser.template fetch_string_raw(parser.get_left_len()); TlBufferParser buffer_parser{&buffer}; input_app_event_out_ = telegram_api::make_object(buffer_parser); } From e8b0983d7b287e48681389f6ddba27ced8e2b058 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 20 Jul 2022 10:28:31 +0300 Subject: [PATCH 47/55] Use std::move to restore ActorContext. --- td/telegram/Client.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/td/telegram/Client.cpp b/td/telegram/Client.cpp index b6d9fea60..101f54c23 100644 --- a/td/telegram/Client.cpp +++ b/td/telegram/Client.cpp @@ -249,8 +249,8 @@ class MultiTd final : public Actor { auto old_context = set_context(context); auto old_tag = set_tag(to_string(td_id)); td = create_actor("Td", std::move(callback), options_); - set_context(old_context); - set_tag(old_tag); + set_context(std::move(old_context)); + set_tag(std::move(old_tag)); } void send(ClientManager::ClientId client_id, ClientManager::RequestId request_id, From 061c4f65d559ae217b7eb444c2b9bf3e62bfd0e6 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 20 Jul 2022 10:56:48 +0300 Subject: [PATCH 48/55] Drop context for SemaphoreActor before creation of the actor. --- td/telegram/net/Session.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/td/telegram/net/Session.cpp b/td/telegram/net/Session.cpp index 9693bfb19..c04ce0f22 100644 --- a/td/telegram/net/Session.cpp +++ b/td/telegram/net/Session.cpp @@ -69,11 +69,6 @@ class SemaphoreActor final : public Actor { size_t capacity_; VectorQueue>> pending_; - void start_up() final { - set_context(std::make_shared()); - set_tag(string()); - } - void finish(Result) { capacity_++; if (!pending_.empty()) { @@ -137,7 +132,11 @@ class GenAuthKeyActor final : public Actor { static TD_THREAD_LOCAL Semaphore *semaphore_; Semaphore &get_handshake_semaphore() { + auto old_context = set_context(std::make_shared()); + auto old_tag = set_tag(string()); init_thread_local(semaphore_, 50); + set_context(std::move(old_context)); + set_tag(std::move(old_tag)); return *semaphore_; } From ac8af37872a4551a3960bfc4a48d1d6e351e0bff Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 20 Jul 2022 13:35:32 +0300 Subject: [PATCH 49/55] Add size() and empty() to WaitFreeHashMap. --- tdutils/td/utils/WaitFreeHashMap.h | 25 +++++++++++++++++++++++++ tdutils/test/WaitFreeHashMap.cpp | 6 ++++++ 2 files changed, 31 insertions(+) diff --git a/tdutils/td/utils/WaitFreeHashMap.h b/tdutils/td/utils/WaitFreeHashMap.h index 0d32e0bd5..14f6af8d3 100644 --- a/tdutils/td/utils/WaitFreeHashMap.h +++ b/tdutils/td/utils/WaitFreeHashMap.h @@ -61,6 +61,31 @@ class WaitFreeHashMap { size_t erase(const KeyT &key) { return get_storage(key).erase(key); } + + size_t size() const { + if (wait_free_storage_ == nullptr) { + return default_map_.size(); + } + + size_t result = 0; + for (size_t i = 0; i < MAX_STORAGE_COUNT; i++) { + result += wait_free_storage_->maps_[i].size(); + } + return result; + } + + bool empty() const { + if (wait_free_storage_ == nullptr) { + return default_map_.empty(); + } + + for (size_t i = 0; i < MAX_STORAGE_COUNT; i++) { + if (!wait_free_storage_->maps_[i].empty()) { + return false; + } + } + return true; + } }; } // namespace td diff --git a/tdutils/test/WaitFreeHashMap.cpp b/tdutils/test/WaitFreeHashMap.cpp index 8865d5ffd..8c1b0c1db 100644 --- a/tdutils/test/WaitFreeHashMap.cpp +++ b/tdutils/test/WaitFreeHashMap.cpp @@ -30,6 +30,8 @@ TEST(WaitFreeHashMap, stress_test) { reference[key] = value; map.set(key, value); ASSERT_EQ(reference[key], map.get(key)); + ASSERT_EQ(reference.size(), map.size()); + ASSERT_EQ(reference.empty(), map.empty()); }); add_step(2000, [&] { @@ -37,6 +39,8 @@ TEST(WaitFreeHashMap, stress_test) { auto ref_it = reference.find(key); auto ref_value = ref_it == reference.end() ? 0 : ref_it->second; ASSERT_EQ(ref_value, map.get(key)); + ASSERT_EQ(reference.size(), map.size()); + ASSERT_EQ(reference.empty(), map.empty()); }); add_step(500, [&] { @@ -44,6 +48,8 @@ TEST(WaitFreeHashMap, stress_test) { size_t reference_erased_count = reference.erase(key); size_t map_erased_count = map.erase(key); ASSERT_EQ(reference_erased_count, map_erased_count); + ASSERT_EQ(reference.size(), map.size()); + ASSERT_EQ(reference.empty(), map.empty()); }); td::RandomSteps runner(std::move(steps)); From 0f87447ffcdabca63f572ece90d1fcfd91e3445b Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 20 Jul 2022 13:40:14 +0300 Subject: [PATCH 50/55] Asynchronously destroy some big data storages. --- td/telegram/AnimationsManager.cpp | 4 +++ td/telegram/AnimationsManager.h | 5 ++++ td/telegram/AudiosManager.cpp | 4 +++ td/telegram/AudiosManager.h | 5 ++++ td/telegram/ContactsManager.cpp | 40 +++++++++++++++++++++++++- td/telegram/DocumentsManager.cpp | 4 +++ td/telegram/DocumentsManager.h | 5 ++++ td/telegram/FileReferenceManager.cpp | 4 +++ td/telegram/FileReferenceManager.h | 7 +++++ td/telegram/MessagesManager.cpp | 37 +++++++++++++++++++----- td/telegram/MessagesManager.h | 2 +- td/telegram/PollManager.cpp | 8 +++++- td/telegram/SecretChatsManager.h | 2 +- td/telegram/StickersManager.cpp | 15 ++++++++++ td/telegram/StickersManager.h | 5 ++++ td/telegram/VideoNotesManager.cpp | 4 +++ td/telegram/VideoNotesManager.h | 5 ++++ td/telegram/VideosManager.cpp | 4 +++ td/telegram/VideosManager.h | 5 ++++ td/telegram/VoiceNotesManager.cpp | 6 ++++ td/telegram/VoiceNotesManager.h | 5 ++++ td/telegram/WebPagesManager.cpp | 8 +++++- td/telegram/files/FileManager.cpp | 8 ++++++ tdactor/td/actor/impl/Scheduler-decl.h | 3 ++ tdactor/td/actor/impl/Scheduler.h | 24 +++++++++++++++- tdutils/td/utils/Enumerator.h | 4 +++ 26 files changed, 210 insertions(+), 13 deletions(-) diff --git a/td/telegram/AnimationsManager.cpp b/td/telegram/AnimationsManager.cpp index 253da4671..78dd3856f 100644 --- a/td/telegram/AnimationsManager.cpp +++ b/td/telegram/AnimationsManager.cpp @@ -139,6 +139,10 @@ AnimationsManager::AnimationsManager(Td *td, ActorShared<> parent) : td_(td), pa next_saved_animations_load_time_ = Time::now(); } +AnimationsManager::~AnimationsManager() { + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), animations_); +} + void AnimationsManager::tear_down() { parent_.reset(); } diff --git a/td/telegram/AnimationsManager.h b/td/telegram/AnimationsManager.h index 17c5f9559..41f25b218 100644 --- a/td/telegram/AnimationsManager.h +++ b/td/telegram/AnimationsManager.h @@ -29,6 +29,11 @@ class Td; class AnimationsManager final : public Actor { public: AnimationsManager(Td *td, ActorShared<> parent); + AnimationsManager(const AnimationsManager &) = delete; + AnimationsManager &operator=(const AnimationsManager &) = delete; + AnimationsManager(AnimationsManager &&) = delete; + AnimationsManager &operator=(AnimationsManager &&) = delete; + ~AnimationsManager() final; int32 get_animation_duration(FileId file_id) const; diff --git a/td/telegram/AudiosManager.cpp b/td/telegram/AudiosManager.cpp index bb689279c..9e097d46b 100644 --- a/td/telegram/AudiosManager.cpp +++ b/td/telegram/AudiosManager.cpp @@ -24,6 +24,10 @@ namespace td { AudiosManager::AudiosManager(Td *td) : td_(td) { } +AudiosManager::~AudiosManager() { + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), audios_); +} + int32 AudiosManager::get_audio_duration(FileId file_id) const { auto it = audios_.find(file_id); if (it == audios_.end()) { diff --git a/td/telegram/AudiosManager.h b/td/telegram/AudiosManager.h index 53efcdeda..27b5ca664 100644 --- a/td/telegram/AudiosManager.h +++ b/td/telegram/AudiosManager.h @@ -23,6 +23,11 @@ class Td; class AudiosManager { public: explicit AudiosManager(Td *td); + AudiosManager(const AudiosManager &) = delete; + AudiosManager &operator=(const AudiosManager &) = delete; + AudiosManager(AudiosManager &&) = delete; + AudiosManager &operator=(AudiosManager &&) = delete; + ~AudiosManager(); int32 get_audio_duration(FileId file_id) const; diff --git a/td/telegram/ContactsManager.cpp b/td/telegram/ContactsManager.cpp index 18e37079a..12b14ae82 100644 --- a/td/telegram/ContactsManager.cpp +++ b/td/telegram/ContactsManager.cpp @@ -3397,7 +3397,45 @@ ContactsManager::ContactsManager(Td *td, ActorShared<> parent) : td_(td), parent channel_participant_cache_timeout_.set_callback_data(static_cast(this)); } -ContactsManager::~ContactsManager() = default; +ContactsManager::~ContactsManager() { + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), users_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), users_full_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), user_photos_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), unknown_users_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), pending_user_photos_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), user_profile_photo_file_source_ids_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), my_photo_file_id_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), chats_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), chats_full_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), unknown_chats_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), chat_full_file_source_ids_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), min_channels_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), channels_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), channels_full_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), unknown_channels_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), invalidated_channels_full_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), channel_full_file_source_ids_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), secret_chats_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), unknown_secret_chats_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), secret_chats_with_user_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), invite_link_infos_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), dialog_access_by_invite_link_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), loaded_from_database_users_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), unavailable_user_fulls_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), loaded_from_database_chats_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), unavailable_chat_fulls_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), loaded_from_database_channels_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), unavailable_channel_fulls_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), loaded_from_database_secret_chats_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), dialog_administrators_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), cached_channel_participants_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), resolved_phone_numbers_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), channel_participants_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), all_imported_contacts_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), linked_channel_ids_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), restricted_user_ids_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), restricted_channel_ids_); +} void ContactsManager::tear_down() { parent_.reset(); diff --git a/td/telegram/DocumentsManager.cpp b/td/telegram/DocumentsManager.cpp index 7213bdb98..f9e729683 100644 --- a/td/telegram/DocumentsManager.cpp +++ b/td/telegram/DocumentsManager.cpp @@ -48,6 +48,10 @@ namespace td { DocumentsManager::DocumentsManager(Td *td) : td_(td) { } +DocumentsManager::~DocumentsManager() { + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), documents_); +} + tl_object_ptr DocumentsManager::get_document_object(FileId file_id, PhotoFormat thumbnail_format) const { if (!file_id.is_valid()) { diff --git a/td/telegram/DocumentsManager.h b/td/telegram/DocumentsManager.h index 28716cca7..fd692dc30 100644 --- a/td/telegram/DocumentsManager.h +++ b/td/telegram/DocumentsManager.h @@ -31,6 +31,11 @@ class Td; class DocumentsManager { public: explicit DocumentsManager(Td *td); + DocumentsManager(const DocumentsManager &) = delete; + DocumentsManager &operator=(const DocumentsManager &) = delete; + DocumentsManager(DocumentsManager &&) = delete; + DocumentsManager &operator=(DocumentsManager &&) = delete; + ~DocumentsManager(); class RemoteDocument { public: diff --git a/td/telegram/FileReferenceManager.cpp b/td/telegram/FileReferenceManager.cpp index 8e409fa05..f757e6c78 100644 --- a/td/telegram/FileReferenceManager.cpp +++ b/td/telegram/FileReferenceManager.cpp @@ -32,6 +32,10 @@ namespace td { int VERBOSITY_NAME(file_references) = VERBOSITY_NAME(INFO); +FileReferenceManager::~FileReferenceManager() { + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), file_sources_); +} + bool FileReferenceManager::is_file_reference_error(const Status &error) { return error.is_error() && error.code() == 400 && begins_with(error.message(), "FILE_REFERENCE_"); } diff --git a/td/telegram/FileReferenceManager.h b/td/telegram/FileReferenceManager.h index 340040db6..944f7e173 100644 --- a/td/telegram/FileReferenceManager.h +++ b/td/telegram/FileReferenceManager.h @@ -35,6 +35,13 @@ extern int VERBOSITY_NAME(file_references); class FileReferenceManager final : public Actor { public: + FileReferenceManager() = default; + FileReferenceManager(const FileReferenceManager &) = delete; + FileReferenceManager &operator=(const FileReferenceManager &) = delete; + FileReferenceManager(FileReferenceManager &&) = delete; + FileReferenceManager &operator=(FileReferenceManager &&) = delete; + ~FileReferenceManager() final; + static bool is_file_reference_error(const Status &error); static size_t get_file_reference_error_pos(const Status &error); diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index a0720032c..883aa5b8f 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -5754,12 +5754,6 @@ void MessagesManager::save_calls_db_state() { G()->td_db()->get_sqlite_pmc()->set("calls_db_state", log_event_store(calls_db_state_).as_slice().str(), Auto()); } -MessagesManager::Dialog::~Dialog() { - if (!G()->close_flag()) { - LOG(ERROR) << "Destroy " << dialog_id; - } -} - MessagesManager::MessagesManager(Td *td, ActorShared<> parent) : recently_found_dialogs_{td, "recently_found", MAX_RECENT_DIALOGS} , recently_opened_dialogs_{td, "recently_opened", MAX_RECENT_DIALOGS} @@ -5814,7 +5808,36 @@ MessagesManager::MessagesManager(Td *td, ActorShared<> parent) update_viewed_messages_timeout_.set_callback_data(static_cast(this)); } -MessagesManager::~MessagesManager() = default; +MessagesManager::~MessagesManager() { + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), ttl_nodes_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), ttl_heap_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), being_sent_messages_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), update_message_ids_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), update_scheduled_message_ids_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), message_id_to_dialog_id_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), last_clear_history_message_id_to_dialog_id_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), dialogs_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), postponed_chat_read_inbox_updates_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), found_public_dialogs_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), found_on_server_dialogs_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), found_common_dialogs_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), message_embedding_codes_[0]); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), message_embedding_codes_[1]); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), replied_by_media_timestamp_messages_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), notification_group_id_to_dialog_id_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), active_get_channel_differencies_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), get_channel_difference_to_log_event_id_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), channel_get_difference_retry_timeouts_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), is_channel_difference_finished_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), resolved_usernames_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), inaccessible_resolved_usernames_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), dialog_bot_command_message_ids_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), full_message_id_to_file_source_id_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), last_outgoing_forwarded_message_date_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), dialog_viewed_messages_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), dialog_online_member_counts_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), previous_repaired_read_inbox_max_message_id_); +} void MessagesManager::on_channel_get_difference_timeout_callback(void *messages_manager_ptr, int64 dialog_id_int) { if (G()->close_flag()) { diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index ef26e0ce7..8ccf8c58d 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -1431,7 +1431,7 @@ class MessagesManager final : public Actor { Dialog &operator=(const Dialog &) = delete; Dialog(Dialog &&other) = delete; Dialog &operator=(Dialog &&other) = delete; - ~Dialog(); + ~Dialog() = default; template void store(StorerT &storer) const; diff --git a/td/telegram/PollManager.cpp b/td/telegram/PollManager.cpp index c62aacc7c..341d8c613 100644 --- a/td/telegram/PollManager.cpp +++ b/td/telegram/PollManager.cpp @@ -265,7 +265,13 @@ void PollManager::tear_down() { parent_.reset(); } -PollManager::~PollManager() = default; +PollManager::~PollManager() { + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), polls_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), server_poll_messages_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), other_poll_messages_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), poll_voters_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), loaded_from_database_polls_); +} void PollManager::on_update_poll_timeout_callback(void *poll_manager_ptr, int64 poll_id_int) { if (G()->close_flag()) { diff --git a/td/telegram/SecretChatsManager.h b/td/telegram/SecretChatsManager.h index cacd5263c..554456ebc 100644 --- a/td/telegram/SecretChatsManager.h +++ b/td/telegram/SecretChatsManager.h @@ -30,7 +30,7 @@ class SecretChatsManager final : public Actor { public: explicit SecretChatsManager(ActorShared<> parent); - // proxy query to corrensponding SecretChatActor + // proxy query to corresponding SecretChatActor void on_update_chat(tl_object_ptr update); void on_new_message(tl_object_ptr &&message_ptr, Promise &&promise); diff --git a/td/telegram/StickersManager.cpp b/td/telegram/StickersManager.cpp index 278b280b6..820e15a74 100644 --- a/td/telegram/StickersManager.cpp +++ b/td/telegram/StickersManager.cpp @@ -1304,6 +1304,21 @@ StickersManager::StickersManager(Td *td, ActorShared<> parent) : td_(td), parent next_update_animated_emoji_clicked_time_ = Time::now(); } +StickersManager::~StickersManager() { + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), stickers_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), sticker_sets_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), short_name_to_sticker_set_id_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), attached_sticker_sets_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), found_stickers_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), found_sticker_sets_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), emoji_language_codes_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), emoji_language_code_versions_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), emoji_language_code_last_difference_times_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), reloaded_emoji_keywords_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), dice_messages_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), emoji_messages_); +} + void StickersManager::start_up() { init(); } diff --git a/td/telegram/StickersManager.h b/td/telegram/StickersManager.h index 1b2058fa6..b8094b19d 100644 --- a/td/telegram/StickersManager.h +++ b/td/telegram/StickersManager.h @@ -50,6 +50,11 @@ class StickersManager final : public Actor { static vector convert_sticker_set_ids(const vector &sticker_set_ids); StickersManager(Td *td, ActorShared<> parent); + StickersManager(const StickersManager &) = delete; + StickersManager &operator=(const StickersManager &) = delete; + StickersManager(StickersManager &&) = delete; + StickersManager &operator=(StickersManager &&) = delete; + ~StickersManager() final; void init(); diff --git a/td/telegram/VideoNotesManager.cpp b/td/telegram/VideoNotesManager.cpp index a0b654206..f6b46dfbe 100644 --- a/td/telegram/VideoNotesManager.cpp +++ b/td/telegram/VideoNotesManager.cpp @@ -22,6 +22,10 @@ namespace td { VideoNotesManager::VideoNotesManager(Td *td) : td_(td) { } +VideoNotesManager::~VideoNotesManager() { + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), video_notes_); +} + int32 VideoNotesManager::get_video_note_duration(FileId file_id) const { auto it = video_notes_.find(file_id); CHECK(it != video_notes_.end()); diff --git a/td/telegram/VideoNotesManager.h b/td/telegram/VideoNotesManager.h index 850a5e594..4c1f01659 100644 --- a/td/telegram/VideoNotesManager.h +++ b/td/telegram/VideoNotesManager.h @@ -24,6 +24,11 @@ class Td; class VideoNotesManager { public: explicit VideoNotesManager(Td *td); + VideoNotesManager(const VideoNotesManager &) = delete; + VideoNotesManager &operator=(const VideoNotesManager &) = delete; + VideoNotesManager(VideoNotesManager &&) = delete; + VideoNotesManager &operator=(VideoNotesManager &&) = delete; + ~VideoNotesManager(); int32 get_video_note_duration(FileId file_id) const; diff --git a/td/telegram/VideosManager.cpp b/td/telegram/VideosManager.cpp index 782f76467..06c2b3a1b 100644 --- a/td/telegram/VideosManager.cpp +++ b/td/telegram/VideosManager.cpp @@ -23,6 +23,10 @@ namespace td { VideosManager::VideosManager(Td *td) : td_(td) { } +VideosManager::~VideosManager() { + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), videos_); +} + int32 VideosManager::get_video_duration(FileId file_id) const { auto it = videos_.find(file_id); CHECK(it != videos_.end()); diff --git a/td/telegram/VideosManager.h b/td/telegram/VideosManager.h index bc2f5fa99..be051c566 100644 --- a/td/telegram/VideosManager.h +++ b/td/telegram/VideosManager.h @@ -24,6 +24,11 @@ class Td; class VideosManager { public: explicit VideosManager(Td *td); + VideosManager(const VideosManager &) = delete; + VideosManager &operator=(const VideosManager &) = delete; + VideosManager(VideosManager &&) = delete; + VideosManager &operator=(VideosManager &&) = delete; + ~VideosManager(); int32 get_video_duration(FileId file_id) const; diff --git a/td/telegram/VoiceNotesManager.cpp b/td/telegram/VoiceNotesManager.cpp index bfa8869fe..593e0b3e0 100644 --- a/td/telegram/VoiceNotesManager.cpp +++ b/td/telegram/VoiceNotesManager.cpp @@ -100,6 +100,12 @@ VoiceNotesManager::VoiceNotesManager(Td *td, ActorShared<> parent) : td_(td), pa voice_note_transcription_timeout_.set_callback_data(static_cast(this)); } +VoiceNotesManager::~VoiceNotesManager() { + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), voice_notes_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), voice_note_messages_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), message_voice_notes_); +} + void VoiceNotesManager::tear_down() { parent_.reset(); } diff --git a/td/telegram/VoiceNotesManager.h b/td/telegram/VoiceNotesManager.h index 737e8e4cd..722005f57 100644 --- a/td/telegram/VoiceNotesManager.h +++ b/td/telegram/VoiceNotesManager.h @@ -28,6 +28,11 @@ class Td; class VoiceNotesManager final : public Actor { public: VoiceNotesManager(Td *td, ActorShared<> parent); + VoiceNotesManager(const VoiceNotesManager &) = delete; + VoiceNotesManager &operator=(const VoiceNotesManager &) = delete; + VoiceNotesManager(VoiceNotesManager &&) = delete; + VoiceNotesManager &operator=(VoiceNotesManager &&) = delete; + ~VoiceNotesManager() final; int32 get_voice_note_duration(FileId file_id) const; diff --git a/td/telegram/WebPagesManager.cpp b/td/telegram/WebPagesManager.cpp index d1fbf56b2..28dda3bce 100644 --- a/td/telegram/WebPagesManager.cpp +++ b/td/telegram/WebPagesManager.cpp @@ -417,7 +417,13 @@ void WebPagesManager::tear_down() { LOG(DEBUG) << "Have " << web_pages_.size() << " web pages to free"; } -WebPagesManager::~WebPagesManager() = default; +WebPagesManager::~WebPagesManager() { + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), web_pages_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), web_page_messages_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), got_web_page_previews_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), url_to_web_page_id_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), url_to_file_source_id_); +} WebPageId WebPagesManager::on_get_web_page(tl_object_ptr &&web_page_ptr, DialogId owner_dialog_id) { diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index eb8ac4700..eb0dc4191 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -840,6 +840,14 @@ void FileManager::init_actor() { } FileManager::~FileManager() { + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), remote_location_info_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), file_hash_to_file_id_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), local_location_to_file_id_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), generate_location_to_file_id_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), pmc_id_to_file_node_id_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), file_id_info_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), empty_file_ids_); + Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), file_nodes_); } string FileManager::fix_file_extension(Slice file_name, Slice file_type, Slice file_extension) { diff --git a/tdactor/td/actor/impl/Scheduler-decl.h b/tdactor/td/actor/impl/Scheduler-decl.h index 72893e4e8..6980ac609 100644 --- a/tdactor/td/actor/impl/Scheduler-decl.h +++ b/tdactor/td/actor/impl/Scheduler-decl.h @@ -102,6 +102,9 @@ class Scheduler { void run_on_scheduler(int32 sched_id, Promise action); // TODO Action + template + void destroy_on_scheduler(int32 sched_id, T &value); + template void send_lambda(ActorRef actor_ref, EventT &&lambda); diff --git a/tdactor/td/actor/impl/Scheduler.h b/tdactor/td/actor/impl/Scheduler.h index 3cdae11fe..8ead97efe 100644 --- a/tdactor/td/actor/impl/Scheduler.h +++ b/tdactor/td/actor/impl/Scheduler.h @@ -161,7 +161,7 @@ void Scheduler::flush_mailbox(ActorInfo *actor_info, const RunFuncT &run_func, c mailbox.erase(mailbox.begin(), mailbox.begin() + i); } -inline void Scheduler::send_to_scheduler(int32 sched_id, const ActorId<> &actor_id, Event &&event) { +inline void Scheduler::send_to_scheduler(int32 sched_id, const ActorId &actor_id, Event &&event) { if (sched_id == sched_id_) { ActorInfo *actor_info = actor_id.get_actor_info(); pending_events_[actor_info].push_back(std::move(event)); @@ -170,6 +170,28 @@ inline void Scheduler::send_to_scheduler(int32 sched_id, const ActorId<> &actor_ } } +template +inline void Scheduler::destroy_on_scheduler(int32 sched_id, T &value) { + if (sched_id < 0 || sched_id_ == sched_id || value.empty()) { + return; + } + + auto empty_context = std::make_shared(); + empty_context->this_ptr_ = empty_context; + ActorContext *current_context = context_; + context_ = empty_context.get(); + + const char *current_tag = LOG_TAG; + LOG_TAG = nullptr; + + run_on_scheduler(sched_id, PromiseCreator::lambda([value = std::move(value)](Unit) { + // destroy value + })); + + context_ = current_context; + LOG_TAG = current_tag; +} + inline void Scheduler::before_tail_send(const ActorId<> &actor_id) { // TODO } diff --git a/tdutils/td/utils/Enumerator.h b/tdutils/td/utils/Enumerator.h index e89fd6718..1b073ca5d 100644 --- a/tdutils/td/utils/Enumerator.h +++ b/tdutils/td/utils/Enumerator.h @@ -42,6 +42,10 @@ class Enumerator { return arr_.size(); } + bool empty() const { + return size() == 0; + } + private: std::map map_; std::vector arr_; From 51513f1780241f63faede6844b1a80e2d26dd540 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 20 Jul 2022 13:57:05 +0300 Subject: [PATCH 51/55] Add non-template Scheduler::destroy_on_scheduler_impl. --- tdactor/td/actor/impl/Scheduler-decl.h | 2 ++ tdactor/td/actor/impl/Scheduler.cpp | 15 +++++++++++++++ tdactor/td/actor/impl/Scheduler.h | 21 ++++----------------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/tdactor/td/actor/impl/Scheduler-decl.h b/tdactor/td/actor/impl/Scheduler-decl.h index 6980ac609..57f49dbb4 100644 --- a/tdactor/td/actor/impl/Scheduler-decl.h +++ b/tdactor/td/actor/impl/Scheduler-decl.h @@ -153,6 +153,8 @@ class Scheduler { private: static void set_scheduler(Scheduler *scheduler); + void destroy_on_scheduler_impl(int32 sched_id, Promise action); + class ServiceActor final : public Actor { public: void set_queue(std::shared_ptr> queues); diff --git a/tdactor/td/actor/impl/Scheduler.cpp b/tdactor/td/actor/impl/Scheduler.cpp index b29b5b34d..9edcaff40 100644 --- a/tdactor/td/actor/impl/Scheduler.cpp +++ b/tdactor/td/actor/impl/Scheduler.cpp @@ -362,6 +362,21 @@ void Scheduler::run_on_scheduler(int32 sched_id, Promise action) { action.set_value(Unit()); } +void Scheduler::destroy_on_scheduler_impl(int32 sched_id, Promise action) { + auto empty_context = std::make_shared(); + empty_context->this_ptr_ = empty_context; + ActorContext *current_context = context_; + context_ = empty_context.get(); + + const char *current_tag = LOG_TAG; + LOG_TAG = nullptr; + + run_on_scheduler(sched_id, std::move(action)); + + context_ = current_context; + LOG_TAG = current_tag; +} + void Scheduler::add_to_mailbox(ActorInfo *actor_info, Event &&event) { if (!actor_info->is_running()) { auto node = actor_info->get_list_node(); diff --git a/tdactor/td/actor/impl/Scheduler.h b/tdactor/td/actor/impl/Scheduler.h index 8ead97efe..182066145 100644 --- a/tdactor/td/actor/impl/Scheduler.h +++ b/tdactor/td/actor/impl/Scheduler.h @@ -172,24 +172,11 @@ inline void Scheduler::send_to_scheduler(int32 sched_id, const ActorId &a template inline void Scheduler::destroy_on_scheduler(int32 sched_id, T &value) { - if (sched_id < 0 || sched_id_ == sched_id || value.empty()) { - return; + if (!value.empty()) { + destroy_on_scheduler_impl(sched_id, PromiseCreator::lambda([value = std::move(value)](Unit) { + // destroy value + })); } - - auto empty_context = std::make_shared(); - empty_context->this_ptr_ = empty_context; - ActorContext *current_context = context_; - context_ = empty_context.get(); - - const char *current_tag = LOG_TAG; - LOG_TAG = nullptr; - - run_on_scheduler(sched_id, PromiseCreator::lambda([value = std::move(value)](Unit) { - // destroy value - })); - - context_ = current_context; - LOG_TAG = current_tag; } inline void Scheduler::before_tail_send(const ActorId<> &actor_id) { From 4ac8be8122848323d1670b857d5bc2bc78dbc6c1 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 20 Jul 2022 16:46:55 +0300 Subject: [PATCH 52/55] Check local location of files when adding them to download manager. --- td/telegram/DownloadManagerCallback.cpp | 3 ++- td/telegram/files/FileManager.cpp | 7 +++++++ td/telegram/files/FileManager.h | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/td/telegram/DownloadManagerCallback.cpp b/td/telegram/DownloadManagerCallback.cpp index 17dd799c5..b41f119c4 100644 --- a/td/telegram/DownloadManagerCallback.cpp +++ b/td/telegram/DownloadManagerCallback.cpp @@ -69,7 +69,8 @@ FileView DownloadManagerCallback::get_file_view(FileId file_id) { } FileView DownloadManagerCallback::get_sync_file_view(FileId file_id) { - return td_->file_manager_->get_sync_file_view(file_id); + td_->file_manager_->check_local_location(file_id); + return get_file_view(file_id); } td_api::object_ptr DownloadManagerCallback::get_file_download_object( diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index eb0dc4191..b80146d56 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -1016,6 +1016,13 @@ static Status check_partial_local_location(const PartialLocalFileLocation &locat return Status::OK(); } +void FileManager::check_local_location(FileId file_id) { + auto node = get_sync_file_node(file_id); + if (node) { + check_local_location(node).ignore(); + } +} + Status FileManager::check_local_location(FileNodePtr node) { Status status; if (node->local_.type() == LocalFileLocation::Type::Full) { diff --git a/td/telegram/files/FileManager.h b/td/telegram/files/FileManager.h index b03f239f6..e0ac997f7 100644 --- a/td/telegram/files/FileManager.h +++ b/td/telegram/files/FileManager.h @@ -447,6 +447,8 @@ class FileManager final : public FileLoadManager::Callback { bool set_encryption_key(FileId file_id, FileEncryptionKey key); bool set_content(FileId file_id, BufferSlice bytes); + void check_local_location(FileId file_id); + void download(FileId file_id, std::shared_ptr callback, int32 new_priority, int64 offset, int64 limit); void upload(FileId file_id, std::shared_ptr callback, int32 new_priority, uint64 upload_order); From 3ffe770d57b2027b9923be9631bac4432d933c5b Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 21 Jul 2022 13:12:25 +0300 Subject: [PATCH 53/55] Fix generated file parsing. --- td/telegram/files/FileManager.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/td/telegram/files/FileManager.hpp b/td/telegram/files/FileManager.hpp index 520739016..9cf26745c 100644 --- a/td/telegram/files/FileManager.hpp +++ b/td/telegram/files/FileManager.hpp @@ -224,9 +224,9 @@ FileId FileManager::parse_file(ParserT &parser) { if (expected_size < 0) { expected_size += static_cast(1) << 32; } + int32 zero; + parse(zero, parser); } - int32 zero; - parse(zero, parser); DialogId owner_dialog_id; if (parser.version() >= static_cast(Version::StoreFileOwnerId)) { parse(owner_dialog_id, parser); From ffc26eea08050180eaac7af8b1903498c1e24619 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 21 Jul 2022 20:24:03 +0300 Subject: [PATCH 54/55] Fix GIT_EXECUTABLE usage. --- CMake/GetGitRevisionDescription.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMake/GetGitRevisionDescription.cmake b/CMake/GetGitRevisionDescription.cmake index 73fe1cc14..a99610c43 100644 --- a/CMake/GetGitRevisionDescription.cmake +++ b/CMake/GetGitRevisionDescription.cmake @@ -69,10 +69,12 @@ function(get_git_head_revision _refspecvar _hashvar) return() endif() + find_package(Git) + # Check if the current source dir is a git submodule or a worktree. # In both cases .git is a file instead of a directory. # - if (NOT IS_DIRECTORY ${GIT_DIR}) + if ((NOT IS_DIRECTORY ${GIT_DIR}) AND Git_FOUND) # The following git command will return a non empty string that # points to the super project working tree if the current # source dir is inside a git submodule. From 376476481c5fae682ab571f047f32efada3a9343 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 21 Jul 2022 20:44:22 +0300 Subject: [PATCH 55/55] Restore Java example compatibility with Java 1.8. --- .../java/org/drinkless/tdlib/example/Example.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/example/java/org/drinkless/tdlib/example/Example.java b/example/java/org/drinkless/tdlib/example/Example.java index eb71f4e4b..486470f6e 100644 --- a/example/java/org/drinkless/tdlib/example/Example.java +++ b/example/java/org/drinkless/tdlib/example/Example.java @@ -615,7 +615,7 @@ public final class Example { } private static void onFatalError(String errorMessage) { - final static class ThrowError implements Runnable { + final class ThrowError implements Runnable { private final String errorMessage; private final AtomicLong errorThrowTime; @@ -640,19 +640,19 @@ public final class Example { throw new ExternalClientError("Fatal error: " + errorMessage); } - private static final class ClientError extends Error { + final class ClientError extends Error { private ClientError(String message) { super(message); } } - private static final class ExternalClientError extends Error { + final class ExternalClientError extends Error { public ExternalClientError(String message) { super(message); } } - private static boolean isDatabaseBrokenError(String message) { + private boolean isDatabaseBrokenError(String message) { return message.contains("Wrong key or database is corrupted") || message.contains("SQL logic error or missing database") || message.contains("database disk image is malformed") || @@ -661,12 +661,12 @@ public final class Example { message.contains("Database was corrupted and deleted during execution and can't be recreated"); } - private static boolean isDiskFullError(String message) { + private boolean isDiskFullError(String message) { return message.contains("PosixError : No space left on device") || message.contains("database or disk is full"); } - private static boolean isDiskError(String message) { + private boolean isDiskError(String message) { return message.contains("I/O error") || message.contains("Structure needs cleaning"); } }