Merge remote-tracking branch 'Github/master' into fork

GitOrigin-RevId: 06ec5e24b9154e1830fd3d37b833df60609c675a

.clang-format

@@ -3,16 +3,20 @@ Language:        Cpp
 # BasedOnStyle:  Google
 AccessModifierOffset: -1
 AlignAfterOpenBracket: Align
+AlignConsecutiveMacros: false
 AlignConsecutiveAssignments: false
 AlignConsecutiveDeclarations: false
 AlignEscapedNewlines: Left
 AlignOperands:   true
 AlignTrailingComments: true
+AllowAllArgumentsOnNextLine: true
+AllowAllConstructorInitializersOnNextLine: true
 AllowAllParametersOfDeclarationOnNextLine: true
-AllowShortBlocksOnASingleLine: false
+AllowShortBlocksOnASingleLine: Never
 AllowShortCaseLabelsOnASingleLine: false
 AllowShortFunctionsOnASingleLine: None # All
-AllowShortIfStatementsOnASingleLine: false # true
+AllowShortIfStatementsOnASingleLine: Never # WithoutElse
+AllowShortLambdasOnASingleLine: Inline # All
 AllowShortLoopsOnASingleLine: false # true
 AlwaysBreakAfterDefinitionReturnType: None
 AlwaysBreakAfterReturnType: None
@@ -21,6 +25,7 @@ AlwaysBreakTemplateDeclarations: Yes
 BinPackArguments: true
 BinPackParameters: true
 BraceWrapping:
+  AfterCaseLabel:  false
   AfterClass:      false
   AfterControlStatement: false
   AfterEnum:       false
@@ -60,6 +65,7 @@ ForEachMacros:
   - Q_FOREACH_THIS_LIST_MUST_BE_NON_EMPTY
 IncludeBlocks:   Preserve
 IndentCaseLabels: true
+IndentGotoLabels: true
 IndentPPDirectives: None
 IndentWidth:     2
 IndentWrappedFunctionNames: false
@@ -87,6 +93,7 @@ ReflowComments:  false # true
 SortIncludes: false # disabled, because we need case insensitive sort
 SortUsingDeclarations: false # true
 SpaceAfterCStyleCast: false
+SpaceAfterLogicalNot: false
 SpaceAfterTemplateKeyword: true
 SpaceBeforeAssignmentOperators: true
 SpaceBeforeCpp11BracedList: false
@@ -94,6 +101,7 @@ SpaceBeforeCtorInitializerColon: true
 SpaceBeforeInheritanceColon: true
 SpaceBeforeParens: ControlStatements
 SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyBlock: false
 SpaceInEmptyParentheses: false
 SpacesBeforeTrailingComments: 2
 SpacesInAngles:  false

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
 
-project(TDLib VERSION 1.5.0 LANGUAGES CXX C)
+project(TDLib VERSION 1.5.1 LANGUAGES CXX C)
 
 if (POLICY CMP0054)
   # do not expand quoted arguments

README.md

@@ -146,7 +146,7 @@ target_link_libraries(YourTarget PRIVATE Td::TdStatic)
 
 Or you could install `TDLib` and then reference it in your CMakeLists.txt like this:
 ```
-find_package(Td 1.5.0 REQUIRED)
+find_package(Td 1.5.1 REQUIRED)
 target_link_libraries(YourTarget PRIVATE Td::TdStatic)
 ```
 See [example/cpp/CMakeLists.txt](https://github.com/tdlib/td/tree/master/example/cpp/CMakeLists.txt).
@@ -179,7 +179,7 @@ the [td_api.tl](https://github.com/tdlib/td/blob/master/td/generate/scheme/td_ap
 all available `TDLib` [methods](https://core.telegram.org/tdlib/docs/classtd_1_1td__api_1_1_function.html) and [classes](https://core.telegram.org/tdlib/docs/classtd_1_1td__api_1_1_object.html).
 
 `TDLib` JSON interface adheres to semantic versioning and versions with the same major version number are binary and backward compatible, but the underlying `TDLib` API can be different for different minor and even patch versions.
-If you need to support different `TDLib` versions then you can use a value of the `version` option to find exact `TDLib` version and to use appropriate API then.
+If you need to support different `TDLib` versions, then you can use a value of the `version` option to find exact `TDLib` version to use appropriate API methods.
 
 See [example/python/tdjson_example.py](https://github.com/tdlib/td/tree/master/example/python/tdjson_example.py) for an example of such usage.
 

benchmark/bench_misc.cpp

@@ -78,7 +78,7 @@ BENCH(NewInt, "new int + delete") {
   do_not_optimize_away(res);
 }
 
-BENCH(NewObj, "new struct then delete") {
+BENCH(NewObj, "new struct, then delete") {
   struct A {
     int32 a = 0;
     int32 b = 0;
@@ -99,7 +99,7 @@ BENCH(NewObj, "new struct then delete") {
 }
 
 #if !TD_THREAD_UNSUPPORTED
-BENCH(ThreadNew, "new struct then delete in several threads") {
+BENCH(ThreadNew, "new struct, then delete in several threads") {
   td::NewObjBench a, b;
   thread ta([&] { a.run(n / 2); });
   thread tb([&] { b.run(n - n / 2); });
@@ -222,15 +222,13 @@ class CreateFileBench : public Benchmark {
     }
   }
   void tear_down() override {
-    td::walk_path("A/",
-                  [&](CSlice path, auto type) {
+    td::walk_path("A/", [&](CSlice path, auto type) {
       if (type == td::WalkPath::Type::ExitDir) {
         rmdir(path).ignore();
       } else if (type == td::WalkPath::Type::NotDir) {
         unlink(path).ignore();
       }
-                  })
-        .ignore();
+    }).ignore();
   }
 };
 
@@ -246,26 +244,22 @@ class WalkPathBench : public Benchmark {
   }
   void run(int n) override {
     int cnt = 0;
-    td::walk_path("A/",
-                  [&](CSlice path, auto type) {
+    td::walk_path("A/", [&](CSlice path, auto type) {
       if (type == td::WalkPath::Type::EnterDir) {
         return;
       }
       stat(path).ok();
       cnt++;
-                  })
-        .ignore();
+    }).ignore();
   }
   void tear_down() override {
-    td::walk_path("A/",
-                  [&](CSlice path, auto type) {
+    td::walk_path("A/", [&](CSlice path, auto type) {
       if (type == td::WalkPath::Type::ExitDir) {
         rmdir(path).ignore();
       } else if (type == td::WalkPath::Type::NotDir) {
         unlink(path).ignore();
       }
-                  })
-        .ignore();
+    }).ignore();
   }
 };
 

benchmark/check_proxy.cpp

@@ -8,32 +8,92 @@
 #include "td/telegram/td_api.h"
 
 #include "td/utils/common.h"
+#include "td/utils/filesystem.h"
 #include "td/utils/logging.h"
 #include "td/utils/misc.h"
 
 #include <cstdlib>
+#include <iostream>
+#include <utility>
 
 static void usage() {
-  td::TsCerr() << "Tests specified MTProto-proxy; exits with code 0 on success.\n";
-  td::TsCerr() << "Usage:\n";
-  td::TsCerr() << "check_proxy [-v<N>] [-h] server:port:secret\n";
+  td::TsCerr() << "Tests specified MTProto-proxies, outputs working proxies to stdout; exits with code 0 if a working "
+                  "proxy was found.\n";
+  td::TsCerr() << "Usage: check_proxy [options] server:port:secret [server2:port2:secret2 ...]\n";
+  td::TsCerr() << "Options:\n";
+  td::TsCerr() << "  -v<N>\tSet verbosity level to N\n";
+  td::TsCerr() << "  -h/--help\tDisplay this information\n";
+  td::TsCerr() << "  -d/--dc-id\tIdentifier of a datacenter, to which try to connect (default is 2)\n";
+  td::TsCerr() << "  -l/--proxy-list\tName of a file with proxies to check; one proxy per line\n";
+  td::TsCerr() << "  -t/--timeout\tMaximum overall timeout for the request (default is 10 seconds)\n";
   std::exit(2);
 }
 
 int main(int argc, char **argv) {
   int new_verbosity_level = VERBOSITY_NAME(FATAL);
-  td::string server;
-  td::int32 port = 0;
-  td::string secret;
+
+  td::vector<std::pair<td::string, td::td_api::object_ptr<td::td_api::testProxy>>> requests;
+
+  auto add_proxy = [&requests](const td::string &arg) {
+    if (arg.empty()) {
+      return;
+    }
+
+    auto secret_pos = arg.rfind(':');
+    if (secret_pos == td::string::npos) {
+      td::TsCerr() << "Error: failed to find proxy port and secret in \"" << arg << "\"\n";
+      usage();
+    }
+    auto secret = arg.substr(secret_pos + 1);
+    auto port_pos = arg.substr(0, secret_pos).rfind(':');
+    if (port_pos == td::string::npos) {
+      td::TsCerr() << "Error: failed to find proxy secret in \"" << arg << "\"\n";
+      usage();
+    }
+    auto r_port = td::to_integer_safe<td::int32>(arg.substr(port_pos + 1, secret_pos - port_pos - 1));
+    if (r_port.is_error()) {
+      td::TsCerr() << "Error: failed to parse proxy port in \"" << arg << "\"\n";
+      usage();
+    }
+    auto port = r_port.move_as_ok();
+    auto server = arg.substr(0, port_pos);
+
+    if (server.empty() || port <= 0 || port > 65536 || secret.empty()) {
+      td::TsCerr() << "Error: proxy address to check is in wrong format: \"" << arg << "\"\n";
+      usage();
+    }
+
+    requests.emplace_back(arg,
+                          td::td_api::make_object<td::td_api::testProxy>(
+                              server, port, td::td_api::make_object<td::td_api::proxyTypeMtproto>(secret), -1, -1));
+  };
+
+  td::int32 dc_id = 2;
+  double timeout = 10.0;
 
   for (int i = 1; i < argc; i++) {
     td::string arg(argv[i]);
-    if (arg.substr(0, 2) == "-v") {
-      if (arg.size() == 2 && i + 1 < argc) {
-        arg = argv[++i];
-      } else {
-        arg = arg.substr(2);
+
+    auto get_next_arg = [&i, &arg, argc, argv](bool is_optional = false) {
+      CHECK(arg.size() >= 2);
+      if (arg.size() == 2 || arg[1] == '-') {
+        if (i + 1 < argc && argv[i + 1][0] != '-') {
+          return td::string(argv[++i]);
         }
+      } else {
+        if (arg.size() > 2) {
+          return arg.substr(2);
+        }
+      }
+      if (!is_optional) {
+        td::TsCerr() << "Error: value is required after " << arg << "\n";
+        usage();
+      }
+      return td::string();
+    };
+
+    if (td::begins_with(arg, "-v")) {
+      arg = get_next_arg(true);
       int new_verbosity = 1;
       while (arg[0] == 'v') {
         new_verbosity++;
@@ -43,31 +103,27 @@ int main(int argc, char **argv) {
         new_verbosity += td::to_integer<int>(arg) - (new_verbosity == 1);
       }
       new_verbosity_level = VERBOSITY_NAME(FATAL) + new_verbosity;
-    } else if (arg == "-h" || arg == "--help") {
+    } else if (td::begins_with(arg, "-t") || arg == "--timeout") {
+      timeout = td::to_double(get_next_arg());
+    } else if (td::begins_with(arg, "-d") || arg == "--dc-id") {
+      dc_id = td::to_integer<td::int32>(get_next_arg());
+    } else if (td::begins_with(arg, "-l") || arg == "--proxy-list") {
+      auto r_proxies = td::read_file_str(get_next_arg());
+      if (r_proxies.is_error()) {
+        td::TsCerr() << "Error: wrong file name specified\n";
+        usage();
+      }
+      for (auto &proxy : td::full_split(r_proxies.ok(), '\n')) {
+        add_proxy(td::trim(proxy));
+      }
+    } else if (arg[0] == '-') {
       usage();
     } else {
-      auto secret_pos = arg.rfind(':');
-      if (secret_pos == td::string::npos) {
-        td::TsCerr() << (PSLICE() << "Error: failed to find proxy port and secret in \"" << arg << "\"\n");
-        usage();
-      }
-      secret = arg.substr(secret_pos + 1);
-      auto port_pos = arg.substr(0, secret_pos).rfind(':');
-      if (port_pos == td::string::npos) {
-        td::TsCerr() << (PSLICE() << "Error: failed to find proxy secret in \"" << arg << "\"\n");
-        usage();
-      }
-      auto r_port = td::to_integer_safe<td::int32>(arg.substr(port_pos + 1, secret_pos - port_pos - 1));
-      if (r_port.is_error()) {
-        td::TsCerr() << (PSLICE() << "Error: failed to parse proxy port in \"" << arg << "\"\n");
-        usage();
-      }
-      port = r_port.move_as_ok();
-      server = arg.substr(0, port_pos);
+      add_proxy(arg);
     }
   }
 
-  if (server.empty() || port <= 0 || port > 65536 || secret.empty()) {
+  if (requests.empty()) {
     td::TsCerr() << "Error: proxy address to check is not specified\n";
     usage();
   }
@@ -75,18 +131,30 @@ int main(int argc, char **argv) {
   SET_VERBOSITY_LEVEL(new_verbosity_level);
 
   td::Client client;
-  client.send({1, td::td_api::make_object<td::td_api::testProxy>(
-                      server, port, td::td_api::make_object<td::td_api::proxyTypeMtproto>(secret))});
-  while (true) {
+  for (size_t i = 0; i < requests.size(); i++) {
+    auto &request = requests[i].second;
+    request->dc_id_ = dc_id;
+    request->timeout_ = timeout;
+    client.send({i + 1, std::move(request)});
+  }
+  size_t successful_requests = 0;
+  size_t failed_requests = 0;
+
+  while (successful_requests + failed_requests != requests.size()) {
     auto response = client.receive(100.0);
-    if (response.id == 1) {
+    if (1 <= response.id && response.id <= requests.size()) {
+      auto &proxy = requests[static_cast<size_t>(response.id - 1)].first;
       if (response.object->get_id() == td::td_api::error::ID) {
-        LOG(ERROR) << to_string(response.object);
-        return 1;
+        LOG(ERROR) << proxy << ": " << to_string(response.object);
+        failed_requests++;
       } else {
-        LOG(ERROR) << "Success!";
-        return 0;
+        std::cout << proxy << std::endl;
+        successful_requests++;
       }
     }
   }
+
+  if (successful_requests == 0) {
+    return 1;
+  }
 }

benchmark/check_tls.cpp

@@ -53,7 +53,9 @@ td::Result<TlsInfo> test_tls(const td::string &url) {
 
   td::string request;
 
-  auto add_string = [&](td::Slice data) { request.append(data.data(), data.size()); };
+  auto add_string = [&](td::Slice data) {
+    request.append(data.data(), data.size());
+  };
   auto add_random = [&](size_t length) {
     while (length-- > 0) {
       request += static_cast<char>(td::Random::secure_int32());

build.html

@@ -23,10 +23,12 @@ select.large { font-size: large; }
         <option>JavaScript</option>
         <option>Go</option>
         <option>Java</option>
+        <option>Kotlin</option>
         <option>C#</option>
         <option>C++</option>
         <option>Swift</option>
         <option>Objective-C</option>
+        <option>Dart</option>
         <option>Rust</option>
         <option>Erlang</option>
         <option>PHP</option>
@@ -165,9 +167,12 @@ function getSupportedOs(language) {
     case 'JavaScript':
       return ['Windows (Node.js)', 'Linux (Node.js)', 'macOS (Node.js)', 'FreeBSD (Node.js)', 'OpenBSD (Node.js)', 'NetBSD (Node.js)', 'Browser'];
     case 'Java':
-      return ['Windows', 'Linux', 'macOS', 'FreeBSD', 'OpenBSD', 'Android'];
+    case 'Kotlin':
+      return ['Android', 'Windows', 'Linux', 'macOS', 'FreeBSD', 'OpenBSD'];
     case 'C#':
       return ['Windows (through C++/CLI)', 'Universal Windows Platform (through C++/CX)', 'Windows (.Net Core)', 'Linux (.Net Core)', 'macOS (.Net Core)', 'FreeBSD (.Net Core)'];
+    case 'Dart':
+      return ['Android', 'iOS', 'Windows', 'Linux', 'macOS', 'tvOS', 'watchOS'];
     case 'Swift':
     case 'Objective-C':
       return ['Windows', 'Linux', 'macOS', 'iOS', 'tvOS', 'watchOS'];
@@ -182,8 +187,10 @@ function getExampleAnchor(language) {
     case 'JavaScript':
     case 'Go':
     case 'Java':
+    case 'Kotlin':
     case 'Swift':
     case 'Objective-C':
+    case 'Dart':
     case 'Rust':
     case 'Erlang':
     case 'PHP':
@@ -272,6 +279,7 @@ function getTarget(language, os) {
     case 'C++':
       return 'tdclient';
     case 'Java':
+    case 'Kotlin':
       return 'JNI';
     default:
       return 'tdjson';
@@ -860,6 +868,9 @@ function onOptionsChanged() {
     }
     commands.push('cd ..');
   }
+  if (jni_install_dir) {
+    install_dir = jni_install_dir;
+  }
   if (install_dir) {
     if (install_dir !== '/usr/local') {
       install_dir = 'td/tdlib';

example/README.md

@@ -13,10 +13,12 @@ Choose your preferred programming language to see examples of usage and a detail
 - [JavaScript](#javascript)
 - [Go](#go)
 - [Java](#java)
+- [Kotlin](#kotlin)
 - [C#](#csharp)
 - [C++](#cxx)
 - [Swift](#swift)
 - [Objective-C](#objective-c)
+- [Dart](#dart)
 - [Rust](#rust)
 - [Erlang](#erlang)
 - [PHP](#php)
@@ -41,7 +43,7 @@ If you use modern Python >= 3.6, take a look at [python-telegram](https://github
 The wrapper uses the full power of asyncio, has a good documentation and has several examples. It can be installed through pip or used in a Docker container.
 You can also try a fork [python-telegram](https://github.com/iTeam-co/python-telegram) of this library.
 
-For older Python versions you can use [demo-pytdlib](https://github.com/i-Naji/demo-pytdlib) or [python-tdlib](https://github.com/andrew-ld/python-tdlib).
+For older Python versions you can use [pytdlib](https://github.com/pytdlib/pytdlib) or [python-tdlib](https://github.com/andrew-ld/python-tdlib).
 These wrappers contain generators for TDLib API classes and basic classes for interaction with TDLib.
 
 You can also check out [example/python/tdjson_example.py](https://github.com/tdlib/td/tree/master/example/python/tdjson_example.py) and
@@ -56,10 +58,10 @@ and [telegram-react](https://github.com/evgeny-nadymov/telegram-react) as an exa
 TDLib can be used from Node.js through the [JSON](https://github.com/tdlib/td#using-json) interface.
 
 Convenient Node.js wrappers already exist for our JSON interface.
-For example, take a look at [tdl](https://github.com/Bannerets/tdl), which provides a convenient, fully-asynchronous interface for interaction with TDLib and contains a bunch of examples, or
-at [Airgram](https://github.com/airgram/airgram) - modern TDLib framework for TypeScript/JavaScript.
+For example, take a look at [Airgram](https://github.com/airgram/airgram) – modern TDLib framework for TypeScript/JavaScript, or
+at [tdl](https://github.com/Bannerets/tdl), which provides a convenient, fully-asynchronous interface for interaction with TDLib and contains a bunch of examples.
 
-You can also see [node-tdlib](https://github.com/wfjsw/node-tdlib), [tdlnode](https://github.com/fonbah/tdlnode), [tglib](https://github.com/nodegin/tglib),
+You can also see [tglib](https://github.com/nodegin/tglib), [node-tdlib](https://github.com/wfjsw/node-tdlib), [tdlnode](https://github.com/fonbah/tdlnode),
 [Paper Plane](https://github.com/BlackSuited/paper-plane) and [node-tlg](https://github.com/dilongfa/node-tlg) for other examples of TDLib JSON interface integration with Node.js.
 
 TDLib can be used also from NativeScript through the [JSON](https://github.com/tdlib/td#using-json) interface.
@@ -86,6 +88,13 @@ To use TDLib to create Android Java applications, use our [prebuilt library for
 
 You can also see [JTDLib](https://github.com/ErnyTech/JTDLib) for another example of Java wrapper for TDLib.
 
+<a name="kotlin"></a>
+## Using TDLib in Kotlin projects
+
+TDLib can be used from the Kotlin/JVM programming language through same way as in [JAVA](#java).
+
+You can also use [ktd](https://github.com/whyoleg/ktd) library with kotlin specific bindings.
+
 <a name="csharp"></a>
 ## Using TDLib in C# projects
 
@@ -97,14 +106,14 @@ If you want to write a cross-platform C# application using .Net Core, see [tdsha
 provides an asynchronous interface for interaction with TDLib, automatically generated classes for TDLib API and has some examples.
 
 Also see [Unigram](https://github.com/UnigramDev/Unigram), which is a full-featured client rewritten from scratch in C# using TDLib SDK for Universal Windows Platform in less than 2 months, or
-[egram.tel](https://github.com/egramtel/egram.tel) - a cross-platform Telegram client written in C#, .NET Core, ReactiveUI and Avalonia.
+[egram.tel](https://github.com/egramtel/egram.tel) – a cross-platform Telegram client written in C#, .NET Core, ReactiveUI and Avalonia.
 
 <a name="cxx"></a>
 ## Using TDLib in C++ projects
 
 TDLib has a simple and convenient C++11-interface for sending and receiving requests and can be statically linked to your application.
 
-See [example/cpp](https://github.com/tdlib/td/tree/master/example/cpp) for examples of TDLib usage from C++.
+See [example/cpp](https://github.com/tdlib/td/tree/master/example/cpp) for an example of TDLib usage from C++.
 [td_example.cpp](https://github.com/tdlib/td/tree/master/example/cpp/td_example.cpp) contains an example of authorization, processing new incoming messages, getting a list of chats and sending a text message.
 
 See also the source code of [Depecher](https://github.com/blacksailer/depecher) – a Telegram app for Sailfish OS, and [TELEports](https://gitlab.com/ubports/apps/teleports) – a Qt-client for Ubuntu Touch, both of which are based on TDLib.
@@ -114,6 +123,8 @@ See also the source code of [Depecher](https://github.com/blacksailer/depecher)
 
 TDLib can be used from the Swift programming language through the [JSON](https://github.com/tdlib/td#using-json) interface and can be linked statically or dynamically.
 
+See [TDLib-iOS](https://github.com/leoMehlig/TDLib-iOS), which provide a convenient TDLib client with automatically generated and fully-documented classes for all TDLib API methods and objects.
+
 See [example/swift](https://github.com/tdlib/td/tree/master/example/swift) for an example of a macOS Swift application.
 See [example/ios](https://github.com/tdlib/td/tree/master/example/ios) for an example of building TDLib for iOS, watchOS, tvOS, and macOS.
 
@@ -124,13 +135,22 @@ TDLib can be used from the Objective-C programming language through [JSON](https
 
 See [example/ios](https://github.com/tdlib/td/tree/master/example/ios) for an example of building TDLib for iOS, watchOS, tvOS, and macOS.
 
+<a name="dart"></a>
+## Using TDLib in Dart projects
+
+TDLib can be used from the Dart programming language through the [JSON](https://github.com/tdlib/td#using-json) interface and a Dart Native Extension or Dart FFI.
+
+See [dart_tdlib](https://github.com/periodicaidan/dart_tdlib) or [Dart wrapper for TDLib](https://github.com/tdlib/td/pull/708/commits/237060abd4c205768153180e9f814298d1aa9d49) for an example of a TDLib Dart bindings through FFI.
+
+See [project.scarlet](https://github.com/aaugmentum/project.scarlet) or [tdlib-dart](https://github.com/triedcatched/tdlib-dart) for an example of using TDLib from Dart.
+
 <a name="rust"></a>
 ## Using TDLib in Rust projects
 
 TDLib can be used from the Rust programming language through the [JSON](https://github.com/tdlib/td#using-json) interface.
 
-See [tdlib-rs](https://github.com/d653/tdlib-rs), [tdlib-futures](https://github.com/yuri91/tdlib-futures), [rust-tdlib](https://github.com/lattenwald/rust-tdlib) or
-[tdjson-rs](https://github.com/mersinvald/tdjson-rs) for an example of TDLib Rust bindings.
+See [rtdlib](https://github.com/fewensa/rtdlib), [tdlib-rs](https://github.com/d653/tdlib-rs), [tdlib-futures](https://github.com/yuri91/tdlib-futures),
+[rust-tdlib](https://github.com/lattenwald/rust-tdlib) or [tdjson-rs](https://github.com/mersinvald/tdjson-rs) for an example of TDLib Rust bindings.
 
 <a name="erlang"></a>
 ## Using TDLib in Erlang projects
@@ -145,7 +165,7 @@ See [erl-tdlib](https://github.com/lattenwald/erl-tdlib) for an example of TDLib
 TDLib can be used from the PHP programming language by wrapping its functionality in a PHP extension.
 
 See [phptdlib](https://github.com/yaroslavche/phptdlib) or [PIF-TDPony](https://github.com/danog/pif-tdpony) for examples of such extensions which provide access to TDLib from PHP.
-See [tdlib-bundle](https://github.com/yaroslavche/tdlib-bundle) - a Symfony bundle based on [phptdlib](https://github.com/yaroslavche/phptdlib).
+See [tdlib-bundle](https://github.com/yaroslavche/tdlib-bundle) – a Symfony bundle based on [phptdlib](https://github.com/yaroslavche/phptdlib).
 
 <a name="lua"></a>
 ## Using TDLib in Lua projects

example/cpp/CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
 
 project(TdExample VERSION 1.0 LANGUAGES CXX)
 
-find_package(Td 1.5.0 REQUIRED)
+find_package(Td 1.5.1 REQUIRED)
 
 add_executable(tdjson_example tdjson_example.cpp)
 target_link_libraries(tdjson_example PRIVATE Td::TdJson)

example/ios/build.sh

@@ -15,7 +15,7 @@ do
   openssl_ssl_library="${openssl_path}/lib/libssl.a"
   options="$options -DOPENSSL_FOUND=1"
   options="$options -DOPENSSL_CRYPTO_LIBRARY=${openssl_crypto_library}"
-  #options="$options -DOPENSSL_SSL_LIBRARY=${openssl_ssl_library}"
+  options="$options -DOPENSSL_SSL_LIBRARY=${openssl_ssl_library}"
   options="$options -DOPENSSL_INCLUDE_DIR=${openssl_path}/include"
   options="$options -DOPENSSL_LIBRARIES=${openssl_crypto_library};${openssl_ssl_library}"
   options="$options -DCMAKE_BUILD_TYPE=Release"

example/uwp/extension.vsixmanifest

@@ -1,6 +1,6 @@
 <PackageManifest Version="2.0.0" xmlns="http://schemas.microsoft.com/developer/vsx-schema/2011">
   <Metadata>
-    <Identity Id="Telegram.Td.UWP" Version="1.5.0" Language="en-US" Publisher="Telegram LLC" />
+    <Identity Id="Telegram.Td.UWP" Version="1.5.1" Language="en-US" Publisher="Telegram LLC" />
     <DisplayName>TDLib for Universal Windows Platform</DisplayName>
     <Description>TDLib is a library for building Telegram clients</Description>
     <MoreInfo>https://core.telegram.org/tdlib</MoreInfo>

memprof/memprof.cpp

@@ -41,9 +41,9 @@ extern void *__libc_stack_end;
 static void *get_bp() {
   void *bp;
 #if defined(__i386__)
-  __asm__ volatile("movl %%ebp, %[r]" : [r] "=r"(bp));
+  __asm__ volatile("movl %%ebp, %[r]" : [ r ] "=r"(bp));
 #elif defined(__x86_64__)
-  __asm__ volatile("movq %%rbp, %[r]" : [r] "=r"(bp));
+  __asm__ volatile("movq %%rbp, %[r]" : [ r ] "=r"(bp));
 #endif
   return bp;
 }

td/generate/scheme/td_api.tl

@@ -158,7 +158,9 @@ file id:int32 size:int32 expected_size:int32 local:localFile remote:remoteFile =
 //@description A file defined by its unique ID @id Unique file identifier
 inputFileId id:int32 = InputFile;
 
-//@description A file defined by its remote ID @id Remote file identifier
+//@description A file defined by its remote ID. The remote ID is guaranteed to work only if it was received after TDLib launch and the corresponding file is still accessible to the user.
+//-For example, if the file is from a message, then the message must be not deleted and accessible to the user. If a file database is disabled, then the corresponding object with the file must be preloaded by the client
+//@id Remote file identifier
 inputFileRemote id:string = InputFile;
 
 //@description A file defined by a local path @path Local path to the file
@@ -208,7 +210,7 @@ pollOption text:string voter_count:int32 vote_percentage:int32 is_chosen:Bool is
 //@minithumbnail Animation minithumbnail; may be null @thumbnail Animation thumbnail; may be null @animation File containing the animation
 animation duration:int32 width:int32 height:int32 file_name:string mime_type:string minithumbnail:minithumbnail thumbnail:photoSize animation:file = Animation;
 
-//@description Describes an audio file. Audio is usually in MP3 format @duration Duration of the audio, in seconds; as defined by the sender @title Title of the audio; as defined by the sender @performer Performer of the audio; as defined by the sender
+//@description Describes an audio file. Audio is usually in MP3 or M4A format @duration Duration of the audio, in seconds; as defined by the sender @title Title of the audio; as defined by the sender @performer Performer of the audio; as defined by the sender
 //@file_name Original name of the file; as defined by the sender @mime_type The MIME type of the file; as defined by the sender @album_cover_minithumbnail The minithumbnail of the album cover; may be null @album_cover_thumbnail The thumbnail of the album cover; as defined by the sender. The full size thumbnail should be extracted from the downloaded file; may be null @audio File containing the audio
 audio duration:int32 title:string performer:string file_name:string mime_type:string album_cover_minithumbnail:minithumbnail album_cover_thumbnail:photoSize audio:file = Audio;
 
@@ -224,7 +226,7 @@ photo has_stickers:Bool minithumbnail:minithumbnail sizes:vector<photoSize> = Ph
 sticker set_id:int64 width:int32 height:int32 emoji:string is_animated:Bool is_mask:Bool mask_position:maskPosition thumbnail:photoSize sticker:file = Sticker;
 
 //@description Describes a video file @duration Duration of the video, in seconds; as defined by the sender @width Video width; as defined by the sender @height Video height; as defined by the sender
-//@file_name Original name of the file; as defined by the sender @mime_type MIME type of the file; as defined by the sender @has_stickers True, if stickers were added to the photo
+//@file_name Original name of the file; as defined by the sender @mime_type MIME type of the file; as defined by the sender @has_stickers True, if stickers were added to the video
 //@supports_streaming True, if the video should be tried to be streamed @minithumbnail Video minithumbnail; may be null @thumbnail Video thumbnail; as defined by the sender; may be null @video File containing the video
 video duration:int32 width:int32 height:int32 file_name:string mime_type:string has_stickers:Bool supports_streaming:Bool minithumbnail:minithumbnail thumbnail:photoSize video:file = Video;
 
@@ -729,8 +731,8 @@ richTextStrikethrough text:RichText = RichText;
 //@description A fixed-width rich text @text Text
 richTextFixed text:RichText = RichText;
 
-//@description A rich text URL link @text Text @url URL
-richTextUrl text:RichText url:string = RichText;
+//@description A rich text URL link @text Text @url URL @is_cached True, if the URL has cached instant view server-side
+richTextUrl text:RichText url:string is_cached:Bool = RichText;
 
 //@description A rich text email link @text Text @email_address Email address
 richTextEmailAddress text:RichText email_address:string = RichText;
@@ -787,7 +789,7 @@ pageBlockVerticalAlignmentMiddle = PageBlockVerticalAlignment;
 //@description The content should be bottom-aligned
 pageBlockVerticalAlignmentBottom = PageBlockVerticalAlignment;
 
-//@description Represents a cell of a table @text Cell text @is_header True, if it is a header cell
+//@description Represents a cell of a table @text Cell text; may be null. If the text is null, then the cell should be invisible @is_header True, if it is a header cell
 //@colspan The number of columns the cell should span @rowspan The number of rows the cell should span
 //@align Horizontal cell content alignment @valign Vertical cell content alignment
 pageBlockTableCell text:RichText is_header:Bool colspan:int32 rowspan:int32 align:PageBlockHorizontalAlignment valign:PageBlockVerticalAlignment = PageBlockTableCell;
@@ -853,6 +855,9 @@ pageBlockPhoto photo:photo caption:pageBlockCaption url:string = PageBlock;
 //@description A video @video Video file; may be null @caption Video caption @need_autoplay True, if the video should be played automatically @is_looped True, if the video should be looped
 pageBlockVideo video:video caption:pageBlockCaption need_autoplay:Bool is_looped:Bool = PageBlock;
 
+//@description A voice note @voice_note Voice note; may be null @caption Voice note caption
+pageBlockVoiceNote voice_note:voiceNote caption:pageBlockCaption = PageBlock;
+
 //@description A page cover @cover Cover
 pageBlockCover cover:PageBlock = PageBlock;
 
@@ -1824,6 +1829,13 @@ gameHighScore position:int32 user_id:int32 score:int32 = GameHighScore;
 gameHighScores scores:vector<gameHighScore> = GameHighScores;
 
 
+//@description Contains the response of a request to TON lite server @response The response
+tonLiteServerResponse response:bytes = TonLiteServerResponse;
+
+//@description Contains the salt to be used with locally stored password to access a local TON-based wallet @salt The salt
+tonWalletPasswordSalt salt:bytes = TonWalletPasswordSalt;
+
+
 //@class ChatEventAction @description Represents a chat event
 
 //@description A message was edited @old_message The original message before the edit @new_message The message after it was edited
@@ -2155,8 +2167,9 @@ notificationGroupTypeSecretChat = NotificationGroupType;
 notificationGroupTypeCalls = NotificationGroupType;
 
 
-//@description Contains information about a notification @id Unique persistent identifier of this notification @date Notification date @type Notification type
-notification id:int32 date:int32 type:NotificationType = Notification;
+//@description Contains information about a notification @id Unique persistent identifier of this notification @date Notification date
+//@is_silent True, if the notification was initially silent @type Notification type
+notification id:int32 date:int32 is_silent:Bool type:NotificationType = Notification;
 
 //@description Describes a group of notifications @id Unique persistent auto-incremented from 1 identifier of the notification group @type Type of the group
 //@chat_id Identifier of a chat to which all notifications in the group belong
@@ -2591,7 +2604,7 @@ updateChatPhoto chat_id:int53 photo:chatPhoto = Update;
 //@description Chat permissions was changed @chat_id Chat identifier @permissions The new chat permissions
 updateChatPermissions chat_id:int53 permissions:chatPermissions = Update;
 
-//@description The last message of a chat was changed. If last_message is null then the last message in the chat became unknown. Some new unknown messages might be added to the chat in this case @chat_id Chat identifier @last_message The new last message in the chat; may be null @order New value of the chat order
+//@description The last message of a chat was changed. If last_message is null, then the last message in the chat became unknown. Some new unknown messages might be added to the chat in this case @chat_id Chat identifier @last_message The new last message in the chat; may be null @order New value of the chat order
 updateChatLastMessage chat_id:int53 last_message:message order:int64 = Update;
 
 //@description The order of the chat in the chat list has changed. Instead of this update updateChatLastMessage, updateChatIsPinned or updateChatDraftMessage might be sent @chat_id Chat identifier @order New value of the order
@@ -2956,7 +2969,9 @@ getMessages chat_id:int53 message_ids:vector<int53> = Messages;
 //@description Returns information about a file; this is an offline request @file_id Identifier of the file to get
 getFile file_id:int32 = File;
 
-//@description Returns information about a file by its remote ID; this is an offline request. Can be used to register a URL as a file for further uploading, or sending as a message @remote_file_id Remote identifier of the file to get @file_type File type, if known
+//@description Returns information about a file by its remote ID; this is an offline request. Can be used to register a URL as a file for further uploading, or sending as a message. Even the request succeeds, the file can be used only if it is still accessible to the user.
+//-For example, if the file is from a message, then the message must be not deleted and accessible to the user. If a file database is disabled, then the corresponding object with the file must be preloaded by the client
+//@remote_file_id Remote identifier of the file to get @file_type File type, if known
 getRemoteFile remote_file_id:string file_type:FileType = File;
 
 //@description Returns an ordered list of chats. Chats are sorted by the pair (order, chat_id) in decreasing order. (For example, to get a list of chats from the beginning, the offset_order should be equal to a biggest signed 64-bit number 9223372036854775807 == 2^63 - 1).
@@ -3947,6 +3962,13 @@ sendCustomRequest method:string parameters:string = CustomRequestResult;
 answerCustomQuery custom_query_id:int64 data:string = Ok;
 
 
+//@description Sends a request to TON lite server through Telegram servers @request The request
+sendTonLiteServerRequest request:bytes = TonLiteServerResponse;
+
+//@description Returns a salt to be used with locally stored password to access a local TON-based wallet
+getTonWalletPasswordSalt = TonWalletPasswordSalt;
+
+
 //@description Succeeds after a specified amount of time has passed. Can be called before authorization. Can be called before initialization @seconds Number of seconds before the function returns
 setAlarm seconds:double = Ok;
 
@@ -4040,7 +4062,8 @@ testSquareInt x:int32 = TestInt;
 //@description Sends a simple network request to the Telegram servers; for testing only. Can be called before authorization
 testNetwork = Ok;
 //@description Sends a simple network request to the Telegram servers via proxy; for testing only. Can be called before authorization @server Proxy server IP address @port Proxy server port @type Proxy type
-testProxy server:string port:int32 type:ProxyType = Ok;
+//@dc_id Identifier of a datacenter, with which to test connection @timeout Maximum overall timeout for the request
+testProxy server:string port:int32 type:ProxyType dc_id:int32 timeout:double = Ok;
 //@description Forces an updates.getDifference call to the Telegram servers; for testing only
 testGetDifference = Ok;
 //@description Does nothing and ensures that the Update object is used; for testing only. This is an offline method. Can be called before authorization

td/generate/scheme/telegram_api.tl

@@ -1011,6 +1011,10 @@ emojiURL#a575739d url:string = EmojiURL;
 
 emojiLanguage#b3fb5361 lang_code:string = EmojiLanguage;
 
+wallet.liteResponse#764386d7 response:bytes = wallet.LiteResponse;
+
+wallet.secretSalt#dd484d64 salt:bytes = wallet.KeySecretSalt;
+
 fileLocationToBeDeprecated#bc7fc6cd volume_id:long local_id:int = FileLocation;
 
 ---functions---
@@ -1320,6 +1324,9 @@ phone.discardCall#b2cbc1c0 flags:# video:flags.0?true peer:InputPhoneCall durati
 phone.setCallRating#59ead627 flags:# user_initiative:flags.0?true peer:InputPhoneCall rating:int comment:string = Updates;
 phone.saveCallDebug#277add7e peer:InputPhoneCall debug:DataJSON = Bool;
 
+wallet.sendLiteRequest#e2c9d33e body:bytes = wallet.LiteResponse;
+wallet.getKeySecretSalt#b57f346 revoke:Bool = wallet.KeySecretSalt;
+
 langpack.getLangPack#f2f2330a lang_pack:string lang_code:string = LangPackDifference;
 langpack.getStrings#efea3803 lang_pack:string lang_code:string keys:Vector<string> = Vector<LangPackString>;
 langpack.getDifference#cd984aa5 lang_pack:string lang_code:string from_version:int = LangPackDifference;

td/generate/tl_json_converter.cpp

@@ -96,8 +96,7 @@ void gen_from_json_constructor(StringBuilder &sb, const T *constructor, bool is_
     sb << " {\n";
     for (auto &arg : constructor->args) {
       sb << "  {\n";
-      sb << "    TRY_RESULT(value, get_json_object_field(from, \"" << tl::simple::gen_cpp_name(arg.name)
-         << "\", td::JsonValue::Type::Null, true));\n";
+      sb << "    auto value = get_json_object_field_force(from, \"" << tl::simple::gen_cpp_name(arg.name) << "\");\n";
       sb << "    if (value.type() != td::JsonValue::Type::Null) {\n";
       if (arg.type->type == tl::simple::Type::Bytes) {
         sb << "      TRY_STATUS(from_json_bytes(to." << tl::simple::gen_cpp_field_name(arg.name) << ", value));\n";

td/mtproto/KDF.cpp

@@ -9,12 +9,13 @@
 #include "td/utils/as.h"
 #include "td/utils/common.h"
 #include "td/utils/crypto.h"
+#include "td/utils/logging.h"
 
 namespace td {
 namespace mtproto {
 
 void KDF(Slice auth_key, const UInt128 &msg_key, int X, UInt256 *aes_key, UInt256 *aes_iv) {
-  CHECK(auth_key.size() == 2048 / 8);
+  LOG_CHECK(auth_key.size() == 2048 / 8) << auth_key.size();
   const char *auth_key_raw = auth_key.data();
   uint8 buf[48];
   as<UInt128>(buf) = msg_key;

td/mtproto/SessionConnection.h

@@ -202,7 +202,9 @@ class SessionConnection
   auto set_buffer_slice(BufferSlice *buffer_slice) TD_WARN_UNUSED_RESULT {
     auto old_buffer_slice = current_buffer_slice_;
     current_buffer_slice_ = buffer_slice;
-    return ScopeExit() + [&to = current_buffer_slice_, from = old_buffer_slice] { to = from; };
+    return ScopeExit() + [&to = current_buffer_slice_, from = old_buffer_slice] {
+      to = from;
+    };
   }
 
   Status parse_message(TlParser &parser, MsgInfo *info, Slice *packet, bool crypto_flag = true) TD_WARN_UNUSED_RESULT;

td/mtproto/TlsInit.cpp

@@ -456,7 +456,7 @@ Status TlsInit::wait_hello_response() {
   std::string hash_dest(32, '\0');
   hmac_sha256(password_, PSLICE() << hello_rand_ << response.as_slice(), hash_dest);
   if (hash_dest != response_rand) {
-    return td::Status::Error("Response hash mismatch");
+    return Status::Error("Response hash mismatch");
   }
 
   stop();

td/mtproto/Transport.cpp

@@ -175,7 +175,7 @@ size_t do_calc_crypto_size2_basic(size_t data_size, size_t enc_size, size_t raw_
 }
 
 size_t do_calc_crypto_size2_rand(size_t data_size, size_t enc_size, size_t raw_size) {
-  size_t rand_data_size = td::Random::secure_uint32() & 0xff;
+  size_t rand_data_size = Random::secure_uint32() & 0xff;
   size_t encrypted_size = (enc_size + data_size + rand_data_size + 12 + 15) & ~15;
   return raw_size + encrypted_size;
 }

td/telegram/ConfigManager.cpp

@@ -229,9 +229,9 @@ ActorOwn<> get_simple_config_azure(Promise<SimpleConfigResult> promise, const Co
   return get_simple_config_impl(std::move(promise), scheduler_id, std::move(url), "tcdnb.azureedge.net", prefer_ipv6);
 }
 
-ActorOwn<> get_simple_config_google_dns(Promise<SimpleConfigResult> promise, const ConfigShared *shared_config,
-                                        bool is_test, int32 scheduler_id) {
-  VLOG(config_recoverer) << "Request simple config from Google DNS";
+static ActorOwn<> get_simple_config_dns(Slice address, Slice host, Promise<SimpleConfigResult> promise,
+                                        const ConfigShared *shared_config, bool is_test, int32 scheduler_id) {
+  VLOG(config_recoverer) << "Request simple config from DNS";
 #if TD_EMSCRIPTEN  // FIXME
   return ActorOwn<>();
 #else
@@ -281,12 +281,24 @@ ActorOwn<> get_simple_config_google_dns(Promise<SimpleConfigResult> promise, con
           return std::move(res);
         }());
       }),
-      PSTRING() << "https://www.google.com/resolve?name=" << url_encode(name) << "&type=16",
-      std::vector<std::pair<string, string>>({{"Host", "dns.google.com"}}), timeout, ttl, prefer_ipv6,
-      SslStream::VerifyPeer::Off));
+      PSTRING() << "https://" << address << "?name=" << url_encode(name) << "&type=16",
+      std::vector<std::pair<string, string>>({{"Host", host.str()}, {"Accept", "application/dns-json"}}), timeout, ttl,
+      prefer_ipv6, SslStream::VerifyPeer::Off));
 #endif
 }
 
+ActorOwn<> get_simple_config_google_dns(Promise<SimpleConfigResult> promise, const ConfigShared *shared_config,
+                                        bool is_test, int32 scheduler_id) {
+  return get_simple_config_dns("www.google.com/resolve", "dns.google.com", std::move(promise), shared_config, is_test,
+                               scheduler_id);
+}
+
+ActorOwn<> get_simple_config_mozilla_dns(Promise<SimpleConfigResult> promise, const ConfigShared *shared_config,
+                                         bool is_test, int32 scheduler_id) {
+  return get_simple_config_dns("mozilla.cloudflare-dns.com/dns-query", "mozilla.cloudflare-dns.com", std::move(promise),
+                               shared_config, is_test, scheduler_id);
+}
+
 ActorOwn<> get_full_config(DcOption option, Promise<FullConfig> promise, ActorShared<> parent) {
   class SessionCallback : public Session::Callback {
    public:
@@ -708,12 +720,13 @@ class ConfigRecoverer : public Actor {
           });
       auto get_simple_config = [&]() {
         switch (simple_config_turn_ % 3) {
-          case 1:
+          case 2:
             return get_simple_config_azure;
           case 0:
-          case 2:
-          default:
             return get_simple_config_google_dns;
+          case 1:
+          default:
+            return get_simple_config_mozilla_dns;
         }
       }();
       simple_config_query_ =
@@ -987,7 +1000,9 @@ void ConfigManager::process_config(tl_object_ptr<telegram_api::config> config) {
     shared_config.set_option_string("photo_search_bot_username", config->img_search_username_);
   }
 
-  auto fix_timeout_ms = [](int32 timeout_ms) { return clamp(timeout_ms, 1000, 86400 * 1000); };
+  auto fix_timeout_ms = [](int32 timeout_ms) {
+    return clamp(timeout_ms, 1000, 86400 * 1000);
+  };
 
   shared_config.set_option_integer("online_update_period_ms", fix_timeout_ms(config->online_update_period_ms_));
 

td/telegram/ConfigManager.h

@@ -41,6 +41,9 @@ ActorOwn<> get_simple_config_azure(Promise<SimpleConfigResult> promise, const Co
 ActorOwn<> get_simple_config_google_dns(Promise<SimpleConfigResult> promise, const ConfigShared *shared_config,
                                         bool is_test, int32 scheduler_id);
 
+ActorOwn<> get_simple_config_mozilla_dns(Promise<SimpleConfigResult> promise, const ConfigShared *shared_config,
+                                         bool is_test, int32 scheduler_id);
+
 class HttpDate {
   static bool is_leap(int32 year) {
     return year % 4 == 0 && (year % 100 != 0 || year % 400 == 0);

td/telegram/ContactsManager.cpp

@@ -4025,7 +4025,7 @@ void ContactsManager::set_profile_photo(const tl_object_ptr<td_api::InputFile> &
   CHECK(!file_view.is_encrypted());
   if (file_view.has_remote_location() && !file_view.remote_location().is_web()) {
     td_->create_handler<UpdateProfilePhotoQuery>(std::move(promise))
-        ->send(file_id, file_view.remote_location().as_input_photo());
+        ->send(td_->file_manager_->dup_file_id(file_id), file_view.remote_location().as_input_photo());
     return;
   }
 
@@ -5368,12 +5368,7 @@ void ContactsManager::on_binlog_user_event(BinlogEvent &&event) {
   auto user_id = log_event.user_id;
   LOG(INFO) << "Add " << user_id << " from binlog";
   User *u = add_user(user_id, "on_binlog_user_event");
-  if (!(u->first_name.empty() && u->last_name.empty()) && Slice(u->debug_source) == Slice("on_binlog_user_event")) {
-    LOG(ERROR) << "Skip adding already added " << user_id;
-    binlog_erase(G()->td_db()->get_binlog(), event.id_);
-    return;  // TODO fix bug in Binlog and remove that fix
-  }
-  LOG_CHECK(u->first_name.empty() && u->last_name.empty()) << user_id << " " << u->debug_source;
+  LOG_CHECK(u->first_name.empty() && u->last_name.empty()) << user_id;
   *u = std::move(log_event.u);  // users come from binlog before all other events, so just add them
 
   u->logevent_id = event.id_;
@@ -5859,12 +5854,7 @@ void ContactsManager::on_binlog_channel_event(BinlogEvent &&event) {
   auto channel_id = log_event.channel_id;
   LOG(INFO) << "Add " << channel_id << " from binlog";
   Channel *c = add_channel(channel_id, "on_binlog_channel_event");
-  if (!c->status.is_banned() && Slice(c->debug_source) == Slice("on_binlog_channel_event")) {
-    LOG(ERROR) << "Skip adding already added " << channel_id;
-    binlog_erase(G()->td_db()->get_binlog(), event.id_);
-    return;  // TODO fix bug in Binlog and remove that fix
-  }
-  LOG_CHECK(c->status.is_banned()) << channel_id << " " << c->debug_source;
+  LOG_CHECK(c->status.is_banned()) << channel_id;
   *c = std::move(log_event.c);  // channels come from binlog before all other events, so just add them
 
   c->logevent_id = event.id_;
@@ -7194,7 +7184,7 @@ void ContactsManager::on_update_user_local_was_online(UserId user_id, int32 loca
 
 void ContactsManager::on_update_user_local_was_online(User *u, UserId user_id, int32 local_was_online) {
   CHECK(u != nullptr);
-  if (u->is_deleted || u->is_bot || user_id == get_my_id()) {
+  if (u->is_deleted || u->is_bot || u->is_support || user_id == get_my_id()) {
     return;
   }
   if (u->was_online > G()->unix_time_cached()) {
@@ -8529,7 +8519,7 @@ void ContactsManager::on_update_chat_participant_count(Chat *c, ChatId chat_id,
   }
 
   if (c->participant_count != participant_count) {
-    if (version == c->version) {
+    if (version == c->version && participant_count != 0) {
       // version is not changed when deleted user is removed from the chat
       LOG_IF(ERROR, c->participant_count != participant_count + 1)
           << "Member count of " << chat_id << " has changed from " << c->participant_count << " to "
@@ -8988,11 +8978,7 @@ bool ContactsManager::get_user(UserId user_id, int left_tries, Promise<Unit> &&p
 
 ContactsManager::User *ContactsManager::add_user(UserId user_id, const char *source) {
   CHECK(user_id.is_valid());
-  User *u = &users_[user_id];
-  if (u->debug_source == nullptr) {
-    u->debug_source = source;
-  }
-  return u;
+  return &users_[user_id];
 }
 
 const ContactsManager::UserFull *ContactsManager::get_user_full(UserId user_id) const {
@@ -9509,7 +9495,6 @@ ContactsManager::Channel *ContactsManager::add_channel(ChannelId channel_id, con
     c->photo_source_id = it->second;
     channel_photo_file_source_ids_.erase(it);
   }
-  c->debug_source = source;
   return c;
 }
 

td/telegram/ContactsManager.h

@@ -531,8 +531,6 @@ class ContactsManager : public Actor {
 
     uint64 logevent_id = 0;
 
-    const char *debug_source = nullptr;
-
     template <class StorerT>
     void store(StorerT &storer) const;
 
@@ -665,8 +663,6 @@ class ContactsManager : public Actor {
     bool is_being_saved = false;  // is current channel being saved to the database
     uint64 logevent_id = 0;
 
-    const char *debug_source = nullptr;
-
     template <class StorerT>
     void store(StorerT &storer) const;
 

td/telegram/DocumentsManager.cpp

@@ -13,6 +13,7 @@
 #include "td/telegram/files/FileLocation.h"
 #include "td/telegram/files/FileManager.h"
 #include "td/telegram/files/FileType.h"
+#include "td/telegram/misc.h"
 #include "td/telegram/net/DcId.h"
 #include "td/telegram/Photo.h"
 #include "td/telegram/StickersManager.h"
@@ -282,8 +283,6 @@ Document DocumentsManager::on_get_document(RemoteDocument remote_document, Dialo
       return {};
     }
 
-    fix_animated_sticker_type();
-
     if (document_type != Document::Type::VoiceNote) {
       thumbnail = get_secret_thumbnail_photo_size(td_->file_manager_.get(), std::move(document->thumb_),
                                                   owner_dialog_id, document->thumb_w_, document->thumb_h_);
@@ -331,8 +330,6 @@ Document DocumentsManager::on_get_document(RemoteDocument remote_document, Dialo
       default:
         UNREACHABLE();
     }
-
-    fix_animated_sticker_type();
   }
 
   LOG(DEBUG) << "Receive document with id = " << id << " of type " << document_type;
@@ -341,6 +338,8 @@ Document DocumentsManager::on_get_document(RemoteDocument remote_document, Dialo
     return {};
   }
 
+  file_name = strip_empty_characters(file_name, 255, true);
+
   auto suggested_file_name = file_name;
   if (suggested_file_name.empty()) {
     suggested_file_name = to_string(static_cast<uint64>(id));

td/telegram/FileReferenceManager.cpp

@@ -201,12 +201,20 @@ void FileReferenceManager::run_node(NodeId node_id) {
     node.query = {};
     return;
   }
+  if (node.last_successful_repair_time >= Time::now() - 60) {
+    VLOG(file_references) << "Recently repaired file reference for file " << node_id << ", do not try again";
+    for (auto &p : node.query->promises) {
+      p.set_error(Status::Error(429, "Too Many Requests: retry after 60"));
+    }
+    node.query = {};
+    return;
+  }
   auto file_source_id = node.file_source_ids.next();
   send_query({node_id, node.query->generation}, file_source_id);
 }
 
 void FileReferenceManager::send_query(Destination dest, FileSourceId file_source_id) {
-  VLOG(file_references) << "Send file references repair query for file " << dest.node_id << " with generation "
+  VLOG(file_references) << "Send file reference repair query for file " << dest.node_id << " with generation "
                         << dest.generation << " from " << file_source_id;
   auto &node = nodes_[dest.node_id];
   node.query->active_queries++;
@@ -288,7 +296,7 @@ void FileReferenceManager::send_query(Destination dest, FileSourceId file_source
 
 FileReferenceManager::Destination FileReferenceManager::on_query_result(Destination dest, FileSourceId file_source_id,
                                                                         Status status, int32 sub) {
-  VLOG(file_references) << "Receive result of file references repair query for file " << dest.node_id
+  VLOG(file_references) << "Receive result of file reference repair query for file " << dest.node_id
                         << " with generation " << dest.generation << " from " << file_source_id << ": " << status << " "
                         << sub;
   auto &node = nodes_[dest.node_id];
@@ -313,6 +321,7 @@ FileReferenceManager::Destination FileReferenceManager::on_query_result(Destinat
   }
 
   if (status.is_ok()) {
+    node.last_successful_repair_time = Time::now();
     for (auto &p : query->promises) {
       p.set_value(Unit());
     }

td/telegram/FileReferenceManager.h

@@ -87,6 +87,7 @@ class FileReferenceManager : public Actor {
   struct Node {
     SetWithPosition<FileSourceId> file_source_ids;
     unique_ptr<Query> query;
+    double last_successful_repair_time = -1e10;
   };
 
   struct FileSourceMessage {

td/telegram/MessageContent.cpp

@@ -2102,10 +2102,9 @@ static tl_object_ptr<telegram_api::inputMediaInvoice> get_input_media_invoice(co
       message_invoice->start_parameter);
 }
 
-static tl_object_ptr<telegram_api::InputMedia> get_input_media(const MessageContent *content, Td *td,
-                                                               tl_object_ptr<telegram_api::InputFile> input_file,
-                                                               tl_object_ptr<telegram_api::InputFile> input_thumbnail,
-                                                               int32 ttl) {
+static tl_object_ptr<telegram_api::InputMedia> get_input_media_impl(
+    const MessageContent *content, Td *td, tl_object_ptr<telegram_api::InputFile> input_file,
+    tl_object_ptr<telegram_api::InputFile> input_thumbnail, int32 ttl) {
   switch (content->get_type()) {
     case MessageContentType::Animation: {
       auto m = static_cast<const MessageAnimation *>(content);
@@ -2208,10 +2207,11 @@ static tl_object_ptr<telegram_api::InputMedia> get_input_media(const MessageCont
 tl_object_ptr<telegram_api::InputMedia> get_input_media(const MessageContent *content, Td *td,
                                                         tl_object_ptr<telegram_api::InputFile> input_file,
                                                         tl_object_ptr<telegram_api::InputFile> input_thumbnail,
-                                                        FileId file_id, FileId thumbnail_file_id, int32 ttl) {
+                                                        FileId file_id, FileId thumbnail_file_id, int32 ttl,
+                                                        bool force) {
   bool had_input_file = input_file != nullptr;
   bool had_input_thumbnail = input_thumbnail != nullptr;
-  auto input_media = get_input_media(content, td, std::move(input_file), std::move(input_thumbnail), ttl);
+  auto input_media = get_input_media_impl(content, td, std::move(input_file), std::move(input_thumbnail), ttl);
   auto was_uploaded = FileManager::extract_was_uploaded(input_media);
   if (had_input_file) {
     if (!was_uploaded) {
@@ -2229,7 +2229,7 @@ tl_object_ptr<telegram_api::InputMedia> get_input_media(const MessageContent *co
   }
   if (!was_uploaded) {
     auto file_reference = FileManager::extract_file_reference(input_media);
-    if (file_reference == FileReferenceView::invalid_file_reference()) {
+    if (file_reference == FileReferenceView::invalid_file_reference() && !force) {
       return nullptr;
     }
   }
@@ -2237,7 +2237,7 @@ tl_object_ptr<telegram_api::InputMedia> get_input_media(const MessageContent *co
 }
 
 tl_object_ptr<telegram_api::InputMedia> get_input_media(const MessageContent *content, Td *td, int32 ttl, bool force) {
-  auto input_media = get_input_media(content, td, nullptr, nullptr, ttl);
+  auto input_media = get_input_media_impl(content, td, nullptr, nullptr, ttl);
   auto file_reference = FileManager::extract_file_reference(input_media);
   if (file_reference == FileReferenceView::invalid_file_reference() && !force) {
     return nullptr;

td/telegram/MessageContent.h

@@ -147,7 +147,8 @@ SecretInputMedia get_secret_input_media(const MessageContent *content, Td *td,
 tl_object_ptr<telegram_api::InputMedia> get_input_media(const MessageContent *content, Td *td,
                                                         tl_object_ptr<telegram_api::InputFile> input_file,
                                                         tl_object_ptr<telegram_api::InputFile> input_thumbnail,
-                                                        FileId file_id, FileId thumbnail_file_id, int32 ttl);
+                                                        FileId file_id, FileId thumbnail_file_id, int32 ttl,
+                                                        bool force);
 
 tl_object_ptr<telegram_api::InputMedia> get_input_media(const MessageContent *content, Td *td, int32 ttl, bool force);
 

td/telegram/MessageEntity.cpp

@@ -976,7 +976,7 @@ Slice fix_url(Slice str) {
 }
 
 const std::unordered_set<Slice, SliceHash> &get_valid_short_usernames() {
-  static const std::unordered_set<Slice, SliceHash> valid_usernames{"ya",   "gif",  "wiki", "vid",  "bing", "pic",
+  static const std::unordered_set<Slice, SliceHash> valid_usernames{"gif",  "wiki", "vid",  "bing", "pic",
                                                                     "bold", "imdb", "coub", "like", "vote"};
   return valid_usernames;
 }
@@ -1181,7 +1181,7 @@ string get_first_url(Slice text, const vector<MessageEntity> &entities) {
         break;
       case MessageEntity::Type::Url: {
         Slice url = utf8_utf16_substr(text, entity.offset, entity.length);
-        if (begins_with(url, "tg:") || is_plain_domain(url)) {
+        if (begins_with(url, "ton:") || begins_with(url, "tg:") || is_plain_domain(url)) {
           continue;
         }
         return url.str();
@@ -1199,7 +1199,7 @@ string get_first_url(Slice text, const vector<MessageEntity> &entities) {
       case MessageEntity::Type::PreCode:
         break;
       case MessageEntity::Type::TextUrl:
-        if (begins_with(entity.argument, "tg:")) {
+        if (begins_with(entity.argument, "ton:") || begins_with(entity.argument, "tg:")) {
           continue;
         }
         return entity.argument;

td/telegram/MessagesManager.cpp

@@ -4139,7 +4139,14 @@ void MessagesManager::Dialog::parse(ParserT &parser) {
   if (has_message_count_by_index) {
     int32 size;
     parse(size, parser);
-    CHECK(static_cast<size_t>(size) <= message_count_by_index.size());
+    if (size < 0) {
+      // the logevent is broken
+      // it should be impossible, but has happenned at least once
+      parser.set_error("Wrong message_count_by_index table size");
+      return;
+    }
+    LOG_CHECK(static_cast<size_t>(size) <= message_count_by_index.size())
+        << size << " " << message_count_by_index.size();
     for (int32 i = 0; i < size; i++) {
       parse(message_count_by_index[i], parser);
     }
@@ -4205,7 +4212,7 @@ void MessagesManager::CallsDbState::parse(ParserT &parser) {
     parse(first_calls_database_message_id_by_index[i], parser);
   }
   parse(size, parser);
-  CHECK(static_cast<size_t>(size) <= message_count_by_index.size());
+  LOG_CHECK(static_cast<size_t>(size) <= message_count_by_index.size()) << size << " " << message_count_by_index.size();
   for (int32 i = 0; i < size; i++) {
     parse(message_count_by_index[i], parser);
   }
@@ -5866,7 +5873,7 @@ bool MessagesManager::update_dialog_notification_settings(DialogId dialog_id,
     *current_settings = new_settings;
 
     if (!was_muted && is_dialog_muted(d)) {
-      remove_all_dialog_notifications(d, d->message_notification_group, "save_scope_notification_settings");
+      remove_all_dialog_notifications(d, false, "save_scope_notification_settings");
     }
     if (is_dialog_pinned_message_notifications_disabled(d) && d->mention_notification_group.group_id.is_valid() &&
         d->pinned_message_notification_message_id.is_valid()) {
@@ -6117,7 +6124,7 @@ void MessagesManager::on_update_dialog_notify_settings(
     return;
   }
 
-  const DialogNotificationSettings notification_settings = td::get_dialog_notification_settings(
+  const DialogNotificationSettings notification_settings = ::td::get_dialog_notification_settings(
       std::move(peer_notify_settings), current_settings->use_default_disable_pinned_message_notifications,
       current_settings->disable_pinned_message_notifications,
       current_settings->use_default_disable_mention_notifications, current_settings->disable_mention_notifications);
@@ -6137,7 +6144,7 @@ void MessagesManager::on_update_scope_notify_settings(
   auto old_notification_settings = get_scope_notification_settings(scope);
   CHECK(old_notification_settings != nullptr);
 
-  const ScopeNotificationSettings notification_settings = td::get_scope_notification_settings(
+  const ScopeNotificationSettings notification_settings = ::td::get_scope_notification_settings(
       std::move(peer_notify_settings), old_notification_settings->disable_pinned_message_notifications,
       old_notification_settings->disable_mention_notifications);
   if (!notification_settings.is_synchronized) {
@@ -6490,6 +6497,12 @@ void MessagesManager::do_send_media(DialogId dialog_id, Message *m, FileId file_
                                     tl_object_ptr<telegram_api::InputFile> input_thumbnail) {
   CHECK(m != nullptr);
 
+  bool have_input_file = input_file != nullptr;
+  bool have_input_thumbnail = input_thumbnail != nullptr;
+  LOG(INFO) << "Do send media file " << file_id << " with thumbnail " << thumbnail_file_id
+            << ", have_input_file = " << have_input_file << ", have_input_thumbnail = " << have_input_thumbnail
+            << ", ttl = " << m->ttl;
+
   MessageContent *content = nullptr;
   if (m->message_id.is_server()) {
     content = m->edited_content.get();
@@ -6501,10 +6514,13 @@ void MessagesManager::do_send_media(DialogId dialog_id, Message *m, FileId file_
     content = m->content.get();
   }
 
-  on_message_media_uploaded(dialog_id, m,
-                            get_input_media(content, td_, std::move(input_file), std::move(input_thumbnail), file_id,
-                                            thumbnail_file_id, m->ttl),
-                            file_id, thumbnail_file_id);
+  auto input_media = get_input_media(content, td_, std::move(input_file), std::move(input_thumbnail), file_id,
+                                     thumbnail_file_id, m->ttl, true);
+  LOG_CHECK(input_media != nullptr) << to_string(get_message_object(dialog_id, m)) << ' ' << have_input_file << ' '
+                                    << have_input_thumbnail << ' ' << file_id << ' ' << thumbnail_file_id << ' '
+                                    << m->ttl;
+
+  on_message_media_uploaded(dialog_id, m, std::move(input_media), file_id, thumbnail_file_id);
 }
 
 void MessagesManager::do_send_secret_media(DialogId dialog_id, Message *m, FileId file_id, FileId thumbnail_file_id,
@@ -6513,6 +6529,11 @@ void MessagesManager::do_send_secret_media(DialogId dialog_id, Message *m, FileI
   CHECK(dialog_id.get_type() == DialogType::SecretChat);
   CHECK(m != nullptr);
   CHECK(m->message_id.is_yet_unsent());
+
+  bool have_input_file = input_encrypted_file != nullptr;
+  LOG(INFO) << "Do send secret media file " << file_id << " with thumbnail " << thumbnail_file_id
+            << ", have_input_file = " << have_input_file;
+
   auto layer = td_->contacts_manager_->get_secret_chat_layer(dialog_id.get_secret_chat_id());
   on_secret_message_media_uploaded(
       dialog_id, m,
@@ -8145,7 +8166,7 @@ void MessagesManager::delete_all_dialog_messages(Dialog *d, bool remove_from_dia
   d->mention_notification_group.max_removed_notification_id = NotificationId();  // it is not needed anymore
   d->mention_notification_group.max_removed_message_id = MessageId();            // it is not needed anymore
   std::fill(d->message_count_by_index.begin(), d->message_count_by_index.end(), 0);
-  CHECK(d->notification_id_to_message_id.empty());
+  d->notification_id_to_message_id.clear();
 
   if (has_last_message_id) {
     set_dialog_last_message_id(d, MessageId(), "delete_all_dialog_messages");
@@ -8235,7 +8256,7 @@ void MessagesManager::read_all_dialog_mentions(DialogId dialog_id, Promise<Unit>
       on_dialog_updated(dialog_id, "read_all_dialog_mentions");
     }
   }
-  remove_message_dialog_notifications(d, MessageId::max(), d->mention_notification_group, "read_all_dialog_mentions");
+  remove_message_dialog_notifications(d, MessageId::max(), true, "read_all_dialog_mentions");
 
   read_all_dialog_mentions_on_server(dialog_id, 0, std::move(promise));
 }
@@ -8441,7 +8462,8 @@ void MessagesManager::read_history_inbox(DialogId dialog_id, MessageId max_messa
 
   Dialog *d = get_dialog_force(dialog_id);
   if (d != nullptr) {
-    if (!max_message_id.is_valid() && (max_message_id != MessageId() || d->last_read_inbox_message_id != MessageId())) {
+    // there can be updateReadHistoryInbox up to message 0, if messages where read and then all messages where deleted
+    if (!max_message_id.is_valid() && max_message_id != MessageId()) {
       LOG(ERROR) << "Receive read inbox update in " << dialog_id << " up to " << max_message_id << " from " << source;
       return;
     }
@@ -8475,7 +8497,8 @@ void MessagesManager::read_history_inbox(DialogId dialog_id, MessageId max_messa
     }
 
     if (max_message_id.get() > d->last_new_message_id.get() && dialog_id.get_type() == DialogType::Channel) {
-      get_channel_difference(dialog_id, d->pts, true, "read_history_inbox");
+      LOG(INFO) << "Schedule getDifference in " << dialog_id.get_channel_id();
+      channel_get_difference_retry_timeout_.add_timeout_in(dialog_id.get(), 0.001);
     }
 
     bool is_saved_messages = dialog_id == get_my_dialog_id();
@@ -8720,7 +8743,7 @@ void MessagesManager::set_dialog_max_unavailable_message_id(DialogId dialog_id,
                      << " from " << source << ", but last new message id is " << d->last_new_message_id;
         }
         max_unavailable_message_id = d->last_new_message_id;
-      } else if (max_unavailable_message_id.is_server()) {
+      } else if (max_unavailable_message_id.is_valid() && max_unavailable_message_id.is_server()) {
         set_dialog_last_new_message_id(d, max_unavailable_message_id, source);
       }
     }
@@ -8993,7 +9016,12 @@ void MessagesManager::ttl_unregister_message(DialogId dialog_id, const Message *
 
   TtlNode ttl_node(dialog_id, m->message_id);
   auto it = ttl_nodes_.find(ttl_node);
-  LOG_CHECK(it != ttl_nodes_.end()) << dialog_id << " " << m->message_id << " " << source << " " << G()->close_flag();
+
+  // expect m->ttl == 0, but m->ttl_expires_at > 0 from binlog
+  LOG_CHECK(it != ttl_nodes_.end()) << dialog_id << " " << m->message_id << " " << source << " " << G()->close_flag()
+                                    << " " << m->ttl << " " << m->ttl_expires_at << " " << Time::now() << " "
+                                    << m->from_database;
+
   auto *heap_node = it->as_heap_node();
   if (heap_node->in_heap()) {
     ttl_heap_.erase(heap_node);
@@ -10029,8 +10057,8 @@ std::pair<DialogId, unique_ptr<MessagesManager::Message>> MessagesManager::creat
   }
 
   int32 ttl = message_info.ttl;
-  bool is_content_secret =
-      is_secret_message_content(ttl, message_info.content->get_type());  // should be calculated before TTL is adjusted
+  auto content_type = message_info.content->get_type();
+  bool is_content_secret = is_secret_message_content(ttl, content_type);  // should be calculated before TTL is adjusted
   if (ttl < 0) {
     LOG(ERROR) << "Wrong ttl = " << ttl << " received in " << message_id << " in " << dialog_id;
     ttl = 0;
@@ -10063,7 +10091,8 @@ std::pair<DialogId, unique_ptr<MessagesManager::Message>> MessagesManager::creat
   message->is_outgoing = is_outgoing;
   message->is_channel_post = is_channel_post;
   message->contains_mention =
-      !is_outgoing && dialog_type != DialogType::User && (flags & MESSAGE_FLAG_HAS_MENTION) != 0;
+      !is_outgoing && dialog_type != DialogType::User &&
+      ((flags & MESSAGE_FLAG_HAS_MENTION) != 0 || content_type == MessageContentType::PinMessage);
   message->contains_unread_mention =
       message_id.is_server() && message->contains_mention && (flags & MESSAGE_FLAG_HAS_UNREAD_CONTENT) != 0 &&
       (dialog_type == DialogType::Chat || (dialog_type == DialogType::Channel && !is_broadcast_channel(dialog_id)));
@@ -10075,7 +10104,6 @@ std::pair<DialogId, unique_ptr<MessagesManager::Message>> MessagesManager::creat
   message->reply_markup = get_reply_markup(std::move(message_info.reply_markup), td_->auth_manager_->is_bot(), false,
                                            message->contains_mention || dialog_id.get_type() == DialogType::User);
 
-  auto content_type = message->content->get_type();
   if (content_type == MessageContentType::ExpiredPhoto || content_type == MessageContentType::ExpiredVideo) {
     CHECK(message->ttl == 0);  // ttl is ignored/set to 0 if the message has already been expired
     if (message->reply_markup != nullptr) {
@@ -10414,7 +10442,7 @@ void MessagesManager::set_dialog_is_empty(Dialog *d, const char *source) {
     set_dialog_reply_markup(d, MessageId());
   }
   std::fill(d->message_count_by_index.begin(), d->message_count_by_index.end(), 0);
-  CHECK(d->notification_id_to_message_id.empty());
+  d->notification_id_to_message_id.clear();
 
   if (d->delete_last_message_date != 0) {
     if (d->is_last_message_deleted_locally && d->last_clear_history_date == 0) {
@@ -10504,15 +10532,14 @@ void MessagesManager::set_dialog_pinned_message_notification(Dialog *d, MessageI
   if (old_message_id.is_valid()) {
     auto m = get_message_force(d, old_message_id, "set_dialog_pinned_message_notification");
     if (m != nullptr && m->notification_id.is_valid() && is_message_notification_active(d, m)) {
-      // Because we removing pinned message notification it will call set_dialog_pinned_message_notification recursively
-      // after m->notification_id is removed, so everything will work. Can't remove pinned_message_notification_message_id
-      // before the call, because the notification needs to be still active inside remove_message_notification_id
-      remove_message_notification_id(d, m, true, false);
+      // Can't remove pinned_message_notification_message_id  before the call,
+      // because the notification needs to be still active inside remove_message_notification_id
+      remove_message_notification_id(d, m, true, false, true);
       on_message_changed(d, m, false, "set_dialog_pinned_message_notification");
     } else {
       send_closure_later(G()->notification_manager(), &NotificationManager::remove_temporary_notification_by_message_id,
                          d->mention_notification_group.group_id, old_message_id, false,
-                         "set_dialog_pinned_message_notification");
+                         "set_dialog_pinned_message_notification 2");
     }
   }
   d->pinned_message_notification_message_id = message_id;
@@ -11089,7 +11116,8 @@ void MessagesManager::delete_notification_id_to_message_id_correspondence(Dialog
   }
 }
 
-void MessagesManager::remove_message_notification_id(Dialog *d, Message *m, bool is_permanent, bool force_update) {
+void MessagesManager::remove_message_notification_id(Dialog *d, Message *m, bool is_permanent, bool force_update,
+                                                     bool ignore_pinned_message_notification_removal) {
   CHECK(d != nullptr);
   CHECK(m != nullptr);
   if (!m->notification_id.is_valid()) {
@@ -11111,7 +11139,8 @@ void MessagesManager::remove_message_notification_id(Dialog *d, Message *m, bool
   delete_notification_id_to_message_id_correspondence(d, notification_id, m->message_id);
   m->removed_notification_id = m->notification_id;
   m->notification_id = NotificationId();
-  if (d->pinned_message_notification_message_id == m->message_id && is_permanent) {
+  if (d->pinned_message_notification_message_id == m->message_id && is_permanent &&
+      !ignore_pinned_message_notification_removal) {
     remove_dialog_pinned_message_notification(d);  // must be called after notification_id is removed
   }
   if (group_info.last_notification_id == notification_id) {
@@ -11472,12 +11501,12 @@ unique_ptr<MessagesManager::Message> MessagesManager::do_delete_message(Dialog *
     update_message_count_by_index(d, -1, result.get());
   }
 
-  on_message_deleted(d, result.get());
+  on_message_deleted(d, result.get(), source);
 
   return result;
 }
 
-void MessagesManager::on_message_deleted(Dialog *d, Message *m) {
+void MessagesManager::on_message_deleted(Dialog *d, Message *m, const char *source) {
   switch (d->dialog_id.get_type()) {
     case DialogType::User:
     case DialogType::Chat:
@@ -11493,7 +11522,7 @@ void MessagesManager::on_message_deleted(Dialog *d, Message *m) {
     default:
       UNREACHABLE();
   }
-  ttl_unregister_message(d->dialog_id, m, Time::now(), "on_message_deleted");
+  ttl_unregister_message(d->dialog_id, m, Time::now(), source);
   unregister_message_content(td_, m->content.get(), {d->dialog_id, m->message_id});
   if (m->notification_id.is_valid()) {
     delete_notification_id_to_message_id_correspondence(d, m->notification_id, m->message_id);
@@ -11527,7 +11556,7 @@ void MessagesManager::do_delete_all_dialog_messages(Dialog *d, unique_ptr<Messag
     cancel_edit_message_media(d->dialog_id, m.get(), "Message was deleted");
   }
 
-  on_message_deleted(d, m.get());
+  on_message_deleted(d, m.get(), "do_delete_all_dialog_messages");
 
   m = nullptr;
 }
@@ -16801,7 +16830,9 @@ void MessagesManager::on_upload_message_media_finished(int64 media_album_id, Dia
   auto &request = it->second;
   CHECK(request.dialog_id == dialog_id);
   auto message_it = std::find(request.message_ids.begin(), request.message_ids.end(), message_id);
-  CHECK(message_it != request.message_ids.end());
+  LOG_CHECK(message_it != request.message_ids.end())
+      << dialog_id << ' ' << request.message_ids << ' ' << message_id << ' ' << request.finished_count << ' '
+      << request.is_finished << ' ' << request.results;
   auto pos = static_cast<size_t>(message_it - request.message_ids.begin());
 
   if (request.is_finished[pos]) {
@@ -16888,7 +16919,7 @@ void MessagesManager::do_send_message_group(int64 media_album_id) {
       bool has_remote = file_view.has_remote_location();
       bool is_web = has_remote ? file_view.remote_location().is_web() : false;
       LOG(FATAL) << request.dialog_id << " " << request.finished_count << " " << i << " " << request.message_ids << " "
-                 << request.results << " " << m->ttl << " " << has_remote << " "
+                 << request.is_finished << " " << request.results << " " << m->ttl << " " << has_remote << " "
                  << file_view.has_alive_remote_location() << " " << file_view.has_active_upload_remote_location() << " "
                  << file_view.has_active_download_remote_location() << " " << file_view.is_encrypted() << " " << is_web
                  << " " << file_view.has_url() << " "
@@ -18579,6 +18610,7 @@ Result<vector<MessageId>> MessagesManager::forward_messages(DialogId to_dialog_i
       LOG(INFO) << "Can't find " << message_id << " to forward";
       continue;
     }
+    CHECK(message_id.is_valid());
 
     if (!can_forward_message(from_dialog_id, forwarded_message)) {
       LOG(INFO) << "Can't forward " << message_id;
@@ -19384,11 +19416,11 @@ MessagesManager::MessageNotificationGroup MessagesManager::get_message_notificat
   auto &group_info = from_mentions ? d->mention_notification_group : d->message_notification_group;
 
   MessageNotificationGroup result;
-  VLOG(notifications) << "Found " << group_info.group_id << '/' << d->dialog_id << " by " << group_id << " with "
-                      << d->unread_mention_count << " unread mentions, pinned "
-                      << d->pinned_message_notification_message_id << ", new secret chat "
-                      << d->new_secret_chat_notification_id << " and " << d->server_unread_count + d->local_unread_count
-                      << " unread messages";
+  VLOG(notifications) << "Found " << (from_mentions ? "Mentions " : "Messages ") << group_info.group_id << '/'
+                      << d->dialog_id << " by " << group_id << " with " << d->unread_mention_count
+                      << " unread mentions, pinned " << d->pinned_message_notification_message_id
+                      << ", new secret chat " << d->new_secret_chat_notification_id << " and "
+                      << d->server_unread_count + d->local_unread_count << " unread messages";
   result.dialog_id = d->dialog_id;
   result.total_count = get_dialog_pending_notification_count(d, from_mentions);
   auto pending_notification_count =
@@ -19404,7 +19436,7 @@ MessagesManager::MessageNotificationGroup MessagesManager::get_message_notificat
     result.type = NotificationGroupType::SecretChat;
     result.notifications.emplace_back(d->new_secret_chat_notification_id,
                                       td_->contacts_manager_->get_secret_chat_date(d->dialog_id.get_secret_chat_id()),
-                                      create_new_secret_chat_notification());
+                                      false, create_new_secret_chat_notification());
   } else {
     result.type = from_mentions ? NotificationGroupType::Mentions : NotificationGroupType::Messages;
     result.notifications = get_message_notifications_from_database_force(
@@ -19468,7 +19500,8 @@ void MessagesManager::try_add_pinned_message_notification(Dialog *d, vector<Noti
       }
 
       auto pos = res.size();
-      res.emplace_back(m->notification_id, m->date, create_new_message_notification(message_id));
+      res.emplace_back(m->notification_id, m->date, m->disable_notification,
+                       create_new_message_notification(message_id));
       while (pos > 0 && res[pos - 1].type->get_message_id().get() < message_id.get()) {
         std::swap(res[pos - 1], res[pos]);
         pos--;
@@ -19582,7 +19615,8 @@ vector<Notification> MessagesManager::get_message_notifications_from_database_fo
 
       if (is_correct) {
         // skip mention messages returned among unread messages
-        res.emplace_back(m->notification_id, m->date, create_new_message_notification(m->message_id));
+        res.emplace_back(m->notification_id, m->date, m->disable_notification,
+                         create_new_message_notification(m->message_id));
       } else {
         remove_message_notification_id(d, m, true, false);
         on_message_changed(d, m, false, "get_message_notifications_from_database_force");
@@ -19686,7 +19720,7 @@ void MessagesManager::get_message_notifications_from_database(DialogId dialog_id
     vector<Notification> notifications;
     if (!from_mentions && d->new_secret_chat_notification_id.get() < from_notification_id.get()) {
       notifications.emplace_back(d->new_secret_chat_notification_id,
-                                 td_->contacts_manager_->get_secret_chat_date(d->dialog_id.get_secret_chat_id()),
+                                 td_->contacts_manager_->get_secret_chat_date(d->dialog_id.get_secret_chat_id()), false,
                                  create_new_secret_chat_notification());
     }
     return promise.set_value(std::move(notifications));
@@ -19825,7 +19859,8 @@ void MessagesManager::on_get_message_notifications_from_database(DialogId dialog
 
     if (is_correct) {
       // skip mention messages returned among unread messages
-      res.emplace_back(m->notification_id, m->date, create_new_message_notification(m->message_id));
+      res.emplace_back(m->notification_id, m->date, m->disable_notification,
+                       create_new_message_notification(m->message_id));
     } else {
       remove_message_notification_id(d, m, true, false);
       on_message_changed(d, m, false, "on_get_message_notifications_from_database");
@@ -20254,7 +20289,7 @@ bool MessagesManager::add_new_message_notification(Dialog *d, Message *m, bool f
   bool is_silent = m->disable_notification || m->message_id.get() <= d->max_notification_message_id.get();
   send_closure_later(G()->notification_manager(), &NotificationManager::add_notification, notification_group_id,
                      from_mentions ? NotificationGroupType::Mentions : NotificationGroupType::Messages, d->dialog_id,
-                     m->date, settings_dialog_id, is_silent, min_delay_ms, m->notification_id,
+                     m->date, settings_dialog_id, m->disable_notification, is_silent, min_delay_ms, m->notification_id,
                      create_new_message_notification(m->message_id), "add_new_message_notification");
   return true;
 }
@@ -20295,9 +20330,9 @@ void MessagesManager::flush_pending_new_message_notifications(DialogId dialog_id
   }
 }
 
-void MessagesManager::remove_all_dialog_notifications(Dialog *d, NotificationGroupInfo &group_info,
-                                                      const char *source) {
+void MessagesManager::remove_all_dialog_notifications(Dialog *d, bool from_mentions, const char *source) {
   // removes up to group_info.last_notification_id
+  NotificationGroupInfo &group_info = from_mentions ? d->mention_notification_group : d->message_notification_group;
   if (group_info.group_id.is_valid() && group_info.last_notification_id.is_valid() &&
       group_info.max_removed_notification_id != group_info.last_notification_id) {
     VLOG(notifications) << "Set max_removed_notification_id in " << group_info.group_id << '/' << d->dialog_id << " to "
@@ -20306,6 +20341,12 @@ void MessagesManager::remove_all_dialog_notifications(Dialog *d, NotificationGro
     if (d->max_notification_message_id.get() > group_info.max_removed_message_id.get()) {
       group_info.max_removed_message_id = d->max_notification_message_id.get_prev_server_message_id();
     }
+    if (!d->pending_new_message_notifications.empty()) {
+      for (auto &it : d->pending_new_message_notifications) {
+        it.first = DialogId();
+      }
+      flush_pending_new_message_notifications(d->dialog_id, from_mentions, DialogId(UserId(2)));
+    }
     // remove_message_notifications will be called by NotificationManager
     send_closure_later(G()->notification_manager(), &NotificationManager::remove_notification_group,
                        group_info.group_id, group_info.last_notification_id, MessageId(), 0, true, Promise<Unit>());
@@ -20318,9 +20359,10 @@ void MessagesManager::remove_all_dialog_notifications(Dialog *d, NotificationGro
   }
 }
 
-void MessagesManager::remove_message_dialog_notifications(Dialog *d, MessageId max_message_id,
-                                                          NotificationGroupInfo &group_info, const char *source) {
+void MessagesManager::remove_message_dialog_notifications(Dialog *d, MessageId max_message_id, bool from_mentions,
+                                                          const char *source) {
   // removes up to max_message_id
+  NotificationGroupInfo &group_info = from_mentions ? d->mention_notification_group : d->message_notification_group;
   if (!group_info.group_id.is_valid()) {
     return;
   }
@@ -20328,6 +20370,15 @@ void MessagesManager::remove_message_dialog_notifications(Dialog *d, MessageId m
   VLOG(notifications) << "Remove message dialog notifications in " << group_info.group_id << '/' << d->dialog_id
                       << " up to " << max_message_id << " from " << source;
 
+  if (!d->pending_new_message_notifications.empty()) {
+    for (auto &it : d->pending_new_message_notifications) {
+      if (it.second.get() <= max_message_id.get()) {
+        it.first = DialogId();
+      }
+    }
+    flush_pending_new_message_notifications(d->dialog_id, from_mentions, DialogId(UserId(3)));
+  }
+
   auto max_notification_message_id = max_message_id;
   if (d->last_message_id.is_valid() && max_notification_message_id.get() >= d->last_message_id.get()) {
     max_notification_message_id = d->last_message_id;
@@ -23992,8 +24043,8 @@ void MessagesManager::delete_all_dialog_messages_from_database(Dialog *d, Messag
       d->pinned_message_notification_message_id.get() <= max_message_id.get()) {
     remove_dialog_pinned_message_notification(d);
   }
-  remove_message_dialog_notifications(d, max_message_id, d->message_notification_group, source);
-  remove_message_dialog_notifications(d, max_message_id, d->mention_notification_group, source);
+  remove_message_dialog_notifications(d, max_message_id, false, source);
+  remove_message_dialog_notifications(d, max_message_id, true, source);
 
   if (!G()->parameters().use_message_db) {
     return;
@@ -24742,8 +24793,8 @@ void MessagesManager::force_create_dialog(DialogId dialog_id, const char *source
               VLOG(notifications) << "Create " << d->new_secret_chat_notification_id << " with " << secret_chat_id;
               send_closure_later(G()->notification_manager(), &NotificationManager::add_notification,
                                  notification_group_id, NotificationGroupType::SecretChat, dialog_id, date, dialog_id,
-                                 false, 0, d->new_secret_chat_notification_id, create_new_secret_chat_notification(),
-                                 "add_new_secret_chat_notification");
+                                 false, false, 0, d->new_secret_chat_notification_id,
+                                 create_new_secret_chat_notification(), "add_new_secret_chat_notification");
             }
           }
         }
@@ -25003,6 +25054,10 @@ void MessagesManager::fix_new_dialog(Dialog *d, unique_ptr<Message> &&last_datab
     last_message_id = last_database_message_id;
   }
 
+  if (!d->last_new_message_id.is_valid() && d->last_new_message_id != MessageId()) {
+    LOG(ERROR) << "Drop invalid last_new_message_id " << d->last_new_message_id << " in " << dialog_id;
+    d->last_new_message_id = MessageId();
+  }
   if (!d->last_new_message_id.is_valid() && d->max_unavailable_message_id.is_valid() &&
       d->max_unavailable_message_id.is_server()) {
     LOG(ERROR) << "Bugfixing wrong last_new_message_id with max_unavailable_message_id to "
@@ -25438,8 +25493,8 @@ bool MessagesManager::set_dialog_order(Dialog *d, int64 new_order, bool need_sen
       channel_get_difference_retry_timeout_.add_timeout_in(dialog_id.get(), 0.001);
     }
     if (dialog_type == DialogType::Channel && !has_unread_counter) {
-      remove_all_dialog_notifications(d, d->message_notification_group, "set_dialog_order 1");
-      remove_all_dialog_notifications(d, d->mention_notification_group, "set_dialog_order 2");
+      remove_all_dialog_notifications(d, false, "set_dialog_order 1");
+      remove_all_dialog_notifications(d, true, "set_dialog_order 2");
       clear_active_dialog_actions(dialog_id);
     }
   }
@@ -25455,8 +25510,8 @@ bool MessagesManager::set_dialog_order(Dialog *d, int64 new_order, bool need_sen
     send_update_chat_is_sponsored(d);
     if (!is_loaded_from_database && is_sponsored) {
       // channel is sponsored only if user isn't a channel member
-      remove_all_dialog_notifications(d, d->message_notification_group, "set_dialog_order 3");
-      remove_all_dialog_notifications(d, d->mention_notification_group, "set_dialog_order 4");
+      remove_all_dialog_notifications(d, false, "set_dialog_order 3");
+      remove_all_dialog_notifications(d, true, "set_dialog_order 4");
     }
     need_update = false;
   }
@@ -27065,13 +27120,17 @@ void MessagesManager::suffix_load_add_query(Dialog *d,
 
 void MessagesManager::suffix_load_till_date(Dialog *d, int32 date, Promise<> promise) {
   LOG(INFO) << "Load suffix of " << d->dialog_id << " till date " << date;
-  auto condition = [date](const Message *m) { return m != nullptr && m->date < date; };
+  auto condition = [date](const Message *m) {
+    return m != nullptr && m->date < date;
+  };
   suffix_load_add_query(d, std::make_pair(std::move(promise), std::move(condition)));
 }
 
 void MessagesManager::suffix_load_till_message_id(Dialog *d, MessageId message_id, Promise<> promise) {
   LOG(INFO) << "Load suffix of " << d->dialog_id << " till " << message_id;
-  auto condition = [message_id](const Message *m) { return m != nullptr && m->message_id.get() < message_id.get(); };
+  auto condition = [message_id](const Message *m) {
+    return m != nullptr && m->message_id.get() < message_id.get();
+  };
   suffix_load_add_query(d, std::make_pair(std::move(promise), std::move(condition)));
 }
 
@@ -27150,7 +27209,7 @@ void MessagesManager::get_payment_form(FullMessageId full_message_id,
     return promise.set_error(r_message_id.move_as_error());
   }
 
-  td::get_payment_form(r_message_id.ok(), std::move(promise));
+  ::td::get_payment_form(r_message_id.ok(), std::move(promise));
 }
 
 void MessagesManager::validate_order_info(FullMessageId full_message_id, tl_object_ptr<td_api::orderInfo> order_info,
@@ -27161,7 +27220,7 @@ void MessagesManager::validate_order_info(FullMessageId full_message_id, tl_obje
     return promise.set_error(r_message_id.move_as_error());
   }
 
-  td::validate_order_info(r_message_id.ok(), std::move(order_info), allow_save, std::move(promise));
+  ::td::validate_order_info(r_message_id.ok(), std::move(order_info), allow_save, std::move(promise));
 }
 
 void MessagesManager::send_payment_form(FullMessageId full_message_id, const string &order_info_id,
@@ -27173,7 +27232,7 @@ void MessagesManager::send_payment_form(FullMessageId full_message_id, const str
     return promise.set_error(r_message_id.move_as_error());
   }
 
-  td::send_payment_form(r_message_id.ok(), order_info_id, shipping_option_id, credentials, std::move(promise));
+  ::td::send_payment_form(r_message_id.ok(), order_info_id, shipping_option_id, credentials, std::move(promise));
 }
 
 void MessagesManager::get_payment_receipt(FullMessageId full_message_id,
@@ -27190,7 +27249,7 @@ void MessagesManager::get_payment_receipt(FullMessageId full_message_id,
     return promise.set_error(Status::Error(5, "Wrong message identifier"));
   }
 
-  td::get_payment_receipt(message_id.get_server_message_id(), std::move(promise));
+  ::td::get_payment_receipt(message_id.get_server_message_id(), std::move(promise));
 }
 
 void MessagesManager::on_get_sponsored_dialog_id(tl_object_ptr<telegram_api::Peer> peer,

td/telegram/MessagesManager.h

@@ -959,7 +959,7 @@ class MessagesManager : public Actor {
     MessageId first_database_message_id;  // identifier of the first message in the database, needed
                                           // until there is no gaps in the database
     MessageId last_database_message_id;   // identifier of the last local or server message, if last_database_message_id
-                                          // is known and last_message_id is known then last_database_message_id <=
+                                          // is known and last_message_id is known, then last_database_message_id <=
                                           // last_message_id
 
     std::array<MessageId, search_messages_filter_size()> first_database_message_id_by_index;
@@ -1475,7 +1475,7 @@ class MessagesManager : public Actor {
   unique_ptr<Message> do_delete_message(Dialog *d, MessageId message_id, bool is_permanently_deleted,
                                         bool only_from_memory, bool *need_update_dialog_pos, const char *source);
 
-  void on_message_deleted(Dialog *d, Message *m);
+  void on_message_deleted(Dialog *d, Message *m, const char *source);
 
   int32 get_unload_dialog_delay() const;
 
@@ -1630,7 +1630,8 @@ class MessagesManager : public Actor {
   static void delete_notification_id_to_message_id_correspondence(Dialog *d, NotificationId notification_id,
                                                                   MessageId message_id);
 
-  void remove_message_notification_id(Dialog *d, Message *m, bool is_permanent, bool force_update);
+  void remove_message_notification_id(Dialog *d, Message *m, bool is_permanent, bool force_update,
+                                      bool ignore_pinned_message_notification_removal = false);
 
   void remove_new_secret_chat_notification(Dialog *d, bool is_permanent);
 
@@ -1702,10 +1703,9 @@ class MessagesManager : public Actor {
 
   void flush_pending_new_message_notifications(DialogId dialog_id, bool from_mentions, DialogId settings_dialog_id);
 
-  void remove_all_dialog_notifications(Dialog *d, NotificationGroupInfo &group_info, const char *source);
+  void remove_all_dialog_notifications(Dialog *d, bool from_mentions, const char *source);
 
-  void remove_message_dialog_notifications(Dialog *d, MessageId max_message_id, NotificationGroupInfo &group_info,
-                                           const char *source);
+  void remove_message_dialog_notifications(Dialog *d, MessageId max_message_id, bool from_mentions, const char *source);
 
   void send_update_message_send_succeeded(Dialog *d, MessageId old_message_id, const Message *m) const;
 

td/telegram/Notification.h

@@ -20,10 +20,11 @@ class Notification {
  public:
   NotificationId notification_id;
   int32 date = 0;
+  bool is_silent = false;
   unique_ptr<NotificationType> type;
 
-  Notification(NotificationId notification_id, int32 date, unique_ptr<NotificationType> type)
-      : notification_id(notification_id), date(date), type(std::move(type)) {
+  Notification(NotificationId notification_id, int32 date, bool is_silent, unique_ptr<NotificationType> type)
+      : notification_id(notification_id), date(date), is_silent(is_silent), type(std::move(type)) {
   }
 };
 
@@ -31,12 +32,13 @@ inline td_api::object_ptr<td_api::notification> get_notification_object(DialogId
                                                                         const Notification &notification) {
   CHECK(notification.type != nullptr);
   return td_api::make_object<td_api::notification>(notification.notification_id.get(), notification.date,
+                                                   notification.is_silent,
                                                    notification.type->get_notification_type_object(dialog_id));
 }
 
 inline StringBuilder &operator<<(StringBuilder &sb, const Notification &notification) {
   return sb << "notification[" << notification.notification_id << ", " << notification.date << ", "
-            << *notification.type << ']';
+            << notification.is_silent << ", " << *notification.type << ']';
 }
 
 }  // namespace td

td/telegram/NotificationManager.cpp

@@ -814,7 +814,7 @@ int32 NotificationManager::get_notification_delay_ms(DialogId dialog_id, const P
   auto delay_ms = [&]() {
     auto online_info = td_->contacts_manager_->get_my_online_status();
     if (!online_info.is_online_local && online_info.is_online_remote) {
-      // If we are offline, but online from some other client then delay notification
+      // If we are offline, but online from some other client, then delay notification
       // for 'notification_cloud_delay' seconds.
       return notification_cloud_delay_ms_;
     }
@@ -842,8 +842,9 @@ int32 NotificationManager::get_notification_delay_ms(DialogId dialog_id, const P
 
 void NotificationManager::add_notification(NotificationGroupId group_id, NotificationGroupType group_type,
                                            DialogId dialog_id, int32 date, DialogId notification_settings_dialog_id,
-                                           bool is_silent, int32 min_delay_ms, NotificationId notification_id,
-                                           unique_ptr<NotificationType> type, const char *source) {
+                                           bool initial_is_silent, bool is_silent, int32 min_delay_ms,
+                                           NotificationId notification_id, unique_ptr<NotificationType> type,
+                                           const char *source) {
   if (is_disabled() || max_notification_group_count_ == 0) {
     on_notification_removed(notification_id);
     return;
@@ -890,6 +891,7 @@ void NotificationManager::add_notification(NotificationGroupId group_id, Notific
   PendingNotification notification;
   notification.date = date;
   notification.settings_dialog_id = notification_settings_dialog_id;
+  notification.initial_is_silent = initial_is_silent;
   notification.is_silent = is_silent;
   notification.notification_id = notification_id;
   notification.type = std::move(type);
@@ -999,8 +1001,7 @@ void NotificationManager::flush_pending_updates(int32 group_id, const char *sour
     VLOG(notifications) << "Have " << as_notification_update(update.get());
   }
 
-  updates.erase(std::remove_if(updates.begin(), updates.end(), [](auto &update) { return update == nullptr; }),
-                updates.end());
+  td::remove_if(updates, [](auto &update) { return update == nullptr; });
 
   // if a notification was added, then deleted and then re-added we need to keep
   // first addition, because it can be with sound,
@@ -1032,10 +1033,7 @@ void NotificationManager::flush_pending_updates(int32 group_id, const char *sour
           notification_id = 0;
         }
       }
-      update_ptr->removed_notification_ids_.erase(
-          std::remove_if(update_ptr->removed_notification_ids_.begin(), update_ptr->removed_notification_ids_.end(),
-                         [](auto &notification_id) { return notification_id == 0; }),
-          update_ptr->removed_notification_ids_.end());
+      td::remove_if(update_ptr->removed_notification_ids_, [](auto &notification_id) { return notification_id == 0; });
     } else {
       CHECK(update->get_id() == td_api::updateNotification::ID);
       auto update_ptr = static_cast<td_api::updateNotification *>(update.get());
@@ -1125,10 +1123,7 @@ void NotificationManager::flush_pending_updates(int32 group_id, const char *sour
           // it is a first addition/edit of needed notification
           first_add_notification_pos[notification_id] = cur_pos;
         }
-        update_ptr->added_notifications_.erase(
-            std::remove_if(update_ptr->added_notifications_.begin(), update_ptr->added_notifications_.end(),
-                           [](auto &notification) { return notification == nullptr; }),
-            update_ptr->added_notifications_.end());
+        td::remove_if(update_ptr->added_notifications_, [](auto &notification) { return notification == nullptr; });
         if (update_ptr->added_notifications_.empty() && !update_ptr->is_silent_) {
           update_ptr->is_silent_ = true;
           is_changed = true;
@@ -1167,10 +1162,8 @@ void NotificationManager::flush_pending_updates(int32 group_id, const char *sour
 
           // we need to keep the deletion, because otherwise we will have 2 consequent additions
         }
-        update_ptr->removed_notification_ids_.erase(
-            std::remove_if(update_ptr->removed_notification_ids_.begin(), update_ptr->removed_notification_ids_.end(),
-                           [](auto &notification_id) { return notification_id == 0; }),
-            update_ptr->removed_notification_ids_.end());
+        td::remove_if(update_ptr->removed_notification_ids_,
+                      [](auto &notification_id) { return notification_id == 0; });
 
         if (update_ptr->removed_notification_ids_.empty() && update_ptr->added_notifications_.empty()) {
           for (size_t i = cur_pos - 1; i > 0; i--) {
@@ -1261,8 +1254,7 @@ void NotificationManager::flush_pending_updates(int32 group_id, const char *sour
       CHECK(old_size == update_ptr->removed_notification_ids_.size());
     }
 
-    updates.erase(std::remove_if(updates.begin(), updates.end(), [](auto &update) { return update == nullptr; }),
-                  updates.end());
+    td::remove_if(updates, [](auto &update) { return update == nullptr; });
     if (updates.empty()) {
       VLOG(notifications) << "There are no updates to send in " << NotificationGroupId(group_id);
       break;
@@ -1385,7 +1377,7 @@ bool NotificationManager::do_flush_pending_notifications(NotificationGroupKey &g
   added_notifications.reserve(pending_notifications.size());
   for (auto &pending_notification : pending_notifications) {
     Notification notification(pending_notification.notification_id, pending_notification.date,
-                              std::move(pending_notification.type));
+                              pending_notification.initial_is_silent, std::move(pending_notification.type));
     added_notifications.push_back(get_notification_object(group_key.dialog_id, notification));
     if (added_notifications.back()->type_ == nullptr) {
       added_notifications.pop_back();
@@ -1513,7 +1505,7 @@ void NotificationManager::flush_pending_notifications(NotificationGroupId group_
     group.total_count += narrow_cast<int32>(group.pending_notifications.size());
     for (auto &pending_notification : group.pending_notifications) {
       group.notifications.emplace_back(pending_notification.notification_id, pending_notification.date,
-                                       std::move(pending_notification.type));
+                                       pending_notification.initial_is_silent, std::move(pending_notification.type));
     }
   } else {
     if (!was_updated) {
@@ -1774,12 +1766,9 @@ void NotificationManager::remove_added_notifications_from_pending_updates(
     if (update->get_id() == td_api::updateNotificationGroup::ID) {
       auto update_ptr = static_cast<td_api::updateNotificationGroup *>(update.get());
       if (!removed_notification_ids.empty() && !update_ptr->removed_notification_ids_.empty()) {
-        update_ptr->removed_notification_ids_.erase(
-            std::remove_if(update_ptr->removed_notification_ids_.begin(), update_ptr->removed_notification_ids_.end(),
-                           [&removed_notification_ids](auto &notification_id) {
+        td::remove_if(update_ptr->removed_notification_ids_, [&removed_notification_ids](auto &notification_id) {
           return removed_notification_ids.count(notification_id) == 1;
-                           }),
-            update_ptr->removed_notification_ids_.end());
+        });
       }
       for (auto &notification : update_ptr->added_notifications_) {
         if (is_removed(notification)) {
@@ -1788,10 +1777,7 @@ void NotificationManager::remove_added_notifications_from_pending_updates(
           notification = nullptr;
         }
       }
-      update_ptr->added_notifications_.erase(
-          std::remove_if(update_ptr->added_notifications_.begin(), update_ptr->added_notifications_.end(),
-                         [](auto &notification) { return notification == nullptr; }),
-          update_ptr->added_notifications_.end());
+      td::remove_if(update_ptr->added_notifications_, [](auto &notification) { return notification == nullptr; });
     } else {
       CHECK(update->get_id() == td_api::updateNotification::ID);
       auto update_ptr = static_cast<td_api::updateNotification *>(update.get());
@@ -1818,8 +1804,8 @@ void NotificationManager::remove_notification(NotificationGroupId group_id, Noti
     return promise.set_value(Unit());
   }
 
-  VLOG(notifications) << "Remove " << notification_id << " from " << group_id << " with force_update = " << force_update
-                      << " from " << source;
+  VLOG(notifications) << "Remove " << notification_id << " from " << group_id << " with is_permanent = " << is_permanent
+                      << ", force_update = " << force_update << " from " << source;
 
   auto group_it = get_group_force(group_id);
   if (group_it == groups_.end()) {
@@ -2308,8 +2294,8 @@ void NotificationManager::add_call_notification(DialogId dialog_id, CallId call_
   }
   active_notifications.push_back(ActiveCallNotification{call_id, notification_id});
 
-  add_notification(group_id, NotificationGroupType::Calls, dialog_id, G()->unix_time() + 120, dialog_id, false, 0,
-                   notification_id, create_new_call_notification(call_id), "add_call_notification");
+  add_notification(group_id, NotificationGroupType::Calls, dialog_id, G()->unix_time() + 120, dialog_id, false, false,
+                   0, notification_id, create_new_call_notification(call_id), "add_call_notification");
 }
 
 void NotificationManager::remove_call_notification(DialogId dialog_id, CallId call_id) {
@@ -3375,9 +3361,9 @@ Status NotificationManager::process_push_notification_payload(string payload, bo
   } else {
     bool is_silent = has_json_object_field(custom, "silent");
     add_message_push_notification(dialog_id, MessageId(server_message_id), random_id, sender_user_id,
-                                  std::move(sender_name), sent_date, contains_mention, is_silent, std::move(loc_key),
-                                  std::move(arg), std::move(attached_photo), std::move(attached_document),
-                                  NotificationId(), 0, std::move(promise));
+                                  std::move(sender_name), sent_date, contains_mention, is_silent, is_silent,
+                                  std::move(loc_key), std::move(arg), std::move(attached_photo),
+                                  std::move(attached_document), NotificationId(), 0, std::move(promise));
   }
   return Status::OK();
 }
@@ -3499,8 +3485,8 @@ class NotificationManager::AddMessagePushNotificationLogEvent {
 
 void NotificationManager::add_message_push_notification(DialogId dialog_id, MessageId message_id, int64 random_id,
                                                         UserId sender_user_id, string sender_name, int32 date,
-                                                        bool contains_mention, bool is_silent, string loc_key,
-                                                        string arg, Photo photo, Document document,
+                                                        bool contains_mention, bool initial_is_silent, bool is_silent,
+                                                        string loc_key, string arg, Photo photo, Document document,
                                                         NotificationId notification_id, uint64 logevent_id,
                                                         Promise<Unit> promise) {
   auto is_pinned = begins_with(loc_key, "PINNED_");
@@ -3561,7 +3547,7 @@ void NotificationManager::add_message_push_notification(DialogId dialog_id, Mess
   if (logevent_id == 0 && G()->parameters().use_message_db) {
     AddMessagePushNotificationLogEvent logevent{
         dialog_id,         message_id, random_id, sender_user_id, sender_name, date,           contains_mention,
-        is_silent, loc_key,    arg,       photo,          document,    notification_id};
+        initial_is_silent, loc_key,    arg,       photo,          document,    notification_id};
     auto storer = LogEventStorerImpl<AddMessagePushNotificationLogEvent>(logevent);
     logevent_id = binlog_add(G()->td_db()->get_binlog(), LogEvent::HandlerType::AddMessagePushNotification, storer);
   }
@@ -3584,7 +3570,8 @@ void NotificationManager::add_message_push_notification(DialogId dialog_id, Mess
                       << " with arg " << arg << ", photo " << photo << " and document " << document << " to "
                       << group_id << " of type " << group_type << " with settings from " << settings_dialog_id;
 
-  add_notification(group_id, group_type, dialog_id, date, settings_dialog_id, is_silent, 0, notification_id,
+  add_notification(group_id, group_type, dialog_id, date, settings_dialog_id, initial_is_silent, is_silent, 0,
+                   notification_id,
                    create_new_push_message_notification(sender_user_id, message_id, std::move(loc_key), std::move(arg),
                                                         std::move(photo), std::move(document)),
                    "add_message_push_notification");
@@ -4030,9 +4017,9 @@ void NotificationManager::on_binlog_events(vector<BinlogEvent> &&events) {
 
         add_message_push_notification(
             log_event.dialog_id_, log_event.message_id_, log_event.random_id_, log_event.sender_user_id_,
-            log_event.sender_name_, log_event.date_, log_event.contains_mention_, true, log_event.loc_key_,
-            log_event.arg_, log_event.photo_, log_event.document_, log_event.notification_id_, event.id_,
-            PromiseCreator::lambda([](Result<Unit> result) {
+            log_event.sender_name_, log_event.date_, log_event.contains_mention_, log_event.is_silent_, true,
+            log_event.loc_key_, log_event.arg_, log_event.photo_, log_event.document_, log_event.notification_id_,
+            event.id_, PromiseCreator::lambda([](Result<Unit> result) {
               if (result.is_error() && result.error().code() != 200 && result.error().code() != 406) {
                 LOG(ERROR) << "Receive error " << result.error() << ", while processing message push notification";
               }

td/telegram/NotificationManager.h

@@ -65,8 +65,9 @@ class NotificationManager : public Actor {
   void load_group_force(NotificationGroupId group_id);
 
   void add_notification(NotificationGroupId group_id, NotificationGroupType group_type, DialogId dialog_id, int32 date,
-                        DialogId notification_settings_dialog_id, bool is_silent, int32 min_delay_ms,
-                        NotificationId notification_id, unique_ptr<NotificationType> type, const char *source);
+                        DialogId notification_settings_dialog_id, bool initial_is_silent, bool is_silent,
+                        int32 min_delay_ms, NotificationId notification_id, unique_ptr<NotificationType> type,
+                        const char *source);
 
   void edit_notification(NotificationGroupId group_id, NotificationId notification_id,
                          unique_ptr<NotificationType> type);
@@ -154,6 +155,7 @@ class NotificationManager : public Actor {
   struct PendingNotification {
     int32 date = 0;
     DialogId settings_dialog_id;
+    bool initial_is_silent = false;
     bool is_silent = false;
     NotificationId notification_id;
     unique_ptr<NotificationType> type;
@@ -303,8 +305,8 @@ class NotificationManager : public Actor {
   Status process_push_notification_payload(string payload, bool was_encrypted, Promise<Unit> &promise);
 
   void add_message_push_notification(DialogId dialog_id, MessageId message_id, int64 random_id, UserId sender_user_id,
-                                     string sender_name, int32 date, bool contains_mention, bool is_silent,
-                                     string loc_key, string arg, Photo photo, Document document,
+                                     string sender_name, int32 date, bool contains_mention, bool initial_is_silent,
+                                     bool is_silent, string loc_key, string arg, Photo photo, Document document,
                                      NotificationId notification_id, uint64 logevent_id, Promise<Unit> promise);
 
   void edit_message_push_notification(DialogId dialog_id, MessageId message_id, int32 edit_date, string loc_key,

td/telegram/PasswordManager.cpp

@@ -726,6 +726,40 @@ void PasswordManager::drop_cached_secret() {
   secret_ = optional<secure_storage::Secret>();
 }
 
+void PasswordManager::get_ton_wallet_password_salt(Promise<td_api::object_ptr<td_api::tonWalletPasswordSalt>> promise) {
+  if (!ton_wallet_password_salt_.empty()) {
+    return promise.set_value(td_api::make_object<td_api::tonWalletPasswordSalt>(ton_wallet_password_salt_));
+  }
+
+  get_ton_wallet_password_salt_queries_.push_back(std::move(promise));
+  if (get_ton_wallet_password_salt_queries_.size() == 1) {
+    send_with_promise(G()->net_query_creator().create(create_storer(telegram_api::wallet_getKeySecretSalt())),
+                      PromiseCreator::lambda([actor_id = actor_id(this)](Result<NetQueryPtr> r_query) mutable {
+                        auto r_result = fetch_result<telegram_api::wallet_getKeySecretSalt>(std::move(r_query));
+                        send_closure(actor_id, &PasswordManager::on_get_ton_wallet_password_salt, std::move(r_result));
+                      }));
+  }
+}
+
+void PasswordManager::on_get_ton_wallet_password_salt(
+    Result<telegram_api::object_ptr<telegram_api::wallet_secretSalt>> result) {
+  auto promises = std::move(get_ton_wallet_password_salt_queries_);
+  reset_to_empty(get_ton_wallet_password_salt_queries_);
+  CHECK(!promises.empty());
+
+  if (result.is_ok()) {
+    ton_wallet_password_salt_ = result.ok()->salt_.as_slice().str();
+
+    for (auto &promise : promises) {
+      promise.set_value(td_api::make_object<td_api::tonWalletPasswordSalt>(ton_wallet_password_salt_));
+    }
+  } else {
+    for (auto &promise : promises) {
+      promise.set_error(result.error().clone());
+    }
+  }
+}
+
 void PasswordManager::timeout_expired() {
   if (Time::now() >= secret_expire_date_) {
     drop_cached_secret();

td/telegram/PasswordManager.h

@@ -87,6 +87,8 @@ class PasswordManager : public NetQueryCallback {
 
   static TempPasswordState get_temp_password_state_sync();
 
+  void get_ton_wallet_password_salt(Promise<td_api::object_ptr<td_api::tonWalletPasswordSalt>> promise);
+
  private:
   static constexpr size_t MIN_NEW_SALT_SIZE = 8;
   static constexpr size_t MIN_NEW_SECURE_SALT_SIZE = 8;
@@ -158,6 +160,9 @@ class PasswordManager : public NetQueryCallback {
 
   int32 last_code_length_ = 0;
 
+  string ton_wallet_password_salt_;
+  vector<Promise<td_api::object_ptr<td_api::tonWalletPasswordSalt>>> get_ton_wallet_password_salt_queries_;
+
   static Result<secure_storage::Secret> decrypt_secure_secret(
       Slice password, tl_object_ptr<telegram_api::SecurePasswordKdfAlgo> algo_ptr, Slice secret, int64 secret_id);
 
@@ -184,6 +189,8 @@ class PasswordManager : public NetQueryCallback {
                                Promise<TempPasswordState> promise);
   void on_finish_create_temp_password(Result<TempPasswordState> result, bool dummy);
 
+  void on_get_ton_wallet_password_salt(Result<telegram_api::object_ptr<telegram_api::wallet_secretSalt>> result);
+
   void on_result(NetQueryPtr query) override;
 
   void start_up() override;

td/telegram/Payments.cpp

@@ -487,6 +487,34 @@ class ClearSavedInfoQuery : public Td::ResultHandler {
   }
 };
 
+class SendLiteRequestQuery : public Td::ResultHandler {
+  Promise<td_api::object_ptr<td_api::tonLiteServerResponse>> promise_;
+
+ public:
+  explicit SendLiteRequestQuery(Promise<td_api::object_ptr<td_api::tonLiteServerResponse>> &&promise)
+      : promise_(std::move(promise)) {
+  }
+
+  void send(BufferSlice request) {
+    send_query(
+        G()->net_query_creator().create(create_storer(telegram_api::wallet_sendLiteRequest(std::move(request)))));
+  }
+
+  void on_result(uint64 id, BufferSlice packet) override {
+    auto result_ptr = fetch_result<telegram_api::wallet_sendLiteRequest>(packet);
+    if (result_ptr.is_error()) {
+      return on_error(id, result_ptr.move_as_error());
+    }
+
+    auto response = result_ptr.move_as_ok();
+    promise_.set_value(td_api::make_object<td_api::tonLiteServerResponse>(response->response_.as_slice().str()));
+  }
+
+  void on_error(uint64 id, Status status) override {
+    promise_.set_error(std::move(status));
+  }
+};
+
 bool operator==(const LabeledPricePart &lhs, const LabeledPricePart &rhs) {
   return lhs.label == rhs.label && lhs.amount == rhs.amount;
 }
@@ -869,4 +897,8 @@ void delete_saved_credentials(Promise<Unit> &&promise) {
   G()->td().get_actor_unsafe()->create_handler<ClearSavedInfoQuery>(std::move(promise))->send(true, false);
 }
 
+void send_ton_lite_server_request(Slice request, Promise<td_api::object_ptr<td_api::tonLiteServerResponse>> &&promise) {
+  G()->td().get_actor_unsafe()->create_handler<SendLiteRequestQuery>(std::move(promise))->send(BufferSlice{request});
+}
+
 }  // namespace td

td/telegram/Payments.h

@@ -154,4 +154,6 @@ void delete_saved_order_info(Promise<Unit> &&promise);
 
 void delete_saved_credentials(Promise<Unit> &&promise);
 
+void send_ton_lite_server_request(Slice request, Promise<td_api::object_ptr<td_api::tonLiteServerResponse>> &&promise);
+
 }  // namespace td

td/telegram/SecretChatActor.cpp

@@ -1662,7 +1662,9 @@ void SecretChatActor::on_outbound_send_message_result(NetQueryPtr query, Promise
                               [&](telegram_api::encryptedFileEmpty &) {
                                 state->message->file = logevent::EncryptedInputFile::from_input_encrypted_file(
                                     telegram_api::inputEncryptedFileEmpty());
-                                get_file = [] { return telegram_api::make_object<telegram_api::encryptedFileEmpty>(); };
+                                get_file = [] {
+                                  return telegram_api::make_object<telegram_api::encryptedFileEmpty>();
+                                };
                               },
                               [&](telegram_api::encryptedFile &file) {
                                 state->message->file = logevent::EncryptedInputFile::from_input_encrypted_file(

td/telegram/SecureStorage.cpp

@@ -181,7 +181,7 @@ Result<Secret> Secret::create(Slice secret) {
     return Status::Error(PSLICE() << "Wrong checksum " << checksum);
   }
   UInt256 res;
-  td::as_slice(res).copy_from(secret);
+  ::td::as_slice(res).copy_from(secret);
 
   UInt256 secret_sha256;
   sha256(secret, ::td::as_slice(secret_sha256));
@@ -191,7 +191,7 @@ Result<Secret> Secret::create(Slice secret) {
 
 Secret Secret::create_new() {
   UInt256 secret;
-  auto secret_slice = td::as_slice(secret);
+  auto secret_slice = ::td::as_slice(secret);
   Random::secure_bytes(secret_slice);
   auto checksum_diff = secret_checksum(secret_slice);
   uint8 new_byte = static_cast<uint8>((static_cast<uint32>(secret_slice.ubegin()[0]) + checksum_diff) % 255);
@@ -200,8 +200,7 @@ Secret Secret::create_new() {
 }
 
 Slice Secret::as_slice() const {
-  using td::as_slice;
-  return as_slice(secret_);
+  return ::td::as_slice(secret_);
 }
 
 int64 Secret::get_hash() const {
@@ -226,8 +225,8 @@ EncryptedSecret Secret::encrypt(Slice key, Slice salt, EnryptionAlgorithm algori
   }();
 
   UInt256 res;
-  aes_cbc_state.encrypt(as_slice(), td::as_slice(res));
-  return EncryptedSecret::create(td::as_slice(res)).move_as_ok();
+  aes_cbc_state.encrypt(as_slice(), ::td::as_slice(res));
+  return EncryptedSecret::create(::td::as_slice(res)).move_as_ok();
 }
 
 Secret::Secret(UInt256 secret, int64 hash) : secret_(secret), hash_(hash) {
@@ -238,7 +237,7 @@ Result<EncryptedSecret> EncryptedSecret::create(Slice encrypted_secret) {
     return Status::Error("Wrong encrypted secret size");
   }
   UInt256 res;
-  td::as_slice(res).copy_from(encrypted_secret);
+  ::td::as_slice(res).copy_from(encrypted_secret);
   return EncryptedSecret{res};
 }
 
@@ -256,12 +255,12 @@ Result<Secret> EncryptedSecret::decrypt(Slice key, Slice salt, EnryptionAlgorith
   }();
 
   UInt256 res;
-  aes_cbc_state.decrypt(td::as_slice(encrypted_secret_), td::as_slice(res));
-  return Secret::create(td::as_slice(res));
+  aes_cbc_state.decrypt(::td::as_slice(encrypted_secret_), ::td::as_slice(res));
+  return Secret::create(::td::as_slice(res));
 }
 
 Slice EncryptedSecret::as_slice() const {
-  return td::as_slice(encrypted_secret_);
+  return ::td::as_slice(encrypted_secret_);
 }
 
 EncryptedSecret::EncryptedSecret(UInt256 encrypted_secret) : encrypted_secret_(encrypted_secret) {

td/telegram/SecureStorage.h

@@ -55,7 +55,7 @@ class ValueHash {
   }
   static Result<ValueHash> create(Slice data);
   Slice as_slice() const {
-    return td::as_slice(hash_);
+    return ::td::as_slice(hash_);
   }
 
  private:

td/telegram/StickersManager.cpp

@@ -2905,11 +2905,13 @@ void StickersManager::on_load_sticker_set_from_database(int64 sticker_set_id, bo
     return;
   }
 
+  // it is possible that a server reload_sticker_set request has failed and cleared requests list with an error
   if (with_stickers) {
-    CHECK(!sticker_set->load_requests.empty());
+    // CHECK(!sticker_set->load_requests.empty());
   } else {
-    CHECK(!sticker_set->load_without_stickers_requests.empty());
+    // CHECK(!sticker_set->load_without_stickers_requests.empty());
   }
+
   if (value.empty()) {
     return do_reload_sticker_set(sticker_set_id, get_input_sticker_set(sticker_set), Auto());
   }
@@ -3093,7 +3095,7 @@ void StickersManager::on_get_archived_sticker_sets(
     LOG(ERROR) << "Receive " << total_count << " as total count of archived sticker sets";
   }
 
-  // if 0 sticker sets are received then set offset_sticker_set_id was found and there is no stickers after it
+  // if 0 sticker sets are received, then set offset_sticker_set_id was found and there is no stickers after it
   // or it wasn't found and there is no archived sets at all
   bool is_last =
       sticker_sets.empty() &&

td/telegram/StickersManager.hpp

@@ -262,6 +262,9 @@ void StickersManager::parse_sticker_set(StickerSet *sticker_set, ParserT &parser
       if (parser.get_error() != nullptr) {
         return;
       }
+      if (!sticker_id.is_valid()) {
+        return parser.set_error("Receive invalid sticker in a sticker set");
+      }
       sticker_set->sticker_ids.push_back(sticker_id);
 
       Sticker *sticker = get_sticker(sticker_id);

td/telegram/StorageManager.cpp

@@ -67,7 +67,7 @@ void StorageManager::get_storage_stats(bool need_all_files, int32 dialog_limit,
     //TODO group same queries
     close_stats_worker();
   }
-  if (pending_run_gc_.size() != 0) {
+  if (!pending_run_gc_.empty()) {
     close_gc_worker();
   }
   stats_dialog_limit_ = dialog_limit;
@@ -106,20 +106,21 @@ void StorageManager::run_gc(FileGcParameters parameters, Promise<FileStats> prom
     promise.set_error(Status::Error(500, "Request aborted"));
     return;
   }
-  if (pending_run_gc_.size() != 0) {
+  if (!pending_run_gc_.empty()) {
     close_gc_worker();
   }
 
-  get_storage_stats(true /*need_all_file*/,
-                    !gc_parameters_.owner_dialog_ids.empty() || !gc_parameters_.exclude_owner_dialog_ids.empty() ||
-                        gc_parameters_.dialog_limit != 0 /*split_by_owner_dialog_id*/,
-                    PromiseCreator::lambda([actor_id = actor_id(this)](Result<FileStats> file_stats) {
-                      send_closure(actor_id, &StorageManager::on_all_files, std::move(file_stats), false);
+  bool split_by_owner_dialog_id = !parameters.owner_dialog_ids.empty() ||
+                                  !parameters.exclude_owner_dialog_ids.empty() || parameters.dialog_limit != 0;
+  get_storage_stats(true /*need_all_files*/, split_by_owner_dialog_id,
+                    PromiseCreator::lambda(
+                        [actor_id = actor_id(this), parameters = std::move(parameters)](Result<FileStats> file_stats) {
+                          send_closure(actor_id, &StorageManager::on_all_files, std::move(parameters),
+                                       std::move(file_stats), false);
                         }));
 
-  //NB: get_storage stats will cancel all gc queries
+  //NB: get_storage_stats will cancel all gc queries, so promise needs to be added after the call
   pending_run_gc_.emplace_back(std::move(promise));
-  gc_parameters_ = std::move(parameters);
 }
 
 void StorageManager::on_file_stats(Result<FileStats> r_file_stats, uint32 generation) {
@@ -146,19 +147,20 @@ void StorageManager::create_stats_worker() {
   }
 }
 
-void StorageManager::on_all_files(Result<FileStats> r_file_stats, bool dummy) {
+void StorageManager::on_all_files(FileGcParameters gc_parameters, Result<FileStats> r_file_stats, bool dummy) {
+  int32 dialog_limit = gc_parameters.dialog_limit;
   if (is_closed_ && r_file_stats.is_ok()) {
     r_file_stats = Status::Error(500, "Request aborted");
   }
   if (r_file_stats.is_error()) {
-    return on_gc_finished(std::move(r_file_stats), false);
+    return on_gc_finished(dialog_limit, std::move(r_file_stats), false);
   }
 
   create_gc_worker();
 
-  send_closure(gc_worker_, &FileGcWorker::run_gc, gc_parameters_, r_file_stats.move_as_ok().all_files,
-               PromiseCreator::lambda([actor_id = actor_id(this)](Result<FileStats> r_file_stats) {
-                 send_closure(actor_id, &StorageManager::on_gc_finished, std::move(r_file_stats), false);
+  send_closure(gc_worker_, &FileGcWorker::run_gc, std::move(gc_parameters), r_file_stats.move_as_ok().all_files,
+               PromiseCreator::lambda([actor_id = actor_id(this), dialog_limit](Result<FileStats> r_file_stats) {
+                 send_closure(actor_id, &StorageManager::on_gc_finished, dialog_limit, std::move(r_file_stats), false);
                }));
 }
 
@@ -204,7 +206,7 @@ void StorageManager::create_gc_worker() {
   }
 }
 
-void StorageManager::on_gc_finished(Result<FileStats> r_file_stats, bool dummy) {
+void StorageManager::on_gc_finished(int32 dialog_limit, Result<FileStats> r_file_stats, bool dummy) {
   if (r_file_stats.is_error()) {
     if (r_file_stats.error().code() != 500) {
       LOG(ERROR) << "GC failed: " << r_file_stats.error();
@@ -216,7 +218,7 @@ void StorageManager::on_gc_finished(Result<FileStats> r_file_stats, bool dummy)
     return;
   }
 
-  send_stats(r_file_stats.move_as_ok(), gc_parameters_.dialog_limit, std::move(pending_run_gc_));
+  send_stats(r_file_stats.move_as_ok(), dialog_limit, std::move(pending_run_gc_));
 }
 
 void StorageManager::save_fast_stat() {
@@ -279,7 +281,6 @@ void StorageManager::close_gc_worker() {
   for (auto &promise : promises) {
     promise.set_error(Status::Error(500, "Request aborted"));
   }
-  gc_generation_++;
   gc_worker_.reset();
   gc_cancellation_token_source_.cancel();
 }

td/telegram/StorageManager.h

@@ -83,15 +83,13 @@ class StorageManager : public Actor {
   // Gc
   ActorOwn<FileGcWorker> gc_worker_;
   std::vector<Promise<FileStats>> pending_run_gc_;
-  uint32 gc_generation_{0};
-  FileGcParameters gc_parameters_;
 
   uint32 last_gc_timestamp_ = 0;
   double next_gc_at_ = 0;
 
-  void on_all_files(Result<FileStats> r_file_stats, bool dummy);
+  void on_all_files(FileGcParameters gc_parameters, Result<FileStats> r_file_stats, bool dummy);
   void create_gc_worker();
-  void on_gc_finished(Result<FileStats> r_file_stats, bool dummy);
+  void on_gc_finished(int32 dialog_limit, Result<FileStats> r_file_stats, bool dummy);
 
   void close_stats_worker();
   void close_gc_worker();

td/telegram/Td.cpp

@@ -552,8 +552,9 @@ class TestQuery : public Td::ResultHandler {
 };
 
 class TestProxyRequest : public RequestOnceActor {
-  int16 dc_id_ = 2;
   Proxy proxy_;
+  int16 dc_id_;
+  double timeout_;
   ActorOwn<> child_;
   Promise<> promise_;
 
@@ -562,6 +563,8 @@ class TestProxyRequest : public RequestOnceActor {
   }
 
   void do_run(Promise<Unit> &&promise) override {
+    set_timeout_in(timeout_);
+
     promise_ = std::move(promise);
     IPAddress ip;
     auto status = ip.init_host_port(proxy_.server(), proxy_.port());
@@ -641,9 +644,17 @@ class TestProxyRequest : public RequestOnceActor {
     promise_.set_value(Unit());
   }
 
+  void timeout_expired() override {
+    send_error(Status::Error(400, "Timeout expired"));
+    stop();
+  }
+
  public:
-  TestProxyRequest(ActorShared<Td> td, uint64 request_id, Proxy proxy)
-      : RequestOnceActor(std::move(td), request_id), proxy_(std::move(proxy)) {
+  TestProxyRequest(ActorShared<Td> td, uint64 request_id, Proxy proxy, int32 dc_id, double timeout)
+      : RequestOnceActor(std::move(td), request_id)
+      , proxy_(std::move(proxy))
+      , dc_id_(static_cast<int16>(dc_id))
+      , timeout_(timeout) {
   }
 };
 
@@ -7116,6 +7127,18 @@ void Td::on_request(uint64 id, const td_api::deleteSavedCredentials &request) {
   delete_saved_credentials(std::move(promise));
 }
 
+void Td::on_request(uint64 id, const td_api::sendTonLiteServerRequest &request) {
+  CHECK_IS_USER();
+  CREATE_REQUEST_PROMISE();
+  send_ton_lite_server_request(request.request_, std::move(promise));
+}
+
+void Td::on_request(uint64 id, const td_api::getTonWalletPasswordSalt &request) {
+  CHECK_IS_USER();
+  CREATE_REQUEST_PROMISE();
+  send_closure(password_manager_, &PasswordManager::get_ton_wallet_password_salt, std::move(promise));
+}
+
 void Td::on_request(uint64 id, td_api::getPassportElement &request) {
   CHECK_IS_USER();
   CLEAN_INPUT_STRING(request.password_);
@@ -7707,7 +7730,7 @@ void Td::on_request(uint64 id, td_api::testProxy &request) {
   if (r_proxy.is_error()) {
     return send_closure(actor_id(this), &Td::send_error, id, r_proxy.move_as_error());
   }
-  CREATE_REQUEST(TestProxyRequest, r_proxy.move_as_ok());
+  CREATE_REQUEST(TestProxyRequest, r_proxy.move_as_ok(), request.dc_id_, request.timeout_);
 }
 
 void Td::on_request(uint64 id, const td_api::testGetDifference &request) {

td/telegram/Td.h

@@ -224,7 +224,7 @@ class Td final : public NetQueryCallback {
   static td_api::object_ptr<td_api::Object> static_request(td_api::object_ptr<td_api::Function> function);
 
  private:
-  static constexpr const char *TDLIB_VERSION = "1.5.0";
+  static constexpr const char *TDLIB_VERSION = "1.5.1";
   static constexpr int64 ONLINE_ALARM_ID = 0;
   static constexpr int64 PING_SERVER_ALARM_ID = -1;
   static constexpr int32 PING_SERVER_TIMEOUT = 300;
@@ -898,6 +898,10 @@ class Td final : public NetQueryCallback {
 
   void on_request(uint64 id, const td_api::deleteSavedCredentials &request);
 
+  void on_request(uint64 id, const td_api::sendTonLiteServerRequest &request);
+
+  void on_request(uint64 id, const td_api::getTonWalletPasswordSalt &request);
+
   void on_request(uint64 id, td_api::getPassportElement &request);
 
   void on_request(uint64 id, td_api::getAllPassportElements &request);

td/telegram/TdDb.cpp

@@ -283,20 +283,20 @@ Status TdDb::init_sqlite(int32 scheduler_id, const TdParameters &parameters, DbK
   CHECK(!parameters.use_message_db || parameters.use_chat_info_db);
   CHECK(!parameters.use_chat_info_db || parameters.use_file_db);
 
-  const string sql_db_name = get_sqlite_path(parameters);
+  const string sql_database_path = get_sqlite_path(parameters);
 
   bool use_sqlite = parameters.use_file_db;
   bool use_file_db = parameters.use_file_db;
   bool use_dialog_db = parameters.use_message_db;
   bool use_message_db = parameters.use_message_db;
   if (!use_sqlite) {
-    unlink(sql_db_name).ignore();
+    unlink(sql_database_path).ignore();
     return Status::OK();
   }
 
-  sqlite_path_ = sql_db_name;
+  sqlite_path_ = sql_database_path;
   TRY_STATUS(SqliteDb::change_key(sqlite_path_, key, old_key));
-  sql_connection_ = std::make_shared<SqliteConnectionSafe>(sql_db_name, key);
+  sql_connection_ = std::make_shared<SqliteConnectionSafe>(sql_database_path, key);
   auto &db = sql_connection_->get();
 
   TRY_STATUS(init_db(db));
@@ -418,6 +418,9 @@ Status TdDb::init(int32 scheduler_id, const TdParameters &parameters, DbKey key,
   VLOG(td_init) << "Finish to init database";
   if (init_sqlite_status.is_error()) {
     LOG(ERROR) << "Destroy bad SQLite database because of " << init_sqlite_status;
+    if (sql_connection_ != nullptr) {
+      sql_connection_->get().close();
+    }
     SqliteDb::destroy(get_sqlite_path(parameters)).ignore();
     TRY_STATUS(init_sqlite(scheduler_id, parameters, new_sqlite_key, old_sqlite_key, *binlog_pmc));
   }
@@ -485,7 +488,7 @@ void TdDb::with_db_path(std::function<void(CSlice)> callback) {
 }
 
 Result<string> TdDb::get_stats() {
-  auto sb = td::StringBuilder({}, true);
+  auto sb = StringBuilder({}, true);
   auto &sql = sql_connection_->get();
   auto run_query = [&](CSlice query, Slice desc) -> Status {
     TRY_RESULT(stmt, sql.get_statement(query));

td/telegram/TdDb.h

@@ -126,4 +126,5 @@ class TdDb {
 
   void do_close(Promise<> on_finished, bool destroy_flag);
 };
+
 }  // namespace td

td/telegram/UpdatesManager.cpp

@@ -677,7 +677,7 @@ void UpdatesManager::on_get_updates(tl_object_ptr<telegram_api::Updates> &&updat
       auto update = move_tl_object_as<telegram_api::updateShort>(updates_ptr);
       LOG(DEBUG) << "Receive " << oneline(to_string(update));
       if (!is_acceptable_update(update->update_.get())) {
-        LOG(ERROR) << "Receive unacceptable short update: " << td::oneline(to_string(update));
+        LOG(ERROR) << "Receive unacceptable short update: " << oneline(to_string(update));
         return get_difference("unacceptable short update");
       }
       short_update_date_ = update->date_;
@@ -1440,8 +1440,10 @@ void UpdatesManager::on_update(tl_object_ptr<telegram_api::updateChannelTooLong>
   td_->messages_manager_->on_update_channel_too_long(std::move(update), force_apply);
 }
 
-void UpdatesManager::on_update(tl_object_ptr<telegram_api::updateChannel> update, bool /*force_apply*/) {
-  // nothing to do
+void UpdatesManager::on_update(tl_object_ptr<telegram_api::updateChannel> update, bool force_apply) {
+  if (!force_apply) {
+    td_->contacts_manager_->invalidate_channel_full(ChannelId(update->channel_id_), false);
+  }
 }
 
 void UpdatesManager::on_update(tl_object_ptr<telegram_api::updateEditChannelMessage> update, bool /*force_apply*/) {
@@ -1549,7 +1551,9 @@ int32 UpdatesManager::get_short_update_date() const {
 
 tl_object_ptr<td_api::ChatAction> UpdatesManager::convert_send_message_action(
     tl_object_ptr<telegram_api::SendMessageAction> action) {
-  auto fix_progress = [](int32 progress) { return progress <= 0 || progress > 100 ? 0 : progress; };
+  auto fix_progress = [](int32 progress) {
+    return progress <= 0 || progress > 100 ? 0 : progress;
+  };
 
   switch (action->get_id()) {
     case telegram_api::sendMessageCancelAction::ID:

td/telegram/WebPageBlock.cpp

@@ -25,6 +25,8 @@
 #include "td/telegram/Version.h"
 #include "td/telegram/VideosManager.h"
 #include "td/telegram/VideosManager.hpp"
+#include "td/telegram/VoiceNotesManager.h"
+#include "td/telegram/VoiceNotesManager.hpp"
 #include "td/telegram/WebPageId.h"
 
 #include "td/utils/common.h"
@@ -97,7 +99,7 @@ class RichText {
       case RichText::Type::Fixed:
         return make_tl_object<td_api::richTextFixed>(texts[0].get_rich_text_object());
       case RichText::Type::Url:
-        return make_tl_object<td_api::richTextUrl>(texts[0].get_rich_text_object(), content);
+        return make_tl_object<td_api::richTextUrl>(texts[0].get_rich_text_object(), content, web_page_id.is_valid());
       case RichText::Type::EmailAddress:
         return make_tl_object<td_api::richTextEmailAddress>(texts[0].get_rich_text_object(), content);
       case RichText::Type::Concatenation:
@@ -234,8 +236,9 @@ class WebPageBlockTableCell {
       UNREACHABLE();
       return nullptr;
     }();
-    return td_api::make_object<td_api::pageBlockTableCell>(text.get_rich_text_object(), is_header, colspan, rowspan,
-                                                           std::move(align), std::move(valign));
+    return td_api::make_object<td_api::pageBlockTableCell>(text.empty() ? nullptr : text.get_rich_text_object(),
+                                                           is_header, colspan, rowspan, std::move(align),
+                                                           std::move(valign));
   }
 
   template <class StorerT>
@@ -1401,8 +1404,10 @@ class WebPageBlockAudio : public WebPageBlock {
     using ::td::store;
 
     bool has_empty_audio = !audio_file_id.is_valid();
+    bool is_voice_note_repaired = true;
     BEGIN_STORE_FLAGS();
     STORE_FLAG(has_empty_audio);
+    STORE_FLAG(is_voice_note_repaired);
     END_STORE_FLAGS();
 
     if (!has_empty_audio) {
@@ -1416,17 +1421,23 @@ class WebPageBlockAudio : public WebPageBlock {
     using ::td::parse;
 
     bool has_empty_audio;
+    bool is_voice_note_repaired;
     if (parser.version() >= static_cast<int32>(Version::FixPageBlockAudioEmptyFile)) {
       BEGIN_PARSE_FLAGS();
       PARSE_FLAG(has_empty_audio);
+      PARSE_FLAG(is_voice_note_repaired);
       END_PARSE_FLAGS();
     } else {
       has_empty_audio = false;
+      is_voice_note_repaired = false;
     }
 
     if (!has_empty_audio) {
       audio_file_id = parser.context()->td().get_actor_unsafe()->audios_manager_->parse_audio(parser);
     } else {
+      if (!is_voice_note_repaired) {
+        parser.set_error("Trying to repair WebPageBlockVoiceNote");
+      }
       audio_file_id = FileId();
     }
     parse(caption, parser);
@@ -1631,6 +1642,64 @@ class WebPageBlockMap : public WebPageBlock {
   }
 };
 
+class WebPageBlockVoiceNote : public WebPageBlock {
+  FileId voice_note_file_id;
+  WebPageBlockCaption caption;
+
+ public:
+  WebPageBlockVoiceNote() = default;
+  WebPageBlockVoiceNote(FileId voice_note_file_id, WebPageBlockCaption &&caption)
+      : voice_note_file_id(voice_note_file_id), caption(std::move(caption)) {
+  }
+
+  Type get_type() const override {
+    return Type::VoiceNote;
+  }
+
+  void append_file_ids(vector<FileId> &file_ids) const override {
+    Document(Document::Type::VoiceNote, voice_note_file_id).append_file_ids(G()->td().get_actor_unsafe(), file_ids);
+    caption.append_file_ids(file_ids);
+  }
+
+  td_api::object_ptr<td_api::PageBlock> get_page_block_object() const override {
+    return make_tl_object<td_api::pageBlockVoiceNote>(
+        G()->td().get_actor_unsafe()->voice_notes_manager_->get_voice_note_object(voice_note_file_id),
+        caption.get_page_block_caption_object());
+  }
+
+  template <class StorerT>
+  void store(StorerT &storer) const {
+    using ::td::store;
+
+    bool has_empty_voice_note = !voice_note_file_id.is_valid();
+    BEGIN_STORE_FLAGS();
+    STORE_FLAG(has_empty_voice_note);
+    END_STORE_FLAGS();
+
+    if (!has_empty_voice_note) {
+      storer.context()->td().get_actor_unsafe()->voice_notes_manager_->store_voice_note(voice_note_file_id, storer);
+    }
+    store(caption, storer);
+  }
+
+  template <class ParserT>
+  void parse(ParserT &parser) {
+    using ::td::parse;
+
+    bool has_empty_voice_note;
+    BEGIN_PARSE_FLAGS();
+    PARSE_FLAG(has_empty_voice_note);
+    END_PARSE_FLAGS();
+
+    if (!has_empty_voice_note) {
+      voice_note_file_id = parser.context()->td().get_actor_unsafe()->voice_notes_manager_->parse_voice_note(parser);
+    } else {
+      voice_note_file_id = FileId();
+    }
+    parse(caption, parser);
+  }
+};
+
 vector<RichText> get_rich_texts(vector<tl_object_ptr<telegram_api::RichText>> &&rich_text_ptrs,
                                 const std::unordered_map<int64, FileId> &documents);
 
@@ -1770,7 +1839,8 @@ unique_ptr<WebPageBlock> get_web_page_block(Td *td, tl_object_ptr<telegram_api::
                                             const std::unordered_map<int64, FileId> &audios,
                                             const std::unordered_map<int64, FileId> &documents,
                                             const std::unordered_map<int64, Photo> &photos,
-                                            const std::unordered_map<int64, FileId> &videos) {
+                                            const std::unordered_map<int64, FileId> &videos,
+                                            const std::unordered_map<int64, FileId> &voice_notes) {
   CHECK(page_block_ptr != nullptr);
   switch (page_block_ptr->get_id()) {
     case telegram_api::pageBlockUnsupported::ID:
@@ -1833,8 +1903,8 @@ unique_ptr<WebPageBlock> get_web_page_block(Td *td, tl_object_ptr<telegram_api::
           }
           case telegram_api::pageListItemBlocks::ID: {
             auto list_item = telegram_api::move_object_as<telegram_api::pageListItemBlocks>(list_item_ptr);
-            item.page_blocks =
-                get_web_page_blocks(td, std::move(list_item->blocks_), animations, audios, documents, photos, videos);
+            item.page_blocks = get_web_page_blocks(td, std::move(list_item->blocks_), animations, audios, documents,
+                                                   photos, videos, voice_notes);
             break;
           }
         }
@@ -1861,8 +1931,8 @@ unique_ptr<WebPageBlock> get_web_page_block(Td *td, tl_object_ptr<telegram_api::
           case telegram_api::pageListOrderedItemBlocks::ID: {
             auto list_item = telegram_api::move_object_as<telegram_api::pageListOrderedItemBlocks>(list_item_ptr);
             item.label = std::move(list_item->num_);
-            item.page_blocks =
-                get_web_page_blocks(td, std::move(list_item->blocks_), animations, audios, documents, photos, videos);
+            item.page_blocks = get_web_page_blocks(td, std::move(list_item->blocks_), animations, audios, documents,
+                                                   photos, videos, voice_notes);
             break;
           }
         }
@@ -1928,7 +1998,8 @@ unique_ptr<WebPageBlock> get_web_page_block(Td *td, tl_object_ptr<telegram_api::
     }
     case telegram_api::pageBlockCover::ID: {
       auto page_block = move_tl_object_as<telegram_api::pageBlockCover>(page_block_ptr);
-      auto cover = get_web_page_block(td, std::move(page_block->cover_), animations, audios, documents, photos, videos);
+      auto cover = get_web_page_block(td, std::move(page_block->cover_), animations, audios, documents, photos, videos,
+                                      voice_notes);
       if (cover == nullptr) {
         return nullptr;
       }
@@ -1967,19 +2038,20 @@ unique_ptr<WebPageBlock> get_web_page_block(Td *td, tl_object_ptr<telegram_api::
       }
       return td::make_unique<WebPageBlockEmbeddedPost>(
           std::move(page_block->url_), std::move(page_block->author_), std::move(author_photo), page_block->date_,
-          get_web_page_blocks(td, std::move(page_block->blocks_), animations, audios, documents, photos, videos),
+          get_web_page_blocks(td, std::move(page_block->blocks_), animations, audios, documents, photos, videos,
+                              voice_notes),
           get_page_block_caption(std::move(page_block->caption_), documents));
     }
     case telegram_api::pageBlockCollage::ID: {
       auto page_block = move_tl_object_as<telegram_api::pageBlockCollage>(page_block_ptr);
-      return td::make_unique<WebPageBlockCollage>(
-          get_web_page_blocks(td, std::move(page_block->items_), animations, audios, documents, photos, videos),
+      return td::make_unique<WebPageBlockCollage>(get_web_page_blocks(td, std::move(page_block->items_), animations,
+                                                                      audios, documents, photos, videos, voice_notes),
                                                   get_page_block_caption(std::move(page_block->caption_), documents));
     }
     case telegram_api::pageBlockSlideshow::ID: {
       auto page_block = move_tl_object_as<telegram_api::pageBlockSlideshow>(page_block_ptr);
-      return td::make_unique<WebPageBlockSlideshow>(
-          get_web_page_blocks(td, std::move(page_block->items_), animations, audios, documents, photos, videos),
+      return td::make_unique<WebPageBlockSlideshow>(get_web_page_blocks(td, std::move(page_block->items_), animations,
+                                                                        audios, documents, photos, videos, voice_notes),
                                                     get_page_block_caption(std::move(page_block->caption_), documents));
     }
     case telegram_api::pageBlockChannel::ID: {
@@ -2014,6 +2086,12 @@ unique_ptr<WebPageBlock> get_web_page_block(Td *td, tl_object_ptr<telegram_api::
     }
     case telegram_api::pageBlockAudio::ID: {
       auto page_block = move_tl_object_as<telegram_api::pageBlockAudio>(page_block_ptr);
+      auto voice_note_it = voice_notes.find(page_block->audio_id_);
+      if (voice_note_it != voice_notes.end()) {
+        return make_unique<WebPageBlockVoiceNote>(voice_note_it->second,
+                                                  get_page_block_caption(std::move(page_block->caption_), documents));
+      }
+
       auto it = audios.find(page_block->audio_id_);
       FileId audio_file_id;
       if (it != audios.end()) {
@@ -2063,9 +2141,9 @@ unique_ptr<WebPageBlock> get_web_page_block(Td *td, tl_object_ptr<telegram_api::
     case telegram_api::pageBlockDetails::ID: {
       auto page_block = move_tl_object_as<telegram_api::pageBlockDetails>(page_block_ptr);
       auto is_open = (page_block->flags_ & telegram_api::pageBlockDetails::OPEN_MASK) != 0;
-      return td::make_unique<WebPageBlockDetails>(
-          get_rich_text(std::move(page_block->title_), documents),
-          get_web_page_blocks(td, std::move(page_block->blocks_), animations, audios, documents, photos, videos),
+      return td::make_unique<WebPageBlockDetails>(get_rich_text(std::move(page_block->title_), documents),
+                                                  get_web_page_blocks(td, std::move(page_block->blocks_), animations,
+                                                                      audios, documents, photos, videos, voice_notes),
                                                   is_open);
     }
     case telegram_api::pageBlockRelatedArticles::ID: {
@@ -2181,6 +2259,8 @@ void WebPageBlock::call_impl(Type type, const WebPageBlock *ptr, F &&f) {
       return f(static_cast<const WebPageBlockRelatedArticles *>(ptr));
     case Type::Map:
       return f(static_cast<const WebPageBlockMap *>(ptr));
+    case Type::VoiceNote:
+      return f(static_cast<const WebPageBlockVoiceNote *>(ptr));
     default:
       UNREACHABLE();
   }
@@ -2233,17 +2313,16 @@ void parse(unique_ptr<WebPageBlock> &block, LogEventParser &parser) {
   parse_web_page_block(block, parser);
 }
 
-vector<unique_ptr<WebPageBlock>> get_web_page_blocks(Td *td,
-                                                     vector<tl_object_ptr<telegram_api::PageBlock>> page_block_ptrs,
-                                                     const std::unordered_map<int64, FileId> &animations,
-                                                     const std::unordered_map<int64, FileId> &audios,
-                                                     const std::unordered_map<int64, FileId> &documents,
-                                                     const std::unordered_map<int64, Photo> &photos,
-                                                     const std::unordered_map<int64, FileId> &videos) {
+vector<unique_ptr<WebPageBlock>> get_web_page_blocks(
+    Td *td, vector<tl_object_ptr<telegram_api::PageBlock>> page_block_ptrs,
+    const std::unordered_map<int64, FileId> &animations, const std::unordered_map<int64, FileId> &audios,
+    const std::unordered_map<int64, FileId> &documents, const std::unordered_map<int64, Photo> &photos,
+    const std::unordered_map<int64, FileId> &videos, const std::unordered_map<int64, FileId> &voice_notes) {
   vector<unique_ptr<WebPageBlock>> result;
   result.reserve(page_block_ptrs.size());
   for (auto &page_block_ptr : page_block_ptrs) {
-    auto page_block = get_web_page_block(td, std::move(page_block_ptr), animations, audios, documents, photos, videos);
+    auto page_block =
+        get_web_page_block(td, std::move(page_block_ptr), animations, audios, documents, photos, videos, voice_notes);
     if (page_block != nullptr) {
       result.push_back(std::move(page_block));
     }

td/telegram/WebPageBlock.h

@@ -52,6 +52,7 @@ class WebPageBlock {
     Details,
     RelatedArticles,
     Map,
+    VoiceNote,
     Size
   };
 
@@ -91,13 +92,11 @@ void store(const unique_ptr<WebPageBlock> &block, LogEventStorerUnsafe &storer);
 
 void parse(unique_ptr<WebPageBlock> &block, LogEventParser &parser);
 
-vector<unique_ptr<WebPageBlock>> get_web_page_blocks(Td *td,
-                                                     vector<tl_object_ptr<telegram_api::PageBlock>> page_block_ptrs,
-                                                     const std::unordered_map<int64, FileId> &animations,
-                                                     const std::unordered_map<int64, FileId> &audios,
-                                                     const std::unordered_map<int64, FileId> &documents,
-                                                     const std::unordered_map<int64, Photo> &photos,
-                                                     const std::unordered_map<int64, FileId> &videos);
+vector<unique_ptr<WebPageBlock>> get_web_page_blocks(
+    Td *td, vector<tl_object_ptr<telegram_api::PageBlock>> page_block_ptrs,
+    const std::unordered_map<int64, FileId> &animations, const std::unordered_map<int64, FileId> &audios,
+    const std::unordered_map<int64, FileId> &documents, const std::unordered_map<int64, Photo> &photos,
+    const std::unordered_map<int64, FileId> &videos, const std::unordered_map<int64, FileId> &voice_notes);
 
 vector<td_api::object_ptr<td_api::PageBlock>> get_page_block_objects(
     const vector<unique_ptr<WebPageBlock>> &page_blocks);

td/telegram/WebPagesManager.cpp

@@ -834,10 +834,11 @@ void WebPagesManager::on_load_web_page_instant_view_from_database(WebPageId web_
 
   WebPageInstantView result;
   if (!value.empty()) {
-    if (log_event_parse(result, value).is_error()) {
+    auto status = log_event_parse(result, value);
+    if (status.is_error()) {
       result = WebPageInstantView();
 
-      LOG(ERROR) << "Erase instant view in " << web_page_id << " from database";
+      LOG(ERROR) << "Erase instant view in " << web_page_id << " from database because of " << status.message();
       G()->td_db()->get_sqlite_pmc()->erase(get_web_page_instant_view_database_key(web_page_id), Auto());
     }
   }
@@ -1199,6 +1200,7 @@ void WebPagesManager::on_get_web_page_instant_view(WebPage *web_page, tl_object_
   std::unordered_map<int64, FileId> audios;
   std::unordered_map<int64, FileId> documents;
   std::unordered_map<int64, FileId> videos;
+  std::unordered_map<int64, FileId> voice_notes;
   for (auto &document_ptr : page->documents_) {
     if (document_ptr->get_id() == telegram_api::document::ID) {
       auto document = move_tl_object_as<telegram_api::document>(document_ptr);
@@ -1212,6 +1214,8 @@ void WebPagesManager::on_get_web_page_instant_view(WebPage *web_page, tl_object_
         documents.emplace(document_id, parsed_document.file_id);
       } else if (parsed_document.type == Document::Type::Video) {
         videos.emplace(document_id, parsed_document.file_id);
+      } else if (parsed_document.type == Document::Type::VoiceNote) {
+        voice_notes.emplace(document_id, parsed_document.file_id);
       } else {
         LOG(ERROR) << "Receive document of the wrong type: " << parsed_document;
       }
@@ -1249,12 +1253,20 @@ void WebPagesManager::on_get_web_page_instant_view(WebPage *web_page, tl_object_
       LOG(ERROR) << "Video has no remote location";
     }
   }
+  if (web_page->document.type == Document::Type::VoiceNote) {
+    auto file_view = td_->file_manager_->get_file_view(web_page->document.file_id);
+    if (file_view.has_remote_location()) {
+      voice_notes.emplace(file_view.remote_location().get_id(), web_page->document.file_id);
+    } else {
+      LOG(ERROR) << "Voice note has no remote location";
+    }
+  }
 
   LOG(INFO) << "Receive a web page instant view with " << page->blocks_.size() << " blocks, " << animations.size()
             << " animations, " << audios.size() << " audios, " << documents.size() << " documents, " << photos.size()
-            << " photos and " << videos.size() << " videos";
+            << " photos, " << videos.size() << " videos and " << voice_notes.size() << " voice notes";
   web_page->instant_view.page_blocks =
-      get_web_page_blocks(td_, std::move(page->blocks_), animations, audios, documents, photos, videos);
+      get_web_page_blocks(td_, std::move(page->blocks_), animations, audios, documents, photos, videos, voice_notes);
   web_page->instant_view.is_v2 = (page->flags_ & telegram_api::page::V2_MASK) != 0;
   web_page->instant_view.is_rtl = (page->flags_ & telegram_api::page::RTL_MASK) != 0;
   web_page->instant_view.hash = hash;
@@ -1405,10 +1417,12 @@ void WebPagesManager::on_load_web_page_from_database(WebPageId web_page_id, stri
       auto result = make_unique<WebPage>();
       auto status = log_event_parse(*result, value);
       if (status.is_error()) {
-        LOG(FATAL) << status << ": " << format::as_hex_dump<4>(Slice(value));
-      }
+        LOG(ERROR) << "Failed to parse web page loaded from database: " << status
+                   << ", value = " << format::as_hex_dump<4>(Slice(value));
+      } else {
         update_web_page(std::move(result), web_page_id, true, true);
       }
+    }
   } else {
     // web page has already been loaded from the server
   }

td/telegram/cli.cpp

@@ -77,14 +77,14 @@ static void dump_memory_usage() {
     int cnt = 0;
     for (auto &info : v) {
       if (cnt++ < 50) {
-        LOG(WARNING) << td::format::as_size(info.size) << td::format::as_array(info.backtrace);
+        LOG(WARNING) << format::as_size(info.size) << format::as_array(info.backtrace);
       } else {
         other_size += info.size;
       }
       total_size += info.size;
     }
-    LOG(WARNING) << tag("other", td::format::as_size(other_size));
-    LOG(WARNING) << tag("total", td::format::as_size(total_size));
+    LOG(WARNING) << tag("other", format::as_size(other_size));
+    LOG(WARNING) << tag("total", format::as_size(total_size));
     LOG(WARNING) << tag("total traces", get_ht_size());
     LOG(WARNING) << tag("fast_backtrace_success_rate", get_fast_backtrace_success_rate());
   }
@@ -301,7 +301,7 @@ class CliClient final : public Actor {
       for (auto &m : messages.messages_) {
         // LOG(PLAIN) << to_string(m);
         if (m->content_->get_id() == td_api::messageText::ID) {
-          LOG(PLAIN) << td::oneline(static_cast<const td_api::messageText *>(m->content_.get())->text_->text_) << "\n";
+          LOG(PLAIN) << oneline(static_cast<const td_api::messageText *>(m->content_.get())->text_->text_) << "\n";
         }
         last_message_id = m->id_;
       }
@@ -452,7 +452,7 @@ class CliClient final : public Actor {
   }
 
   vector<int64> as_chat_ids(Slice chat_ids, char delimiter = ' ') const {
-    return transform(full_split(chat_ids, delimiter), [this](Slice str) { return as_chat_id(str); });
+    return transform(full_split(trim(chat_ids), delimiter), [this](Slice str) { return as_chat_id(str); });
   }
 
   static int64 as_message_id(Slice str) {
@@ -464,7 +464,7 @@ class CliClient final : public Actor {
   }
 
   static vector<int64> as_message_ids(Slice message_ids, char delimiter = ' ') {
-    return transform(full_split(message_ids, delimiter), as_message_id);
+    return transform(full_split(trim(message_ids), delimiter), as_message_id);
   }
 
   int32 as_user_id(Slice str) const {
@@ -516,11 +516,11 @@ class CliClient final : public Actor {
     return static_cast<int32>(result);
   }
 
-  static int32 as_file_id(string str) {
-    return to_integer<int32>(trim(std::move(str)));
+  static int32 as_file_id(Slice str) {
+    return to_integer<int32>(trim(str));
   }
 
-  static td_api::object_ptr<td_api::InputFile> as_input_file_id(string str) {
+  static td_api::object_ptr<td_api::InputFile> as_input_file_id(Slice str) {
     return td_api::make_object<td_api::inputFileId>(as_file_id(str));
   }
 
@@ -538,10 +538,11 @@ class CliClient final : public Actor {
   }
 
   static tl_object_ptr<td_api::InputFile> as_input_file(string str) {
+    str = trim(str);
     if ((str.size() >= 20 && is_base64url(str)) || begins_with(str, "http")) {
       return as_remote_file(str);
     }
-    auto r_id = to_integer_safe<int32>(trim(str));
+    auto r_id = to_integer_safe<int32>(str);
     if (r_id.is_ok()) {
       return as_input_file_id(str);
     }
@@ -557,6 +558,16 @@ class CliClient final : public Actor {
     return td_api::make_object<td_api::inputThumbnail>(std::move(input_file), width, height);
   }
 
+  static tl_object_ptr<td_api::inputThumbnail> as_input_thumbnail(const string &thumbnail, int32 width = 0,
+                                                                  int32 height = 0) {
+    return as_input_thumbnail(as_input_file(thumbnail), width, height);
+  }
+
+  static tl_object_ptr<td_api::inputThumbnail> as_input_thumbnail(const string &original_path, const string &conversion,
+                                                                  int32 width = 0, int32 height = 0) {
+    return as_input_thumbnail(as_generated_file(original_path, conversion), width, height);
+  }
+
   static int32 as_call_id(string str) {
     return to_integer<int32>(trim(std::move(str)));
   }
@@ -573,7 +584,7 @@ class CliClient final : public Actor {
   }
 
   static bool as_bool(string str) {
-    str = to_lower(str);
+    str = to_lower(trim(str));
     return str == "true" || str == "1";
   }
 
@@ -1529,6 +1540,10 @@ class CliClient final : public Actor {
       send_request(td_api::make_object<td_api::deleteSavedOrderInfo>());
     } else if (op == "dsc") {
       send_request(td_api::make_object<td_api::deleteSavedCredentials>());
+    } else if (op == "stlsr") {
+      send_request(td_api::make_object<td_api::sendTonLiteServerRequest>());
+    } else if (op == "gtwps") {
+      send_request(td_api::make_object<td_api::getTonWalletPasswordSalt>());
     } else if (op == "gpr") {
       send_request(td_api::make_object<td_api::getUserPrivacySettingRules>(get_user_privacy_setting(args)));
     } else if (op == "spr") {
@@ -2765,8 +2780,8 @@ class CliClient final : public Actor {
       std::tie(message_id, photo) = split(args);
       send_request(td_api::make_object<td_api::editMessageMedia>(
           as_chat_id(chat_id), as_message_id(message_id), nullptr,
-          td_api::make_object<td_api::inputMessagePhoto>(as_input_file(photo), as_input_thumbnail(as_input_file(photo)),
-                                                         Auto(), 0, 0, as_caption(""), 0)));
+          td_api::make_object<td_api::inputMessagePhoto>(as_input_file(photo), as_input_thumbnail(photo), Auto(), 0, 0,
+                                                         as_caption(""), 0)));
     } else if (op == "empttl") {
       string chat_id;
       string message_id;
@@ -2775,8 +2790,8 @@ class CliClient final : public Actor {
       std::tie(message_id, photo) = split(args);
       send_request(td_api::make_object<td_api::editMessageMedia>(
           as_chat_id(chat_id), as_message_id(message_id), nullptr,
-          td_api::make_object<td_api::inputMessagePhoto>(as_input_file(photo), as_input_thumbnail(as_input_file(photo)),
-                                                         Auto(), 0, 0, as_caption(""), 10)));
+          td_api::make_object<td_api::inputMessagePhoto>(as_input_file(photo), as_input_thumbnail(photo), Auto(), 0, 0,
+                                                         as_caption(""), 10)));
     } else if (op == "emvt") {
       string chat_id;
       string message_id;
@@ -2787,9 +2802,8 @@ class CliClient final : public Actor {
       std::tie(video, thumbnail) = split(args);
       send_request(td_api::make_object<td_api::editMessageMedia>(
           as_chat_id(chat_id), as_message_id(message_id), nullptr,
-          td_api::make_object<td_api::inputMessageVideo>(as_input_file(video),
-                                                         as_input_thumbnail(as_input_file(thumbnail)), Auto(), 1, 2, 3,
-                                                         true, as_caption(""), 0)));
+          td_api::make_object<td_api::inputMessageVideo>(as_input_file(video), as_input_thumbnail(thumbnail), Auto(), 1,
+                                                         2, 3, true, as_caption(""), 0)));
     } else if (op == "emll") {
       string chat_id;
       string message_id;
@@ -2953,9 +2967,9 @@ class CliClient final : public Actor {
       string thumbnail_path;
       std::tie(chat_id, args) = split(args);
       std::tie(document_path, thumbnail_path) = split(args);
-      send_message(chat_id, td_api::make_object<td_api::inputMessageDocument>(
-                                as_input_file(document_path), as_input_thumbnail(as_local_file(thumbnail_path)),
-                                as_caption("test caption")));
+      send_message(chat_id,
+                   td_api::make_object<td_api::inputMessageDocument>(
+                       as_input_file(document_path), as_input_thumbnail(thumbnail_path), as_caption("test caption")));
     } else if (op == "sdg" || op == "sdgu") {
       string chat_id;
       string document_path;
@@ -2978,8 +2992,7 @@ class CliClient final : public Actor {
       std::tie(document_path, args) = split(args);
       std::tie(thumbnail_path, thumbnail_conversion) = split(args);
       send_message(chat_id, td_api::make_object<td_api::inputMessageDocument>(
-                                as_input_file(document_path),
-                                as_input_thumbnail(as_generated_file(thumbnail_path, thumbnail_conversion)),
+                                as_input_file(document_path), as_input_thumbnail(thumbnail_path, thumbnail_conversion),
                                 as_caption("test caption")));
     } else if (op == "sdgtg") {
       string chat_id;
@@ -2993,8 +3006,7 @@ class CliClient final : public Actor {
       std::tie(thumbnail_path, thumbnail_conversion) = split(args);
       send_message(chat_id, td_api::make_object<td_api::inputMessageDocument>(
                                 as_generated_file(document_path, document_conversion),
-                                as_input_thumbnail(as_generated_file(thumbnail_path, thumbnail_conversion)),
-                                as_caption("test caption")));
+                                as_input_thumbnail(thumbnail_path, thumbnail_conversion), as_caption("test caption")));
     } else if (op == "sdid") {
       string chat_id;
       string file_id;
@@ -3043,7 +3055,7 @@ class CliClient final : public Actor {
       auto options = full_split(args);
 
       send_message(chat_id, td_api::make_object<td_api::inputMessagePoll>(question, std::move(options)));
-    } else if (op == "sp" || op == "spcaption") {
+    } else if (op == "sp" || op == "spcaption" || op == "spttl") {
       string chat_id;
       string photo_path;
       string sticker_file_ids_str;
@@ -3058,14 +3070,7 @@ class CliClient final : public Actor {
 
       send_message(chat_id, td_api::make_object<td_api::inputMessagePhoto>(
                                 as_input_file(photo_path), nullptr, std::move(sticker_file_ids), 0, 0,
-                                as_caption(op == "spcaption" ? "cap \n\n\n\n tion " : ""), 0));
-    } else if (op == "spttl") {
-      string chat_id;
-      string photo_path;
-      std::tie(chat_id, photo_path) = split(args);
-
-      send_message(chat_id, td_api::make_object<td_api::inputMessagePhoto>(as_input_file(photo_path), nullptr, Auto(),
-                                                                           0, 0, as_caption(""), 10));
+                                as_caption(op == "spcaption" ? "cap \n\n\n\n tion " : ""), op == "spttl" ? 10 : 0));
     } else if (op == "spg") {
       string chat_id;
       string photo_path;
@@ -3085,8 +3090,8 @@ class CliClient final : public Actor {
       std::tie(chat_id, args) = split(args);
       std::tie(photo_path, thumbnail_path) = split(args);
 
-      send_message(chat_id, td_api::make_object<td_api::inputMessagePhoto>(
-                                as_input_file(photo_path), as_input_thumbnail(as_local_file(thumbnail_path), 90, 89),
+      send_message(chat_id, td_api::make_object<td_api::inputMessagePhoto>(as_input_file(photo_path),
+                                                                           as_input_thumbnail(thumbnail_path, 90, 89),
                                                                            vector<int32>(), 0, 0, as_caption(""), 0));
     } else if (op == "sptg") {
       string chat_id;
@@ -3097,9 +3102,9 @@ class CliClient final : public Actor {
       std::tie(photo_path, args) = split(args);
       std::tie(thumbnail_path, thumbnail_conversion) = split(args);
 
-      send_message(chat_id, td_api::make_object<td_api::inputMessagePhoto>(
-                                as_input_file(photo_path),
-                                as_input_thumbnail(as_generated_file(thumbnail_path, thumbnail_conversion), 90, 89),
+      send_message(chat_id,
+                   td_api::make_object<td_api::inputMessagePhoto>(
+                       as_input_file(photo_path), as_input_thumbnail(thumbnail_path, thumbnail_conversion, 90, 89),
                        vector<int32>(), 0, 0, as_caption(""), 0));
     } else if (op == "spgtg") {
       string chat_id;
@@ -3115,8 +3120,8 @@ class CliClient final : public Actor {
 
       send_message(chat_id, td_api::make_object<td_api::inputMessagePhoto>(
                                 as_generated_file(photo_path, conversion),
-                                as_input_thumbnail(as_generated_file(thumbnail_path, thumbnail_conversion), 90, 89),
-                                vector<int32>(), 0, 0, as_caption(""), 0));
+                                as_input_thumbnail(thumbnail_path, thumbnail_conversion, 90, 89), vector<int32>(), 0, 0,
+                                as_caption(""), 0));
     } else if (op == "spid") {
       string chat_id;
       string file_id;
@@ -3137,15 +3142,15 @@ class CliClient final : public Actor {
       std::tie(chat_id, args) = split(args);
       std::tie(sticker_path, thumbnail_path) = split(args);
 
-      send_message(chat_id, td_api::make_object<td_api::inputMessageSticker>(
-                                as_input_file(sticker_path), as_input_thumbnail(as_local_file(thumbnail_path)), 0, 0));
+      send_message(chat_id, td_api::make_object<td_api::inputMessageSticker>(as_input_file(sticker_path),
+                                                                             as_input_thumbnail(thumbnail_path), 0, 0));
     } else if (op == "ssid") {
       string chat_id;
       string file_id;
       std::tie(chat_id, file_id) = split(args);
 
       send_message(chat_id, td_api::make_object<td_api::inputMessageSticker>(as_input_file_id(file_id), nullptr, 0, 0));
-    } else if (op == "sv") {
+    } else if (op == "sv" || op == "svttl") {
       string chat_id;
       string video_path;
       string sticker_file_ids_str;
@@ -3160,7 +3165,17 @@ class CliClient final : public Actor {
 
       send_message(chat_id, td_api::make_object<td_api::inputMessageVideo>(as_input_file(video_path), nullptr,
                                                                            std::move(sticker_file_ids), 1, 2, 3, true,
-                                                                           as_caption(""), 0));
+                                                                           as_caption(""), op == "svttl" ? 10 : 0));
+    } else if (op == "svt" || op == "svtttl") {
+      string chat_id;
+      string video;
+      string thumbnail;
+      std::tie(chat_id, args) = split(args);
+      std::tie(video, thumbnail) = split(args);
+
+      send_message(chat_id, td_api::make_object<td_api::inputMessageVideo>(
+                                as_input_file(video), as_input_thumbnail(thumbnail), vector<int32>(), 0, 0, 0, true,
+                                as_caption(""), op == "svtttl" ? 10 : 0));
     } else if (op == "svn") {
       string chat_id;
       string video_path;
@@ -3646,7 +3661,7 @@ class CliClient final : public Actor {
         send_request(
             td_api::make_object<td_api::editProxy>(as_proxy_id(proxy_id), server, port_int, enable, std::move(type)));
       } else if (op == "tproxy") {
-        send_request(td_api::make_object<td_api::testProxy>(server, port_int, std::move(type)));
+        send_request(td_api::make_object<td_api::testProxy>(server, port_int, std::move(type), 2, 10.0));
       } else {
         send_request(td_api::make_object<td_api::addProxy>(server, port_int, enable, std::move(type)));
       }
@@ -3670,11 +3685,11 @@ class CliClient final : public Actor {
       fd.seek(size).ignore();
       fd.truncate_to_current_position(size).ignore();
     } else if (op == "SetVerbosity" || op == "SV") {
-      td::Log::set_verbosity_level(to_integer<int>(args));
+      Log::set_verbosity_level(to_integer<int>(args));
     } else if (op[0] == 'v' && op[1] == 'v') {
-      td::Log::set_verbosity_level(static_cast<int>(op.size()));
+      Log::set_verbosity_level(static_cast<int>(op.size()));
     } else if (op[0] == 'v' && ('0' <= op[1] && op[1] <= '9')) {
-      td::Log::set_verbosity_level(to_integer<int>(op.substr(1)));
+      Log::set_verbosity_level(to_integer<int>(op.substr(1)));
     } else if (op == "slse") {
       execute(td_api::make_object<td_api::setLogStream>(td_api::make_object<td_api::logStreamEmpty>()));
     } else if (op == "slsd") {
@@ -3880,7 +3895,7 @@ void main(int argc, char **argv) {
   ignore_signal(SignalType::Pipe).ensure();
   set_signal_handler(SignalType::Error, fail_signal).ensure();
   set_signal_handler(SignalType::Abort, fail_signal).ensure();
-  td::Log::set_fatal_error_callback(on_fatal_error);
+  Log::set_fatal_error_callback(on_fatal_error);
 
   const char *locale_name = (std::setlocale(LC_ALL, "fr-FR") == nullptr ? "" : "fr-FR");
   std::locale new_locale(locale_name);
@@ -3894,8 +3909,8 @@ void main(int argc, char **argv) {
   CliLog cli_log;
   log_interface = &cli_log;
 
-  td::FileLog file_log;
-  td::TsLog ts_log(&file_log);
+  FileLog file_log;
+  TsLog ts_log(&file_log);
 
   int new_verbosity_level = VERBOSITY_NAME(INFO);
   bool use_test_dc = false;
@@ -3903,7 +3918,7 @@ void main(int argc, char **argv) {
   bool disable_network = false;
   auto api_id = [](auto x) -> int32 {
     if (x) {
-      return td::to_integer<int32>(Slice(x));
+      return to_integer<int32>(Slice(x));
     }
     return 0;
   }(std::getenv("TD_API_ID"));
@@ -3947,7 +3962,7 @@ void main(int argc, char **argv) {
       if (i + 1 >= argc) {
         return usage();
       }
-      api_id = td::to_integer<int32>(Slice(argv[++i]));
+      api_id = to_integer<int32>(Slice(argv[++i]));
     } else if (!std::strcmp(argv[i], "--api_hash") || !std::strcmp(argv[i], "--api-hash")) {
       if (i + 1 >= argc) {
         return usage();
@@ -4001,7 +4016,7 @@ void main(int argc, char **argv) {
         .release();
 
     scheduler.start();
-    while (scheduler.run_main(td::Timestamp::in(100))) {
+    while (scheduler.run_main(Timestamp::in(100))) {
     }
     scheduler.finish();
   }

td/telegram/files/FileBitmask.cpp

@@ -37,7 +37,7 @@ Bitmask Bitmask::compress(int k) const {
 
 std::string Bitmask::encode(int32 prefix_count) {
   // remove zeroes in the end to make encoding deterministic
-  td::Slice data(data_);
+  Slice data(data_);
 
   int save_i = -1;
   char save_c;

td/telegram/files/FileEncryptionKey.cpp

@@ -92,7 +92,7 @@ StringBuilder &operator<<(StringBuilder &string_builder, const FileEncryptionKey
   if (key.is_secret()) {
     return string_builder << "SecretKey{" << key.size() << "}";
   }
-  if (key.is_secret()) {
+  if (key.is_secure()) {
     return string_builder << "SecureKey{" << key.size() << "}";
   }
   return string_builder << "NoKey{}";

td/telegram/files/FileEncryptionKey.h

@@ -86,14 +86,15 @@ struct FileEncryptionKey {
     }
   }
 
+  friend bool operator==(const FileEncryptionKey &lhs, const FileEncryptionKey &rhs) {
+    return lhs.key_iv_ == rhs.key_iv_;
+  }
+
+ private:
   string key_iv_;  // TODO wrong alignment is possible
   Type type_ = Type::None;
 };
 
-inline bool operator==(const FileEncryptionKey &lhs, const FileEncryptionKey &rhs) {
-  return lhs.key_iv_ == rhs.key_iv_;
-}
-
 inline bool operator!=(const FileEncryptionKey &lhs, const FileEncryptionKey &rhs) {
   return !(lhs == rhs);
 }

td/telegram/files/FileManager.cpp

@@ -463,6 +463,7 @@ const FullLocalFileLocation &FileView::local_location() const {
 bool FileView::has_remote_location() const {
   return static_cast<bool>(node_->remote_.full);
 }
+
 bool FileView::has_alive_remote_location() const {
   return node_->remote_.is_full_alive;
 }
@@ -662,6 +663,10 @@ bool FileView::can_download_from_server() const {
   if (remote_location().get_dc_id().is_empty()) {
     return false;
   }
+  if (!remote_location().is_encrypted_any() && !remote_location().has_file_reference() &&
+      ((node_->download_id_ == 0 && node_->download_was_update_file_reference_) || !node_->remote_.is_full_alive)) {
+    return false;
+  }
   return true;
 }
 
@@ -1273,7 +1278,7 @@ static int merge_choose_encryption_key(const FileEncryptionKey &a, const FileEnc
   if (a.empty() != b.empty()) {
     return a.empty() > b.empty();
   }
-  if (a.key_iv_ != b.key_iv_) {
+  if (a != b) {
     return -1;
   }
   return 2;
@@ -2049,13 +2054,16 @@ void FileManager::download(FileId file_id, std::shared_ptr<DownloadCallback> cal
 
 void FileManager::run_download(FileNodePtr node) {
   if (node->need_load_from_pmc_) {
+    LOG(INFO) << "Skip run_download, because file " << node->main_file_id_ << " needs to be loaded from PMC";
     return;
   }
   if (node->generate_id_) {
+    LOG(INFO) << "Skip run_download, because file " << node->main_file_id_ << " is being generated";
     return;
   }
   auto file_view = FileView(node);
   if (!file_view.can_download_from_server()) {
+    LOG(INFO) << "Skip run_download, because file " << node->main_file_id_ << " can't be downloaded from server";
     return;
   }
   int8 priority = 0;
@@ -2070,6 +2078,7 @@ void FileManager::run_download(FileNodePtr node) {
   node->set_download_priority(priority);
 
   if (priority == 0) {
+    LOG(INFO) << "Cancel downloading of file " << node->main_file_id_;
     if (old_priority != 0) {
       do_cancel_download(node);
     }
@@ -2082,6 +2091,7 @@ void FileManager::run_download(FileNodePtr node) {
   node->is_download_limit_dirty_ = false;
 
   if (old_priority != 0) {
+    LOG(INFO) << "Update download offset and limits of file " << node->main_file_id_;
     CHECK(node->download_id_ != 0);
     send_closure(file_load_manager_, &FileLoadManager::update_priority, node->download_id_, priority);
     if (need_update_limit) {
@@ -2100,6 +2110,7 @@ void FileManager::run_download(FileNodePtr node) {
   auto file_id = node->main_file_id_;
 
   if (node->need_reload_photo_ && file_view.may_reload_photo()) {
+    LOG(INFO) << "Reload photo from file " << node->main_file_id_;
     QueryId id = queries_container_.create(Query{file_id, Query::DownloadReloadDialog});
     node->download_id_ = id;
     context_->reload_photo(file_view.remote_location().get_source(),
@@ -2146,7 +2157,7 @@ void FileManager::run_download(FileNodePtr node) {
   QueryId id = queries_container_.create(Query{file_id, Query::Download});
   node->download_id_ = id;
   node->is_download_started_ = false;
-  LOG(DEBUG) << "Run download of file " << file_id << " of size " << node->size_ << " from "
+  LOG(INFO) << "Run download of file " << file_id << " of size " << node->size_ << " from "
             << node->remote_.full.value() << " with suggested name " << node->suggested_name() << " and encyption key "
             << node->encryption_key_;
   auto download_offset = file_view.is_encrypted_any() ? 0 : node->download_offset_;
@@ -2322,6 +2333,14 @@ void FileManager::resume_upload(FileId file_id, std::vector<int> bad_parts, std:
     }
     return;
   }
+  if (file_view.get_type() == FileType::Thumbnail &&
+      (!file_view.has_local_location() && file_view.can_download_from_server())) {
+    // TODO
+    if (callback) {
+      callback->on_upload_error(file_id, Status::Error("Failed to upload thumbnail without local location"));
+    }
+    return;
+  }
 
   LOG(INFO) << "Change upload priority of file " << file_id << " to " << new_priority;
   auto *file_info = get_file_id_info(file_id);
@@ -2409,10 +2428,20 @@ void FileManager::external_file_generate_finish(int64 id, Status status, Promise
 
 void FileManager::run_generate(FileNodePtr node) {
   if (node->need_load_from_pmc_) {
+    LOG(INFO) << "Skip run_generate, because file " << node->main_file_id_ << " needs to be loaded from PMC";
     return;
   }
   FileView file_view(node);
-  if (file_view.has_local_location() || file_view.can_download_from_server() || !file_view.can_generate()) {
+  if (file_view.has_local_location()) {
+    LOG(INFO) << "Skip run_generate, because file " << node->main_file_id_ << " has local location";
+    return;
+  }
+  if (file_view.can_download_from_server()) {
+    LOG(INFO) << "Skip run_generate, because file " << node->main_file_id_ << " can be downloaded from server";
+    return;
+  }
+  if (!file_view.can_generate()) {
+    LOG(INFO) << "Skip run_generate, because file " << node->main_file_id_ << " can't be generated";
     return;
   }
 
@@ -2620,7 +2649,7 @@ string FileManager::get_persistent_id(const FullRemoteFileLocation &location) {
   auto binary = serialize(location_copy);
 
   binary = zero_encode(binary);
-  binary.push_back(static_cast<char>(narrow_cast<td::uint8>(Version::Next) - 1));
+  binary.push_back(static_cast<char>(narrow_cast<uint8>(Version::Next) - 1));
   binary.push_back(PERSISTENT_ID_VERSION);
   return base64url_encode(binary);
 }

td/telegram/files/FileStatsWorker.cpp

@@ -106,8 +106,7 @@ void scan_fs(CancellationToken &token, CallbackT &&callback) {
       continue;
     }
     auto files_dir = get_files_dir(file_type);
-    td::walk_path(files_dir,
-                  [&](CSlice path, WalkPath::Type type) {
+    walk_path(files_dir, [&](CSlice path, WalkPath::Type type) {
       if (token) {
         return WalkPath::Action::Abort;
       }
@@ -133,8 +132,7 @@ void scan_fs(CancellationToken &token, CallbackT &&callback) {
       info.mtime_nsec = stat.mtime_nsec_;
       callback(info);
       return WalkPath::Action::Continue;
-                  })
-        .ignore();
+    }).ignore();
   }
 }
 }  // namespace

td/telegram/files/PartsManager.cpp

@@ -228,7 +228,9 @@ bool PartsManager::is_part_in_streaming_limit(int part_i) const {
     return true;
   }
 
-  auto is_intersect_with = [&](int64 begin, int64 end) { return max(begin, offset_begin) < min(end, offset_end); };
+  auto is_intersect_with = [&](int64 begin, int64 end) {
+    return max(begin, offset_begin) < min(end, offset_end);
+  };
 
   auto streaming_begin = streaming_offset_;
   auto streaming_end = streaming_offset_ + streaming_limit_;

td/telegram/misc.cpp

@@ -136,11 +136,11 @@ bool clean_input_string(string &str) {
   return true;
 }
 
-string strip_empty_characters(string str, size_t max_length) {
+string strip_empty_characters(string str, size_t max_length, bool strip_rtlo) {
   static const char *space_characters[] = {u8"\u1680", u8"\u180E", u8"\u2000", u8"\u2001", u8"\u2002",
                                            u8"\u2003", u8"\u2004", u8"\u2005", u8"\u2006", u8"\u2007",
-                                           u8"\u2008", u8"\u2009", u8"\u200A", u8"\u200B", u8"\u202F",
-                                           u8"\u205F", u8"\u3000", u8"\uFEFF", u8"\uFFFC"};
+                                           u8"\u2008", u8"\u2009", u8"\u200A", u8"\u200B", u8"\u202E",
+                                           u8"\u202F", u8"\u205F", u8"\u3000", u8"\uFEFF", u8"\uFFFC"};
   static bool can_be_first[std::numeric_limits<unsigned char>::max() + 1];
   static bool can_be_first_inited = [&] {
     for (auto space_ch : space_characters) {
@@ -162,7 +162,10 @@ string strip_empty_characters(string str, size_t max_length) {
       bool found = false;
       for (auto space_ch : space_characters) {
         if (space_ch[0] == str[i] && space_ch[1] == str[i + 1] && space_ch[2] == str[i + 2]) {
+          if (static_cast<unsigned char>(str[i + 2]) != 0xAE || static_cast<unsigned char>(str[i + 1]) != 0x80 ||
+              static_cast<unsigned char>(str[i]) != 0xE2 || strip_rtlo) {
             found = true;
+          }
           break;
         }
       }
@@ -277,20 +280,25 @@ string get_emoji_fingerprint(uint64 num) {
 
 Result<string> check_url(Slice url) {
   bool is_tg = false;
+  bool is_ton = false;
   if (begins_with(url, "tg://")) {
     url.remove_prefix(5);
     is_tg = true;
   } else if (begins_with(url, "tg:")) {
     url.remove_prefix(3);
     is_tg = true;
-  } else {
-    is_tg = false;
+  } else if (begins_with(url, "ton://")) {
+    url.remove_prefix(6);
+    is_ton = true;
+  } else if (begins_with(url, "ton:")) {
+    url.remove_prefix(4);
+    is_ton = true;
   }
   TRY_RESULT(http_url, parse_url(url));
-  if (is_tg) {
+  if (is_tg || is_ton) {
     if (begins_with(url, "http://") || http_url.protocol_ == HttpUrl::Protocol::HTTPS || !http_url.userinfo_.empty() ||
         http_url.specified_port_ != 0 || http_url.is_ipv6_) {
-      return Status::Error("Wrong tg URL");
+      return Status::Error(is_tg ? Slice("Wrong tg URL") : Slice("Wrong ton URL"));
     }
 
     Slice query(http_url.query_);
@@ -298,7 +306,7 @@ Result<string> check_url(Slice url) {
     if (query[1] == '?') {
       query.remove_prefix(1);
     }
-    return PSTRING() << "tg://" << http_url.host_ << query;
+    return PSTRING() << (is_tg ? "tg" : "ton") << "://" << http_url.host_ << query;
   }
 
   if (url.find('.') == string::npos) {

td/telegram/misc.h

@@ -22,7 +22,7 @@ string clean_username(string str) TD_WARN_UNUSED_RESULT;
 bool clean_input_string(string &str) TD_WARN_UNUSED_RESULT;
 
 // strips empty characters and ensures that string length is no more than max_length
-string strip_empty_characters(string str, size_t max_length) TD_WARN_UNUSED_RESULT;
+string strip_empty_characters(string str, size_t max_length, bool strip_rtlo = false) TD_WARN_UNUSED_RESULT;
 
 // checks if string is empty after strip_empty_characters
 bool is_empty_string(const string &str) TD_WARN_UNUSED_RESULT;
@@ -33,7 +33,7 @@ int32 get_vector_hash(const vector<uint32> &numbers) TD_WARN_UNUSED_RESULT;
 // returns emoji corresponding to the specified number
 string get_emoji_fingerprint(uint64 num);
 
-// checks whether url is a valid tg or HTTP(S) URL and returns its in a canonical form
+// checks whether url is a valid tg, ton or HTTP(S) URL and returns its in a canonical form
 Result<string> check_url(Slice url);
 
 }  // namespace td

td/telegram/net/ConnectionCreator.cpp

@@ -663,7 +663,7 @@ Result<mtproto::TransportType> ConnectionCreator::get_transport_type(const Proxy
     CHECK(info.option != nullptr);
     string proxy_authorization;
     if (!proxy.user().empty() || !proxy.password().empty()) {
-      proxy_authorization = "|basic " + td::base64_encode(PSLICE() << proxy.user() << ':' << proxy.password());
+      proxy_authorization = "|basic " + base64_encode(PSLICE() << proxy.user() << ':' << proxy.password());
     }
     return mtproto::TransportType{
         mtproto::TransportType::Http, 0,

td/telegram/net/DcOptionsSet.cpp

@@ -135,11 +135,9 @@ Result<DcOptionsSet::ConnectionInfo> DcOptionsSet::find_connection(DcId dc_id, b
                                   << tag("prefer_ipv6", prefer_ipv6));
   }
 
-  auto last_error_at = std::min_element(options.begin(), options.end(),
-                                        [](const auto &a_option, const auto &b_option) {
+  auto last_error_at = std::min_element(options.begin(), options.end(), [](const auto &a_option, const auto &b_option) {
                          return a_option.stat->error_at > b_option.stat->error_at;
-                                        })
-                           ->stat->error_at;
+                       })->stat->error_at;
 
   auto result = *std::min_element(options.begin(), options.end(), [](const auto &a_option, const auto &b_option) {
     auto &a = *a_option.stat;

td/telegram/net/NetQuery.cpp

@@ -77,10 +77,10 @@ void dump_pending_network_queries() {
         was_gap = false;
       }
       auto nq = &static_cast<NetQuery &>(*cur);
-      LOG(WARNING) << tag("id", nq->my_id_) << *nq << tag("total_flood", td::format::as_time(nq->total_timeout)) << " "
-                   << tag("since start", td::format::as_time(td::Time::now_cached() - nq->start_timestamp_))
+      LOG(WARNING) << tag("id", nq->my_id_) << *nq << tag("total_flood", format::as_time(nq->total_timeout)) << " "
+                   << tag("since start", format::as_time(Time::now_cached() - nq->start_timestamp_))
                    << tag("state", nq->debug_str_)
-                   << tag("since state", td::format::as_time(td::Time::now_cached() - nq->debug_timestamp_))
+                   << tag("since state", format::as_time(Time::now_cached() - nq->debug_timestamp_))
                    << tag("resend_cnt", nq->debug_resend_cnt_) << tag("fail_cnt", nq->debug_send_failed_cnt_)
                    << tag("ack", nq->debug_ack) << tag("unknown", nq->debug_unknown);
     } else {

tdactor/td/actor/impl/ConcurrentScheduler.cpp

@@ -80,7 +80,7 @@ void ConcurrentScheduler::start() {
     auto &sched = schedulers_[i];
     threads_.push_back(td::thread([&]() {
 #if TD_PORT_WINDOWS
-      td::detail::Iocp::Guard iocp_guard(iocp_.get());
+      detail::Iocp::Guard iocp_guard(iocp_.get());
 #endif
       while (!is_finished()) {
         sched->run(Timestamp::in(10));
@@ -105,7 +105,7 @@ bool ConcurrentScheduler::run_main(Timestamp timeout) {
   auto &main_sched = schedulers_[0];
   if (!is_finished()) {
 #if TD_PORT_WINDOWS
-    td::detail::Iocp::Guard iocp_guard(iocp_.get());
+    detail::Iocp::Guard iocp_guard(iocp_.get());
 #endif
     main_sched->run(timeout);
   }
@@ -137,7 +137,7 @@ void ConcurrentScheduler::finish() {
   SCOPE_EXIT {
     iocp_->clear();
   };
-  td::detail::Iocp::Guard iocp_guard(iocp_.get());
+  detail::Iocp::Guard iocp_guard(iocp_.get());
 #endif
 
 #if !TD_THREAD_UNSUPPORTED && !TD_EVENTFD_UNSUPPORTED

tdactor/td/actor/impl/Scheduler.h

@@ -229,7 +229,8 @@ void Scheduler::send_impl(const ActorId<> &actor_id, const RunFuncT &run_func, c
 
 template <ActorSendType send_type, class EventT>
 void Scheduler::send_lambda(ActorRef actor_ref, EventT &&lambda) {
-  return send_impl<send_type>(actor_ref.get(),
+  return send_impl<send_type>(
+      actor_ref.get(),
       [&](ActorInfo *actor_info) {
         event_context_ptr_->link_token = actor_ref.token();
         lambda();
@@ -243,7 +244,8 @@ void Scheduler::send_lambda(ActorRef actor_ref, EventT &&lambda) {
 
 template <ActorSendType send_type, class EventT>
 void Scheduler::send_closure(ActorRef actor_ref, EventT &&closure) {
-  return send_impl<send_type>(actor_ref.get(),
+  return send_impl<send_type>(
+      actor_ref.get(),
       [&](ActorInfo *actor_info) {
         event_context_ptr_->link_token = actor_ref.token();
         closure.run(static_cast<typename EventT::ActorType *>(actor_info->get_actor_unsafe()));
@@ -258,7 +260,8 @@ void Scheduler::send_closure(ActorRef actor_ref, EventT &&closure) {
 template <ActorSendType send_type>
 void Scheduler::send(ActorRef actor_ref, Event &&event) {
   event.set_link_token(actor_ref.token());
-  return send_impl<send_type>(actor_ref.get(), [&](ActorInfo *actor_info) { do_event(actor_info, std::move(event)); },
+  return send_impl<send_type>(
+      actor_ref.get(), [&](ActorInfo *actor_info) { do_event(actor_info, std::move(event)); },
       [&]() { return std::move(event); });
 }
 

tddb/CMakeLists.txt

@@ -8,17 +8,18 @@ set(TDDB_SOURCE
   td/db/binlog/detail/BinlogEventsBuffer.cpp
   td/db/binlog/detail/BinlogEventsProcessor.cpp
 
+  td/db/SqliteConnectionSafe.cpp
   td/db/SqliteDb.cpp
-  td/db/SqliteStatement.cpp
   td/db/SqliteKeyValue.cpp
   td/db/SqliteKeyValueAsync.cpp
+  td/db/SqliteStatement.cpp
 
   td/db/detail/RawSqliteDb.cpp
 
   td/db/binlog/Binlog.h
-  td/db/binlog/BinlogInterface.h
   td/db/binlog/BinlogEvent.h
   td/db/binlog/BinlogHelper.h
+  td/db/binlog/BinlogInterface.h
   td/db/binlog/ConcurrentBinlog.h
   td/db/binlog/detail/BinlogEventsBuffer.h
   td/db/binlog/detail/BinlogEventsProcessor.h

tddb/td/db/BinlogKeyValue.h

@@ -77,7 +77,8 @@ class BinlogKeyValue : public KeyValueSyncInterface {
     }
 
     binlog_ = std::make_shared<BinlogT>();
-    TRY_STATUS(binlog_->init(name,
+    TRY_STATUS(binlog_->init(
+        name,
         [&](const BinlogEvent &binlog_event) {
           Event event;
           event.parse(TlParser(binlog_event.data_));

tddb/td/db/SqliteConnectionSafe.cpp

@@ -0,0 +1,51 @@
+//
+// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+#include "td/db/SqliteConnectionSafe.h"
+
+#include "td/utils/common.h"
+#include "td/utils/format.h"
+#include "td/utils/logging.h"
+#include "td/utils/port/Stat.h"
+
+namespace td {
+
+SqliteConnectionSafe::SqliteConnectionSafe(string path, DbKey key)
+    : path_(std::move(path)), lsls_connection_([path = path_, key = std::move(key)] {
+      auto r_db = SqliteDb::open_with_key(path, key);
+      if (r_db.is_error()) {
+        auto r_stat = stat(path);
+        if (r_stat.is_error()) {
+          LOG(FATAL) << "Can't open database " << path << " (" << r_stat.error() << "): " << r_db.error();
+        } else {
+          LOG(FATAL) << "Can't open database " << path << " of size " << r_stat.ok().size_ << ": " << r_db.error();
+        }
+      }
+      auto db = r_db.move_as_ok();
+      db.exec("PRAGMA synchronous=NORMAL").ensure();
+      db.exec("PRAGMA temp_store=MEMORY").ensure();
+      db.exec("PRAGMA secure_delete=1").ensure();
+      db.exec("PRAGMA recursive_triggers=1").ensure();
+      return db;
+    }) {
+}
+
+SqliteDb &SqliteConnectionSafe::get() {
+  return lsls_connection_.get();
+}
+
+void SqliteConnectionSafe::close() {
+  LOG(INFO) << "Close SQLite database " << tag("path", path_);
+  lsls_connection_.clear_values();
+}
+
+void SqliteConnectionSafe::close_and_destroy() {
+  close();
+  LOG(INFO) << "Destroy SQLite database " << tag("path", path_);
+  SqliteDb::destroy(path_).ignore();
+}
+
+}  // namespace td

tddb/td/db/SqliteConnectionSafe.h

@@ -8,50 +8,27 @@
 
 #include "td/actor/SchedulerLocalStorage.h"
 
+#include "td/db/DbKey.h"
 #include "td/db/SqliteDb.h"
 
 #include "td/utils/common.h"
-#include "td/utils/format.h"
-#include "td/utils/logging.h"
 
 namespace td {
 
 class SqliteConnectionSafe {
  public:
   SqliteConnectionSafe() = default;
-  explicit SqliteConnectionSafe(string name, DbKey key = DbKey::empty())
-      : lsls_connection_([name = name, key = std::move(key)] {
-        auto r_db = SqliteDb::open_with_key(name, key);
-        if (r_db.is_error()) {
-          LOG(FATAL) << "Can't open database " << name << ": " << r_db.error();
-        }
-        auto db = r_db.move_as_ok();
-        db.exec("PRAGMA synchronous=NORMAL").ensure();
-        db.exec("PRAGMA temp_store=MEMORY").ensure();
-        db.exec("PRAGMA secure_delete=1").ensure();
-        db.exec("PRAGMA recursive_triggers=1").ensure();
-        return db;
-      })
-      , name_(std::move(name)) {
-  }
+  explicit SqliteConnectionSafe(string path, DbKey key = DbKey::empty());
 
-  SqliteDb &get() {
-    return lsls_connection_.get();
-  }
+  SqliteDb &get();
 
-  void close() {
-    LOG(INFO) << "Close SQLite database " << tag("path", name_);
-    lsls_connection_.clear_values();
-  }
-  void close_and_destroy() {
-    close();
-    LOG(INFO) << "Destroy SQLite database " << tag("path", name_);
-    SqliteDb::destroy(name_).ignore();
-  }
+  void close();
+
+  void close_and_destroy();
 
  private:
+  string path_;
   LazySchedulerLocalStorage<SqliteDb> lsls_connection_;
-  string name_;
 };
 
 }  // namespace td

tddb/td/db/SqliteKeyValue.cpp

@@ -24,7 +24,9 @@ Result<bool> SqliteKeyValue::init(string path) {
 }
 
 Status SqliteKeyValue::init_with_connection(SqliteDb connection, string table_name) {
-  auto init_guard = ScopeExit() + [&]() { close(); };
+  auto init_guard = ScopeExit() + [&]() {
+    close();
+  };
   db_ = std::move(connection);
   table_name_ = std::move(table_name);
   TRY_STATUS(init(db_, table_name_));

tddb/td/db/SqliteStatement.h

@@ -56,7 +56,9 @@ class SqliteStatement {
   void reset();
 
   auto guard() {
-    return ScopeExit{} + [this] { this->reset(); };
+    return ScopeExit{} + [this] {
+      this->reset();
+    };
   }
 
   // TODO get row

tddb/td/db/binlog/binlog_dump.cpp

@@ -28,7 +28,8 @@ int main(int argc, char *argv[]) {
   SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR));
   td::Binlog binlog;
   binlog
-      .init(argv[1],
+      .init(
+          argv[1],
           [&](auto &event) {
             info[0].compressed_size += event.raw_event_.size();
             info[event.type_].compressed_size += event.raw_event_.size();

tddb/td/db/detail/RawSqliteDb.h

@@ -29,8 +29,8 @@ class RawSqliteDb {
   static void with_db_path(Slice main_path, F &&f) {
     f(PSLICE() << main_path);
     f(PSLICE() << main_path << "-journal");
-    f(PSLICE() << main_path << "-shm");
     f(PSLICE() << main_path << "-wal");
+    f(PSLICE() << main_path << "-shm");
   }
   static Status destroy(Slice path) TD_WARN_UNUSED_RESULT;
 

tdnet/td/net/HttpProxy.cpp

@@ -24,7 +24,7 @@ void HttpProxy::send_connect() {
   string proxy_authorization;
   if (!username_.empty() || !password_.empty()) {
     auto userinfo = PSTRING() << username_ << ':' << password_;
-    proxy_authorization = PSTRING() << "Proxy-Authorization: basic " << td::base64_encode(userinfo) << "\r\n";
+    proxy_authorization = PSTRING() << "Proxy-Authorization: basic " << base64_encode(userinfo) << "\r\n";
   }
   fd_.output_buffer().append(PSLICE() << "CONNECT " << host << " HTTP/1.1\r\n"
                                       << "Host: " << host << "\r\n"

tdnet/td/net/SslStream.cpp

@@ -224,7 +224,9 @@ class SslStreamImpl {
     if (ssl_ctx == nullptr) {
       return create_openssl_error(-7, "Failed to create an SSL context");
     }
-    auto ssl_ctx_guard = ScopeExit() + [&]() { SSL_CTX_free(ssl_ctx); };
+    auto ssl_ctx_guard = ScopeExit() + [&]() {
+      SSL_CTX_free(ssl_ctx);
+    };
     long options = 0;
 #ifdef SSL_OP_NO_SSLv2
     options |= SSL_OP_NO_SSLv2;

tdutils/td/utils/DecTree.h

@@ -180,7 +180,7 @@ class DecTree {
     }
   }
   void insert(KeyType key, ValueType value) {
-    root_ = insert_node(std::move(root_), std::move(key), std::move(value), td::Random::fast_uint32());
+    root_ = insert_node(std::move(root_), std::move(key), std::move(value), Random::fast_uint32());
   }
   void remove(const KeyType &key) {
     root_ = remove_node(std::move(root_), key);
@@ -195,7 +195,7 @@ class DecTree {
     if (size() == 0) {
       return nullptr;
     } else {
-      return get_node_by_idx(root_, td::Random::fast_uint32() % size());
+      return get_node_by_idx(root_, Random::fast_uint32() % size());
     }
   }
   const ValueType *get(const KeyType &key) const {
@@ -205,7 +205,7 @@ class DecTree {
     if (size() == 0) {
       return nullptr;
     } else {
-      return get_node_by_idx(root_, td::Random::fast_uint32() % size());
+      return get_node_by_idx(root_, Random::fast_uint32() % size());
     }
   }
   bool exists(const KeyType &key) const {

tdutils/td/utils/FileLog.cpp

@@ -18,13 +18,13 @@
 namespace td {
 
 Status FileLog::init(string path, int64 rotate_threshold, bool redirect_stderr) {
+  if (path.empty()) {
+    return Status::Error("Log file path can't be empty");
+  }
   if (path == path_) {
     set_rotate_threshold(rotate_threshold);
     return Status::OK();
   }
-  if (path.empty()) {
-    return Status::Error("Log file path can't be empty");
-  }
 
   TRY_RESULT(fd, FileFd::open(path, FileFd::Create | FileFd::Write | FileFd::Append));
 

tdutils/td/utils/JsonBuilder.cpp

@@ -593,6 +593,15 @@ bool has_json_object_field(const JsonObject &object, Slice name) {
   return false;
 }
 
+JsonValue get_json_object_field_force(JsonObject &object, Slice name) {
+  for (auto &field_value : object) {
+    if (field_value.first == name) {
+      return std::move(field_value.second);
+    }
+  }
+  return JsonValue();
+}
+
 Result<JsonValue> get_json_object_field(JsonObject &object, Slice name, JsonValue::Type type, bool is_optional) {
   for (auto &field_value : object) {
     if (field_value.first == name) {

tdutils/td/utils/JsonBuilder.h

@@ -860,6 +860,8 @@ auto json_array(const A &a, F &&f) {
 
 bool has_json_object_field(const JsonObject &object, Slice name);
 
+JsonValue get_json_object_field_force(JsonObject &object, Slice name) TD_WARN_UNUSED_RESULT;
+
 Result<JsonValue> get_json_object_field(JsonObject &object, Slice name, JsonValue::Type type,
                                         bool is_optional = true) TD_WARN_UNUSED_RESULT;
 

tdutils/td/utils/MpscLinkQueue.h

@@ -154,7 +154,7 @@ class MpscLinkQueueUniquePtrNode {
   MpscLinkQueueImpl::Node *to_mpsc_link_queue_node() {
     return ptr_.release()->to_mpsc_link_queue_node();
   }
-  static MpscLinkQueueUniquePtrNode<Value> from_mpsc_link_queue_node(td::MpscLinkQueueImpl::Node *node) {
+  static MpscLinkQueueUniquePtrNode<Value> from_mpsc_link_queue_node(MpscLinkQueueImpl::Node *node) {
     return MpscLinkQueueUniquePtrNode<Value>(unique_ptr<Value>(Value::from_mpsc_link_queue_node(node)));
   }
 

tdutils/td/utils/ObjectPool.h

@@ -45,7 +45,7 @@ class ObjectPool {
     // It is not very usual case of acquire/release use.
     // Instead of publishing an object via some flag we do the opposite.
     // We publish new generation via destruction of the data.
-    // In usual case if we see a flag then we are able to use an object.
+    // In usual case if we see a flag, then we are able to use an object.
     // In our case if we have used an object and it is already invalid, then generation will mismatch
     bool is_alive() const {
       if (!storage_) {

tdutils/td/utils/Time.h

@@ -55,7 +55,7 @@ class Timestamp {
     return Timestamp{timeout};
   }
   static Timestamp at_unix(double timeout) {
-    return Timestamp{timeout - td::Clocks::system() + Time::now()};
+    return Timestamp{timeout - Clocks::system() + Time::now()};
   }
 
   static Timestamp in(double timeout) {

tdutils/td/utils/format.h

@@ -227,6 +227,19 @@ StringBuilder &operator<<(StringBuilder &stream, const Array<ArrayT> &array) {
   return stream << Slice("}");
 }
 
+inline StringBuilder &operator<<(StringBuilder &stream, const Array<vector<bool>> &array) {
+  bool first = true;
+  stream << Slice("{");
+  for (bool x : array.ref) {
+    if (!first) {
+      stream << Slice(", ");
+    }
+    stream << x;
+    first = false;
+  }
+  return stream << Slice("}");
+}
+
 template <class ArrayT>
 Array<ArrayT> as_array(const ArrayT &array) {
   return Array<ArrayT>{array};

tdutils/td/utils/misc.h

@@ -83,6 +83,25 @@ auto transform(T &&v, const Func &f) {
   return detail::transform_helper<std::decay_t<T>>().transform(std::forward<T>(v), f);
 }
 
+template <class T, class Func>
+void remove_if(vector<T> &v, const Func &f) {
+  size_t i = 0;
+  while (i != v.size() && !f(v[i])) {
+    i++;
+  }
+  if (i == v.size()) {
+    return;
+  }
+
+  size_t j = i;
+  while (++i != v.size()) {
+    if (!f(v[i])) {
+      v[j++] = std::move(v[i]);
+    }
+  }
+  v.erase(v.begin() + j, v.end());
+}
+
 template <class T>
 void reset_to_empty(T &value) {
   using std::swap;

tdutils/td/utils/port/StdStreams.cpp

@@ -22,7 +22,9 @@ namespace td {
 template <int id>
 static FileFd &get_file_fd() {
   static FileFd result = FileFd::from_native_fd(NativeFd(id, true));
-  static auto guard = td::ScopeExit() + [&] { result.move_as_native_fd().release(); };
+  static auto guard = ScopeExit() + [&] {
+    result.move_as_native_fd().release();
+  };
   return result;
 }
 
@@ -42,7 +44,9 @@ static FileFd &get_file_fd() {
   static auto handle = GetStdHandle(id);
   LOG_IF(FATAL, handle == INVALID_HANDLE_VALUE) << "Failed to GetStdHandle " << id;
   static FileFd result = FileFd::from_native_fd(NativeFd(handle, true));
-  static auto guard = td::ScopeExit() + [&] { result.move_as_native_fd().release(); };
+  static auto guard = ScopeExit() + [&] {
+    result.move_as_native_fd().release();
+  };
 #else
   static FileFd result;
 #endif
@@ -67,7 +71,7 @@ class BufferedStdinImpl : public Iocp::Callback {
 #if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM)
   BufferedStdinImpl() : info_(NativeFd(GetStdHandle(STD_INPUT_HANDLE), true)) {
     iocp_ref_ = Iocp::get()->get_ref();
-    read_thread_ = td::thread([this] { this->read_loop(); });
+    read_thread_ = thread([this] { this->read_loop(); });
   }
 #else
   BufferedStdinImpl() {
@@ -107,7 +111,7 @@ class BufferedStdinImpl : public Iocp::Callback {
   PollableFdInfo info_;
   ChainBufferWriter writer_;
   ChainBufferReader reader_ = writer_.extract_reader();
-  td::thread read_thread_;
+  thread read_thread_;
   std::atomic<bool> close_flag_{false};
   IocpRef iocp_ref_;
   std::atomic<int> refcnt_{1};
@@ -130,7 +134,7 @@ class BufferedStdinImpl : public Iocp::Callback {
     }
   }
   void on_iocp(Result<size_t> r_size, WSAOVERLAPPED *overlapped) override {
-    info_.add_flags_from_poll(td::PollFlags::Read());
+    info_.add_flags_from_poll(PollFlags::Read());
     dec_refcnt();
   }
 

tdutils/td/utils/port/detail/ThreadPthread.cpp

@@ -14,7 +14,9 @@ char disable_linker_warning_about_empty_file_thread_pthread_cpp TD_UNUSED;
 
 #include <pthread.h>
 #include <sched.h>
+#if TD_FREEBSD || TD_OPENBSD || TD_NETBSD
 #include <sys/sysctl.h>
+#endif
 #include <unistd.h>
 
 namespace td {
@@ -30,7 +32,7 @@ unsigned ThreadPthread::hardware_concurrency() {
   }
 #endif
 
-// *BSD
+#if TD_FREEBSD || TD_OPENBSD || TD_NETBSD
 #if defined(HW_AVAILCPU) && defined(CTL_HW)
   {
     int mib[2] = {CTL_HW, HW_AVAILCPU};
@@ -51,6 +53,7 @@ unsigned ThreadPthread::hardware_concurrency() {
       return res;
     }
   }
+#endif
 #endif
 
   // Just in case

tdutils/td/utils/port/detail/ThreadPthread.h

@@ -31,9 +31,7 @@ class ThreadPthread {
   ThreadPthread() = default;
   ThreadPthread(const ThreadPthread &other) = delete;
   ThreadPthread &operator=(const ThreadPthread &other) = delete;
-  ThreadPthread(ThreadPthread &&other) noexcept
-      : is_inited_(std::move(other.is_inited_))
-      , thread_(other.thread_) {
+  ThreadPthread(ThreadPthread &&other) noexcept : is_inited_(std::move(other.is_inited_)), thread_(other.thread_) {
   }
   ThreadPthread &operator=(ThreadPthread &&other) {
     join();

tdutils/td/utils/port/signals.cpp

@@ -289,7 +289,7 @@ void signal_safe_write_pointer(void *p, bool add_header) {
   char *ptr = end;
   *--ptr = '\n';
   do {
-    *--ptr = td::format::hex_digit(addr % 16);
+    *--ptr = format::hex_digit(addr % 16);
     addr /= 16;
   } while (addr != 0);
   *--ptr = 'x';

tdutils/td/utils/port/wstring_convert.cpp

@@ -21,10 +21,7 @@ Result<std::wstring> to_wstring(CSlice slice) {
     return Status::Error("Wrong encoding");
   }
 
-  size_t wstring_len = 0;
-  for (auto c : slice) {
-    wstring_len += ((c & 0xc0) != 0x80) + ((c & 0xf8) == 0xf0);
-  }
+  size_t wstring_len = utf8_utf16_length(slice);
 
   std::wstring result(wstring_len, static_cast<wchar_t>(0));
   if (wstring_len) {

tdutils/td/utils/translit.cpp

@@ -47,7 +47,7 @@ static const std::vector<std::pair<string, string>> &get_ru_to_en_complex_rules(
   return rules;
 }
 
-void add_word_transliterations(vector<string> &result, Slice word, bool allow_partial,
+static void add_word_transliterations(vector<string> &result, Slice word, bool allow_partial,
                                       const std::unordered_map<uint32, string> &simple_rules,
                                       const std::vector<std::pair<string, string>> &complex_rules) {
   string s;

tdutils/td/utils/utf8.h

@@ -28,6 +28,15 @@ inline size_t utf8_length(Slice str) {
   return result;
 }
 
+/// returns length of UTF-8 string in UTF-16 code units
+inline size_t utf8_utf16_length(Slice str) {
+  size_t result = 0;
+  for (auto c : str) {
+    result += is_utf8_character_first_code_unit(c) + ((c & 0xf8) == 0xf0);
+  }
+  return result;
+}
+
 /// appends a Unicode character using UTF-8 encoding
 void append_utf8_character(string &str, uint32 ch);
 

tdutils/test/filesystem.cpp

@@ -36,7 +36,7 @@ TEST(Misc, clean_filename) {
   ASSERT_STREQ(clean_filename("....test"), "test");
   ASSERT_STREQ(clean_filename("test.exe...."), "test.exe");  // extension has changed
   ASSERT_STREQ(clean_filename("test.exe01234567890123456789...."),
-               "test.exe01234567890123456789");  // extension may be more then 20 characters
+               "test.exe01234567890123456789");  // extension may be more than 20 characters
   ASSERT_STREQ(clean_filename("....test....asdf"), "test.asdf");
   ASSERT_STREQ(clean_filename("കറുപ്പ്.txt"), "കറപപ.txt");
 }

tdutils/test/misc.cpp

@@ -153,7 +153,8 @@ TEST(Misc, get_last_argument) {
 }
 
 TEST(Misc, call_n_arguments) {
-  auto f = [](int, int) {};
+  auto f = [](int, int) {
+  };
   call_n_arguments<2>(f, 1, 3, 4);
 }
 
@@ -208,6 +209,65 @@ TEST(Misc, base64) {
               "Ojo7ISUiOw==");
 }
 
+template <class T>
+static void test_remove_if(vector<int> v, const T &func, vector<int> expected) {
+  remove_if(v, func);
+  if (expected != v) {
+    LOG(FATAL) << "Receive " << v << ", expected " << expected << " in remove_if";
+  }
+}
+
+TEST(Misc, remove_if) {
+  auto odd = [](int x) {
+    return x % 2 == 1;
+  };
+  auto even = [](int x) {
+    return x % 2 == 0;
+  };
+  auto all = [](int x) {
+    return true;
+  };
+  auto none = [](int x) {
+    return false;
+  };
+
+  vector<int> v{1, 2, 3, 4, 5, 6};
+  test_remove_if(v, odd, {2, 4, 6});
+  test_remove_if(v, even, {1, 3, 5});
+  test_remove_if(v, all, {});
+  test_remove_if(v, none, v);
+
+  v = vector<int>{1, 3, 5, 2, 4, 6};
+  test_remove_if(v, odd, {2, 4, 6});
+  test_remove_if(v, even, {1, 3, 5});
+  test_remove_if(v, all, {});
+  test_remove_if(v, none, v);
+
+  v.clear();
+  test_remove_if(v, odd, v);
+  test_remove_if(v, even, v);
+  test_remove_if(v, all, v);
+  test_remove_if(v, none, v);
+
+  v.push_back(-1);
+  test_remove_if(v, odd, v);
+  test_remove_if(v, even, v);
+  test_remove_if(v, all, {});
+  test_remove_if(v, none, v);
+
+  v[0] = 1;
+  test_remove_if(v, odd, {});
+  test_remove_if(v, even, v);
+  test_remove_if(v, all, {});
+  test_remove_if(v, none, v);
+
+  v[0] = 2;
+  test_remove_if(v, odd, v);
+  test_remove_if(v, even, {});
+  test_remove_if(v, all, {});
+  test_remove_if(v, none, v);
+}
+
 TEST(Misc, to_integer) {
   ASSERT_EQ(to_integer<int32>("-1234567"), -1234567);
   ASSERT_EQ(to_integer<int64>("-1234567"), -1234567);
@@ -757,8 +817,12 @@ TEST(Misc, uint128) {
                                      static_cast<int64>(std::numeric_limits<int32>::min()) - 1};
 
 #if TD_HAVE_INT128
-  auto to_intrinsic = [](uint128_emulated num) { return uint128_intrinsic(num.hi(), num.lo()); };
-  auto eq = [](uint128_emulated a, uint128_intrinsic b) { return a.hi() == b.hi() && a.lo() == b.lo(); };
+  auto to_intrinsic = [](uint128_emulated num) {
+    return uint128_intrinsic(num.hi(), num.lo());
+  };
+  auto eq = [](uint128_emulated a, uint128_intrinsic b) {
+    return a.hi() == b.hi() && a.lo() == b.lo();
+  };
   auto ensure_eq = [&](uint128_emulated a, uint128_intrinsic b) {
     if (!eq(a, b)) {
       LOG(FATAL) << "[" << a.hi() << ";" << a.lo() << "] vs [" << b.hi() << ";" << b.lo() << "]";

tdutils/test/port.cpp

@@ -41,33 +41,28 @@ TEST(Port, files) {
   int cnt = 0;
   const int ITER_COUNT = 1000;
   for (int i = 0; i < ITER_COUNT; i++) {
-    walk_path(main_dir,
-              [&](CSlice name, WalkPath::Type type) {
+    walk_path(main_dir, [&](CSlice name, WalkPath::Type type) {
       if (type == WalkPath::Type::NotDir) {
         ASSERT_TRUE(name == fd_path || name == fd2_path);
       }
       cnt++;
-              })
-        .ensure();
+    }).ensure();
   }
   ASSERT_EQ((5 * 2 + 2) * ITER_COUNT, cnt);
   bool was_abort = false;
-  walk_path(main_dir,
-            [&](CSlice name, WalkPath::Type type) {
+  walk_path(main_dir, [&](CSlice name, WalkPath::Type type) {
     CHECK(!was_abort);
     if (type == WalkPath::Type::EnterDir && ends_with(name, PSLICE() << TD_DIR_SLASH << "B")) {
       was_abort = true;
       return WalkPath::Action::Abort;
     }
     return WalkPath::Action::Continue;
-            })
-      .ensure();
+  }).ensure();
   CHECK(was_abort);
 
   cnt = 0;
   bool is_first_dir = true;
-  walk_path(main_dir,
-            [&](CSlice name, WalkPath::Type type) {
+  walk_path(main_dir, [&](CSlice name, WalkPath::Type type) {
     cnt++;
     if (type == WalkPath::Type::EnterDir) {
       if (is_first_dir) {
@@ -77,8 +72,7 @@ TEST(Port, files) {
       }
     }
     return WalkPath::Action::Continue;
-            })
-      .ensure();
+  }).ensure();
   ASSERT_EQ(6, cnt);
 
   ASSERT_EQ(0u, fd.get_size().move_as_ok());

test/db.cpp

@@ -47,11 +47,17 @@ TEST(DB, binlog_encryption_bug) {
   auto empty = DbKey::empty();
   {
     Binlog binlog;
-    binlog.init(binlog_name.str(), [&](const BinlogEvent &x) {}, cucumber).ensure();
+    binlog
+        .init(
+            binlog_name.str(), [&](const BinlogEvent &x) {}, cucumber)
+        .ensure();
   }
   {
     Binlog binlog;
-    binlog.init(binlog_name.str(), [&](const BinlogEvent &x) {}, cucumber).ensure();
+    binlog
+        .init(
+            binlog_name.str(), [&](const BinlogEvent &x) {}, cucumber)
+        .ensure();
   }
 }
 
@@ -92,7 +98,10 @@ TEST(DB, binlog_encryption) {
     std::vector<string> v;
     LOG(INFO) << "RESTART";
     Binlog binlog;
-    binlog.init(binlog_name.str(), [&](const BinlogEvent &x) { v.push_back(x.data_.str()); }, hello).ensure();
+    binlog
+        .init(
+            binlog_name.str(), [&](const BinlogEvent &x) { v.push_back(x.data_.str()); }, hello)
+        .ensure();
     CHECK(v == std::vector<string>({"AAAA", "BBBB", long_data, "CCCC"}));
   }
 
@@ -102,7 +111,8 @@ TEST(DB, binlog_encryption) {
     std::vector<string> v;
     LOG(INFO) << "RESTART";
     Binlog binlog;
-    auto status = binlog.init(binlog_name.str(), [&](const BinlogEvent &x) { v.push_back(x.data_.str()); }, cucumber);
+    auto status = binlog.init(
+        binlog_name.str(), [&](const BinlogEvent &x) { v.push_back(x.data_.str()); }, cucumber);
     CHECK(status.is_error());
   }
 
@@ -112,8 +122,8 @@ TEST(DB, binlog_encryption) {
     std::vector<string> v;
     LOG(INFO) << "RESTART";
     Binlog binlog;
-    auto status =
-        binlog.init(binlog_name.str(), [&](const BinlogEvent &x) { v.push_back(x.data_.str()); }, cucumber, hello);
+    auto status = binlog.init(
+        binlog_name.str(), [&](const BinlogEvent &x) { v.push_back(x.data_.str()); }, cucumber, hello);
     CHECK(v == std::vector<string>({"AAAA", "BBBB", long_data, "CCCC"}));
   }
 };

test/message_entities.cpp

@@ -41,9 +41,8 @@ TEST(MessageEntities, mention) {
   check_mention("@abcdefghijklmnopqrstuvwxyz123456", {"@abcdefghijklmnopqrstuvwxyz123456"});
   check_mention("@abcdefghijklmnopqrstuvwxyz1234567", {});
   check_mention("нет@mention", {});
-  check_mention(
-      "@ya @gif @wiki @vid @bing @pic @bold @imdb @coub @like @vote @giff @cap ya cap @y @yar @bingg @bin",
-      {"@ya", "@gif", "@wiki", "@vid", "@bing", "@pic", "@bold", "@imdb", "@coub", "@like", "@vote", "@bingg"});
+  check_mention("@ya @gif @wiki @vid @bing @pic @bold @imdb @coub @like @vote @giff @cap ya cap @y @yar @bingg @bin",
+                {"@gif", "@wiki", "@vid", "@bing", "@pic", "@bold", "@imdb", "@coub", "@like", "@vote", "@bingg"});
 };
 
 static void check_bot_command(string str, std::vector<string> expected) {