From 09f906f192b6c5f42f7a26e64fb26b0880028bc5 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 11 Oct 2020 01:59:27 +0300 Subject: [PATCH 01/23] Protect access to some static variables using ExitGuard. GitOrigin-RevId: 65c7510c60d585b90e90d09067c7dfdaf79c4cd3 --- td/telegram/Client.cpp | 7 ++++--- td/telegram/ClientJson.cpp | 2 ++ tdutils/td/utils/port/FileFd.cpp | 13 ++++++++----- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/td/telegram/Client.cpp b/td/telegram/Client.cpp index 78b8d9a1e..ae7efbf60 100644 --- a/td/telegram/Client.cpp +++ b/td/telegram/Client.cpp @@ -13,6 +13,7 @@ #include "td/utils/common.h" #include "td/utils/crypto.h" +#include "td/utils/ExitGuard.h" #include "td/utils/logging.h" #include "td/utils/misc.h" #include "td/utils/MpscPollableQueue.h" @@ -174,7 +175,7 @@ class ClientManager::Impl final { td.second.reset(); } } - while (!tds_.empty()) { + while (!tds_.empty() && !ExitGuard::is_exited()) { receive(10); } concurrent_scheduler_->finish(); @@ -521,7 +522,7 @@ class ClientManager::Impl final { for (auto &it : impls_) { close_impl(it.first); } - while (!impls_.empty()) { + while (!impls_.empty() && !ExitGuard::is_exited()) { receive(10); } } @@ -569,7 +570,7 @@ class Client::Impl final { Impl &operator=(Impl &&) = delete; ~Impl() { multi_impl_->close(td_id_); - while (true) { + while (!ExitGuard::is_exited()) { auto response = receiver_.receive(10.0); if (response.object == nullptr && response.client_id != 0 && response.request_id == 0) { break; diff --git a/td/telegram/ClientJson.cpp b/td/telegram/ClientJson.cpp index d5beab12d..8f946986a 100644 --- a/td/telegram/ClientJson.cpp +++ b/td/telegram/ClientJson.cpp @@ -10,6 +10,7 @@ #include "td/telegram/td_api_json.h" #include "td/utils/common.h" +#include "td/utils/ExitGuard.h" #include "td/utils/JsonBuilder.h" #include "td/utils/logging.h" #include "td/utils/port/thread_local.h" @@ -116,6 +117,7 @@ const char *ClientJson::execute(Slice request) { static ClientManager *get_manager() { static ClientManager client_manager; + static ExitGuard exit_guard; return &client_manager; } diff --git a/tdutils/td/utils/port/FileFd.cpp b/tdutils/td/utils/port/FileFd.cpp index 1a6965eca..634dacb64 100644 --- a/tdutils/td/utils/port/FileFd.cpp +++ b/tdutils/td/utils/port/FileFd.cpp @@ -12,6 +12,7 @@ #endif #include "td/utils/common.h" +#include "td/utils/ExitGuard.h" #include "td/utils/logging.h" #include "td/utils/misc.h" #include "td/utils/port/detail/PollableFd.h" @@ -356,6 +357,7 @@ Result FileFd::pread(MutableSlice slice, int64 offset) const { static std::mutex in_process_lock_mutex; static std::unordered_set locked_files; +static ExitGuard exit_guard; static Status create_local_lock(const string &path, int32 &max_tries) { while (true) { @@ -469,12 +471,13 @@ Status FileFd::lock(const LockFlags flags, const string &path, int32 max_tries) } void FileFd::remove_local_lock(const string &path) { - if (!path.empty()) { - VLOG(fd) << "Unlock file \"" << path << '"'; - std::unique_lock lock(in_process_lock_mutex); - auto erased_count = locked_files.erase(path); - CHECK(erased_count > 0); + if (path.empty() || ExitGuard::is_exited()) { + return; } + VLOG(fd) << "Unlock file \"" << path << '"'; + std::unique_lock lock(in_process_lock_mutex); + auto erased_count = locked_files.erase(path); + CHECK(erased_count > 0 || ExitGuard::is_exited()); } void FileFd::close() { From 714f037f15e84100ca4765747b78260ce3b22a58 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 11 Oct 2020 11:08:56 +0300 Subject: [PATCH 02/23] Move ClientManager singleton creation to ClientManager::get_manager_singleton. GitOrigin-RevId: 071b947ff072186c70387cbd00f1c6b1c17d0e6b --- td/telegram/Client.cpp | 6 ++++++ td/telegram/Client.h | 6 ++++++ td/telegram/ClientJson.cpp | 5 +---- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/td/telegram/Client.cpp b/td/telegram/Client.cpp index ae7efbf60..e8aa89870 100644 --- a/td/telegram/Client.cpp +++ b/td/telegram/Client.cpp @@ -631,4 +631,10 @@ ClientManager::~ClientManager() = default; ClientManager::ClientManager(ClientManager &&other) = default; ClientManager &ClientManager::operator=(ClientManager &&other) = default; +ClientManager *ClientManager::get_manager_singleton() { + static ClientManager client_manager; + static ExitGuard exit_guard; + return &client_manager; +} + } // namespace td diff --git a/td/telegram/Client.h b/td/telegram/Client.h index 879fe4890..7cb386def 100644 --- a/td/telegram/Client.h +++ b/td/telegram/Client.h @@ -247,6 +247,12 @@ class ClientManager final { */ ClientManager &operator=(ClientManager &&other); + /** + * Returns a pointer to a singleton ClientManager instance. + * \return A unique singleton ClientManager instance. + */ + static ClientManager *get_manager_singleton(); + private: friend class Client; class Impl; diff --git a/td/telegram/ClientJson.cpp b/td/telegram/ClientJson.cpp index 8f946986a..80ef8450b 100644 --- a/td/telegram/ClientJson.cpp +++ b/td/telegram/ClientJson.cpp @@ -10,7 +10,6 @@ #include "td/telegram/td_api_json.h" #include "td/utils/common.h" -#include "td/utils/ExitGuard.h" #include "td/utils/JsonBuilder.h" #include "td/utils/logging.h" #include "td/utils/port/thread_local.h" @@ -116,9 +115,7 @@ const char *ClientJson::execute(Slice request) { } static ClientManager *get_manager() { - static ClientManager client_manager; - static ExitGuard exit_guard; - return &client_manager; + return ClientManager::get_manager_singleton(); } static std::mutex extra_mutex; From 29cd47f01a41cf4dc06918c560b3195a5a15c653 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 11 Oct 2020 11:13:30 +0300 Subject: [PATCH 03/23] Fix Client close waiting in Java example. GitOrigin-RevId: e5fa246b26b6477da4b3e3ec7444aa5e3e6c7bb2 --- .../java/org/drinkless/tdlib/example/Example.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/example/java/org/drinkless/tdlib/example/Example.java b/example/java/org/drinkless/tdlib/example/Example.java index 8e52fda92..7c2e5649e 100644 --- a/example/java/org/drinkless/tdlib/example/Example.java +++ b/example/java/org/drinkless/tdlib/example/Example.java @@ -29,7 +29,8 @@ public final class Example { private static TdApi.AuthorizationState authorizationState = null; private static volatile boolean haveAuthorization = false; - private static volatile boolean quiting = false; + private static volatile boolean needQuit = false; + private static volatile boolean canQuit = false; private static final Client.ResultHandler defaultHandler = new DefaultHandler(); @@ -160,8 +161,10 @@ public final class Example { break; case TdApi.AuthorizationStateClosed.CONSTRUCTOR: print("Closed"); - if (!quiting) { + if (!needQuit) { client = Client.create(new UpdateHandler(), null, null); // recreate client after previous has closed + } else { + canQuit = true; } break; default: @@ -230,7 +233,7 @@ public final class Example { client.send(new TdApi.LogOut(), defaultHandler); break; case "q": - quiting = true; + needQuit = true; haveAuthorization = false; client.send(new TdApi.Close(), defaultHandler); break; @@ -316,7 +319,7 @@ public final class Example { defaultHandler.onResult(Client.execute(new TdApi.GetTextEntities("@telegram /test_command https://telegram.org telegram.me @gif @test"))); // main loop - while (!quiting) { + while (!needQuit) { // await authorization authorizationLock.lock(); try { @@ -331,6 +334,9 @@ public final class Example { getCommand(); } } + while (!canQuit) { + Thread.sleep(1); + } } private static class OrderedChat implements Comparable { From 4ad90cecc6c869a9c562d7b9366bff5661d9bb71 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 11 Oct 2020 11:48:42 +0300 Subject: [PATCH 04/23] Fix Client close waiting in C# example. GitOrigin-RevId: 411fb84b3f7b4840381962db4e761a04db464182 --- example/csharp/TdExample.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/example/csharp/TdExample.cs b/example/csharp/TdExample.cs index eb5c0c0f5..7c5b4d255 100644 --- a/example/csharp/TdExample.cs +++ b/example/csharp/TdExample.cs @@ -23,7 +23,8 @@ namespace TdExample private static TdApi.AuthorizationState _authorizationState = null; private static volatile bool _haveAuthorization = false; - private static volatile bool _quiting = false; + private static volatile bool _needQuit = false; + private static volatile bool _canQuit = false; private static volatile AutoResetEvent _gotAuthorization = new AutoResetEvent(false); @@ -133,9 +134,11 @@ namespace TdExample { Print("Closed"); _client.Dispose(); // _client is closed and native resources can be disposed now - if (!_quiting) + if (!_needQuit) { _client = CreateTdClient(); // recreate _client after previous has closed + } else { + _canQuit = true; } } else @@ -187,7 +190,7 @@ namespace TdExample _client.Send(new TdApi.Close(), _defaultHandler); break; case "q": - _quiting = true; + _needQuit = true; _haveAuthorization = false; _client.Send(new TdApi.Close(), _defaultHandler); break; @@ -228,7 +231,7 @@ namespace TdExample _defaultHandler.OnResult(Td.Client.Execute(new TdApi.GetTextEntities("@telegram /test_command https://telegram.org telegram.me @gif @test"))); // main loop - while (!_quiting) + while (!_needQuit) { // await authorization _gotAuthorization.Reset(); @@ -240,6 +243,9 @@ namespace TdExample GetCommand(); } } + while (!_canQuit) { + Thread.Sleep(1); + } } private class DefaultHandler : Td.ClientResultHandler From 78b19d949fa9b3c9368307d349d5d7769475b707 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 11 Oct 2020 14:20:26 +0300 Subject: [PATCH 05/23] Fix on_fatal_error in Java example. GitOrigin-RevId: 09b6463518e391a70918685205236ffa59a9a121 --- example/java/td_jni.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/example/java/td_jni.cpp b/example/java/td_jni.cpp index 34ad9c836..6571c8c69 100644 --- a/example/java/td_jni.cpp +++ b/example/java/td_jni.cpp @@ -103,8 +103,11 @@ static jclass log_class; static void on_fatal_error(const char *error_message) { auto env = td::jni::get_jni_env(java_vm, JAVA_VERSION); + if (env == nullptr) { + return; + } jmethodID on_fatal_error_method = env->GetStaticMethodID(log_class, "onFatalError", "(Ljava/lang/String;)V"); - if (env && on_fatal_error_method) { + if (on_fatal_error_method) { jstring error_str = td::jni::to_jstring(env.get(), error_message); env->CallStaticVoidMethod(log_class, on_fatal_error_method, error_str); if (error_str) { From a67225d358f1a48b7b1ef0f33ee38d0cabb8e7bb Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 11 Oct 2020 14:44:42 +0300 Subject: [PATCH 06/23] Make get_jni_env safer. GitOrigin-RevId: 15d1d788e9f2811e769554044992ae1d7db7fb9a --- td/tl/tl_jni_object.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/td/tl/tl_jni_object.cpp b/td/tl/tl_jni_object.cpp index 640635c32..169e0dd3b 100644 --- a/td/tl/tl_jni_object.cpp +++ b/td/tl/tl_jni_object.cpp @@ -79,7 +79,7 @@ void register_native_method(JNIEnv *env, jclass clazz, std::string name, std::st std::unique_ptr get_jni_env(JavaVM *java_vm, jint jni_version) { JNIEnv *env = nullptr; - if (java_vm->GetEnv(reinterpret_cast(&env), jni_version) == JNI_EDETACHED) { + if (java_vm->GetEnv(reinterpret_cast(&env), jni_version) == JNI_EDETACHED && env != nullptr) { #ifdef JDK1_2 // if not Android JNI auto p_env = reinterpret_cast(&env); #else From 10a7edfbea98fd34dee5ff1c3190b8fe934c5407 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 11 Oct 2020 14:45:22 +0300 Subject: [PATCH 07/23] Disable logging after program exit. GitOrigin-RevId: 8ca24e1999fe95f171e18477baa2e56912a2e895 --- td/telegram/Logging.cpp | 2 ++ td/telegram/cli.cpp | 2 ++ tdutils/td/utils/ExitGuard.cpp | 7 +++++++ tdutils/td/utils/ExitGuard.h | 4 +--- tdutils/td/utils/logging.cpp | 21 ++++++++++++++++++++- tdutils/td/utils/logging.h | 10 ++-------- 6 files changed, 34 insertions(+), 12 deletions(-) diff --git a/td/telegram/Logging.cpp b/td/telegram/Logging.cpp index 8694ab498..131303a67 100644 --- a/td/telegram/Logging.cpp +++ b/td/telegram/Logging.cpp @@ -29,6 +29,7 @@ #include "td/actor/actor.h" +#include "td/utils/ExitGuard.h" #include "td/utils/FileLog.h" #include "td/utils/logging.h" #include "td/utils/misc.h" @@ -44,6 +45,7 @@ static std::mutex logging_mutex; static FileLog file_log; static TsLog ts_log(&file_log); static NullLog null_log; +static ExitGuard exit_guard; #define ADD_TAG(tag) \ { #tag, &VERBOSITY_NAME(tag) } diff --git a/td/telegram/cli.cpp b/td/telegram/cli.cpp index 2d2e1fd0a..a4cd7bd66 100644 --- a/td/telegram/cli.cpp +++ b/td/telegram/cli.cpp @@ -20,6 +20,7 @@ #include "td/utils/buffer.h" #include "td/utils/common.h" #include "td/utils/crypto.h" +#include "td/utils/ExitGuard.h" #include "td/utils/FileLog.h" #include "td/utils/format.h" #include "td/utils/JsonBuilder.h" @@ -4442,6 +4443,7 @@ static void on_fatal_error(const char *error) { } void main(int argc, char **argv) { + ExitGuard exit_guard; ignore_signal(SignalType::HangUp).ensure(); ignore_signal(SignalType::Pipe).ensure(); set_signal_handler(SignalType::Error, fail_signal).ensure(); diff --git a/tdutils/td/utils/ExitGuard.cpp b/tdutils/td/utils/ExitGuard.cpp index 1c62dec5e..758f85f6d 100644 --- a/tdutils/td/utils/ExitGuard.cpp +++ b/tdutils/td/utils/ExitGuard.cpp @@ -6,8 +6,15 @@ // #include "td/utils/ExitGuard.h" +#include "td/utils/logging.h" + namespace td { std::atomic ExitGuard::is_exited_{false}; +ExitGuard::~ExitGuard() { + is_exited_.store(true, std::memory_order_relaxed); + set_verbosity_level(VERBOSITY_NAME(FATAL)); +} + } // namespace td diff --git a/tdutils/td/utils/ExitGuard.h b/tdutils/td/utils/ExitGuard.h index 1a91a731f..62a5bdaa9 100644 --- a/tdutils/td/utils/ExitGuard.h +++ b/tdutils/td/utils/ExitGuard.h @@ -17,9 +17,7 @@ class ExitGuard { ExitGuard &operator=(ExitGuard &&) = delete; ExitGuard(const ExitGuard &) = delete; ExitGuard &operator=(const ExitGuard &) = delete; - ~ExitGuard() { - is_exited_.store(true, std::memory_order_relaxed); - } + ~ExitGuard(); static bool is_exited() { return is_exited_.load(std::memory_order_relaxed); diff --git a/tdutils/td/utils/logging.cpp b/tdutils/td/utils/logging.cpp index 7fac06f33..06c38b69f 100644 --- a/tdutils/td/utils/logging.cpp +++ b/tdutils/td/utils/logging.cpp @@ -6,6 +6,7 @@ // #include "td/utils/logging.h" +#include "td/utils/ExitGuard.h" #include "td/utils/port/Clocks.h" #include "td/utils/port/StdStreams.h" #include "td/utils/port/thread_local.h" @@ -43,6 +44,9 @@ Logger::Logger(LogInterface &log, const LogOptions &options, int log_level, Slic if (!options_.add_info) { return; } + if (ExitGuard::is_exited()) { + return; + } // log level sb_ << '['; @@ -104,6 +108,9 @@ Logger::Logger(LogInterface &log, const LogOptions &options, int log_level, Slic } Logger::~Logger() { + if (ExitGuard::is_exited()) { + return; + } if (options_.fix_newlines) { sb_ << '\n'; auto slice = as_cslice(); @@ -152,7 +159,7 @@ TsCerr &TsCerr::operator<<(Slice slice) { } void TsCerr::enterCritical() { - while (lock_.test_and_set(std::memory_order_acquire)) { + while (lock_.test_and_set(std::memory_order_acquire) && !ExitGuard::is_exited()) { // spin } } @@ -162,6 +169,16 @@ void TsCerr::exitCritical() { } TsCerr::Lock TsCerr::lock_ = ATOMIC_FLAG_INIT; +void TsLog::enter_critical() { + while (lock_.test_and_set(std::memory_order_acquire) && !ExitGuard::is_exited()) { + // spin + } +} + +void TsLog::exit_critical() { + lock_.clear(std::memory_order_release); +} + class DefaultLog : public LogInterface { public: void append(CSlice slice, int log_level) override { @@ -290,4 +307,6 @@ ScopedDisableLog::~ScopedDisableLog() { } } +static ExitGuard exit_guard; + } // namespace td diff --git a/tdutils/td/utils/logging.h b/tdutils/td/utils/logging.h index 1202f0083..00c62169b 100644 --- a/tdutils/td/utils/logging.h +++ b/tdutils/td/utils/logging.h @@ -321,14 +321,8 @@ class TsLog : public LogInterface { private: LogInterface *log_ = nullptr; std::atomic_flag lock_ = ATOMIC_FLAG_INIT; - void enter_critical() { - while (lock_.test_and_set(std::memory_order_acquire)) { - // spin - } - } - void exit_critical() { - lock_.clear(std::memory_order_release); - } + void enter_critical(); + void exit_critical(); }; } // namespace td From 66d8ee522810a047090f26d03a9cc20ea4f30eed Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 11 Oct 2020 14:53:12 +0300 Subject: [PATCH 08/23] Fix get_jni_env. GitOrigin-RevId: 5da8136b7c96affd3a39194624dc35c415bfad68 --- td/tl/tl_jni_object.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/td/tl/tl_jni_object.cpp b/td/tl/tl_jni_object.cpp index 169e0dd3b..31af7508d 100644 --- a/td/tl/tl_jni_object.cpp +++ b/td/tl/tl_jni_object.cpp @@ -79,13 +79,16 @@ void register_native_method(JNIEnv *env, jclass clazz, std::string name, std::st std::unique_ptr get_jni_env(JavaVM *java_vm, jint jni_version) { JNIEnv *env = nullptr; - if (java_vm->GetEnv(reinterpret_cast(&env), jni_version) == JNI_EDETACHED && env != nullptr) { + if (java_vm->GetEnv(reinterpret_cast(&env), jni_version) == JNI_EDETACHED) { #ifdef JDK1_2 // if not Android JNI auto p_env = reinterpret_cast(&env); #else auto p_env = &env; #endif - java_vm->AttachCurrentThread(p_env, nullptr); + if (java_vm->AttachCurrentThread(p_env, nullptr) != JNI_OK) { + java_vm = nullptr; + env = nullptr; + } } else { java_vm = nullptr; } From 9856b0e46e8081ddf7ae889ccdd902d997dec1db Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 11 Oct 2020 21:21:38 +0300 Subject: [PATCH 09/23] Add some workariunds for crashes on exit without closing all clients. GitOrigin-RevId: 5c74e9fe6951b6a8eb65d6c1e5ddf7bf8e0c8163 --- td/telegram/Client.cpp | 17 ++++++++++++----- tdactor/td/actor/impl/Scheduler.cpp | 1 + 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/td/telegram/Client.cpp b/td/telegram/Client.cpp index e8aa89870..b9b428e78 100644 --- a/td/telegram/Client.cpp +++ b/td/telegram/Client.cpp @@ -176,9 +176,11 @@ class ClientManager::Impl final { } } while (!tds_.empty() && !ExitGuard::is_exited()) { - receive(10); + receive(0.1); + } + if (!ExitGuard::is_exited()) { // prevent closing of schedulers from already killed by OS threads + concurrent_scheduler_->finish(); } - concurrent_scheduler_->finish(); } private: @@ -380,7 +382,9 @@ class MultiImpl { Scheduler::instance()->finish(); } scheduler_thread_.join(); - concurrent_scheduler_->finish(); + if (!ExitGuard::is_exited()) { // prevent closing of schedulers from already killed by OS threads + concurrent_scheduler_->finish(); + } } private: @@ -519,11 +523,14 @@ class ClientManager::Impl final { Impl(Impl &&) = delete; Impl &operator=(Impl &&) = delete; ~Impl() { + if (ExitGuard::is_exited()) { + return; + } for (auto &it : impls_) { close_impl(it.first); } while (!impls_.empty() && !ExitGuard::is_exited()) { - receive(10); + receive(0.1); } } @@ -571,7 +578,7 @@ class Client::Impl final { ~Impl() { multi_impl_->close(td_id_); while (!ExitGuard::is_exited()) { - auto response = receiver_.receive(10.0); + auto response = receiver_.receive(0.1); if (response.object == nullptr && response.client_id != 0 && response.request_id == 0) { break; } diff --git a/tdactor/td/actor/impl/Scheduler.cpp b/tdactor/td/actor/impl/Scheduler.cpp index 39cb4661e..ca8f58fb6 100644 --- a/tdactor/td/actor/impl/Scheduler.cpp +++ b/tdactor/td/actor/impl/Scheduler.cpp @@ -111,6 +111,7 @@ void Scheduler::ServiceActor::tear_down() { /*** SchedlerGuard ***/ SchedulerGuard::SchedulerGuard(Scheduler *scheduler, bool lock) : scheduler_(scheduler) { if (lock) { + // the next check can fail if OS killed the scheduler's thread without releasing the guard CHECK(!scheduler_->has_guard_); scheduler_->has_guard_ = true; } From 919848f0fe043e87d0514cb8a73e8d60b8b8e69d Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 11 Oct 2020 21:28:33 +0300 Subject: [PATCH 10/23] Use ClientManager for Java example interface implementation. GitOrigin-RevId: 4280b6407a1c1a18bf2a6e952f6761847b69cb83 --- example/java/org/drinkless/tdlib/Client.java | 209 +++++++------------ example/java/td_jni.cpp | 45 ++-- 2 files changed, 98 insertions(+), 156 deletions(-) diff --git a/example/java/org/drinkless/tdlib/Client.java b/example/java/org/drinkless/tdlib/Client.java index 986d5b3c7..d95196f17 100644 --- a/example/java/org/drinkless/tdlib/Client.java +++ b/example/java/org/drinkless/tdlib/Client.java @@ -8,14 +8,11 @@ package org.drinkless.tdlib; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Main class for interaction with the TDLib. */ -public final class Client implements Runnable { +public final class Client { /** * Interface for handler for results of queries to TDLib and incoming updates from TDLib. */ @@ -55,25 +52,11 @@ public final class Client implements Runnable { * @throws NullPointerException if query is null. */ public void send(TdApi.Function query, ResultHandler resultHandler, ExceptionHandler exceptionHandler) { - if (query == null) { - throw new NullPointerException("query is null"); - } - - readLock.lock(); - try { - if (isClientDestroyed) { - if (resultHandler != null) { - handleResult(new TdApi.Error(500, "Client is closed"), resultHandler, exceptionHandler); - } - return; - } - - long queryId = currentQueryId.incrementAndGet(); + long queryId = currentQueryId.incrementAndGet(); + if (resultHandler != null) { handlers.put(queryId, new Handler(resultHandler, exceptionHandler)); - nativeClientSend(nativeClientId, queryId, query); - } finally { - readLock.unlock(); } + nativeClientSend(nativeClientId, queryId, query); } /** @@ -97,22 +80,9 @@ public final class Client implements Runnable { * @throws NullPointerException if query is null. */ public static TdApi.Object execute(TdApi.Function query) { - if (query == null) { - throw new NullPointerException("query is null"); - } return nativeClientExecute(query); } - /** - * Overridden method from Runnable, do not call it directly. - */ - @Override - public void run() { - while (!stopFlag) { - receiveQueries(300.0 /*seconds*/); - } - } - /** * Creates new Client. * @@ -123,54 +93,79 @@ public final class Client implements Runnable { */ public static Client create(ResultHandler updateHandler, ExceptionHandler updateExceptionHandler, ExceptionHandler defaultExceptionHandler) { Client client = new Client(updateHandler, updateExceptionHandler, defaultExceptionHandler); - new Thread(client, "TDLib thread").start(); + synchronized (responseReceiver) { + if (!responseReceiver.isRun) { + responseReceiver.isRun = true; + + Thread receiverThread = new Thread(responseReceiver, "TDLib thread"); + receiverThread.setDaemon(true); + receiverThread.start(); + } + } return client; } - /** - * Closes Client. - */ - public void close() { - writeLock.lock(); - try { - if (isClientDestroyed) { - return; + private static class ResponseReceiver implements Runnable { + public boolean isRun = false; + + @Override + public void run() { + while (true) { + int resultN = nativeClientReceive(clientIds, eventIds, events, 100000.0 /*seconds*/); + for (int i = 0; i < resultN; i++) { + processResult(clientIds[i], eventIds[i], events[i]); + events[i] = null; + } } - if (!stopFlag) { - send(new TdApi.Close(), null); - } - isClientDestroyed = true; - while (!stopFlag) { - Thread.yield(); - } - while (!handlers.isEmpty()) { - receiveQueries(300.0); - } - updateHandlers.remove(nativeClientId); - defaultExceptionHandlers.remove(nativeClientId); - destroyNativeClient(nativeClientId); - } finally { - writeLock.unlock(); } + + private void processResult(int clientId, long id, TdApi.Object object) { + boolean isClosed = false; + if (id == 0 && object instanceof TdApi.UpdateAuthorizationState) { + TdApi.AuthorizationState authorizationState = ((TdApi.UpdateAuthorizationState) object).authorizationState; + if (authorizationState instanceof TdApi.AuthorizationStateClosed) { + isClosed = true; + } + } + + Handler handler = id == 0 ? updateHandlers.get(clientId) : handlers.remove(id); + if (handler != null) { + try { + handler.resultHandler.onResult(object); + } catch (Throwable cause) { + ExceptionHandler exceptionHandler = handler.exceptionHandler; + if (exceptionHandler == null) { + exceptionHandler = defaultExceptionHandlers.get(clientId); + } + if (exceptionHandler != null) { + try { + exceptionHandler.onException(cause); + } catch (Throwable ignored) { + } + } + } + } + + if (isClosed) { + updateHandlers.remove(clientId); // there will be no more updates + defaultExceptionHandlers.remove(clientId); // ignore further exceptions + } + } + + private static final int MAX_EVENTS = 1000; + private final int[] clientIds = new int[MAX_EVENTS]; + private final long[] eventIds = new long[MAX_EVENTS]; + private final TdApi.Object[] events = new TdApi.Object[MAX_EVENTS]; } - private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); - private final Lock readLock = readWriteLock.readLock(); - private final Lock writeLock = readWriteLock.writeLock(); + private final int nativeClientId; - private volatile boolean stopFlag = false; - private volatile boolean isClientDestroyed = false; - private final long nativeClientId; + private static final ConcurrentHashMap defaultExceptionHandlers = new ConcurrentHashMap(); + private static final ConcurrentHashMap updateHandlers = new ConcurrentHashMap(); + private static final ConcurrentHashMap handlers = new ConcurrentHashMap(); + private static final AtomicLong currentQueryId = new AtomicLong(); - private static final ConcurrentHashMap defaultExceptionHandlers = new ConcurrentHashMap(); - private static final ConcurrentHashMap updateHandlers = new ConcurrentHashMap(); - - private final ConcurrentHashMap handlers = new ConcurrentHashMap(); - private final AtomicLong currentQueryId = new AtomicLong(); - - private static final int MAX_EVENTS = 1000; - private final long[] eventIds = new long[MAX_EVENTS]; - private final TdApi.Object[] events = new TdApi.Object[MAX_EVENTS]; + private static final ResponseReceiver responseReceiver = new ResponseReceiver(); private static class Handler { final ResultHandler resultHandler; @@ -184,76 +179,24 @@ public final class Client implements Runnable { private Client(ResultHandler updateHandler, ExceptionHandler updateExceptionHandler, ExceptionHandler defaultExceptionHandler) { nativeClientId = createNativeClient(); - updateHandlers.put(nativeClientId, new Handler(updateHandler, updateExceptionHandler)); + if (updateHandler != null) { + updateHandlers.put(nativeClientId, new Handler(updateHandler, updateExceptionHandler)); + } if (defaultExceptionHandler != null) { - defaultExceptionHandlers.put(nativeClientId, defaultExceptionHandler); + defaultExceptionHandlers.put(nativeClientId, defaultExceptionHandler); } } @Override protected void finalize() throws Throwable { - try { - close(); - } finally { - super.finalize(); - } + send(new TdApi.Close(), null, null); } - private void processResult(long id, TdApi.Object object) { - if (object instanceof TdApi.UpdateAuthorizationState) { - if (((TdApi.UpdateAuthorizationState) object).authorizationState instanceof TdApi.AuthorizationStateClosed) { - stopFlag = true; - } - } - Handler handler; - if (id == 0) { - // update handler stays forever - handler = updateHandlers.get(nativeClientId); - } else { - handler = handlers.remove(id); - } - if (handler == null) { - return; - } + private static native int createNativeClient(); - handleResult(object, handler.resultHandler, handler.exceptionHandler); - } + private static native void nativeClientSend(int nativeClientId, long eventId, TdApi.Function function); - private void handleResult(TdApi.Object object, ResultHandler resultHandler, ExceptionHandler exceptionHandler) { - if (resultHandler == null) { - return; - } - - try { - resultHandler.onResult(object); - } catch (Throwable cause) { - if (exceptionHandler == null) { - exceptionHandler = defaultExceptionHandlers.get(nativeClientId); - } - if (exceptionHandler != null) { - try { - exceptionHandler.onException(cause); - } catch (Throwable ignored) { - } - } - } - } - - private void receiveQueries(double timeout) { - int resultN = nativeClientReceive(nativeClientId, eventIds, events, timeout); - for (int i = 0; i < resultN; i++) { - processResult(eventIds[i], events[i]); - events[i] = null; - } - } - - private static native long createNativeClient(); - - private static native void nativeClientSend(long nativeClientId, long eventId, TdApi.Function function); - - private static native int nativeClientReceive(long nativeClientId, long[] eventIds, TdApi.Object[] events, double timeout); + private static native int nativeClientReceive(int[] clientIds, long[] eventIds, TdApi.Object[] events, double timeout); private static native TdApi.Object nativeClientExecute(TdApi.Function function); - - private static native void destroyNativeClient(long nativeClientId); } diff --git a/example/java/td_jni.cpp b/example/java/td_jni.cpp index 6571c8c69..e0106e258 100644 --- a/example/java/td_jni.cpp +++ b/example/java/td_jni.cpp @@ -26,31 +26,35 @@ static td::td_api::object_ptr fetch_function(JNIEnv *env, return result; } -static td::Client *get_client(jlong client_id) { - return reinterpret_cast(static_cast(client_id)); +static td::ClientManager *get_manager() { + return td::ClientManager::get_manager_singleton(); } -static jlong Client_createNativeClient(JNIEnv *env, jclass clazz) { - return static_cast(reinterpret_cast(new td::Client())); +static jint Client_createNativeClient(JNIEnv *env, jclass clazz) { + return static_cast(get_manager()->create_client()); } -static void Client_nativeClientSend(JNIEnv *env, jclass clazz, jlong client_id, jlong id, jobject function) { - get_client(client_id)->send({static_cast(id), fetch_function(env, function)}); +static void Client_nativeClientSend(JNIEnv *env, jclass clazz, jint client_id, jlong id, jobject function) { + get_manager()->send(static_cast(client_id), static_cast(id), + fetch_function(env, function)); } -static jint Client_nativeClientReceive(JNIEnv *env, jclass clazz, jlong client_id, jlongArray ids, jobjectArray events, - jdouble timeout) { - auto client = get_client(client_id); - jsize events_size = env->GetArrayLength(ids); // ids and events size must be of equal size +static jint Client_nativeClientReceive(JNIEnv *env, jclass clazz, jintArray client_ids, jlongArray ids, + jobjectArray events, jdouble timeout) { + jsize events_size = env->GetArrayLength(ids); // client_ids, ids and events must be of equal size if (events_size == 0) { return 0; } jsize result_size = 0; - auto response = client->receive(timeout); + auto *manager = get_manager(); + auto response = manager->receive(timeout); while (response.object) { - jlong result_id = static_cast(response.id); - env->SetLongArrayRegion(ids, result_size, 1, &result_id); + jint client_id = static_cast(response.client_id); + env->SetIntArrayRegion(client_ids, result_size, 1, &client_id); + + jlong request_id = static_cast(response.request_id); + env->SetLongArrayRegion(ids, result_size, 1, &request_id); jobject object; response.object->store(env, object); @@ -62,21 +66,17 @@ static jint Client_nativeClientReceive(JNIEnv *env, jclass clazz, jlong client_i break; } - response = client->receive(0); + response = manager->receive(0); } return result_size; } static jobject Client_nativeClientExecute(JNIEnv *env, jclass clazz, jobject function) { jobject result; - td::Client::execute({0, fetch_function(env, function)}).object->store(env, result); + td::ClientManager::execute(fetch_function(env, function))->store(env, result); return result; } -static void Client_destroyNativeClient(JNIEnv *env, jclass clazz, jlong client_id) { - delete get_client(client_id); -} - static void Log_setVerbosityLevel(JNIEnv *env, jclass clazz, jint new_log_verbosity_level) { td::Log::set_verbosity_level(static_cast(new_log_verbosity_level)); } @@ -136,11 +136,10 @@ static jint register_native(JavaVM *vm) { #define TD_OBJECT "L" PACKAGE_NAME "/TdApi$Object;" #define TD_FUNCTION "L" PACKAGE_NAME "/TdApi$Function;" - register_method(client_class, "createNativeClient", "()J", Client_createNativeClient); - register_method(client_class, "nativeClientSend", "(JJ" TD_FUNCTION ")V", Client_nativeClientSend); - register_method(client_class, "nativeClientReceive", "(J[J[" TD_OBJECT "D)I", Client_nativeClientReceive); + register_method(client_class, "createNativeClient", "()I", Client_createNativeClient); + register_method(client_class, "nativeClientSend", "(IJ" TD_FUNCTION ")V", Client_nativeClientSend); + register_method(client_class, "nativeClientReceive", "([I[J[" TD_OBJECT "D)I", Client_nativeClientReceive); register_method(client_class, "nativeClientExecute", "(" TD_FUNCTION ")" TD_OBJECT, Client_nativeClientExecute); - register_method(client_class, "destroyNativeClient", "(J)V", Client_destroyNativeClient); register_method(log_class, "setVerbosityLevel", "(I)V", Log_setVerbosityLevel); register_method(log_class, "setFilePath", "(Ljava/lang/String;)Z", Log_setFilePath); From 22be9b95bfb787abc99ba5400cc1fc24665c3c5d Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 11 Oct 2020 21:31:01 +0300 Subject: [PATCH 11/23] Do not include set in broadly used format.h. GitOrigin-RevId: 81c2113ef5bfe48868a3a2968efd6cadb1cfccf3 --- tdutils/td/utils/format.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tdutils/td/utils/format.h b/tdutils/td/utils/format.h index a3b5b3a46..8798d0e49 100644 --- a/tdutils/td/utils/format.h +++ b/tdutils/td/utils/format.h @@ -11,7 +11,6 @@ #include "td/utils/Slice.h" #include "td/utils/StringBuilder.h" -#include #include #include @@ -335,9 +334,5 @@ template StringBuilder &operator<<(StringBuilder &stream, const vector &vec) { return stream << format::as_array(vec); } -template -StringBuilder &operator<<(StringBuilder &stream, const std::set &vec) { - return stream << format::as_array(vec); -} } // namespace td From 8b40856d6e3d4f0929c41ccd08d4a7ab02d29888 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 11 Oct 2020 23:48:17 +0300 Subject: [PATCH 12/23] Always link OpenSSL with Crypt32 on Windows. GitOrigin-RevId: 56dd9e21c8175f291222c8d42153dcd50cf57ed9 --- CMakeLists.txt | 4 ++-- benchmark/CMakeLists.txt | 4 ++-- sqlite/CMakeLists.txt | 4 ++-- tdutils/CMakeLists.txt | 6 ++++++ 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 86cdcabcd..5d5fc1639 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -809,9 +809,9 @@ target_include_directories(tdcore SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR}) target_link_libraries(tdcore PUBLIC tdapi tdactor tdutils tdnet tddb PRIVATE ${OPENSSL_CRYPTO_LIBRARY} ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES}) if (WIN32) if (MINGW) - target_link_libraries(tdcore PRIVATE ws2_32 mswsock) + target_link_libraries(tdcore PRIVATE ws2_32 mswsock crypt32) else() - target_link_libraries(tdcore PRIVATE ws2_32 Mswsock) + target_link_libraries(tdcore PRIVATE ws2_32 Mswsock Crypt32) endif() endif() diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index e175ee414..78bb649f5 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -10,9 +10,9 @@ add_executable(bench_crypto bench_crypto.cpp) target_link_libraries(bench_crypto PRIVATE tdutils ${OPENSSL_CRYPTO_LIBRARY} ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES}) if (WIN32) if (MINGW) - target_link_libraries(bench_crypto PRIVATE ws2_32 mswsock) + target_link_libraries(bench_crypto PRIVATE ws2_32 mswsock crypt32) else() - target_link_libraries(bench_crypto PRIVATE ws2_32 Mswsock) + target_link_libraries(bench_crypto PRIVATE ws2_32 Mswsock Crypt32) endif() endif() target_include_directories(bench_crypto SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR}) diff --git a/sqlite/CMakeLists.txt b/sqlite/CMakeLists.txt index 67fdeaad1..a11f5c75f 100644 --- a/sqlite/CMakeLists.txt +++ b/sqlite/CMakeLists.txt @@ -23,9 +23,9 @@ target_include_directories(tdsqlite SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR}) target_link_libraries(tdsqlite PRIVATE ${OPENSSL_CRYPTO_LIBRARY} ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES}) if (WIN32) if (MINGW) - target_link_libraries(tdsqlite PRIVATE ws2_32 mswsock) + target_link_libraries(tdsqlite PRIVATE ws2_32 mswsock crypt32) else() - target_link_libraries(tdsqlite PRIVATE ws2_32 Mswsock) + target_link_libraries(tdsqlite PRIVATE ws2_32 Mswsock Crypt32) endif() endif() diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index d3b12e6f3..1312c1dae 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -316,6 +316,12 @@ target_include_directories(tdutils PUBLIC $ Date: Mon, 12 Oct 2020 01:10:55 +0300 Subject: [PATCH 13/23] Update recommended Emscripten version. GitOrigin-RevId: 99502278d1de3bdc78eae343bffd991505cacfa9 --- example/web/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/web/README.md b/example/web/README.md index 6f137ec9c..15a4964db 100644 --- a/example/web/README.md +++ b/example/web/README.md @@ -6,7 +6,7 @@ You need a Unix shell with `sed`, `tar` and `wget` utilities to run the provided ## Building tdweb NPM package -* Install the 1.39.5 [emsdk](https://kripken.github.io/emscripten-site/docs/getting_started/downloads.html), which is known to work. Do not use the system-provided `emscripten` package, because it contains a version that is too old. +* Install the 2.0.6 [emsdk](https://kripken.github.io/emscripten-site/docs/getting_started/downloads.html), which is known to work. Do not use the system-provided `emscripten` package, because it contains a too old emsdk version. * Install all `TDLib` build dependencies as described in [Building](https://github.com/tdlib/td#building). * Run `source ./emsdk_env.sh` from `emsdk` directory to set up the correct build environment. * On `macOS`, install the `coreutils` [Homebrew](https://brew.sh) package and replace `realpath` in scripts with `grealpath`: From fb1307ff60c5b57b24407de81362f61507a919d9 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 12 Oct 2020 01:48:13 +0300 Subject: [PATCH 14/23] Check for unneeded chat actions only while trying to send it. GitOrigin-RevId: 7067f21e42ba0d3c3dc797f72341681e32998e6a --- td/telegram/MessagesManager.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 78b6792e2..cc05c2e4a 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -28905,7 +28905,7 @@ void MessagesManager::send_dialog_action(DialogId dialog_id, MessageId top_threa auto &query_ref = set_typing_query_[dialog_id]; if (!query_ref.empty() && !td_->auth_manager_->is_bot()) { - LOG(INFO) << "Cancel previous set typing query"; + LOG(INFO) << "Cancel previous send chat action query"; cancel_query(query_ref); } query_ref = td_->create_handler(std::move(promise)) @@ -28913,7 +28913,7 @@ void MessagesManager::send_dialog_action(DialogId dialog_id, MessageId top_threa } void MessagesManager::on_send_dialog_action_timeout(DialogId dialog_id) { - LOG(INFO) << "Receive send_dialog_action timeout in " << dialog_id; + LOG(INFO) << "Receive send_chat_action timeout in " << dialog_id; Dialog *d = get_dialog(dialog_id); CHECK(d != nullptr); @@ -30749,7 +30749,7 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, uniq if (queue_id & 1) { LOG(INFO) << "Add " << message_id << " from " << source << " to queue " << queue_id; yet_unsent_media_queues_[queue_id][message_id.get()]; // reserve place for promise - if (!td_->auth_manager_->is_bot() && !is_dialog_action_unneeded(dialog_id)) { + if (!td_->auth_manager_->is_bot()) { pending_send_dialog_action_timeout_.add_timeout_in(dialog_id.get(), 1.0); } } From 4da9f9d334d61281c70c439aa4af204794831aa6 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 12 Oct 2020 10:17:02 +0300 Subject: [PATCH 15/23] Use stable sort for photo sizes to keep "i" the last. GitOrigin-RevId: 7feab3e73a3a03bc9fd3a0e76e05ab4cbb0f10d2 --- td/telegram/Photo.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/td/telegram/Photo.cpp b/td/telegram/Photo.cpp index 0a3c6cee8..d99dcb8f6 100644 --- a/td/telegram/Photo.cpp +++ b/td/telegram/Photo.cpp @@ -578,7 +578,7 @@ static vector> get_photo_sizes_object(File auto sizes = transform(photo_sizes, [file_manager](const PhotoSize &photo_size) { return get_photo_size_object(file_manager, &photo_size); }); - std::sort(sizes.begin(), sizes.end(), [](const auto &lhs, const auto &rhs) { + std::stable_sort(sizes.begin(), sizes.end(), [](const auto &lhs, const auto &rhs) { if (lhs->photo_->expected_size_ != rhs->photo_->expected_size_) { return lhs->photo_->expected_size_ < rhs->photo_->expected_size_; } From a4b71466e0492facec6258577f9087e2cf6702a4 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 12 Oct 2020 10:24:30 +0300 Subject: [PATCH 16/23] Minor improvements. GitOrigin-RevId: 4f0c8840ee69c7a690cb12c5e36231cf9da6d91c --- td/telegram/MessageReplyInfo.h | 2 +- tdutils/test/bitmask.cpp | 53 ++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/td/telegram/MessageReplyInfo.h b/td/telegram/MessageReplyInfo.h index d1c3a75da..0c1b4e8d9 100644 --- a/td/telegram/MessageReplyInfo.h +++ b/td/telegram/MessageReplyInfo.h @@ -9,7 +9,7 @@ #include "td/telegram/ChannelId.h" #include "td/telegram/DialogId.h" #include "td/telegram/MessageId.h" -#include "Td/telegram/td_api.h" +#include "td/telegram/td_api.h" #include "td/telegram/telegram_api.h" #include "td/utils/common.h" diff --git a/tdutils/test/bitmask.cpp b/tdutils/test/bitmask.cpp index b3fa109c7..3729a2a46 100644 --- a/tdutils/test/bitmask.cpp +++ b/tdutils/test/bitmask.cpp @@ -4,11 +4,17 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // -#include "td/utils/tests.h" +#include "td/utils/common.h" #include "td/utils/misc.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/tests.h" #include "td/utils/utf8.h" +#include + namespace td { + class RangeSet { template static auto find(T &ranges, int64 begin) { @@ -28,17 +34,14 @@ class RangeSet { int64 end; }; - static constexpr int64 BitSize = 1024; - static constexpr int64 MaxPartSize = 16 * 1024 * 1024; - - RangeSet() = default; + static constexpr int64 BIT_SIZE = 1024; static RangeSet create_one_range(int64 end, int64 begin = 0) { RangeSet res; res.ranges_.push_back({begin, end}); return res; } - static td::Result decode(CSlice data) { + static Result decode(CSlice data) { if (!check_utf8(data)) { return Status::Error("Invalid encoding"); } @@ -50,7 +53,7 @@ class RangeSet { begin = next_utf8_unsafe(begin, &size, "RangeSet"); if (!is_empty && size != 0) { - res.ranges_.push_back({curr * BitSize, (curr + size) * BitSize}); + res.ranges_.push_back({curr * BIT_SIZE, (curr + size) * BIT_SIZE}); } curr += size; is_empty = !is_empty; @@ -58,12 +61,12 @@ class RangeSet { return res; } - std::string encode(int64 prefix_size = -1) const { - std::vector sizes; + string encode(int64 prefix_size = -1) const { + vector sizes; uint32 all_end = 0; if (prefix_size != -1) { - prefix_size = (prefix_size + BitSize - 1) / BitSize * BitSize; + prefix_size = (prefix_size + BIT_SIZE - 1) / BIT_SIZE * BIT_SIZE; } for (auto it : ranges_) { if (prefix_size != -1 && it.begin >= prefix_size) { @@ -73,10 +76,10 @@ class RangeSet { it.end = prefix_size; } - CHECK(it.begin % BitSize == 0); - CHECK(it.end % BitSize == 0); - uint32 begin = narrow_cast(it.begin / BitSize); - uint32 end = narrow_cast(it.end / BitSize); + CHECK(it.begin % BIT_SIZE == 0); + CHECK(it.end % BIT_SIZE == 0); + uint32 begin = narrow_cast(it.begin / BIT_SIZE); + uint32 end = narrow_cast(it.end / BIT_SIZE); if (sizes.empty()) { if (begin != 0) { sizes.push_back(0); @@ -89,7 +92,7 @@ class RangeSet { all_end = end; } - std::string res; + string res; for (auto c : sizes) { append_utf8_character(res, c); } @@ -149,8 +152,8 @@ class RangeSet { } void set(int64 begin, int64 end) { - CHECK(begin % BitSize == 0); - CHECK(end % BitSize == 0); + CHECK(begin % BIT_SIZE == 0); + CHECK(end % BIT_SIZE == 0); // 1. skip all with r.end < begin auto it_begin = find(begin); @@ -162,16 +165,16 @@ class RangeSet { if (it_begin == it_end) { ranges_.insert(it_begin, Range{begin, end}); } else { - begin = std::min(begin, it_begin->begin); + begin = td::min(begin, it_begin->begin); --it_end; - end = std::max(end, it_end->end); + end = td::max(end, it_end->end); *it_end = Range{begin, end}; ranges_.erase(it_begin, it_end); } } - std::vector as_vector(int32 part_size) const { - std::vector res; + vector as_vector(int32 part_size) const { + vector res; for (auto it : ranges_) { auto begin = narrow_cast((it.begin + part_size - 1) / part_size); auto end = narrow_cast(it.end / part_size); @@ -183,13 +186,12 @@ class RangeSet { } private: - std::vector ranges_; + vector ranges_; }; TEST(Bitmask, simple) { auto validate_encoding = [](auto &rs) { auto str = rs.encode(); - LOG(ERROR) << str.size(); RangeSet rs2 = RangeSet::decode(str).move_as_ok(); auto str2 = rs2.encode(); rs = std::move(rs2); @@ -238,9 +240,10 @@ TEST(Bitmask, simple) { ASSERT_EQ(8, get(3)); ASSERT_EQ(10, rs.get_ready_prefix_size(S * 3, S * 3 + 10)); - ASSERT_TRUE(!rs.is_ready(S*11, S *12)); + ASSERT_TRUE(!rs.is_ready(S * 11, S * 12)); ASSERT_EQ(3, rs.get_ready_parts(2, S * 2)); - ASSERT_EQ(std::vector({2, 3, 4, 7}), rs.as_vector(S * 2) ); + ASSERT_EQ(vector({2, 3, 4, 7}), rs.as_vector(S * 2)); } } + } // namespace td From 3c5b09bad705625945255b31c5f445002b6e3db1 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 12 Oct 2020 10:31:57 +0300 Subject: [PATCH 17/23] Hide photo sizes, which can't be downloaded. GitOrigin-RevId: 420860824f41d5fecbb935c756d9b6f0a2564cb7 --- td/telegram/Photo.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/td/telegram/Photo.cpp b/td/telegram/Photo.cpp index d99dcb8f6..2a8326294 100644 --- a/td/telegram/Photo.cpp +++ b/td/telegram/Photo.cpp @@ -585,6 +585,9 @@ static vector> get_photo_sizes_object(File return static_cast(lhs->width_) * static_cast(lhs->height_) < static_cast(rhs->width_) * static_cast(rhs->height_); }); + td::remove_if(sizes, [](const auto &size) { + return !size->photo_->local_->can_be_downloaded_ && !size->photo_->local_->is_downloading_completed_; + }); return sizes; } From fbeea0b10862134fd6b4721aed68dc0b1e380e47 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 12 Oct 2020 10:56:01 +0300 Subject: [PATCH 18/23] Add is_deleted_secret_chat method. GitOrigin-RevId: eae2a7bcdf92587bbd77cf7c42c210b74107d8a4 --- td/telegram/MessagesManager.cpp | 27 ++++++++++++++++++++++----- td/telegram/MessagesManager.h | 2 ++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index cc05c2e4a..8fea014fb 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -23714,6 +23714,26 @@ bool MessagesManager::is_broadcast_channel(DialogId dialog_id) const { return td_->contacts_manager_->get_channel_type(dialog_id.get_channel_id()) == ChannelType::Broadcast; } +bool MessagesManager::is_deleted_secret_chat(const Dialog *d) const { + if (d == nullptr) { + return true; + } + if (d->dialog_id.get_type() != DialogType::SecretChat) { + return false; + } + + if (d->order != DEFAULT_ORDER || d->messages != nullptr) { + return false; + } + + auto state = td_->contacts_manager_->get_secret_chat_state(d->dialog_id.get_secret_chat_id()); + if (state != SecretChatState::Closed) { + return false; + } + + return true; +} + int32 MessagesManager::get_message_schedule_date(const Message *m) { if (!m->message_id.is_scheduled()) { return 0; @@ -32897,11 +32917,8 @@ void MessagesManager::update_dialog_pos(Dialog *d, const char *source, bool need } } if (dialog_type == DialogType::SecretChat) { - auto secret_chat_id = d->dialog_id.get_secret_chat_id(); - auto date = td_->contacts_manager_->get_secret_chat_date(secret_chat_id); - auto state = td_->contacts_manager_->get_secret_chat_state(secret_chat_id); - // do not return removed from the chat list closed secret chats - if (date != 0 && (d->order != DEFAULT_ORDER || state != SecretChatState::Closed || d->messages != nullptr)) { + auto date = td_->contacts_manager_->get_secret_chat_date(d->dialog_id.get_secret_chat_id()); + if (date != 0 && !is_deleted_secret_chat(d)) { LOG(INFO) << "Creation of secret chat at " << date << " found"; int64 creation_order = get_dialog_order(MessageId(), date); if (creation_order > new_order) { diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index 2b2fbb28e..f6a0333c2 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -2916,6 +2916,8 @@ class MessagesManager : public Actor { bool is_broadcast_channel(DialogId dialog_id) const; + bool is_deleted_secret_chat(const Dialog *d) const; + static int32 get_message_schedule_date(const Message *m); int32 recently_found_dialogs_loaded_ = 0; // 0 - not loaded, 1 - load request was sent, 2 - loaded From c1309d7657b2e1d9920c7ab5f8f3450ac2471f55 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 12 Oct 2020 11:33:16 +0300 Subject: [PATCH 19/23] Do not return deleted secret chats and replace migrated basic groups with corresponding supergroups in recently found chats. GitOrigin-RevId: e50390583e3956a4eb8fdb92ec8a82f04cc7914f --- td/telegram/ContactsManager.cpp | 8 ++++++ td/telegram/ContactsManager.h | 1 + td/telegram/MessagesManager.cpp | 49 ++++++++++++++++++++++++++++++++- td/telegram/MessagesManager.h | 5 +++- 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/td/telegram/ContactsManager.cpp b/td/telegram/ContactsManager.cpp index deec1976e..3afc4822f 100644 --- a/td/telegram/ContactsManager.cpp +++ b/td/telegram/ContactsManager.cpp @@ -12815,6 +12815,14 @@ bool ContactsManager::get_chat_is_active(ChatId chat_id) const { return c->is_active; } +ChannelId ContactsManager::get_chat_migrated_to_channel_id(ChatId chat_id) const { + auto c = get_chat(chat_id); + if (c == nullptr) { + return ChannelId(); + } + return c->migrated_to_channel_id; +} + DialogParticipantStatus ContactsManager::get_chat_status(ChatId chat_id) const { auto c = get_chat(chat_id); if (c == nullptr) { diff --git a/td/telegram/ContactsManager.h b/td/telegram/ContactsManager.h index c43beeeee..197fed8c8 100644 --- a/td/telegram/ContactsManager.h +++ b/td/telegram/ContactsManager.h @@ -464,6 +464,7 @@ class ContactsManager : public Actor { void reload_chat_full(ChatId chat_id, Promise &&promise); bool get_chat_is_active(ChatId chat_id) const; + ChannelId get_chat_migrated_to_channel_id(ChatId chat_id) const; DialogParticipantStatus get_chat_status(ChatId chat_id) const; DialogParticipantStatus get_chat_permissions(ChatId chat_id) const; bool is_appointed_chat_administrator(ChatId chat_id) const; diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 8fea014fb..08c3d7cb8 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -15404,6 +15404,9 @@ std::pair> MessagesManager::search_dialogs(const string } promise.set_value(Unit()); + + update_recently_found_dialogs(); + size_t result_size = min(static_cast(limit), recently_found_dialog_ids_.size()); return {narrow_cast(recently_found_dialog_ids_.size()), vector(recently_found_dialog_ids_.begin(), recently_found_dialog_ids_.begin() + result_size)}; @@ -35219,6 +35222,7 @@ void MessagesManager::save_recently_found_dialogs() { } value += to_string(dialog_id.get()); } + LOG(DEBUG) << "Save recently found chats " << value; G()->td_db()->get_binlog_pmc()->set("recently_found_dialog_usernames_and_ids", value); } @@ -35237,6 +35241,7 @@ bool MessagesManager::load_recently_found_dialogs(Promise &promise) { return true; } + LOG(DEBUG) << "Loaded recently found chats " << found_dialogs_str; auto found_dialogs = full_split(found_dialogs_str, ','); if (recently_found_dialogs_loaded_ == 1 && resolve_recently_found_dialogs_multipromise_.promise_count() == 0) { // queries was sent and have already been finished @@ -35342,7 +35347,7 @@ bool MessagesManager::add_recently_found_dialog_internal(DialogId dialog_id) { // TODO create function auto it = std::find(recently_found_dialog_ids_.begin(), recently_found_dialog_ids_.end(), dialog_id); if (it == recently_found_dialog_ids_.end()) { - if (narrow_cast(recently_found_dialog_ids_.size()) == MAX_RECENT_FOUND_DIALOGS) { + if (narrow_cast(recently_found_dialog_ids_.size()) == MAX_RECENTLY_FOUND_DIALOGS) { CHECK(!recently_found_dialog_ids_.empty()); recently_found_dialog_ids_.back() = dialog_id; } else { @@ -35359,6 +35364,48 @@ bool MessagesManager::remove_recently_found_dialog_internal(DialogId dialog_id) return td::remove(recently_found_dialog_ids_, dialog_id); } +void MessagesManager::update_recently_found_dialogs() { + vector dialog_ids; + for (auto dialog_id : recently_found_dialog_ids_) { + const Dialog *d = get_dialog(dialog_id); + if (d == nullptr) { + continue; + } + switch (dialog_id.get_type()) { + case DialogType::User: + // always keep + break; + case DialogType::Chat: { + auto channel_id = td_->contacts_manager_->get_chat_migrated_to_channel_id(dialog_id.get_chat_id()); + if (channel_id.is_valid() && get_dialog(DialogId(channel_id)) != nullptr) { + dialog_id = DialogId(channel_id); + } + break; + } + case DialogType::Channel: + // always keep + break; + case DialogType::SecretChat: + if (is_deleted_secret_chat(d)) { + dialog_id = DialogId(); + } + break; + case DialogType::None: + default: + UNREACHABLE(); + break; + } + if (dialog_id.is_valid()) { + dialog_ids.push_back(dialog_id); + } + } + + if (dialog_ids != recently_found_dialog_ids_) { + recently_found_dialog_ids_ = std::move(dialog_ids); + save_recently_found_dialogs(); + } +} + void MessagesManager::suffix_load_loop(Dialog *d) { if (d->suffix_load_has_query_) { return; diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index f6a0333c2..6ee4299a4 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -1644,7 +1644,7 @@ class MessagesManager : public Actor { static constexpr int32 MIN_CHANNEL_DIFFERENCE = 10; static constexpr int32 MAX_CHANNEL_DIFFERENCE = 100; static constexpr int32 MAX_BOT_CHANNEL_DIFFERENCE = 100000; // server side limit - static constexpr int32 MAX_RECENT_FOUND_DIALOGS = 20; // some reasonable value + static constexpr int32 MAX_RECENTLY_FOUND_DIALOGS = 30; // some reasonable value static constexpr size_t MAX_TITLE_LENGTH = 128; // server side limit for chat title static constexpr size_t MAX_DESCRIPTION_LENGTH = 255; // server side limit for chat description static constexpr size_t MAX_DIALOG_FILTER_TITLE_LENGTH = 12; // server side limit for dialog filter title @@ -2831,7 +2831,10 @@ class MessagesManager : public Actor { bool remove_recently_found_dialog_internal(DialogId dialog_id); + void update_recently_found_dialogs(); + void save_recently_found_dialogs(); + bool load_recently_found_dialogs(Promise &promise); void reget_message_from_server_if_needed(DialogId dialog_id, const Message *m); From 706555502b3599812e19c5c9a24c65675396dc20 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 12 Oct 2020 11:49:23 +0300 Subject: [PATCH 20/23] Build tests only once. GitOrigin-RevId: 386e5708eab6b806bd62a06e854187f4988303d3 --- test/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index dce6e85f6..6fab03211 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -26,13 +26,13 @@ set(TESTS_MAIN main.cpp ) -add_library(all_tests STATIC ${TD_TEST_SOURCE}) -target_include_directories(all_tests PUBLIC $) -target_link_libraries(all_tests PRIVATE tdcore tdclient) +#add_library(all_tests STATIC ${TD_TEST_SOURCE}) +#target_include_directories(all_tests PUBLIC $) +#target_link_libraries(all_tests PRIVATE tdcore tdclient) if (NOT CMAKE_CROSSCOMPILING OR EMSCRIPTEN) #Tests - add_executable(test-tdutils ${TESTS_MAIN} ${TDUTILS_TEST_SOURCE}) + add_executable(test-tdutils EXCLUDE_FROM_ALL ${TESTS_MAIN} ${TDUTILS_TEST_SOURCE}) add_executable(run_all_tests ${TESTS_MAIN} ${TD_TEST_SOURCE}) if (CLANG AND NOT CYGWIN AND NOT EMSCRIPTEN AND NOT (CMAKE_HOST_SYSTEM_NAME MATCHES "OpenBSD")) target_compile_options(test-tdutils PUBLIC -fsanitize=undefined -fno-sanitize=vptr) From 966621376b87fda5b9eb29dd2f4da7732bf1c654 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 12 Oct 2020 13:58:01 +0300 Subject: [PATCH 21/23] Use new JSON interface in cpp example. GitOrigin-RevId: b92842c8ca490877a3154cd04fe9c4cea0e76ee2 --- example/cpp/tdjson_example.cpp | 36 ++++++++++++++++------------------ 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/example/cpp/tdjson_example.cpp b/example/cpp/tdjson_example.cpp index c22320ee8..5d69c4e21 100644 --- a/example/cpp/tdjson_example.cpp +++ b/example/cpp/tdjson_example.cpp @@ -13,32 +13,32 @@ int main() { // disable TDLib logging - td_json_client_execute(nullptr, "{\"@type\":\"setLogVerbosityLevel\", \"new_verbosity_level\":0}"); + td_execute("{\"@type\":\"setLogVerbosityLevel\", \"new_verbosity_level\":0}"); - void *client = td_json_client_create(); - // somehow share the client with other threads, which will be able to send requests via td_json_client_send + int client_id = td_create_client(); + // somehow share the client_id with other threads, which will be able to send requests via td_send const bool test_incorrect_queries = false; if (test_incorrect_queries) { - td_json_client_execute(nullptr, "{\"@type\":\"setLogVerbosityLevel\", \"new_verbosity_level\":3}"); - td_json_client_execute(nullptr, ""); - td_json_client_execute(nullptr, "test"); - td_json_client_execute(nullptr, "\"test\""); - td_json_client_execute(nullptr, "{\"@type\":\"test\", \"@extra\":1}"); + td_execute("{\"@type\":\"setLogVerbosityLevel\", \"new_verbosity_level\":1}"); + td_execute(""); + td_execute("test"); + td_execute("\"test\""); + td_execute("{\"@type\":\"test\", \"@extra\":1}"); - td_json_client_send(client, "{\"@type\":\"getFileMimeType\"}"); - td_json_client_send(client, "{\"@type\":\"getFileMimeType\", \"@extra\":1}"); - td_json_client_send(client, "{\"@type\":\"getFileMimeType\", \"@extra\":null}"); - td_json_client_send(client, "{\"@type\":\"test\"}"); - td_json_client_send(client, "[]"); - td_json_client_send(client, "{\"@type\":\"test\", \"@extra\":1}"); - td_json_client_send(client, "{\"@type\":\"sendMessage\", \"chat_id\":true, \"@extra\":1}"); - td_json_client_send(client, "test"); + td_send(client_id, "{\"@type\":\"getFileMimeType\"}"); + td_send(client_id, "{\"@type\":\"getFileMimeType\", \"@extra\":1}"); + td_send(client_id, "{\"@type\":\"getFileMimeType\", \"@extra\":null}"); + td_send(client_id, "{\"@type\":\"test\"}"); + td_send(client_id, "[]"); + td_send(client_id, "{\"@type\":\"test\", \"@extra\":1}"); + td_send(client_id, "{\"@type\":\"sendMessage\", \"chat_id\":true, \"@extra\":1}"); + td_send(client_id, "test"); } const double WAIT_TIMEOUT = 10.0; // seconds while (true) { - const char *result = td_json_client_receive(client, WAIT_TIMEOUT); + const char *result = td_receive(WAIT_TIMEOUT); if (result != nullptr) { // parse the result as a JSON object and process it as an incoming update or an answer to a previously sent request @@ -49,6 +49,4 @@ int main() { std::cout << result << std::endl; } } - - td_json_client_destroy(client); } From f05b0a833d72c8f8349dfecb5be5e568628977de Mon Sep 17 00:00:00 2001 From: Ruan Diego Lacerda Menezes Date: Mon, 12 Oct 2020 08:34:16 -0300 Subject: [PATCH 22/23] Add Object Pascal language to readme (#1229) --- example/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/example/README.md b/example/README.md index 44090b565..eab03bbf2 100644 --- a/example/README.md +++ b/example/README.md @@ -18,6 +18,7 @@ Choose your preferred programming language to see examples of usage and a detail - [C++](#cxx) - [Swift](#swift) - [Objective-C](#objective-c) +- [Object Pascal](#object-pascal) - [Dart](#dart) - [Rust](#rust) - [Erlang](#erlang) @@ -135,6 +136,14 @@ TDLib can be used from the Objective-C programming language through [JSON](https See [example/ios](https://github.com/tdlib/td/tree/master/example/ios) for an example of building TDLib for iOS, watchOS, tvOS, and macOS. + +## Using TDLib in Object Pascal projects with Delphi and Lazarus + +TDLib can be used from the Object Pascal programming language. + +See [tdlib-delphi](https://github.com/dieletro/tdlib-delphi) or [tdlib-lazarus](https://github.com/dieletro/tdlib-lazarus) for examples of such use and a Delphi or Lazarus client for TDLib. +This Example was developed for use and consumption of the Official [*Telegram TDLib API*](https://core.telegram.org/tdlib) for use in Delphi + ## Using TDLib in Dart projects From 6b1f6b829d0d71f5d50662b1ff58b7869c1e1d7f Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 12 Oct 2020 14:50:41 +0300 Subject: [PATCH 23/23] Add Object Pascal to build instructions generator. GitOrigin-RevId: 2c05c0571a01caab103e04d588c8b3aca38c3139 --- build.html | 3 +++ example/README.md | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/build.html b/build.html index cc4910b6b..289fd2823 100644 --- a/build.html +++ b/build.html @@ -28,6 +28,7 @@ select.large { font-size: large; } + @@ -202,6 +203,8 @@ function getExampleAnchor(language) { case 'Elixir': case 'C': return language.toLowerCase(); + case 'Object Pascal': + return 'object-pascal'; case 'C#': return 'csharp'; case 'C++': diff --git a/example/README.md b/example/README.md index eab03bbf2..d9081f9b8 100644 --- a/example/README.md +++ b/example/README.md @@ -139,10 +139,11 @@ See [example/ios](https://github.com/tdlib/td/tree/master/example/ios) for an ex ## Using TDLib in Object Pascal projects with Delphi and Lazarus -TDLib can be used from the Object Pascal programming language. +TDLib can be used from the Object Pascal programming language through the [JSON](https://github.com/tdlib/td#using-json). -See [tdlib-delphi](https://github.com/dieletro/tdlib-delphi) or [tdlib-lazarus](https://github.com/dieletro/tdlib-lazarus) for examples of such use and a Delphi or Lazarus client for TDLib. -This Example was developed for use and consumption of the Official [*Telegram TDLib API*](https://core.telegram.org/tdlib) for use in Delphi +See [tdlib-delphi](https://github.com/dieletro/tdlib-delphi) for an example of TDLib usage from Delphi. + +See [tdlib-lazarus](https://github.com/dieletro/tdlib-lazarus) for an example of TDLib usage from Lazarus. ## Using TDLib in Dart projects