From eac1f0a0960744978c3144cc18b6e4064447551a Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 10 Jan 2023 20:03:05 +0300 Subject: [PATCH 01/30] Improve config recovery source order. --- td/telegram/ConfigManager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/td/telegram/ConfigManager.cpp b/td/telegram/ConfigManager.cpp index 5fc20f8a6..c92ecfbd6 100644 --- a/td/telegram/ConfigManager.cpp +++ b/td/telegram/ConfigManager.cpp @@ -823,9 +823,9 @@ class ConfigRecoverer final : public Actor { case 2: return get_simple_config_firebase_remote_config; case 4: - return get_simple_config_firebase_realtime; - case 9: return get_simple_config_firebase_firestore; + case 9: + return get_simple_config_firebase_realtime; case 0: case 3: case 8: From d7ea7dfea74ea8126e50bff3578554d5e8ea1f0e Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 10 Jan 2023 22:03:17 +0300 Subject: [PATCH 02/30] Remove never used AbridgedTransport. --- td/mtproto/TcpTransport.cpp | 60 --------------------------------- td/mtproto/TcpTransport.h | 67 +++++++++++-------------------------- 2 files changed, 19 insertions(+), 108 deletions(-) diff --git a/td/mtproto/TcpTransport.cpp b/td/mtproto/TcpTransport.cpp index b2270a376..d4da59a5b 100644 --- a/td/mtproto/TcpTransport.cpp +++ b/td/mtproto/TcpTransport.cpp @@ -77,64 +77,6 @@ void IntermediateTransport::init_output_stream(ChainBufferWriter *stream) { stream->append(Slice(reinterpret_cast(&magic), 4)); } -size_t AbridgedTransport::read_from_stream(ChainBufferReader *stream, BufferSlice *message, uint32 *quick_ack) { - if (stream->empty()) { - return 1; - } - uint8 byte = 0; - stream->clone().advance(1, MutableSlice(&byte, 1)); - size_t header_size; - uint32 data_size; - if (byte < 0x7f) { - header_size = 1; - data_size = byte * 4u; - } else { - if (stream->size() < 4) { - return 4; - } - header_size = 4; - stream->clone().advance(4, MutableSlice(reinterpret_cast(&data_size), sizeof(data_size))); - data_size >>= 8; - data_size = data_size * 4; - } - - size_t total_size = header_size + data_size; - if (stream->size() < total_size) { - // optimization - // stream->make_solid(total_size); - return total_size; - } - - stream->advance(header_size); - *message = stream->cut_head(data_size).move_as_buffer_slice(); - return 0; -} - -void AbridgedTransport::write_prepare_inplace(BufferWriter *message, bool quick_ack) { - CHECK(!quick_ack); - size_t size = message->size() / 4; - CHECK(size % 4 == 0); - CHECK(size < 1 << 24); - - size_t prepend_size = size >= 0x7f ? 4 : 1; - - MutableSlice prepend = message->prepare_prepend(); - CHECK(prepend.size() >= prepend_size); - message->confirm_prepend(prepend_size); - - MutableSlice data = message->as_slice(); - if (size >= 0x7f) { - uint32 size_encoded = 0x7f + (static_cast(size) << 8); - as(data.begin()) = size_encoded; - } else { - as(data.begin()) = static_cast(size); - } -} - -void AbridgedTransport::init_output_stream(ChainBufferWriter *stream) { - stream->append("\xef"); -} - void ObfuscatedTransport::init(ChainBufferReader *input, ChainBufferWriter *output) { input_ = input; output_ = output; @@ -164,8 +106,6 @@ void ObfuscatedTransport::init(ChainBufferReader *input, ChainBufferWriter *outp } break; } - // TODO: It is actually IntermediateTransport::init_output_stream, so it will work only with - // TransportImpl==IntermediateTransport as(header_slice.begin() + 56) = impl_.with_padding() ? 0xdddddddd : 0xeeeeeeee; if (dc_id_ != 0) { as(header_slice.begin() + 60) = dc_id_; diff --git a/td/mtproto/TcpTransport.h b/td/mtproto/TcpTransport.h index a4f0c9ac5..8d9d5664b 100644 --- a/td/mtproto/TcpTransport.h +++ b/td/mtproto/TcpTransport.h @@ -24,51 +24,24 @@ namespace td { namespace mtproto { namespace tcp { -class ITransport { - // Writes packet into message. - // Returns 0 if everything is ok, and [expected_size] otherwise. - // There is no sense to call this function when [stream->size > expected_size] - // - // (tpc is stream-base protocol. So the input message is a stream, not a slice) - virtual size_t read_from_stream(ChainBufferReader *stream, BufferSlice *message, uint32 *quick_ack) = 0; - - // Writes header inplace. - virtual void write_prepare_inplace(BufferWriter *message, bool quick_ack) = 0; - - // Writes first several bytes into output stream. - virtual void init_output_stream(ChainBufferWriter *stream) = 0; - - virtual bool support_quick_ack() const = 0; - - public: - ITransport() = default; - ITransport(const ITransport &) = delete; - ITransport &operator=(const ITransport &) = delete; - ITransport(ITransport &&) = delete; - ITransport &operator=(ITransport &&) = delete; - virtual ~ITransport() = default; -}; - -class AbridgedTransport final : public ITransport { - public: - size_t read_from_stream(ChainBufferReader *stream, BufferSlice *message, uint32 *quick_ack) final; - void write_prepare_inplace(BufferWriter *message, bool quick_ack) final; - void init_output_stream(ChainBufferWriter *stream) final; - bool support_quick_ack() const final { - return false; - } -}; - -class IntermediateTransport final : public ITransport { +class IntermediateTransport { public: explicit IntermediateTransport(bool with_padding) : with_padding_(with_padding) { } - size_t read_from_stream(ChainBufferReader *stream, BufferSlice *message, uint32 *quick_ack) final; - void write_prepare_inplace(BufferWriter *message, bool quick_ack) final; - void init_output_stream(ChainBufferWriter *stream) final; - bool support_quick_ack() const final { - return true; - } + + // Writes a packet into message. + // Returns 0 if everything is ok, and [expected_size] otherwise. + // There is no sense to call this function when [stream->size > expected_size] + // + // (TCP is a stream-oriented protocol, so the input message is a stream, not a slice) + size_t read_from_stream(ChainBufferReader *stream, BufferSlice *message, uint32 *quick_ack); + + // Writes header inplace. + void write_prepare_inplace(BufferWriter *message, bool quick_ack); + + // Writes first several bytes into output stream. + void init_output_stream(ChainBufferWriter *stream); + bool with_padding() const { return with_padding_; } @@ -77,8 +50,6 @@ class IntermediateTransport final : public ITransport { bool with_padding_; }; -using TransportImpl = IntermediateTransport; - class OldTransport final : public IStreamTransport { public: OldTransport() = default; @@ -86,7 +57,7 @@ class OldTransport final : public IStreamTransport { return impl_.read_from_stream(input_, message, quick_ack); } bool support_quick_ack() const final { - return impl_.support_quick_ack(); + return true; } void write(BufferWriter &&message, bool quick_ack) final { impl_.write_prepare_inplace(&message, quick_ack); @@ -121,7 +92,7 @@ class OldTransport final : public IStreamTransport { } private: - TransportImpl impl_{false}; + IntermediateTransport impl_{false}; ChainBufferReader *input_{nullptr}; ChainBufferWriter *output_{nullptr}; }; @@ -135,7 +106,7 @@ class ObfuscatedTransport final : public IStreamTransport { Result read_next(BufferSlice *message, uint32 *quick_ack) final TD_WARN_UNUSED_RESULT; bool support_quick_ack() const final { - return impl_.support_quick_ack(); + return true; } void write(BufferWriter &&message, bool quick_ack) final; @@ -182,7 +153,7 @@ class ObfuscatedTransport final : public IStreamTransport { bool is_first_tls_packet_{true}; ProxySecret secret_; std::string header_; - TransportImpl impl_; + IntermediateTransport impl_; TlsReaderByteFlow tls_reader_byte_flow_; AesCtrByteFlow aes_ctr_byte_flow_; ByteFlowSink byte_flow_sink_; From 853030e9995d173ddf99bd7ac2624df6cc18a3cc Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 10 Jan 2023 22:56:11 +0300 Subject: [PATCH 03/30] Don't use FILE_FLAG_OPEN_REPARSE_POINT with UWP. --- tdutils/td/utils/port/FileFd.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tdutils/td/utils/port/FileFd.cpp b/tdutils/td/utils/port/FileFd.cpp index 829b6c05f..86d57c5c9 100644 --- a/tdutils/td/utils/port/FileFd.cpp +++ b/tdutils/td/utils/port/FileFd.cpp @@ -199,7 +199,9 @@ Result FileFd::open(CSlice filepath, int32 flags, int32 mode) { } else { creation_disposition = OPEN_EXISTING; } +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM) native_flags |= FILE_FLAG_OPEN_REPARSE_POINT; +#endif } if (flags & Direct) { From 64406035a72d523421fef6d0c07bdf56fee0761e Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 10 Jan 2023 22:59:37 +0300 Subject: [PATCH 04/30] Update rust-tdlib link. --- example/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/README.md b/example/README.md index 137f80e97..5acd8343b 100644 --- a/example/README.md +++ b/example/README.md @@ -184,7 +184,7 @@ See also [telegram-flutter](https://github.com/ivk1800/telegram-flutter) - Teleg TDLib can be used from the Rust programming language through the [JSON](https://github.com/tdlib/td#using-json) interface. -See [rust-tdlib](https://github.com/aCLr/rust-tdlib), or [tdlib](https://github.com/melix99/tdlib-rs), which provide convenient TDLib clients with automatically generated and fully-documented classes for all TDLib API methods and objects. +See [rust-tdlib](https://github.com/antonio-antuan/rust-tdlib), or [tdlib](https://github.com/melix99/tdlib-rs), which provide convenient TDLib clients with automatically generated and fully-documented classes for all TDLib API methods and objects. See [rtdlib](https://github.com/fewensa/rtdlib), [tdlib-rs](https://github.com/d653/tdlib-rs), [tdlib-futures](https://github.com/yuri91/tdlib-futures), [tdlib-sys](https://github.com/nuxeh/tdlib-sys), [tdjson-rs](https://github.com/mersinvald/tdjson-rs), [rust-tdlib](https://github.com/vhaoran/rust-tdlib), or [tdlib-json-sys](https://github.com/aykxt/tdlib-json-sys) for examples of TDLib Rust bindings. From 3b6874818a5b8955eedbf6f638d02503519e98a0 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 11 Jan 2023 01:08:22 +0300 Subject: [PATCH 05/30] Always wait for mailbox. --- td/telegram/MessagesManager.cpp | 2 -- td/telegram/Td.cpp | 2 -- tdactor/td/actor/impl/Actor-decl.h | 2 -- tdactor/td/actor/impl/Actor.h | 4 ---- tdactor/td/actor/impl/ActorInfo-decl.h | 2 -- tdactor/td/actor/impl/ActorInfo.h | 6 +----- tdactor/test/actors_simple.cpp | 1 - 7 files changed, 1 insertion(+), 18 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index ea3a9ecaa..cf7cfd3fd 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -13601,8 +13601,6 @@ void MessagesManager::init() { td_->notification_settings_manager_->init(); // load scope notification settings init_stickers_manager(td_); // load available reactions - always_wait_for_mailbox(); - start_time_ = Time::now(); last_channel_pts_jump_warning_time_ = start_time_ - 3600; diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index 44675de09..dda737c73 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -3114,8 +3114,6 @@ void Td::on_connection_state_changed(ConnectionState new_state) { } void Td::start_up() { - always_wait_for_mailbox(); - uint64 check_endianness = 0x0706050403020100; auto check_endianness_raw = reinterpret_cast(&check_endianness); for (unsigned char c = 0; c < 8; c++) { diff --git a/tdactor/td/actor/impl/Actor-decl.h b/tdactor/td/actor/impl/Actor-decl.h index 819a33959..eacc871cd 100644 --- a/tdactor/td/actor/impl/Actor-decl.h +++ b/tdactor/td/actor/impl/Actor-decl.h @@ -79,8 +79,6 @@ class Actor : public ObserverBase { std::shared_ptr set_context(std::shared_ptr context); string set_tag(string tag); - void always_wait_for_mailbox(); - // for ActorInfo mostly void init(ObjectPool::OwnerPtr &&info); ActorInfo *get_info(); diff --git a/tdactor/td/actor/impl/Actor.h b/tdactor/td/actor/impl/Actor.h index 34f5c5bd2..81f9f41a9 100644 --- a/tdactor/td/actor/impl/Actor.h +++ b/tdactor/td/actor/impl/Actor.h @@ -164,8 +164,4 @@ inline Slice Actor::get_name() const { return info_->get_name(); } -inline void Actor::always_wait_for_mailbox() { - info_->always_wait_for_mailbox(); -} - } // namespace td diff --git a/tdactor/td/actor/impl/ActorInfo-decl.h b/tdactor/td/actor/impl/ActorInfo-decl.h index f3de1a4c0..870e26dff 100644 --- a/tdactor/td/actor/impl/ActorInfo-decl.h +++ b/tdactor/td/actor/impl/ActorInfo-decl.h @@ -110,14 +110,12 @@ class ActorInfo final void set_wait_generation(uint32 wait_generation); bool must_wait(uint32 wait_generation) const; - void always_wait_for_mailbox(); private: Deleter deleter_ = Deleter::None; bool need_context_ = true; bool need_start_up_ = true; bool is_running_ = false; - bool always_wait_for_mailbox_{false}; uint32 wait_generation_{0}; std::atomic sched_id_{0}; diff --git a/tdactor/td/actor/impl/ActorInfo.h b/tdactor/td/actor/impl/ActorInfo.h index 19ba303bc..714ce43bd 100644 --- a/tdactor/td/actor/impl/ActorInfo.h +++ b/tdactor/td/actor/impl/ActorInfo.h @@ -66,11 +66,7 @@ inline void ActorInfo::set_wait_generation(uint32 wait_generation) { } inline bool ActorInfo::must_wait(uint32 wait_generation) const { - return wait_generation_ == wait_generation || (always_wait_for_mailbox_ && !mailbox_.empty()); -} - -inline void ActorInfo::always_wait_for_mailbox() { - always_wait_for_mailbox_ = true; + return wait_generation_ == wait_generation || !mailbox_.empty(); } inline void ActorInfo::on_actor_moved(Actor *actor_new_ptr) { diff --git a/tdactor/test/actors_simple.cpp b/tdactor/test/actors_simple.cpp index e741b6da0..be1a78b25 100644 --- a/tdactor/test/actors_simple.cpp +++ b/tdactor/test/actors_simple.cpp @@ -571,7 +571,6 @@ TEST(Actors, stop_in_teardown) { class AlwaysWaitForMailbox final : public td::Actor { public: void start_up() final { - always_wait_for_mailbox(); td::create_actor("Sleep", 0.1, td::PromiseCreator::lambda([actor_id = actor_id(this), ptr = this](td::Unit) { td::send_closure(actor_id, &AlwaysWaitForMailbox::g); From 62720bde3ccf2f969ea506381ce0dc2d555f3af6 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 11 Jan 2023 12:49:04 +0300 Subject: [PATCH 06/30] Remove now unneeded wait_generation. --- tdactor/td/actor/impl/ActorInfo-decl.h | 4 +--- tdactor/td/actor/impl/ActorInfo.h | 9 ++------- tdactor/td/actor/impl/Scheduler-decl.h | 3 --- tdactor/td/actor/impl/Scheduler.cpp | 2 -- tdactor/td/actor/impl/Scheduler.h | 9 +-------- 5 files changed, 4 insertions(+), 23 deletions(-) diff --git a/tdactor/td/actor/impl/ActorInfo-decl.h b/tdactor/td/actor/impl/ActorInfo-decl.h index 870e26dff..9af073f02 100644 --- a/tdactor/td/actor/impl/ActorInfo-decl.h +++ b/tdactor/td/actor/impl/ActorInfo-decl.h @@ -108,15 +108,13 @@ class ActorInfo final bool need_context() const; bool need_start_up() const; - void set_wait_generation(uint32 wait_generation); - bool must_wait(uint32 wait_generation) const; + bool must_wait() const; private: Deleter deleter_ = Deleter::None; bool need_context_ = true; bool need_start_up_ = true; bool is_running_ = false; - uint32 wait_generation_{0}; std::atomic sched_id_{0}; Actor *actor_ = nullptr; diff --git a/tdactor/td/actor/impl/ActorInfo.h b/tdactor/td/actor/impl/ActorInfo.h index 714ce43bd..16e1cfe77 100644 --- a/tdactor/td/actor/impl/ActorInfo.h +++ b/tdactor/td/actor/impl/ActorInfo.h @@ -50,7 +50,6 @@ inline void ActorInfo::init(int32 sched_id, Slice name, ObjectPool::O need_context_ = need_context; need_start_up_ = need_start_up; is_running_ = false; - wait_generation_ = 0; } inline bool ActorInfo::need_context() const { @@ -61,12 +60,8 @@ inline bool ActorInfo::need_start_up() const { return need_start_up_; } -inline void ActorInfo::set_wait_generation(uint32 wait_generation) { - wait_generation_ = wait_generation; -} - -inline bool ActorInfo::must_wait(uint32 wait_generation) const { - return wait_generation_ == wait_generation || !mailbox_.empty(); +inline bool ActorInfo::must_wait() const { + return !mailbox_.empty(); } inline void ActorInfo::on_actor_moved(Actor *actor_new_ptr) { diff --git a/tdactor/td/actor/impl/Scheduler-decl.h b/tdactor/td/actor/impl/Scheduler-decl.h index dd1816a77..c4f827768 100644 --- a/tdactor/td/actor/impl/Scheduler-decl.h +++ b/tdactor/td/actor/impl/Scheduler-decl.h @@ -200,8 +200,6 @@ class Scheduler { template void send_impl(const ActorId<> &actor_id, const RunFuncT &run_func, const EventFuncT &event_func); - void inc_wait_generation(); - Timestamp run_timeout(); void run_mailbox(); Timestamp run_events(Timestamp timeout); @@ -231,7 +229,6 @@ class Scheduler { bool has_guard_ = false; bool close_flag_ = false; - uint32 wait_generation_ = 1; int32 sched_id_ = 0; int32 sched_n_ = 0; std::shared_ptr> inbound_queue_; diff --git a/tdactor/td/actor/impl/Scheduler.cpp b/tdactor/td/actor/impl/Scheduler.cpp index f38ef1def..64a1e1423 100644 --- a/tdactor/td/actor/impl/Scheduler.cpp +++ b/tdactor/td/actor/impl/Scheduler.cpp @@ -497,7 +497,6 @@ void Scheduler::run_mailbox() { ListNode *node = actors_list.get(); CHECK(node); auto actor_info = ActorInfo::from_list_node(node); - inc_wait_generation(); flush_mailbox(actor_info, static_cast(nullptr), static_cast(nullptr)); } VLOG(actor) << "Run mailbox : finish " << actor_count_; @@ -526,7 +525,6 @@ Timestamp Scheduler::run_timeout() { while (!timeout_queue_.empty() && timeout_queue_.top_key() < now) { HeapNode *node = timeout_queue_.pop(); ActorInfo *actor_info = ActorInfo::from_heap_node(node); - inc_wait_generation(); send(actor_info->actor_id(), Event::timeout()); } return get_timeout(); diff --git a/tdactor/td/actor/impl/Scheduler.h b/tdactor/td/actor/impl/Scheduler.h index 176dbe425..298c3a05c 100644 --- a/tdactor/td/actor/impl/Scheduler.h +++ b/tdactor/td/actor/impl/Scheduler.h @@ -191,10 +191,6 @@ inline void Scheduler::before_tail_send(const ActorId<> &actor_id) { // TODO } -inline void Scheduler::inc_wait_generation() { - wait_generation_ += 2; -} - template void Scheduler::send_impl(const ActorId<> &actor_id, const RunFuncT &run_func, const EventFuncT &event_func) { ActorInfo *actor_info = actor_id.get_actor_info(); @@ -210,7 +206,7 @@ void Scheduler::send_impl(const ActorId<> &actor_id, const RunFuncT &run_func, c CHECK(has_guard_ || !on_current_sched); if (likely(send_type == ActorSendType::Immediate && on_current_sched && !actor_info->is_running() && - !actor_info->must_wait(wait_generation_))) { // run immediately + !actor_info->must_wait())) { // run immediately if (likely(actor_info->mailbox_.empty())) { EventGuard guard(this, actor_info); run_func(actor_info); @@ -220,9 +216,6 @@ void Scheduler::send_impl(const ActorId<> &actor_id, const RunFuncT &run_func, c } else { if (on_current_sched) { add_to_mailbox(actor_info, event_func()); - if (send_type == ActorSendType::Later) { - actor_info->set_wait_generation(wait_generation_); - } } else { send_to_scheduler(actor_sched_id, actor_id, event_func()); } From 7a233f25ebd99d90fc46e50c980265b84844fd44 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 11 Jan 2023 13:27:53 +0300 Subject: [PATCH 07/30] Simplify Scheduler::send_impl. --- tdactor/td/actor/impl/ActorInfo-decl.h | 2 -- tdactor/td/actor/impl/ActorInfo.h | 4 ---- tdactor/td/actor/impl/Scheduler.h | 10 +++------- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/tdactor/td/actor/impl/ActorInfo-decl.h b/tdactor/td/actor/impl/ActorInfo-decl.h index 9af073f02..6726c0408 100644 --- a/tdactor/td/actor/impl/ActorInfo-decl.h +++ b/tdactor/td/actor/impl/ActorInfo-decl.h @@ -108,8 +108,6 @@ class ActorInfo final bool need_context() const; bool need_start_up() const; - bool must_wait() const; - private: Deleter deleter_ = Deleter::None; bool need_context_ = true; diff --git a/tdactor/td/actor/impl/ActorInfo.h b/tdactor/td/actor/impl/ActorInfo.h index 16e1cfe77..f734f4c0b 100644 --- a/tdactor/td/actor/impl/ActorInfo.h +++ b/tdactor/td/actor/impl/ActorInfo.h @@ -60,10 +60,6 @@ inline bool ActorInfo::need_start_up() const { return need_start_up_; } -inline bool ActorInfo::must_wait() const { - return !mailbox_.empty(); -} - inline void ActorInfo::on_actor_moved(Actor *actor_new_ptr) { actor_ = actor_new_ptr; } diff --git a/tdactor/td/actor/impl/Scheduler.h b/tdactor/td/actor/impl/Scheduler.h index 298c3a05c..e41e31374 100644 --- a/tdactor/td/actor/impl/Scheduler.h +++ b/tdactor/td/actor/impl/Scheduler.h @@ -206,13 +206,9 @@ void Scheduler::send_impl(const ActorId<> &actor_id, const RunFuncT &run_func, c CHECK(has_guard_ || !on_current_sched); if (likely(send_type == ActorSendType::Immediate && on_current_sched && !actor_info->is_running() && - !actor_info->must_wait())) { // run immediately - if (likely(actor_info->mailbox_.empty())) { - EventGuard guard(this, actor_info); - run_func(actor_info); - } else { - flush_mailbox(actor_info, &run_func, &event_func); - } + actor_info->mailbox_.empty())) { // run immediately + EventGuard guard(this, actor_info); + run_func(actor_info); } else { if (on_current_sched) { add_to_mailbox(actor_info, event_func()); From e87e18c8be3f11439f91d163a8fb30cea1e3e75f Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 11 Jan 2023 13:52:46 +0300 Subject: [PATCH 08/30] Use busy-waiting instead of short Sleep on Windows. --- tdutils/td/utils/port/sleep.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tdutils/td/utils/port/sleep.cpp b/tdutils/td/utils/port/sleep.cpp index 448f9bf8e..ee693537c 100644 --- a/tdutils/td/utils/port/sleep.cpp +++ b/tdutils/td/utils/port/sleep.cpp @@ -14,14 +14,23 @@ #else #include #endif +#else +#include "td/utils/port/Clocks.h" #endif namespace td { void usleep_for(int32 microseconds) { #if TD_PORT_WINDOWS - int32 milliseconds = microseconds / 1000 + (microseconds % 1000 ? 1 : 0); - Sleep(milliseconds); + if (microseconds < 2000) { + auto end_time = Clocks::monotonic() + microseconds * 1e-6; + do { + SwitchToThread(); + } while (Clocks::monotonic() < end_time); + } else { + int32 milliseconds = microseconds / 1000 + (microseconds % 1000 ? 1 : 0); + Sleep(milliseconds); + } #else #if _POSIX_C_SOURCE >= 199309L timespec ts; From 75bdc6292b488c195645a01b9242c7dab9d01fd1 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 12 Jan 2023 22:25:47 +0300 Subject: [PATCH 09/30] Remove some getAuthorizationState usages. --- example/python/tdjson_example.py | 4 ++-- example/uwp/app/MainPage.xaml.cs | 5 ----- td/generate/DoxygenTlDocumentationGenerator.php | 2 +- td/tl/TlObject.h | 2 +- test/online.cpp | 9 ++------- test/tdclient.cpp | 9 ++------- 6 files changed, 8 insertions(+), 23 deletions(-) diff --git a/example/python/tdjson_example.py b/example/python/tdjson_example.py index 4ee23f2c3..ce8d9f9ba 100644 --- a/example/python/tdjson_example.py +++ b/example/python/tdjson_example.py @@ -75,8 +75,8 @@ def td_receive(): # another test for TDLib execute method print(str(td_execute({'@type': 'getTextEntities', 'text': '@telegram /test_command https://telegram.org telegram.me', '@extra': ['5', 7.0, 'a']})).encode('utf-8')) -# start the client by sending request to it -td_send({'@type': 'getAuthorizationState', '@extra': 1.01234}) +# start the client by sending a request to it +td_send({'@type': 'getOption', 'name': 'version', '@extra': 1.01234}) # main events cycle while True: diff --git a/example/uwp/app/MainPage.xaml.cs b/example/uwp/app/MainPage.xaml.cs index 82c7fdd0d..e4218386e 100644 --- a/example/uwp/app/MainPage.xaml.cs +++ b/example/uwp/app/MainPage.xaml.cs @@ -85,11 +85,6 @@ namespace TdApp AcceptCommand("LogOut"); _client.Send(new TdApi.LogOut(), _handler); } - else if (command.StartsWith("gas")) - { - AcceptCommand(command); - _client.Send(new TdApi.GetAuthorizationState(), _handler); - } else if (command.StartsWith("sap")) { var args = command.Split(" ".ToCharArray(), 2); diff --git a/td/generate/DoxygenTlDocumentationGenerator.php b/td/generate/DoxygenTlDocumentationGenerator.php index dc89536b2..2d5a37b2c 100644 --- a/td/generate/DoxygenTlDocumentationGenerator.php +++ b/td/generate/DoxygenTlDocumentationGenerator.php @@ -231,7 +231,7 @@ EOT * A function to create a dynamically allocated TDLib API object. Can be treated as an analogue of std::make_unique. * Usage example: * \\code - * auto get_authorization_state_request = td::td_api::make_object(); + * auto get_me_request = td::td_api::make_object(); * auto message_text = td::td_api::make_object("Hello, world!!!", * td::td_api::array>()); * auto send_message_request = td::td_api::make_object(chat_id, 0, 0, nullptr, nullptr, diff --git a/td/tl/TlObject.h b/td/tl/TlObject.h index 399637d09..5c5e76d2c 100644 --- a/td/tl/TlObject.h +++ b/td/tl/TlObject.h @@ -187,7 +187,7 @@ using tl_object_ptr = tl::unique_ptr; * A function to create a dynamically allocated TL-object. Can be treated as an analogue of std::make_unique. * Usage example: * \code - * auto get_authorization_state_request = td::make_tl_object(); + * auto get_me_request = td::make_tl_object(); * auto message_text = td::make_tl_object("Hello, world!!!", * td::td_api::array>()); * auto send_message_request = td::make_tl_object(chat_id, 0, 0, nullptr, nullptr, diff --git a/test/online.cpp b/test/online.cpp index 1ffc5523f..fd758129d 100644 --- a/test/online.cpp +++ b/test/online.cpp @@ -221,14 +221,12 @@ class InitTask : public Task { private: Options options_; td::Promise<> promise_; - bool start_flag_{false}; void start_up() override { - send_query(td::make_tl_object(), - [this](auto res) { this->process_authorization_state(res.move_as_ok()); }); + send_query(td::make_tl_object("version"), + [](auto res) { LOG(INFO) << td::td_api::to_string(res.ok()); }); } void process_authorization_state(td::tl_object_ptr authorization_state) { - start_flag_ = true; td::tl_object_ptr function; switch (authorization_state->get_id()) { case td::td_api::authorizationStateReady::ID: @@ -267,9 +265,6 @@ class InitTask : public Task { }); } void process_update(std::shared_ptr update) override { - if (!start_flag_) { - return; - } if (!update->object) { return; } diff --git a/test/tdclient.cpp b/test/tdclient.cpp index 6d8dd383c..3514089b7 100644 --- a/test/tdclient.cpp +++ b/test/tdclient.cpp @@ -204,11 +204,10 @@ class DoAuthentication final : public TestClinetTask { : name_(std::move(name)), phone_(std::move(phone)), code_(std::move(code)), promise_(std::move(promise)) { } void start_up() final { - send_query(td::make_tl_object(), - [this](auto res) { this->process_authorization_state(std::move(res)); }); + send_query(td::make_tl_object("version"), + [](auto res) { LOG(INFO) << td::td_api::to_string(res); }); } void process_authorization_state(td::tl_object_ptr authorization_state) { - start_flag_ = true; td::tl_object_ptr function; switch (authorization_state->get_id()) { case td::td_api::authorizationStateWaitPhoneNumber::ID: @@ -261,12 +260,8 @@ class DoAuthentication final : public TestClinetTask { td::string phone_; td::string code_; td::Promise<> promise_; - bool start_flag_{false}; void process_update(std::shared_ptr update) final { - if (!start_flag_) { - return; - } if (!update->object) { return; } From b1883d357c8b5d00bbcfb4aa157d94f9aadef6d2 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 13 Jan 2023 13:09:38 +0300 Subject: [PATCH 10/30] Add QueryMerger. --- CMakeLists.txt | 2 + td/telegram/QueryMerger.cpp | 82 +++++++++++++++++++++++++++++++++++++ td/telegram/QueryMerger.h | 54 ++++++++++++++++++++++++ 3 files changed, 138 insertions(+) create mode 100644 td/telegram/QueryMerger.cpp create mode 100644 td/telegram/QueryMerger.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 5cd9be657..fb7c0aff8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -429,6 +429,7 @@ set(TDLIB_SOURCE td/telegram/Premium.cpp td/telegram/PremiumGiftOption.cpp td/telegram/QueryCombiner.cpp + td/telegram/QueryMerger.cpp td/telegram/RecentDialogList.cpp td/telegram/ReplyMarkup.cpp td/telegram/ReportReason.cpp @@ -690,6 +691,7 @@ set(TDLIB_SOURCE td/telegram/PtsManager.h td/telegram/PublicDialogType.h td/telegram/QueryCombiner.h + td/telegram/QueryMerger.h td/telegram/RecentDialogList.h td/telegram/ReplyMarkup.h td/telegram/ReportReason.h diff --git a/td/telegram/QueryMerger.cpp b/td/telegram/QueryMerger.cpp new file mode 100644 index 000000000..0154fbaac --- /dev/null +++ b/td/telegram/QueryMerger.cpp @@ -0,0 +1,82 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2023 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/QueryMerger.h" + +#include "td/utils/logging.h" +#include "td/utils/Time.h" + +namespace td { + +QueryMerger::QueryMerger(Slice name, size_t max_concurrent_query_count, size_t max_merged_query_count) + : max_concurrent_query_count_(max_concurrent_query_count), max_merged_query_count_(max_merged_query_count) { + register_actor(name, this).release(); +} + +void QueryMerger::add_query(int64 query_id, Promise &&promise) { + LOG(INFO) << "Add query " << query_id << " with" << (promise ? "" : "out") << " promise"; + CHECK(query_id != 0); + auto &query = queries_[query_id]; + query.promises_.push_back(std::move(promise)); + if (query.promises_.size() != 1) { + // duplicate query, just wait + return; + } + pending_queries_.push(query_id); + loop(); +} + +void QueryMerger::send_query(vector query_ids) { + CHECK(merge_function_ != nullptr); + LOG(INFO) << "Send queries " << query_ids; + query_count_++; + merge_function_(query_ids, PromiseCreator::lambda([actor_id = actor_id(this), query_ids](Result &&result) { + send_closure(actor_id, &QueryMerger::on_get_query_result, std::move(query_ids), std::move(result)); + })); +} + +void QueryMerger::on_get_query_result(vector query_ids, Result &&result) { + LOG(INFO) << "Get result of queries " << query_ids << (result.is_error() ? " error" : " success"); + query_count_--; + for (auto query_id : query_ids) { + auto it = queries_.find(query_id); + CHECK(it != queries_.end()); + auto promises = std::move(it->second.promises_); + queries_.erase(it); + + if (result.is_ok()) { + set_promises(promises); + } else { + fail_promises(promises, result.move_as_error()); + } + } + loop(); +} + +void QueryMerger::loop() { + if (query_count_ == max_concurrent_query_count_) { + return; + } + + vector query_ids; + while (!pending_queries_.empty()) { + auto query_id = pending_queries_.front(); + pending_queries_.pop(); + query_ids.push_back(query_id); + if (query_ids.size() == max_merged_query_count_) { + send_query(std::move(query_ids)); + query_ids.clear(); + if (query_count_ == max_concurrent_query_count_) { + break; + } + } + } + if (!query_ids.empty()) { + send_query(std::move(query_ids)); + } +} + +} // namespace td diff --git a/td/telegram/QueryMerger.h b/td/telegram/QueryMerger.h new file mode 100644 index 000000000..e87f81666 --- /dev/null +++ b/td/telegram/QueryMerger.h @@ -0,0 +1,54 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2023 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/actor.h" + +#include "td/utils/common.h" +#include "td/utils/FlatHashMap.h" +#include "td/utils/Promise.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +#include +#include + +namespace td { + +// merges queries into a single request +class QueryMerger final : public Actor { + public: + QueryMerger(Slice name, size_t max_concurrent_query_count, size_t max_merged_query_count); + + using MergeFunction = std::function query_ids, Promise &&promise)>; + void set_merge_function(MergeFunction merge_function) { + merge_function_ = std::move(merge_function); + } + + void add_query(int64 query_id, Promise &&promise); + + private: + struct QueryInfo { + vector> promises_; + }; + + size_t query_count_ = 0; + size_t max_concurrent_query_count_; + size_t max_merged_query_count_; + + MergeFunction merge_function_; + std::queue pending_queries_; + FlatHashMap queries_; + + void send_query(vector query_ids); + + void on_get_query_result(vector query_ids, Result &&result); + + void loop() final; +}; + +} // namespace td From b514162c7ea8081b3240f349030dbd05e28ff63d Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 13 Jan 2023 13:53:56 +0300 Subject: [PATCH 11/30] Add QueryMerger test. --- test/CMakeLists.txt | 1 + test/query_merger.cpp | 96 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 test/query_merger.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2ac9ad32d..2a140a3af 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -11,6 +11,7 @@ set(TD_TEST_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/message_entities.cpp ${CMAKE_CURRENT_SOURCE_DIR}/mtproto.cpp ${CMAKE_CURRENT_SOURCE_DIR}/poll.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/query_merger.cpp ${CMAKE_CURRENT_SOURCE_DIR}/secret.cpp ${CMAKE_CURRENT_SOURCE_DIR}/secure_storage.cpp ${CMAKE_CURRENT_SOURCE_DIR}/set_with_position.cpp diff --git a/test/query_merger.cpp b/test/query_merger.cpp new file mode 100644 index 000000000..cdda2390f --- /dev/null +++ b/test/query_merger.cpp @@ -0,0 +1,96 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2023 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/QueryMerger.h" + +#include "td/actor/actor.h" +#include "td/actor/ConcurrentScheduler.h" +#include "td/actor/SleepActor.h" + +#include "td/utils/common.h" +#include "td/utils/FlatHashSet.h" +#include "td/utils/Random.h" +#include "td/utils/tests.h" + +#include + +class TestQueryMerger : public td::Actor { + void start_up() override { + query_merger_.set_merge_function([this](td::vector query_ids, td::Promise &&promise) { + ASSERT_TRUE(!query_ids.empty()); + ASSERT_EQ(query_ids.size(), td::min(next_query_ids_.size(), MAX_MERGED_QUERY_COUNT)); + for (auto query_id : query_ids) { + auto next_query_id = next_query_ids_.front(); + next_query_ids_.pop(); + ASSERT_EQ(query_id, next_query_id); + } + current_query_count_++; + ASSERT_TRUE(current_query_count_ <= MAX_CONCURRENT_QUERY_COUNT); + if (!next_query_ids_.empty()) { + ASSERT_EQ(current_query_count_, MAX_CONCURRENT_QUERY_COUNT); + } + td::create_actor("CompleteMergeQuery", 0.02, + td::PromiseCreator::lambda([this, query_ids, promise = std::move(promise)]( + td::Result result) mutable { + for (auto query_id : query_ids) { + LOG(INFO) << "Complete " << query_id; + bool is_erased = pending_query_ids_.erase(query_id); + ASSERT_TRUE(is_erased); + } + current_query_count_--; + promise.set_result(std::move(result)); + })) + .release(); + yield(); + }); + loop(); + } + + void loop() { + std::size_t query_count = 0; + std::size_t added_queries = td::Random::fast(1, 3); + while (query_count++ < added_queries && total_query_count_++ < MAX_QUERY_COUNT) { + td::int64 query_id = td::Random::fast(1, 20); + if (pending_query_ids_.insert(query_id).second) { + next_query_ids_.push(query_id); + } + query_merger_.add_query(query_id, + td::PromiseCreator::lambda([this, query_id](td::Result result) mutable { + completed_query_count_++; + if (completed_query_count_ == MAX_QUERY_COUNT) { + ASSERT_EQ(current_query_count_, 0u); + ASSERT_TRUE(next_query_ids_.empty()); + ASSERT_TRUE(pending_query_ids_.empty()); + td::Scheduler::instance()->finish(); + } else { + yield(); + } + })); + } + } + + static constexpr std::size_t MAX_CONCURRENT_QUERY_COUNT = 5; + static constexpr std::size_t MAX_MERGED_QUERY_COUNT = 3; + static constexpr std::size_t MAX_QUERY_COUNT = 1000; + + td::QueryMerger query_merger_{"QueryMerger", MAX_CONCURRENT_QUERY_COUNT, MAX_MERGED_QUERY_COUNT}; + std::size_t current_query_count_ = 0; + std::size_t total_query_count_ = 0; + std::size_t completed_query_count_ = 0; + + std::queue next_query_ids_; + td::FlatHashSet pending_query_ids_; +}; + +TEST(QueryMerger, stress) { + td::ConcurrentScheduler sched(0, 0); + sched.create_actor_unsafe(0, "TestQueryMerger").release(); + sched.start(); + while (sched.run_main(10)) { + // empty + } + sched.finish(); +} From eed410ce0acd56220b013b98c8b4524923b81484 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 13 Jan 2023 14:29:47 +0300 Subject: [PATCH 12/30] Merge getUser queries. --- td/telegram/ContactsManager.cpp | 19 +++++++++++-------- td/telegram/ContactsManager.h | 3 +++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/td/telegram/ContactsManager.cpp b/td/telegram/ContactsManager.cpp index 5a22922ac..4ae23247e 100644 --- a/td/telegram/ContactsManager.cpp +++ b/td/telegram/ContactsManager.cpp @@ -3915,6 +3915,15 @@ ContactsManager::ContactsManager(Td *td, ActorShared<> parent) : td_(td), parent channel_participant_cache_timeout_.set_callback(on_channel_participant_cache_timeout_callback); channel_participant_cache_timeout_.set_callback_data(static_cast(this)); + + get_user_queries_.set_merge_function([this](vector query_ids, Promise &&promise) { + auto users = transform(query_ids, [this](int64 query_id) { + auto r_input_user = get_input_user(UserId(query_id)); + CHECK(r_input_user.is_ok()); + return r_input_user.move_as_ok(); + }); + td_->create_handler(std::move(promise))->send(std::move(users)); + }); } ContactsManager::~ContactsManager() { @@ -15787,7 +15796,6 @@ bool ContactsManager::get_user(UserId user_id, int left_tries, Promise &&p } if (td_->auth_manager_->is_bot() ? !have_user(user_id) : !have_min_user(user_id)) { - // TODO UserLoader if (left_tries > 2 && G()->parameters().use_chat_info_db) { send_closure_later(actor_id(this), &ContactsManager::load_user_from_database, nullptr, user_id, std::move(promise)); @@ -15803,9 +15811,7 @@ bool ContactsManager::get_user(UserId user_id, int left_tries, Promise &&p return false; } - vector> users; - users.push_back(r_input_user.move_as_ok()); - td_->create_handler(std::move(promise))->send(std::move(users)); + get_user_queries_.add_query(user_id.get(), std::move(promise)); return false; } @@ -15850,10 +15856,7 @@ void ContactsManager::reload_user(UserId user_id, Promise &&promise) { return promise.set_error(r_input_user.move_as_error()); } - // there is no much reason to combine different requests into one request - vector> users; - users.push_back(r_input_user.move_as_ok()); - td_->create_handler(std::move(promise))->send(std::move(users)); + get_user_queries_.add_query(user_id.get(), std::move(promise)); } void ContactsManager::load_user_full(UserId user_id, bool force, Promise &&promise, const char *source) { diff --git a/td/telegram/ContactsManager.h b/td/telegram/ContactsManager.h index b813068b8..b9b73b4d3 100644 --- a/td/telegram/ContactsManager.h +++ b/td/telegram/ContactsManager.h @@ -32,6 +32,7 @@ #include "td/telegram/PremiumGiftOption.h" #include "td/telegram/PublicDialogType.h" #include "td/telegram/QueryCombiner.h" +#include "td/telegram/QueryMerger.h" #include "td/telegram/RestrictionReason.h" #include "td/telegram/SecretChatId.h" #include "td/telegram/StickerSetId.h" @@ -1876,6 +1877,8 @@ class ContactsManager final : public Actor { FlatHashMap>, SecretChatIdHash> load_secret_chat_from_database_queries_; FlatHashSet loaded_from_database_secret_chats_; + QueryMerger get_user_queries_{"GetUserMerger", 3, 50}; + QueryCombiner get_user_full_queries_{"GetUserFullCombiner", 2.0}; QueryCombiner get_chat_full_queries_{"GetChatFullCombiner", 2.0}; From 13e8094aaacdcf2f6224035c40ae3a0659932a27 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 13 Jan 2023 14:38:36 +0300 Subject: [PATCH 13/30] Reload contacts if failed to load the from database. --- td/telegram/ContactsManager.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/td/telegram/ContactsManager.cpp b/td/telegram/ContactsManager.cpp index 4ae23247e..59bf825a7 100644 --- a/td/telegram/ContactsManager.cpp +++ b/td/telegram/ContactsManager.cpp @@ -6048,7 +6048,8 @@ int64 ContactsManager::get_contacts_hash() { } void ContactsManager::reload_contacts(bool force) { - if (!td_->auth_manager_->is_bot() && next_contacts_sync_date_ != std::numeric_limits::max() && + if (!G()->close_flag() && !td_->auth_manager_->is_bot() && + next_contacts_sync_date_ != std::numeric_limits::max() && (next_contacts_sync_date_ < G()->unix_time() || force)) { next_contacts_sync_date_ = std::numeric_limits::max(); td_->create_handler()->send(get_contacts_hash()); @@ -9542,6 +9543,9 @@ void ContactsManager::on_load_contacts_from_database(string value) { [actor_id = actor_id(this), expected_contact_count = user_ids.size()](Result result) { if (result.is_ok()) { send_closure(actor_id, &ContactsManager::on_get_contacts_finished, expected_contact_count); + } else { + LOG(INFO) << "Failed to load contact users from database: " << result.error(); + send_closure(actor_id, &ContactsManager::reload_contacts, true); } })); From d3690e15022fc04ad491d4b22ddb60f94809036d Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 13 Jan 2023 14:45:53 +0300 Subject: [PATCH 14/30] Merge GetChat queries. --- td/telegram/ContactsManager.cpp | 8 +++++--- td/telegram/ContactsManager.h | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/td/telegram/ContactsManager.cpp b/td/telegram/ContactsManager.cpp index 59bf825a7..6d757ac07 100644 --- a/td/telegram/ContactsManager.cpp +++ b/td/telegram/ContactsManager.cpp @@ -3924,6 +3924,9 @@ ContactsManager::ContactsManager(Td *td, ActorShared<> parent) : td_(td), parent }); td_->create_handler(std::move(promise))->send(std::move(users)); }); + get_chat_queries_.set_merge_function([this](vector query_ids, Promise &&promise) { + td_->create_handler(std::move(promise))->send(std::move(query_ids)); + }); } ContactsManager::~ContactsManager() { @@ -16114,7 +16117,7 @@ bool ContactsManager::get_chat(ChatId chat_id, int left_tries, Promise &&p } if (left_tries > 1) { - td_->create_handler(std::move(promise))->send(vector{chat_id.get()}); + get_chat_queries_.add_query(chat_id.get(), std::move(promise)); return false; } @@ -16131,8 +16134,7 @@ void ContactsManager::reload_chat(ChatId chat_id, Promise &&promise) { return promise.set_error(Status::Error(400, "Invalid basic group identifier")); } - // there is no much reason to combine different requests into one request - td_->create_handler(std::move(promise))->send(vector{chat_id.get()}); + get_chat_queries_.add_query(chat_id.get(), std::move(promise)); } const ContactsManager::ChatFull *ContactsManager::get_chat_full(ChatId chat_id) const { diff --git a/td/telegram/ContactsManager.h b/td/telegram/ContactsManager.h index b9b73b4d3..1d0d854dd 100644 --- a/td/telegram/ContactsManager.h +++ b/td/telegram/ContactsManager.h @@ -1878,6 +1878,7 @@ class ContactsManager final : public Actor { FlatHashSet loaded_from_database_secret_chats_; QueryMerger get_user_queries_{"GetUserMerger", 3, 50}; + QueryMerger get_chat_queries_{"GetChatMerger", 3, 50}; QueryCombiner get_user_full_queries_{"GetUserFullCombiner", 2.0}; QueryCombiner get_chat_full_queries_{"GetChatFullCombiner", 2.0}; From fb79543d4136e6fb5a8ca80c43263bd4c9ba7177 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 13 Jan 2023 15:42:15 +0300 Subject: [PATCH 15/30] Combine getChannel queries with the same channel_id. --- td/telegram/ContactsManager.cpp | 17 +++++++++++------ td/telegram/ContactsManager.h | 1 + 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/td/telegram/ContactsManager.cpp b/td/telegram/ContactsManager.cpp index 6d757ac07..f15d5af12 100644 --- a/td/telegram/ContactsManager.cpp +++ b/td/telegram/ContactsManager.cpp @@ -3917,16 +3917,22 @@ ContactsManager::ContactsManager(Td *td, ActorShared<> parent) : td_(td), parent channel_participant_cache_timeout_.set_callback_data(static_cast(this)); get_user_queries_.set_merge_function([this](vector query_ids, Promise &&promise) { - auto users = transform(query_ids, [this](int64 query_id) { + auto input_users = transform(query_ids, [this](int64 query_id) { auto r_input_user = get_input_user(UserId(query_id)); CHECK(r_input_user.is_ok()); return r_input_user.move_as_ok(); }); - td_->create_handler(std::move(promise))->send(std::move(users)); + td_->create_handler(std::move(promise))->send(std::move(input_users)); }); get_chat_queries_.set_merge_function([this](vector query_ids, Promise &&promise) { td_->create_handler(std::move(promise))->send(std::move(query_ids)); }); + get_channel_queries_.set_merge_function([this](vector query_ids, Promise &&promise) { + CHECK(query_ids.size() == 1); + auto input_channel = get_input_channel(ChannelId(query_ids[0])); + CHECK(input_channel != nullptr); + td_->create_handler(std::move(promise))->send(std::move(input_channel)); + }); } ContactsManager::~ContactsManager() { @@ -16529,7 +16535,7 @@ bool ContactsManager::get_channel(ChannelId channel_id, int left_tries, Promise< } if (left_tries > 1 && td_->auth_manager_->is_bot()) { - td_->create_handler(std::move(promise))->send(get_input_channel(channel_id)); + get_channel_queries_.add_query(channel_id.get(), std::move(promise)); return false; } @@ -16552,9 +16558,8 @@ void ContactsManager::reload_channel(ChannelId channel_id, Promise &&promi input_channel = make_tl_object(channel_id.get(), 0); } - // there is no much reason to combine different requests into one request // requests with 0 access_hash must not be merged - td_->create_handler(std::move(promise))->send(std::move(input_channel)); + get_channel_queries_.add_query(channel_id.get(), std::move(promise)); } const ContactsManager::ChannelFull *ContactsManager::get_channel_full_const(ChannelId channel_id) const { @@ -17434,7 +17439,7 @@ void ContactsManager::on_chat_update(telegram_api::chat &chat, const char *sourc update_channel(c, migrated_to_channel_id); // get info about the channel - td_->create_handler(Promise<>())->send(std::move(input_channel)); + get_channel_queries_.add_query(migrated_to_channel_id.get(), Promise()); } } break; diff --git a/td/telegram/ContactsManager.h b/td/telegram/ContactsManager.h index 1d0d854dd..fd2daca22 100644 --- a/td/telegram/ContactsManager.h +++ b/td/telegram/ContactsManager.h @@ -1879,6 +1879,7 @@ class ContactsManager final : public Actor { QueryMerger get_user_queries_{"GetUserMerger", 3, 50}; QueryMerger get_chat_queries_{"GetChatMerger", 3, 50}; + QueryMerger get_channel_queries_{"GetChannelMerger", 100, 1}; // can't merge getChannel queries without access hash QueryCombiner get_user_full_queries_{"GetUserFullCombiner", 2.0}; QueryCombiner get_chat_full_queries_{"GetChatFullCombiner", 2.0}; From 1a732db19e10c85eb9ddad0b6710a8d91bfd8dd3 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 14 Jan 2023 22:01:35 +0300 Subject: [PATCH 16/30] Explicitly fail some promises on closing. --- td/telegram/StickersManager.cpp | 39 ++++++++++++++++++++------------- td/telegram/StickersManager.h | 2 ++ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/td/telegram/StickersManager.cpp b/td/telegram/StickersManager.cpp index c5c5a58da..cff6562a0 100644 --- a/td/telegram/StickersManager.cpp +++ b/td/telegram/StickersManager.cpp @@ -4707,6 +4707,7 @@ void StickersManager::search_stickers(string emoji, int32 limit, void StickersManager::on_load_found_stickers_from_database(string emoji, string value) { if (G()->close_flag()) { + on_search_stickers_failed(emoji, G()->close_status()); return; } if (value.empty()) { @@ -4744,6 +4745,18 @@ void StickersManager::on_search_stickers_finished(const string &emoji, const Fou } } +void StickersManager::on_search_stickers_failed(const string &emoji, Status &&error) { + auto it = search_stickers_queries_.find(emoji); + CHECK(it != search_stickers_queries_.end()); + CHECK(!it->second.empty()); + auto queries = std::move(it->second); + search_stickers_queries_.erase(it); + + for (auto &query : queries) { + query.second.set_error(error.clone()); + } +} + void StickersManager::on_find_stickers_success(const string &emoji, tl_object_ptr &&stickers) { CHECK(stickers != nullptr); @@ -4791,15 +4804,7 @@ void StickersManager::on_find_stickers_fail(const string &emoji, Status &&error) return on_find_stickers_success(emoji, make_tl_object()); } - auto it = search_stickers_queries_.find(emoji); - CHECK(it != search_stickers_queries_.end()); - CHECK(!it->second.empty()); - auto queries = std::move(it->second); - search_stickers_queries_.erase(it); - - for (auto &query : queries) { - query.second.set_error(error.clone()); - } + on_search_stickers_failed(emoji, std::move(error)); } void StickersManager::get_premium_stickers(int32 limit, Promise> &&promise) { @@ -5162,6 +5167,7 @@ void StickersManager::load_installed_sticker_sets(StickerType sticker_type, Prom void StickersManager::on_load_installed_sticker_sets_from_database(StickerType sticker_type, string value) { if (G()->close_flag()) { + on_get_installed_sticker_sets_failed(sticker_type, G()->close_status()); return; } if (value.empty()) { @@ -6134,7 +6140,14 @@ void StickersManager::load_custom_emoji_sticker_from_database(CustomEmojiId cust } void StickersManager::on_load_custom_emoji_from_database(CustomEmojiId custom_emoji_id, string value) { + auto it = custom_emoji_load_queries_.find(custom_emoji_id); + CHECK(it != custom_emoji_load_queries_.end()); + CHECK(!it->second.empty()); + auto promises = std::move(it->second); + custom_emoji_load_queries_.erase(it); + if (G()->close_flag()) { + fail_promises(promises, G()->close_status()); return; } @@ -6149,12 +6162,6 @@ void StickersManager::on_load_custom_emoji_from_database(CustomEmojiId custom_em LOG(INFO) << "Failed to load " << custom_emoji_id << " from database"; } - auto it = custom_emoji_load_queries_.find(custom_emoji_id); - CHECK(it != custom_emoji_load_queries_.end()); - CHECK(!it->second.empty()); - auto promises = std::move(it->second); - custom_emoji_load_queries_.erase(it); - set_promises(promises); } @@ -8377,6 +8384,7 @@ void StickersManager::load_recent_stickers(bool is_attached, Promise &&pro void StickersManager::on_load_recent_stickers_from_database(bool is_attached, string value) { if (G()->close_flag()) { + fail_promises(load_recent_stickers_queries_[is_attached], G()->close_status()); return; } if (value.empty()) { @@ -8777,6 +8785,7 @@ void StickersManager::load_favorite_stickers(Promise &&promise) { void StickersManager::on_load_favorite_stickers_from_database(const string &value) { if (G()->close_flag()) { + fail_promises(load_favorite_stickers_queries_, G()->close_status()); return; } if (value.empty()) { diff --git a/td/telegram/StickersManager.h b/td/telegram/StickersManager.h index 9b20e32a8..908bd543a 100644 --- a/td/telegram/StickersManager.h +++ b/td/telegram/StickersManager.h @@ -627,6 +627,8 @@ class StickersManager final : public Actor { void on_search_stickers_finished(const string &emoji, const FoundStickers &found_stickers); + void on_search_stickers_failed(const string &emoji, Status &&error); + static string get_custom_emoji_database_key(CustomEmojiId custom_emoji_id); void load_custom_emoji_sticker_from_database_force(CustomEmojiId custom_emoji_id); From 973df16c88c7138163d7dcfb3a9213dcd68e1659 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 14 Jan 2023 23:08:50 +0300 Subject: [PATCH 17/30] Improve logging. --- td/telegram/StateManager.cpp | 1 + td/telegram/net/Session.cpp | 2 ++ td/telegram/net/Session.h | 12 ++++++------ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/td/telegram/StateManager.cpp b/td/telegram/StateManager.cpp index 7a1c1cc97..69901cc34 100644 --- a/td/telegram/StateManager.cpp +++ b/td/telegram/StateManager.cpp @@ -68,6 +68,7 @@ void StateManager::add_callback(unique_ptr callback) { callbacks_.push_back(std::move(callback)); } } + void StateManager::wait_first_sync(Promise<> promise) { if (was_sync_) { return promise.set_value(Unit()); diff --git a/td/telegram/net/Session.cpp b/td/telegram/net/Session.cpp index 4610159fa..d594bba83 100644 --- a/td/telegram/net/Session.cpp +++ b/td/telegram/net/Session.cpp @@ -333,12 +333,14 @@ void Session::on_network(bool network_flag, uint32 network_generation) { } void Session::on_online(bool online_flag) { + LOG(DEBUG) << "Set online flag to " << online_flag; online_flag_ = online_flag; connection_online_update(true); loop(); } void Session::on_logging_out(bool logging_out_flag) { + LOG(DEBUG) << "Set logging out flag to " << logging_out_flag; logging_out_flag_ = logging_out_flag; connection_online_update(true); loop(); diff --git a/td/telegram/net/Session.h b/td/telegram/net/Session.h index 6dba9fcfd..3eb61fee4 100644 --- a/td/telegram/net/Session.h +++ b/td/telegram/net/Session.h @@ -104,17 +104,17 @@ class Session final // Just re-ask answer_id each time we get information about it. // Though mtproto::Connection must ensure delivery of such query. - int32 raw_dc_id_; // numerical datacenter ID, i.e. 2 - int32 dc_id_; // unique datacenter ID, i.e. -10002 - enum class Mode : int8 { Tcp, Http } mode_ = Mode::Tcp; - bool is_main_; // true only for the primary Session(s) to the main DC - bool is_cdn_; - bool need_destroy_; + const int32 raw_dc_id_; // numerical datacenter ID, i.e. 2 + const int32 dc_id_; // unique datacenter ID, i.e. -10002 + const bool is_main_; // true only for the primary Session(s) to the main DC + const bool is_cdn_; + const bool need_destroy_; bool was_on_network_ = false; bool network_flag_ = false; bool online_flag_ = false; bool logging_out_flag_ = false; bool connection_online_flag_ = false; + enum class Mode : int8 { Tcp, Http } mode_ = Mode::Tcp; uint32 network_generation_ = 0; uint64 being_binded_tmp_auth_key_id_ = 0; uint64 being_checked_main_auth_key_id_ = 0; From fb935a8de2153fc283c048ec38c3d3a15e54324f Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 15 Jan 2023 00:25:22 +0300 Subject: [PATCH 18/30] Reduce ping timeout in primary non-main-DC connections. --- td/telegram/ConfigManager.cpp | 4 ++-- td/telegram/net/NetQueryDispatcher.cpp | 10 +++++----- td/telegram/net/Session.cpp | 20 +++++++++++++------- td/telegram/net/Session.h | 5 +++-- td/telegram/net/SessionMultiProxy.cpp | 10 +++++----- td/telegram/net/SessionMultiProxy.h | 7 ++++--- td/telegram/net/SessionProxy.cpp | 8 +++++--- td/telegram/net/SessionProxy.h | 5 +++-- 8 files changed, 40 insertions(+), 29 deletions(-) diff --git a/td/telegram/ConfigManager.cpp b/td/telegram/ConfigManager.cpp index c92ecfbd6..fc25db0e9 100644 --- a/td/telegram/ConfigManager.cpp +++ b/td/telegram/ConfigManager.cpp @@ -521,8 +521,8 @@ static ActorOwn<> get_full_config(DcOption option, Promise("ConfigSession", std::move(session_callback), std::move(auth_data), raw_dc_id, - int_dc_id, false /*is_main*/, true /*use_pfs*/, false /*is_cdn*/, - false /*need_destroy_auth_key*/, mtproto::AuthKey(), + int_dc_id, false /*is_primary*/, false /*is_main*/, true /*use_pfs*/, + false /*is_cdn*/, false /*need_destroy_auth_key*/, mtproto::AuthKey(), std::vector()); auto query = G()->net_query_creator().create_unauth(telegram_api::help_getConfig(), DcId::empty()); query->total_timeout_limit_ = 60 * 60 * 24; diff --git a/td/telegram/net/NetQueryDispatcher.cpp b/td/telegram/net/NetQueryDispatcher.cpp index 5d1fb36d2..6409eeb96 100644 --- a/td/telegram/net/NetQueryDispatcher.cpp +++ b/td/telegram/net/NetQueryDispatcher.cpp @@ -172,17 +172,17 @@ Status NetQueryDispatcher::wait_dc_init(DcId dc_id, bool force) { int32 download_session_count = is_premium ? 8 : 2; int32 download_small_session_count = is_premium ? 8 : 2; dc.main_session_ = create_actor(PSLICE() << "SessionMultiProxy:" << raw_dc_id << ":main", - session_count, auth_data, raw_dc_id == main_dc_id_, use_pfs, - false, false, is_cdn, need_destroy_key); + session_count, auth_data, true, raw_dc_id == main_dc_id_, + use_pfs, false, false, is_cdn, need_destroy_key); dc.upload_session_ = create_actor_on_scheduler( PSLICE() << "SessionMultiProxy:" << raw_dc_id << ":upload", slow_net_scheduler_id, upload_session_count, - auth_data, false, use_pfs, false, true, is_cdn, need_destroy_key); + auth_data, false, false, use_pfs, false, true, is_cdn, need_destroy_key); dc.download_session_ = create_actor_on_scheduler( PSLICE() << "SessionMultiProxy:" << raw_dc_id << ":download", slow_net_scheduler_id, download_session_count, - auth_data, false, use_pfs, true, true, is_cdn, need_destroy_key); + auth_data, false, false, use_pfs, true, true, is_cdn, need_destroy_key); dc.download_small_session_ = create_actor_on_scheduler( PSLICE() << "SessionMultiProxy:" << raw_dc_id << ":download_small", slow_net_scheduler_id, - download_small_session_count, auth_data, false, use_pfs, true, true, is_cdn, need_destroy_key); + download_small_session_count, auth_data, false, false, use_pfs, true, true, is_cdn, need_destroy_key); dc.is_inited_ = true; if (dc_id.is_internal()) { send_closure_later(dc_auth_manager_, &DcAuthManager::add_dc, std::move(auth_data)); diff --git a/td/telegram/net/Session.cpp b/td/telegram/net/Session.cpp index d594bba83..2a8d29cb1 100644 --- a/td/telegram/net/Session.cpp +++ b/td/telegram/net/Session.cpp @@ -232,9 +232,14 @@ bool Session::PriorityQueue::empty() const { } Session::Session(unique_ptr callback, std::shared_ptr shared_auth_data, int32 raw_dc_id, - int32 dc_id, bool is_main, bool use_pfs, bool is_cdn, bool need_destroy, + int32 dc_id, bool is_primary, bool is_main, bool use_pfs, bool is_cdn, bool need_destroy, const mtproto::AuthKey &tmp_auth_key, const vector &server_salts) - : raw_dc_id_(raw_dc_id), dc_id_(dc_id), is_main_(is_main), is_cdn_(is_cdn), need_destroy_(need_destroy) { + : raw_dc_id_(raw_dc_id) + , dc_id_(dc_id) + , is_primary_(is_primary) + , is_main_(is_main) + , is_cdn_(is_cdn) + , need_destroy_(need_destroy) { VLOG(dc) << "Start connection " << tag("need_destroy", need_destroy_); if (need_destroy_) { use_pfs = false; @@ -347,18 +352,19 @@ void Session::on_logging_out(bool logging_out_flag) { } void Session::connection_online_update(bool force) { - bool new_connection_online_flag = (online_flag_ || logging_out_flag_) && - (has_queries() || last_activity_timestamp_ + 10 > Time::now_cached() || is_main_); + bool new_connection_online_flag = + (online_flag_ || logging_out_flag_) && + (has_queries() || last_activity_timestamp_ + 10 > Time::now_cached() || is_primary_); if (connection_online_flag_ == new_connection_online_flag && !force) { return; } connection_online_flag_ = new_connection_online_flag; VLOG(dc) << "Set connection_online " << connection_online_flag_; if (main_connection_.connection_) { - main_connection_.connection_->set_online(connection_online_flag_, is_main_); + main_connection_.connection_->set_online(connection_online_flag_, is_primary_); } if (long_poll_connection_.connection_) { - long_poll_connection_.connection_->set_online(connection_online_flag_, is_main_); + long_poll_connection_.connection_->set_online(connection_online_flag_, is_primary_); } } @@ -1287,7 +1293,7 @@ void Session::connection_open_finish(ConnectionInfo *info, if (can_destroy_auth_key()) { info->connection_->destroy_key(); } - info->connection_->set_online(connection_online_flag_, is_main_); + info->connection_->set_online(connection_online_flag_, is_primary_); info->connection_->set_name(name); Scheduler::subscribe(info->connection_->get_poll_info().extract_pollable_fd(this)); info->mode_ = mode_; diff --git a/td/telegram/net/Session.h b/td/telegram/net/Session.h index 3eb61fee4..e16d9cd11 100644 --- a/td/telegram/net/Session.h +++ b/td/telegram/net/Session.h @@ -67,8 +67,8 @@ class Session final }; Session(unique_ptr callback, std::shared_ptr shared_auth_data, int32 raw_dc_id, int32 dc_id, - bool is_main, bool use_pfs, bool is_cdn, bool need_destroy, const mtproto::AuthKey &tmp_auth_key, - const vector &server_salts); + bool is_primary, bool is_main, bool use_pfs, bool is_cdn, bool need_destroy, + const mtproto::AuthKey &tmp_auth_key, const vector &server_salts); void send(NetQueryPtr &&query); @@ -106,6 +106,7 @@ class Session final const int32 raw_dc_id_; // numerical datacenter ID, i.e. 2 const int32 dc_id_; // unique datacenter ID, i.e. -10002 + const bool is_primary_; // true for primary Sessions to all DCs const bool is_main_; // true only for the primary Session(s) to the main DC const bool is_cdn_; const bool need_destroy_; diff --git a/td/telegram/net/SessionMultiProxy.cpp b/td/telegram/net/SessionMultiProxy.cpp index 489b1ece8..7c181804f 100644 --- a/td/telegram/net/SessionMultiProxy.cpp +++ b/td/telegram/net/SessionMultiProxy.cpp @@ -18,14 +18,14 @@ namespace td { -SessionMultiProxy::SessionMultiProxy() = default; SessionMultiProxy::~SessionMultiProxy() = default; SessionMultiProxy::SessionMultiProxy(int32 session_count, std::shared_ptr shared_auth_data, - bool is_main, bool use_pfs, bool allow_media_only, bool is_media, bool is_cdn, - bool need_destroy_auth_key) + bool is_primary, bool is_main, bool use_pfs, bool allow_media_only, bool is_media, + bool is_cdn, bool need_destroy_auth_key) : session_count_(session_count) , auth_data_(std::move(shared_auth_data)) + , is_primary_(is_primary) , is_main_(is_main) , use_pfs_(use_pfs) , allow_media_only_(allow_media_only) @@ -142,8 +142,8 @@ void SessionMultiProxy::init() { int32 session_id_; }; info.proxy = create_actor(name, make_unique(actor_id(this), sessions_generation_, i), - auth_data_, is_main_, allow_media_only_, is_media_, get_pfs_flag(), is_cdn_, - need_destroy_auth_key_ && i == 0); + auth_data_, is_primary_, is_main_, allow_media_only_, is_media_, + get_pfs_flag(), is_cdn_, need_destroy_auth_key_ && i == 0); sessions_.push_back(std::move(info)); } } diff --git a/td/telegram/net/SessionMultiProxy.h b/td/telegram/net/SessionMultiProxy.h index be9bff761..673476375 100644 --- a/td/telegram/net/SessionMultiProxy.h +++ b/td/telegram/net/SessionMultiProxy.h @@ -19,12 +19,12 @@ class SessionProxy; class SessionMultiProxy final : public Actor { public: - SessionMultiProxy(); + SessionMultiProxy(int32 session_count, std::shared_ptr shared_auth_data, bool is_primary, + bool is_main, bool use_pfs, bool allow_media_only, bool is_media, bool is_cdn, + bool need_destroy_auth_key); SessionMultiProxy(const SessionMultiProxy &other) = delete; SessionMultiProxy &operator=(const SessionMultiProxy &other) = delete; ~SessionMultiProxy() final; - SessionMultiProxy(int32 session_count, std::shared_ptr shared_auth_data, bool is_main, bool use_pfs, - bool allow_media_only, bool is_media, bool is_cdn, bool need_destroy_auth_key); void send(NetQueryPtr query); void update_main_flag(bool is_main); @@ -39,6 +39,7 @@ class SessionMultiProxy final : public Actor { private: int32 session_count_ = 0; std::shared_ptr auth_data_; + const bool is_primary_; bool is_main_ = false; bool use_pfs_ = false; bool allow_media_only_ = false; diff --git a/td/telegram/net/SessionProxy.cpp b/td/telegram/net/SessionProxy.cpp index 450f9c313..bb009fe87 100644 --- a/td/telegram/net/SessionProxy.cpp +++ b/td/telegram/net/SessionProxy.cpp @@ -78,10 +78,11 @@ class SessionCallback final : public Session::Callback { }; SessionProxy::SessionProxy(unique_ptr callback, std::shared_ptr shared_auth_data, - bool is_main, bool allow_media_only, bool is_media, bool use_pfs, bool is_cdn, - bool need_destroy) + bool is_primary, bool is_main, bool allow_media_only, bool is_media, bool use_pfs, + bool is_cdn, bool need_destroy) : callback_(std::move(callback)) , auth_data_(std::move(shared_auth_data)) + , is_primary_(is_primary) , is_main_(is_main) , allow_media_only_(allow_media_only) , is_media_(is_media) @@ -212,7 +213,8 @@ void SessionProxy::open_session(bool force) { session_ = create_actor( name, make_unique(actor_shared(this, session_generation_), dc_id, allow_media_only_, is_media_, hash), - auth_data_, raw_dc_id, int_dc_id, is_main_, use_pfs_, is_cdn_, need_destroy_, tmp_auth_key_, server_salts_); + auth_data_, raw_dc_id, int_dc_id, is_primary_, is_main_, use_pfs_, is_cdn_, need_destroy_, tmp_auth_key_, + server_salts_); } void SessionProxy::update_auth_key_state() { diff --git a/td/telegram/net/SessionProxy.h b/td/telegram/net/SessionProxy.h index 5ee652328..eb2f48e89 100644 --- a/td/telegram/net/SessionProxy.h +++ b/td/telegram/net/SessionProxy.h @@ -29,8 +29,8 @@ class SessionProxy final : public Actor { virtual void on_query_finished() = 0; }; - SessionProxy(unique_ptr callback, std::shared_ptr shared_auth_data, bool is_main, - bool allow_media_only, bool is_media, bool use_pfs, bool is_cdn, bool need_destroy); + SessionProxy(unique_ptr callback, std::shared_ptr shared_auth_data, bool is_primary, + bool is_main, bool allow_media_only, bool is_media, bool use_pfs, bool is_cdn, bool need_destroy); void send(NetQueryPtr query); void update_main_flag(bool is_main); @@ -41,6 +41,7 @@ class SessionProxy final : public Actor { unique_ptr callback_; std::shared_ptr auth_data_; AuthKeyState auth_key_state_ = AuthKeyState::Empty; + const bool is_primary_; bool is_main_; bool allow_media_only_; bool is_media_; From 45ff12ca49bf40c8c0fc8ce83bbb99af63b0ff8c Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 15 Jan 2023 11:05:53 +0300 Subject: [PATCH 19/30] Improve PTS/QTS spelling. --- td/telegram/MessagesManager.cpp | 84 ++++++++++++++++----------------- td/telegram/PtsManager.h | 2 +- td/telegram/SecretChatActor.cpp | 8 ++-- td/telegram/SecretChatActor.h | 4 +- td/telegram/Td.cpp | 2 +- td/telegram/UpdatesManager.cpp | 76 ++++++++++++++--------------- td/telegram/UpdatesManager.h | 2 +- 7 files changed, 89 insertions(+), 89 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index cf7cfd3fd..a1e740229 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -4721,7 +4721,7 @@ class GetChannelDifferenceQuery final : public Td::ResultHandler { void on_error(Status status) final { if (!td_->messages_manager_->on_get_dialog_error(dialog_id_, status, "GetChannelDifferenceQuery") && status.message() != "PERSISTENT_TIMESTAMP_INVALID") { - LOG(ERROR) << "Receive error for GetChannelDifferenceQuery for " << dialog_id_ << " with pts " << pts_ + LOG(ERROR) << "Receive error for GetChannelDifferenceQuery for " << dialog_id_ << " with PTS " << pts_ << " and limit " << limit_ << ": " << status; } td_->messages_manager_->on_get_channel_difference(dialog_id_, pts_, limit_, nullptr); @@ -6623,7 +6623,7 @@ void MessagesManager::skip_old_pending_pts_update(tl_object_ptrmessage_, false); if (update_message_ids_.count(full_message_id) > 0) { if (new_pts == old_pts || old_pts == std::numeric_limits::max()) { - // apply sent message anyway if it is definitely non-deleted or being skipped because of pts overflow + // apply sent message anyway if it is definitely non-deleted or being skipped because of PTS overflow auto added_full_message_id = on_get_message(std::move(update_new_message->message_), true, false, false, true, true, "updateNewMessage with an awaited message"); if (added_full_message_id != full_message_id) { @@ -6631,8 +6631,8 @@ void MessagesManager::skip_old_pending_pts_update(tl_object_ptr(update.get()); if (being_sent_messages_.count(update_sent_message->random_id_) > 0) { if (new_pts == old_pts || old_pts == std::numeric_limits::max()) { - // apply sent message anyway if it is definitely non-deleted or being skipped because of pts overflow + // apply sent message anyway if it is definitely non-deleted or being skipped because of PTS overflow on_send_message_success(update_sent_message->random_id_, update_sent_message->message_id_, update_sent_message->date_, update_sent_message->ttl_period_, FileId(), "process old updateSentMessage"); return; } else if (update_sent_message->random_id_ != 0) { - LOG(ERROR) << "Receive awaited sent " << update_sent_message->message_id_ << " from " << source << " with pts " - << new_pts << " and pts_count " << pts_count << ", but current pts is " << old_pts; + LOG(ERROR) << "Receive awaited sent " << update_sent_message->message_id_ << " from " << source << " with PTS " + << new_pts << " and pts_count " << pts_count << ", but current PTS is " << old_pts; dump_debug_message_op(get_dialog(being_sent_messages_[update_sent_message->random_id_].get_dialog_id()), 3); } } @@ -7866,7 +7866,7 @@ void MessagesManager::add_pending_channel_update(DialogId dialog_id, tl_object_p return; } if (pts_count < 0 || new_pts <= pts_count) { - LOG(ERROR) << "Receive channel update from " << source << " with wrong pts = " << new_pts + LOG(ERROR) << "Receive channel update from " << source << " with wrong PTS = " << new_pts << " or pts_count = " << pts_count << ": " << oneline(to_string(update)); promise.set_value(Unit()); return; @@ -7879,7 +7879,7 @@ void MessagesManager::add_pending_channel_update(DialogId dialog_id, tl_object_p return; } - // TODO need to save all updates that can change result of running queries not associated with pts (for example + // TODO need to save all updates that can change result of running queries not associated with PTS (for example // getHistory) and apply them to result of these queries Dialog *d = get_dialog_force(dialog_id, "add_pending_channel_update 2"); @@ -7894,14 +7894,14 @@ void MessagesManager::add_pending_channel_update(DialogId dialog_id, tl_object_p } if (new_pts <= pts && new_pts >= pts - 19999) { - LOG(INFO) << "There is no need to process an update with pts " << new_pts << " in " << dialog_id << " with pts " + LOG(INFO) << "There is no need to process an update with PTS " << new_pts << " in " << dialog_id << " with PTS " << pts; promise.set_value(Unit()); return; } if (new_pts > pts && pts != new_pts - pts_count) { - LOG(INFO) << "Found a gap in unknown " << dialog_id << " with pts = " << pts << ". new_pts = " << new_pts + LOG(INFO) << "Found a gap in unknown " << dialog_id << " with PTS = " << pts << ". new_pts = " << new_pts << ", pts_count = " << pts_count << " in update from " << source; add_postponed_channel_update(dialog_id, std::move(update), new_pts, pts_count, std::move(promise)); get_channel_difference(dialog_id, pts, true, "add_pending_channel_update 3"); @@ -7966,12 +7966,12 @@ void MessagesManager::add_pending_channel_update(DialogId dialog_id, tl_object_p } if (old_pts != new_pts - pts_count) { - LOG(INFO) << "Found a gap in the " << dialog_id << " with pts = " << old_pts << ". new_pts = " << new_pts + LOG(INFO) << "Found a gap in the " << dialog_id << " with PTS = " << old_pts << ". new_pts = " << new_pts << ", pts_count = " << pts_count << " in update from " << source; if (d->was_opened || td_->contacts_manager_->get_channel_status(channel_id).is_member() || is_dialog_sponsored(d)) { add_postponed_channel_update(dialog_id, std::move(update), new_pts, pts_count, std::move(promise)); - get_channel_difference(dialog_id, old_pts, true, "add_pending_channel_update pts mismatch"); + get_channel_difference(dialog_id, old_pts, true, "add_pending_channel_update PTS mismatch"); } else { promise.set_value(Unit()); } @@ -9683,7 +9683,7 @@ void MessagesManager::after_get_difference() { for (auto &it : update_message_ids_) { // there can be unhandled updateMessageId updates after getDifference even for ordinary chats, // because despite updates coming during getDifference have already been applied, - // some of them could be postponed because of pts gap + // some of them could be postponed because of PTS gap auto full_message_id = it.first; auto dialog_id = full_message_id.get_dialog_id(); auto message_id = full_message_id.get_message_id(); @@ -12646,7 +12646,7 @@ void MessagesManager::read_history_outbox(DialogId dialog_id, MessageId max_mess return; } - // it is impossible for just sent outgoing messages because updates are ordered by pts + // it is impossible for just sent outgoing messages because updates are ordered by PTS if (d->last_new_message_id.is_valid() && max_message_id > d->last_new_message_id && dialog_id.get_type() != DialogType::Channel) { LOG(INFO) << "Receive read outbox update about unknown " << max_message_id << " in " << dialog_id @@ -14751,7 +14751,7 @@ std::pair> MessagesManager::creat is_outgoing = supposed_to_be_outgoing; /* - // it is useless to call getChannelDifference, because the channel pts will be increased already + // it is useless to call getChannelDifference, because the channel PTS will be increased already if (dialog_type == DialogType::Channel && !running_get_difference_ && !running_get_channel_difference(dialog_id) && get_channel_difference_to_log_event_id_.count(dialog_id) == 0) { // it is safer to completely ignore the message and re-get it through getChannelDifference @@ -15842,16 +15842,16 @@ void MessagesManager::on_get_dialogs(FolderId folder_id, vector &&promise) { TRY_STATUS_PROMISE(promise, G()->close_status()); LOG(INFO) << "Receive " << (affected_history.is_final_ ? "final " : "partial ") - << "affected history with pts = " << affected_history.pts_ + << "affected history with PTS = " << affected_history.pts_ << " and pts_count = " << affected_history.pts_count_; if (affected_history.pts_count_ > 0) { @@ -27972,8 +27972,8 @@ void MessagesManager::on_message_media_edited(DialogId dialog_id, MessageId mess // updateMessageContent was already sent and needs to be sent again, // only if 'i' and 't' sizes from edited_content were added to the photo auto pts = result.ok(); - LOG(INFO) << "Successfully edited " << message_id << " in " << dialog_id << " with pts = " << pts - << " and last edit pts = " << m->last_edit_pts; + LOG(INFO) << "Successfully edited " << message_id << " in " << dialog_id << " with PTS = " << pts + << " and last edit PTS = " << m->last_edit_pts; std::swap(m->content, m->edited_content); bool need_send_update_message_content = m->edited_content->get_type() == MessageContentType::Photo && m->content->get_type() == MessageContentType::Photo; @@ -32031,7 +32031,7 @@ FullMessageId MessagesManager::on_send_message_success(int64 random_id, MessageI auto it = being_sent_messages_.find(random_id); if (it == being_sent_messages_.end()) { LOG(ERROR) << "Result from sendMessage for " << new_message_id << " with random_id " << random_id << " sent at " - << date << " comes from " << source << " after updateNewMessageId, but was not discarded by pts"; + << date << " comes from " << source << " after updateNewMessageId, but was not discarded by PTS"; return {}; } @@ -38975,7 +38975,7 @@ int32 MessagesManager::load_channel_pts(DialogId dialog_id) const { return 0; } auto pts = to_integer(G()->td_db()->get_binlog_pmc()->get(get_channel_pts_key(dialog_id))); - LOG(INFO) << "Load " << dialog_id << " pts = " << pts; + LOG(INFO) << "Load " << dialog_id << " PTS = " << pts; return pts; } @@ -38984,7 +38984,7 @@ void MessagesManager::set_channel_pts(Dialog *d, int32 new_pts, const char *sour CHECK(d->dialog_id.get_type() == DialogType::Channel); LOG_IF(ERROR, running_get_channel_difference(d->dialog_id)) - << "Set pts of " << d->dialog_id << " to " << new_pts << " from " << source + << "Set PTS of " << d->dialog_id << " to " << new_pts << " from " << source << " while running getChannelDifference"; if (is_debug_message_op_enabled()) { @@ -38993,7 +38993,7 @@ void MessagesManager::set_channel_pts(Dialog *d, int32 new_pts, const char *sour // TODO delete_first_messages support in channels if (new_pts == std::numeric_limits::max()) { - LOG(ERROR) << "Update " << d->dialog_id << " pts to -1 from " << source; + LOG(ERROR) << "Update " << d->dialog_id << " PTS to -1 from " << source; G()->td_db()->get_binlog_pmc()->erase(get_channel_pts_key(d->dialog_id)); d->pts = std::numeric_limits::max(); if (d->pending_read_channel_inbox_pts != 0) { @@ -39001,12 +39001,12 @@ void MessagesManager::set_channel_pts(Dialog *d, int32 new_pts, const char *sour } return; } - if (new_pts > d->pts || (0 < new_pts && new_pts < d->pts - 99999)) { // pts can only go up or drop cardinally + if (new_pts > d->pts || (0 < new_pts && new_pts < d->pts - 99999)) { // PTS can only go up or drop cardinally if (new_pts < d->pts - 99999) { - LOG(WARNING) << "Pts of " << d->dialog_id << " decreases from " << d->pts << " to " << new_pts << " from " + LOG(WARNING) << "PTS of " << d->dialog_id << " decreases from " << d->pts << " to " << new_pts << " from " << source; } else { - LOG(INFO) << "Update " << d->dialog_id << " pts to " << new_pts << " from " << source; + LOG(INFO) << "Update " << d->dialog_id << " PTS to " << new_pts << " from " << source; } d->pts = new_pts; @@ -39025,7 +39025,7 @@ void MessagesManager::set_channel_pts(Dialog *d, int32 new_pts, const char *sour G()->td_db()->get_binlog_pmc()->set(get_channel_pts_key(d->dialog_id), to_string(new_pts)); } } else if (new_pts < d->pts) { - LOG(ERROR) << "Receive wrong pts " << new_pts << " in " << d->dialog_id << " from " << source << ". Current pts is " + LOG(ERROR) << "Receive wrong PTS " << new_pts << " in " << d->dialog_id << " from " << source << ". Current PTS is " << d->pts; } } @@ -39181,7 +39181,7 @@ void MessagesManager::do_get_channel_difference(DialogId dialog_id, int32 pts, b limit = MIN_CHANNEL_DIFFERENCE; } - LOG(INFO) << "-----BEGIN GET CHANNEL DIFFERENCE----- for " << dialog_id << " with pts " << pts << " and limit " + LOG(INFO) << "-----BEGIN GET CHANNEL DIFFERENCE----- for " << dialog_id << " with PTS " << pts << " and limit " << limit << " from " << source; td_->create_handler()->send(dialog_id, std::move(input_channel), pts, limit, force); @@ -39503,7 +39503,7 @@ void MessagesManager::on_get_channel_difference( channel_get_difference_retry_timeouts_.erase(dialog_id); - LOG(INFO) << "Receive result of getChannelDifference for " << dialog_id << " with pts = " << request_pts + LOG(INFO) << "Receive result of getChannelDifference for " << dialog_id << " with PTS = " << request_pts << " and limit = " << request_limit << " from " << source << ": " << to_string(difference_ptr); bool have_new_messages = false; @@ -39557,18 +39557,18 @@ void MessagesManager::on_get_channel_difference( int32 flags = difference->flags_; is_final = (flags & CHANNEL_DIFFERENCE_FLAG_IS_FINAL) != 0; LOG_IF(ERROR, !is_final) << "Receive channelDifferenceEmpty as result of getChannelDifference from " << source - << " with pts = " << request_pts << " and limit = " << request_limit << " in " + << " with PTS = " << request_pts << " and limit = " << request_limit << " in " << dialog_id << ", but it is not final"; if (flags & CHANNEL_DIFFERENCE_FLAG_HAS_TIMEOUT) { timeout = difference->timeout_; } - // bots can receive channelDifferenceEmpty with pts bigger than known pts + // bots can receive channelDifferenceEmpty with PTS bigger than known PTS // also, this can happen for deleted channels if (request_pts != difference->pts_ && !td_->auth_manager_->is_bot() && have_input_peer(dialog_id, AccessRights::Read)) { LOG(ERROR) << "Receive channelDifferenceEmpty as result of getChannelDifference from " << source - << " with pts = " << request_pts << " and limit = " << request_limit << " in " << dialog_id + << " with PTS = " << request_pts << " and limit = " << request_limit << " in " << dialog_id << ", but PTS has changed to " << difference->pts_; } set_channel_pts(d, difference->pts_, "channel difference empty"); @@ -39586,7 +39586,7 @@ void MessagesManager::on_get_channel_difference( auto new_pts = difference->pts_; if (request_pts >= new_pts && request_pts > 1 && (request_pts > new_pts || !td_->auth_manager_->is_bot())) { LOG(ERROR) << "Receive channelDifference as result of getChannelDifference from " << source - << " with pts = " << request_pts << " and limit = " << request_limit << " in " << dialog_id + << " with PTS = " << request_pts << " and limit = " << request_limit << " in " << dialog_id << ", but PTS has changed from " << d->pts << " to " << new_pts << ". Difference: " << oneline(to_string(difference)); new_pts = request_pts + 1; @@ -39599,7 +39599,7 @@ void MessagesManager::on_get_channel_difference( auto message_id = MessageId::get_message_id(message, false); if (message_id <= cur_message_id) { LOG(ERROR) << "Receive " << cur_message_id << " after " << message_id << " in channelDifference of " - << dialog_id << " from " << source << " with pts " << request_pts << " and limit " + << dialog_id << " from " << source << " with PTS " << request_pts << " and limit " << request_limit << ": " << to_string(difference); after_get_channel_difference(dialog_id, false); return; @@ -39637,7 +39637,7 @@ void MessagesManager::on_get_channel_difference( CHECK(dialog != nullptr); if ((dialog->flags_ & telegram_api::dialog::PTS_MASK) == 0) { - LOG(ERROR) << "Receive " << dialog_id << " without pts"; + LOG(ERROR) << "Receive " << dialog_id << " without PTS"; return after_get_channel_difference(dialog_id, false); } @@ -39650,7 +39650,7 @@ void MessagesManager::on_get_channel_difference( auto new_pts = dialog->pts_; if (request_pts > new_pts - request_limit) { LOG(ERROR) << "Receive channelDifferenceTooLong as result of getChannelDifference from " << source - << " with pts = " << request_pts << " and limit = " << request_limit << " in " << dialog_id + << " with PTS = " << request_pts << " and limit = " << request_limit << " in " << dialog_id << ", but PTS has changed from " << d->pts << " to " << new_pts << ". Difference: " << oneline(to_string(difference)); if (request_pts >= new_pts) { @@ -39747,13 +39747,13 @@ void MessagesManager::after_get_channel_difference(DialogId dialog_id, bool succ } if (updates.size() != old_size || running_get_channel_difference(dialog_id)) { if (success && update_pts - 10000 < pts && update_pts_count == 1) { - // if getChannelDifference was successful and update pts is near channel pts, + // if getChannelDifference was successful and update PTS is near channel PTS, // we hope that the update eventually can be applied LOG(INFO) << "Can't apply postponed channel updates"; } else { // otherwise protect from getChannelDifference repeating calls by dropping postponed updates LOG(WARNING) << "Failed to apply postponed updates of type " << update_id << " in " << dialog_id - << " with pts " << pts << ", update pts is " << update_pts << ", update pts count is " + << " with PTS " << pts << ", update PTS is " << update_pts << ", update PTS count is " << update_pts_count; vector> update_promises; for (auto &postponed_update : updates) { diff --git a/td/telegram/PtsManager.h b/td/telegram/PtsManager.h index 1c6307b2a..6a508fa34 100644 --- a/td/telegram/PtsManager.h +++ b/td/telegram/PtsManager.h @@ -12,7 +12,7 @@ namespace td { // It is not about handling gaps. -// It is about finding mem processed pts. +// It is about finding mem processed PTS. // All checks must be done before. class PtsManager { diff --git a/td/telegram/SecretChatActor.cpp b/td/telegram/SecretChatActor.cpp index f3b33547e..b1bbfc624 100644 --- a/td/telegram/SecretChatActor.cpp +++ b/td/telegram/SecretChatActor.cpp @@ -1158,7 +1158,7 @@ void SecretChatActor::do_inbound_message_decrypted_pending(unique_ptrlog_event_id(); - // qts + // QTS auto qts_promise = std::move(message->promise); if (log_event_id == 0) { @@ -1185,9 +1185,9 @@ Status SecretChatActor::do_inbound_message_decrypted(unique_ptr Add message to MessageManager [save_message] // Note: if we are able to add message by random_id, we may not wait for (log event). Otherwise, we should force // binlog flush. - // 4. [save_log_event] => Update qts [qts] + // 4. [save_log_event] => Update QTS [qts] // 5. [save_changes; save_message; ?qts) => Remove log event [remove_log_event] - // Note: It is easier not to wait for qts. In the worst case old update will be handled again after restart. + // Note: It is easier not to wait for QTS. In the worst case old update will be handled again after restart. auto state_id = inbound_message_states_.create(); InboundMessageState &state = *inbound_message_states_.get(state_id); @@ -1236,7 +1236,7 @@ Status SecretChatActor::do_inbound_message_decrypted(unique_ptrpromise); // process message diff --git a/td/telegram/SecretChatActor.h b/td/telegram/SecretChatActor.h index f42779fd8..478b5a4fc 100644 --- a/td/telegram/SecretChatActor.h +++ b/td/telegram/SecretChatActor.h @@ -111,7 +111,7 @@ class SecretChatActor final : public NetQueryCallback { void cancel_chat(bool delete_history, bool is_already_discarded, Promise<> promise); // Inbound messages - // Logevent is created by SecretChatsManager, because it must contain qts + // Logevent is created by SecretChatsManager, because it must contain QTS void add_inbound_message(unique_ptr message); // Outbound messages @@ -479,7 +479,7 @@ class SecretChatActor final : public NetQueryCallback { // This is completly flawed. // (A-start_save_to_binlog ----> B-start_save_to_binlog+change_memory ----> A-finish_save_to_binlog+surprise) // - // Instead, I suggest general solution that is already used with SeqNoState and qts + // Instead, I suggest general solution that is already used with SeqNoState and QTS // 1. We APPLY CHANGE to memory immediately AFTER corresponding EVENT is SENT to the binlog. // 2. We SEND CHANGE to database only after corresponding EVENT is SAVED to the binlog. // 3. Then, we are able to ERASE EVENT just AFTER the CHANGE is SAVED to the binlog. diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index dda737c73..f05ae9e7d 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -3464,7 +3464,7 @@ void Td::close_impl(bool destroy_flag) { close_flag_ = 1; G()->set_close_flag(); send_closure(auth_manager_actor_, &AuthManager::on_closing, destroy_flag); - updates_manager_->timeout_expired(); // save pts and qts + updates_manager_->timeout_expired(); // save PTS and QTS // wait till all request_actors will stop request_actors_.clear(); diff --git a/td/telegram/UpdatesManager.cpp b/td/telegram/UpdatesManager.cpp index fee041b45..f609fc56d 100644 --- a/td/telegram/UpdatesManager.cpp +++ b/td/telegram/UpdatesManager.cpp @@ -250,7 +250,7 @@ void UpdatesManager::fill_pts_gap(void *td) { min_pts = min(min_pts, updates_manager->postponed_pts_updates_.begin()->first); max_pts = max(max_pts, updates_manager->postponed_pts_updates_.rbegin()->first); } - string source = PSTRING() << "pts from " << updates_manager->get_pts() << " to " << min_pts << '-' << max_pts; + string source = PSTRING() << "PTS from " << updates_manager->get_pts() << " to " << min_pts << '-' << max_pts; fill_gap(td, source.c_str()); } @@ -284,7 +284,7 @@ void UpdatesManager::fill_qts_gap(void *td) { min_qts = updates_manager->pending_qts_updates_.begin()->first; max_qts = updates_manager->pending_qts_updates_.rbegin()->first; } - string source = PSTRING() << "qts from " << updates_manager->get_qts() << " to " << min_qts << '-' << max_qts; + string source = PSTRING() << "QTS from " << updates_manager->get_qts() << " to " << min_qts << '-' << max_qts; fill_gap(td, source.c_str()); } @@ -354,7 +354,7 @@ void UpdatesManager::run_get_difference(bool is_recursive, const char *source) { pts = 0; } - VLOG(get_difference) << "-----BEGIN GET DIFFERENCE----- from " << source << " with pts = " << pts << ", qts = " << qts + VLOG(get_difference) << "-----BEGIN GET DIFFERENCE----- from " << source << " with PTS = " << pts << ", QTS = " << qts << ", date = " << date; before_get_difference(false); @@ -471,27 +471,27 @@ void UpdatesManager::timeout_expired() { Promise<> UpdatesManager::set_pts(int32 pts, const char *source) { if (pts == std::numeric_limits::max()) { - LOG(WARNING) << "Update pts from " << get_pts() << " to -1 from " << source; + LOG(WARNING) << "Update PTS from " << get_pts() << " to -1 from " << source; save_pts(pts); auto result = add_pts(pts); init_state(); return result; } Promise<> result; - if (pts > get_pts() || (0 < pts && pts < get_pts() - 399999)) { // pts can only go up or drop cardinally + if (pts > get_pts() || (0 < pts && pts < get_pts() - 399999)) { // PTS can only go up or drop cardinally if (pts < get_pts() - 399999) { LOG(WARNING) << "PTS decreases from " << get_pts() << " to " << pts << " from " << source; } else { - LOG(INFO) << "Update pts from " << get_pts() << " to " << pts << " from " << source; + LOG(INFO) << "Update PTS from " << get_pts() << " to " << pts << " from " << source; } result = add_pts(pts); if (last_get_difference_pts_ < get_pts() - FORCED_GET_DIFFERENCE_PTS_DIFF) { last_get_difference_pts_ = get_pts(); - schedule_get_difference("rare pts getDifference"); + schedule_get_difference("rare PTS getDifference"); } } else if (pts < get_pts() && (pts > 1 || td_->option_manager_->get_option_integer("session_count") <= 1)) { - LOG(ERROR) << "Receive wrong pts = " << pts << " from " << source << ". Current pts = " << get_pts(); + LOG(ERROR) << "Receive wrong PTS = " << pts << " from " << source << ". Current PTS = " << get_pts(); } return result; } @@ -1070,8 +1070,8 @@ void UpdatesManager::schedule_get_difference(const char *source) { return; } if (!retry_timeout_.has_timeout()) { - LOG(WARNING) << "Schedule getDifference in " << retry_time_ << " seconds with pts = " << get_pts() - << ", qts = " << get_qts() << ", date = " << get_date() << " from " << source; + LOG(WARNING) << "Schedule getDifference in " << retry_time_ << " seconds with PTS = " << get_pts() + << ", QTS = " << get_qts() << ", date = " << get_date() << " from " << source; retry_timeout_.set_callback(std::move(fill_get_difference_gap)); retry_timeout_.set_callback_data(static_cast(td_)); retry_timeout_.set_timeout_in(retry_time_); @@ -1091,12 +1091,12 @@ void UpdatesManager::on_get_updates_state(tl_object_ptrunread_count; if (get_pts() == std::numeric_limits::max()) { - LOG(WARNING) << "Restore pts to " << state->pts_; - // restoring right pts + LOG(WARNING) << "Restore PTS to " << state->pts_; + // restoring right PTS CHECK(pending_pts_updates_.empty()); auto real_running_get_difference = running_get_difference_; running_get_difference_ = false; - process_postponed_pts_updates(); // drop all updates with old pts + process_postponed_pts_updates(); // drop all updates with old PTS running_get_difference_ = real_running_get_difference; pts_manager_.init(state->pts_); last_get_difference_pts_ = get_pts(); @@ -1583,7 +1583,7 @@ void UpdatesManager::on_get_difference(tl_object_ptr= UPDATE_APPLY_WARNING_TIME) { - LOG(WARNING) << "Applied " << update_count << " pts updates in " << passed_time + LOG(WARNING) << "Applied " << update_count << " PTS updates in " << passed_time << " seconds after postponing them for " << (Time::now() - get_difference_start_time_) << " seconds"; } } @@ -2164,16 +2164,16 @@ void UpdatesManager::add_pending_qts_update(tl_object_ptr Promise &&promise) { CHECK(update != nullptr); if (qts <= 1) { - LOG(ERROR) << "Receive wrong qts " << qts << " in " << oneline(to_string(update)); - schedule_get_difference("wrong qts"); + LOG(ERROR) << "Receive wrong QTS " << qts << " in " << oneline(to_string(update)); + schedule_get_difference("wrong QTS"); promise.set_value(Unit()); return; } int32 old_qts = get_qts(); - LOG(INFO) << "Process update with qts = " << qts << ", current qts = " << old_qts; + LOG(INFO) << "Process update with QTS = " << qts << ", current QTS = " << old_qts; if (qts < old_qts - 100001) { - LOG(WARNING) << "Restore qts after qts overflow from " << old_qts << " to " << qts << " by " + LOG(WARNING) << "Restore QTS after QTS overflow from " << old_qts << " to " << qts << " by " << oneline(to_string(update)); add_qts(qts - 1).set_value(Unit()); CHECK(get_qts() == qts - 1); @@ -2182,19 +2182,19 @@ void UpdatesManager::add_pending_qts_update(tl_object_ptr } if (qts <= old_qts) { - LOG(INFO) << "Skip already applied update with qts = " << qts; + LOG(INFO) << "Skip already applied update with QTS = " << qts; promise.set_value(Unit()); return; } if (running_get_difference_ || (qts - 1 > old_qts && old_qts > 0)) { - LOG(INFO) << "Postpone update with qts = " << qts; + LOG(INFO) << "Postpone update with QTS = " << qts; if (!running_get_difference_ && pending_qts_updates_.empty()) { set_qts_gap_timeout(MAX_UNFILLED_GAP_TIME); } auto &pending_update = pending_qts_updates_[qts]; if (pending_update.update != nullptr) { - LOG(WARNING) << "Receive duplicate update with qts = " << qts; + LOG(WARNING) << "Receive duplicate update with QTS = " << qts; } else { pending_update.receive_time = Time::now(); } @@ -2328,11 +2328,11 @@ void UpdatesManager::process_updates(vector> void UpdatesManager::process_pts_update(tl_object_ptr &&update) { CHECK(update != nullptr); - // TODO need to save all updates that can change result of running queries not associated with pts (for example + // TODO need to save all updates that can change result of running queries not associated with PTS (for example // getHistory) and apply the updates to results of the queries if (!check_pts_update(update)) { - LOG(ERROR) << "Receive wrong pts update: " << oneline(to_string(update)); + LOG(ERROR) << "Receive wrong PTS update: " << oneline(to_string(update)); update = nullptr; return; } @@ -2352,16 +2352,16 @@ void UpdatesManager::add_pending_pts_update(tl_object_ptr CHECK(source != nullptr); LOG(INFO) << "Receive from " << source << " pending " << to_string(update); if (pts_count < 0 || new_pts <= pts_count) { - LOG(ERROR) << "Receive update with wrong pts = " << new_pts << " or pts_count = " << pts_count << " from " << source + LOG(ERROR) << "Receive update with wrong PTS = " << new_pts << " or pts_count = " << pts_count << " from " << source << ": " << oneline(to_string(update)); return promise.set_value(Unit()); } - // TODO need to save all updates that can change result of running queries not associated with pts (for example + // TODO need to save all updates that can change result of running queries not associated with PTS (for example // getHistory) and apply them to result of this queries if (!check_pts_update(update)) { - LOG(ERROR) << "Receive wrong pts update from " << source << ": " << oneline(to_string(update)); + LOG(ERROR) << "Receive wrong PTS update from " << source << ": " << oneline(to_string(update)); return promise.set_value(Unit()); } @@ -2376,7 +2376,7 @@ void UpdatesManager::add_pending_pts_update(tl_object_ptr auto now = Time::now(); if (now > last_pts_jump_warning_time_ + 1 && (need_restore_pts || now < last_pts_jump_warning_time_ + 5) && !(old_pts == std::numeric_limits::max() && running_get_difference_)) { - LOG(ERROR) << "Restore pts after delete_first_messages from " << old_pts << " to " << new_pts + LOG(ERROR) << "Restore PTS after delete_first_messages from " << old_pts << " to " << new_pts << " is disabled, pts_count = " << pts_count << ", update is from " << source << ": " << oneline(to_string(update)); last_pts_jump_warning_time_ = now; @@ -2385,8 +2385,8 @@ void UpdatesManager::add_pending_pts_update(tl_object_ptr set_pts_gap_timeout(0.001); /* - LOG(WARNING) << "Restore pts after delete_first_messages"; - set_pts(new_pts - 1, "restore pts after delete_first_messages"); + LOG(WARNING) << "Restore PTS after delete_first_messages"; + set_pts(new_pts - 1, "restore PTS after delete_first_messages"); old_pts = get_pts(); CHECK(old_pts == new_pts - 1); */ @@ -2408,7 +2408,7 @@ void UpdatesManager::add_pending_pts_update(tl_object_ptr return; } - // is_acceptable_update check was skipped for postponed pts updates + // is_acceptable_update check was skipped for postponed PTS updates if (source == AFTER_GET_DIFFERENCE_SOURCE && !is_acceptable_update(update.get())) { LOG(INFO) << "Postpone again unacceptable pending update"; postpone_pts_update(std::move(update), new_pts, pts_count, receive_time, std::move(promise)); @@ -2507,7 +2507,7 @@ void UpdatesManager::process_qts_update(tl_object_ptr &&up LOG(DEBUG) << "Process " << to_string(update_ptr); if (last_get_difference_qts_ < qts - FORCED_GET_DIFFERENCE_PTS_DIFF) { if (last_get_difference_qts_ != 0) { - schedule_get_difference("rare qts getDifference"); + schedule_get_difference("rare QTS getDifference"); } last_get_difference_qts_ = qts; } @@ -2576,7 +2576,7 @@ void UpdatesManager::process_all_pending_pts_updates() { auto diff = Time::now() - last_pts_gap_time_; last_pts_gap_time_ = 0; if (diff > 0.1) { - VLOG(get_difference) << "Gap in pts from " << accumulated_pts_ - accumulated_pts_count_ << " to " + VLOG(get_difference) << "Gap in PTS from " << accumulated_pts_ - accumulated_pts_count_ << " to " << accumulated_pts_ << " has been filled in " << begin_diff << '-' << diff << " seconds"; } } @@ -2631,7 +2631,7 @@ void UpdatesManager::process_postponed_pts_updates() { if (old_pts > new_pts - pts_count || last_update_it == postponed_pts_updates_.end() || i == GAP_TIMEOUT_UPDATE_COUNT) { // the updates can't be applied - VLOG(get_difference) << "Can't apply " << i << " next postponed updates with pts " << update_it->second.pts + VLOG(get_difference) << "Can't apply " << i << " next postponed updates with PTS " << update_it->second.pts << '-' << new_pts << ", because their pts_count is " << pts_count << " instead of expected " << new_pts - old_pts; last_update_it = update_it; @@ -2795,7 +2795,7 @@ void UpdatesManager::process_pending_qts_updates() { return; } - LOG(DEBUG) << "Process " << pending_qts_updates_.size() << " pending qts updates"; + LOG(DEBUG) << "Process " << pending_qts_updates_.size() << " pending QTS updates"; auto begin_time = Time::now(); auto initial_qts = get_qts(); int32 applied_update_count = 0; diff --git a/td/telegram/UpdatesManager.h b/td/telegram/UpdatesManager.h index 345645ef1..c00e2b969 100644 --- a/td/telegram/UpdatesManager.h +++ b/td/telegram/UpdatesManager.h @@ -225,7 +225,7 @@ class UpdatesManager final : public Actor { std::multimap postponed_updates_; // updates received during getDifference std::multimap pending_seq_updates_; // updates with too big seq - std::map pending_qts_updates_; // updates with too big qts + std::map pending_qts_updates_; // updates with too big QTS Timeout pts_gap_timeout_; From 052d5e120b27cba2e139de0b72f40ab78ba090f3 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 15 Jan 2023 11:25:54 +0300 Subject: [PATCH 20/30] Improve some mixed-case spelling. --- td/telegram/InlineQueriesManager.cpp | 2 +- td/telegram/WebPagesManager.cpp | 2 +- td/telegram/logevent/SecretChatEvent.h | 2 +- td/telegram/net/Session.cpp | 6 +++--- td/telegram/net/Session.h | 2 +- tdutils/td/utils/OrderedEventsProcessor.h | 2 +- tdutils/td/utils/port/UdpSocketFd.h | 1 - 7 files changed, 8 insertions(+), 9 deletions(-) diff --git a/td/telegram/InlineQueriesManager.cpp b/td/telegram/InlineQueriesManager.cpp index 5ce024cc1..a1546fd0f 100644 --- a/td/telegram/InlineQueriesManager.cpp +++ b/td/telegram/InlineQueriesManager.cpp @@ -1682,7 +1682,7 @@ void InlineQueriesManager::on_get_inline_query_results(DialogId dialog_id, UserI article->hide_url_ = true; } else { LOG_IF(ERROR, result->url_ != article->url_) - << "Url has changed from " << article->url_ << " to " << result->url_; + << "URL has changed from " << article->url_ << " to " << result->url_; article->hide_url_ = false; } article->title_ = std::move(result->title_); diff --git a/td/telegram/WebPagesManager.cpp b/td/telegram/WebPagesManager.cpp index 0539b355f..95e91dfe4 100644 --- a/td/telegram/WebPagesManager.cpp +++ b/td/telegram/WebPagesManager.cpp @@ -696,7 +696,7 @@ void WebPagesManager::on_get_web_page_by_url(const string &url, WebPageId web_pa } if (cached_web_page_id.is_valid() && web_page_id.is_valid() && web_page_id != cached_web_page_id) { - LOG(ERROR) << "Url \"" << url << "\" preview is changed from " << cached_web_page_id << " to " << web_page_id; + LOG(ERROR) << "URL \"" << url << "\" preview is changed from " << cached_web_page_id << " to " << web_page_id; } cached_web_page_id = web_page_id; diff --git a/td/telegram/logevent/SecretChatEvent.h b/td/telegram/logevent/SecretChatEvent.h index 45f07bf66..3218a072e 100644 --- a/td/telegram/logevent/SecretChatEvent.h +++ b/td/telegram/logevent/SecretChatEvent.h @@ -219,7 +219,7 @@ inline StringBuilder &operator<<(StringBuilder &sb, const EncryptedInputFile &fi } // LogEvents -// TODO: Qts and SeqNoState could be just Logevents that are updated during regenerate +// TODO: QTS and SeqNoState could be just Logevents that are updated during regenerate class InboundSecretMessage final : public SecretChatLogEventBase { public: static constexpr Type type = SecretChatEvent::Type::InboundSecretMessage; diff --git a/td/telegram/net/Session.cpp b/td/telegram/net/Session.cpp index 2a8d29cb1..b167f457e 100644 --- a/td/telegram/net/Session.cpp +++ b/td/telegram/net/Session.cpp @@ -1277,14 +1277,14 @@ void Session::connection_open_finish(ConnectionInfo *info, Slice mode_name; if (mode_ == Mode::Tcp) { mode = mtproto::SessionConnection::Mode::Tcp; - mode_name = Slice("Tcp"); + mode_name = Slice("TCP"); } else { if (info->connection_id_ == 0) { mode = mtproto::SessionConnection::Mode::Http; - mode_name = Slice("Http"); + mode_name = Slice("HTTP"); } else { mode = mtproto::SessionConnection::Mode::HttpLongPoll; - mode_name = Slice("HttpLongPoll"); + mode_name = Slice("LongPoll"); } } auto name = PSTRING() << get_name() << "::Connect::" << mode_name << "::" << raw_connection->extra().debug_str; diff --git a/td/telegram/net/Session.h b/td/telegram/net/Session.h index e16d9cd11..02d1d4dee 100644 --- a/td/telegram/net/Session.h +++ b/td/telegram/net/Session.h @@ -269,7 +269,7 @@ class Session final void raw_event(const Event::Raw &event) final; friend StringBuilder &operator<<(StringBuilder &sb, Mode mode) { - return sb << (mode == Mode::Http ? "Http" : "Tcp"); + return sb << (mode == Mode::Http ? "HTTP" : "TCP"); } }; diff --git a/tdutils/td/utils/OrderedEventsProcessor.h b/tdutils/td/utils/OrderedEventsProcessor.h index f472fc320..d5c638eb8 100644 --- a/tdutils/td/utils/OrderedEventsProcessor.h +++ b/tdutils/td/utils/OrderedEventsProcessor.h @@ -13,7 +13,7 @@ namespace td { -// Process states in order defined by their Id +// Process states in order defined by their SeqNo template class OrderedEventsProcessor { public: diff --git a/tdutils/td/utils/port/UdpSocketFd.h b/tdutils/td/utils/port/UdpSocketFd.h index 92489f6b1..67cdf801a 100644 --- a/tdutils/td/utils/port/UdpSocketFd.h +++ b/tdutils/td/utils/port/UdpSocketFd.h @@ -20,7 +20,6 @@ #include namespace td { -// Udp and errors namespace detail { class UdpSocketFdImpl; class UdpSocketFdImplDeleter { From 8d8f72a17a34d10281f27d65aa5610838526a071 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 15 Jan 2023 12:00:26 +0300 Subject: [PATCH 21/30] Fix some lower-case spelling. --- td/telegram/InputInvoice.cpp | 4 ++-- td/telegram/LanguagePackManager.cpp | 2 +- td/telegram/WebPagesManager.cpp | 14 +++++++------- td/telegram/files/FileLocation.h | 2 +- td/telegram/net/Session.cpp | 2 +- tdutils/CMakeLists.txt | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/td/telegram/InputInvoice.cpp b/td/telegram/InputInvoice.cpp index ff7789138..7f48f7838 100644 --- a/td/telegram/InputInvoice.cpp +++ b/td/telegram/InputInvoice.cpp @@ -136,13 +136,13 @@ Result InputInvoice::process_input_message_invoice( auto r_http_url = parse_url(input_invoice->photo_url_); if (r_http_url.is_error()) { if (!input_invoice->photo_url_.empty()) { - LOG(INFO) << "Can't register url " << input_invoice->photo_url_; + LOG(INFO) << "Can't register URL " << input_invoice->photo_url_; } } else { auto url = r_http_url.ok().get_url(); auto r_invoice_file_id = td->file_manager_->from_persistent_id(url, FileType::Temp); if (r_invoice_file_id.is_error()) { - LOG(INFO) << "Can't register url " << url; + LOG(INFO) << "Can't register URL " << url; } else { auto invoice_file_id = r_invoice_file_id.move_as_ok(); diff --git a/td/telegram/LanguagePackManager.cpp b/td/telegram/LanguagePackManager.cpp index 154b346fa..345003b5a 100644 --- a/td/telegram/LanguagePackManager.cpp +++ b/td/telegram/LanguagePackManager.cpp @@ -1688,7 +1688,7 @@ Result LanguagePackManager::get_language_info return Status::Error(400, "Language pack plural code must be encoded in UTF-8"); } if (!clean_input_string(language_pack_info->translation_url_)) { - return Status::Error(400, "Language pack translation url must be encoded in UTF-8"); + return Status::Error(400, "Language pack translation URL must be encoded in UTF-8"); } if (language_pack_info->total_string_count_ < 0) { language_pack_info->total_string_count_ = 0; diff --git a/td/telegram/WebPagesManager.cpp b/td/telegram/WebPagesManager.cpp index 95e91dfe4..7cddd3a2c 100644 --- a/td/telegram/WebPagesManager.cpp +++ b/td/telegram/WebPagesManager.cpp @@ -207,7 +207,7 @@ class WebPagesManager::WebPageInstantView { friend StringBuilder &operator<<(StringBuilder &string_builder, const WebPagesManager::WebPageInstantView &instant_view) { - return string_builder << "InstantView(url = " << instant_view.url << ", size = " << instant_view.page_blocks.size() + return string_builder << "InstantView(URL = " << instant_view.url << ", size = " << instant_view.page_blocks.size() << ", view_count = " << instant_view.view_count << ", hash = " << instant_view.hash << ", is_empty = " << instant_view.is_empty << ", is_v2 = " << instant_view.is_v2 << ", is_rtl = " << instant_view.is_rtl << ", is_full = " << instant_view.is_full @@ -781,7 +781,7 @@ void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const st void WebPagesManager::on_get_web_page_preview_fail(int64 request_id, const string &url, Status error, Promise &&promise) { - LOG(INFO) << "Clean up getting of web page preview with url \"" << url << '"'; + LOG(INFO) << "Clean up getting of web page preview with URL \"" << url << '"'; CHECK(error.is_error()); promise.set_error(std::move(error)); } @@ -830,7 +830,7 @@ tl_object_ptr WebPagesManager::get_web_page_preview_result(int6 } void WebPagesManager::get_web_page_instant_view(const string &url, bool force_full, Promise &&promise) { - LOG(INFO) << "Trying to get web page instant view for the url \"" << url << '"'; + LOG(INFO) << "Trying to get web page instant view for the URL \"" << url << '"'; if (url.empty()) { return promise.set_value(WebPageId()); } @@ -1046,16 +1046,16 @@ WebPageId WebPagesManager::get_web_page_by_url(const string &url) const { auto it = url_to_web_page_id_.find(url); if (it != url_to_web_page_id_.end()) { - LOG(INFO) << "Return " << it->second << " for the url \"" << url << '"'; + LOG(INFO) << "Return " << it->second << " for the URL \"" << url << '"'; return it->second; } - LOG(INFO) << "Can't find web page identifier for the url \"" << url << '"'; + LOG(INFO) << "Can't find web page identifier for the URL \"" << url << '"'; return WebPageId(); } void WebPagesManager::get_web_page_by_url(const string &url, Promise &&promise) { - LOG(INFO) << "Trying to get web page identifier for the url \"" << url << '"'; + LOG(INFO) << "Trying to get web page identifier for the URL \"" << url << '"'; if (url.empty()) { return promise.set_value(WebPageId()); } @@ -1089,7 +1089,7 @@ void WebPagesManager::load_web_page_by_url(string url, Promise &&prom void WebPagesManager::on_load_web_page_id_by_url_from_database(string url, string value, Promise &&promise) { TRY_STATUS_PROMISE(promise, G()->close_status()); - LOG(INFO) << "Successfully loaded url \"" << url << "\" of size " << value.size() << " from database"; + LOG(INFO) << "Successfully loaded URL \"" << url << "\" of size " << value.size() << " from database"; // G()->td_db()->get_sqlite_pmc()->erase(get_web_page_url_database_key(web_page_id), Auto()); // value.clear(); diff --git a/td/telegram/files/FileLocation.h b/td/telegram/files/FileLocation.h index 2cf48a115..5d3d07a11 100644 --- a/td/telegram/files/FileLocation.h +++ b/td/telegram/files/FileLocation.h @@ -142,7 +142,7 @@ struct WebRemoteFileLocation { }; inline StringBuilder &operator<<(StringBuilder &string_builder, const WebRemoteFileLocation &location) { - return string_builder << "[url = " << location.url_ << ", access_hash = " << location.access_hash_ << "]"; + return string_builder << "[URL = " << location.url_ << ", access_hash = " << location.access_hash_ << "]"; } struct CommonRemoteFileLocation { diff --git a/td/telegram/net/Session.cpp b/td/telegram/net/Session.cpp index b167f457e..04371800e 100644 --- a/td/telegram/net/Session.cpp +++ b/td/telegram/net/Session.cpp @@ -1265,7 +1265,7 @@ void Session::connection_open_finish(ConnectionInfo *info, VLOG(dc) << "Change mode " << mode_ << "--->" << expected_mode; mode_ = expected_mode; if (info->connection_id_ == 1 && mode_ != Mode::Http) { - LOG(WARNING) << "Got tcp connection for long poll connection"; + LOG(WARNING) << "Receive TCP connection for long poll connection"; connection_add(std::move(raw_connection)); info->state_ = ConnectionInfo::State::Empty; yield(); diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index 7c568fdeb..08d8029cc 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -2,7 +2,7 @@ if ((CMAKE_MAJOR_VERSION LESS 3) OR (CMAKE_VERSION VERSION_LESS "3.0.2")) message(FATAL_ERROR "CMake >= 3.0.2 is required") endif() -option(TDUTILS_MIME_TYPE "Generate mime types conversion; requires gperf" ON) +option(TDUTILS_MIME_TYPE "Generate MIME types conversion; requires gperf" ON) if (NOT DEFINED CMAKE_INSTALL_LIBDIR) set(CMAKE_INSTALL_LIBDIR "lib") From c508e54a49c03a34ac448d3036963aa181b57d73 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 15 Jan 2023 23:27:58 +0300 Subject: [PATCH 22/30] Improve message_id variable names in Session. --- td/telegram/net/Session.cpp | 154 ++++++++++++++++++------------------ td/telegram/net/Session.h | 34 ++++---- 2 files changed, 95 insertions(+), 93 deletions(-) diff --git a/td/telegram/net/Session.cpp b/td/telegram/net/Session.cpp index 04371800e..b24347d4d 100644 --- a/td/telegram/net/Session.cpp +++ b/td/telegram/net/Session.cpp @@ -693,9 +693,9 @@ void Session::on_closed(Status status) { current_info_->state_ = ConnectionInfo::State::Empty; } -void Session::on_session_created(uint64 unique_id, uint64 first_id) { +void Session::on_session_created(uint64 unique_id, uint64 first_message_id) { // TODO: use unique_id - LOG(INFO) << "New session " << unique_id << " created with first message_id " << first_id; + LOG(INFO) << "New session " << unique_id << " created with first message_id " << first_message_id; if (!use_pfs_ && !auth_data_.use_pfs()) { last_success_timestamp_ = Time::now(); } @@ -709,7 +709,7 @@ void Session::on_session_created(uint64 unique_id, uint64 first_id) { for (auto it = sent_queries_.begin(); it != sent_queries_.end();) { Query *query_ptr = &it->second; - if (query_ptr->container_id < first_id) { + if (query_ptr->container_message_id < first_message_id) { // container vector leak otherwise cleanup_container(it->first, &it->second); mark_as_known(it->first, &it->second); @@ -737,30 +737,30 @@ void Session::on_session_failed(Status status) { callback_->on_failed(); } -void Session::on_container_sent(uint64 container_id, vector msg_ids) { - CHECK(container_id != 0); +void Session::on_container_sent(uint64 container_message_id, vector message_ids) { + CHECK(container_message_id != 0); - td::remove_if(msg_ids, [&](uint64 msg_id) { - auto it = sent_queries_.find(msg_id); + td::remove_if(message_ids, [&](uint64 message_id) { + auto it = sent_queries_.find(message_id); if (it == sent_queries_.end()) { return true; // remove } - it->second.container_id = container_id; + it->second.container_message_id = container_message_id; return false; }); - if (msg_ids.empty()) { + if (message_ids.empty()) { return; } - auto size = msg_ids.size(); - sent_containers_.emplace(container_id, ContainerInfo{size, std::move(msg_ids)}); + auto size = message_ids.size(); + sent_containers_.emplace(container_message_id, ContainerInfo{size, std::move(message_ids)}); } -void Session::on_message_ack(uint64 id) { - on_message_ack_impl(id, 1); +void Session::on_message_ack(uint64 message_id) { + on_message_ack_impl(message_id, 1); } -void Session::on_message_ack_impl(uint64 id, int32 type) { - auto cit = sent_containers_.find(id); +void Session::on_message_ack_impl(uint64 container_message_id, int32 type) { + auto cit = sent_containers_.find(container_message_id); if (cit != sent_containers_.end()) { auto container_info = std::move(cit->second); sent_containers_.erase(cit); @@ -771,15 +771,15 @@ void Session::on_message_ack_impl(uint64 id, int32 type) { return; } - on_message_ack_impl_inner(id, type, false); + on_message_ack_impl_inner(container_message_id, type, false); } -void Session::on_message_ack_impl_inner(uint64 id, int32 type, bool in_container) { - auto it = sent_queries_.find(id); +void Session::on_message_ack_impl_inner(uint64 message_id, int32 type, bool in_container) { + auto it = sent_queries_.find(message_id); if (it == sent_queries_.end()) { return; } - VLOG(net_query) << "Ack " << tag("msg_id", id) << it->second.query; + VLOG(net_query) << "Ack " << tag("message_id", message_id) << it->second.query; it->second.ack = true; { auto lock = it->second.query->lock(); @@ -787,17 +787,17 @@ void Session::on_message_ack_impl_inner(uint64 id, int32 type, bool in_container } it->second.query->quick_ack_promise_.set_value(Unit()); if (!in_container) { - cleanup_container(id, &it->second); + cleanup_container(message_id, &it->second); } mark_as_known(it->first, &it->second); } -void Session::dec_container(uint64 message_id, Query *query) { - if (query->container_id == message_id) { +void Session::dec_container(uint64 container_message_id, Query *query) { + if (query->container_message_id == container_message_id) { // message was sent without any container return; } - auto it = sent_containers_.find(query->container_id); + auto it = sent_containers_.find(query->container_message_id); if (it == sent_containers_.end()) { return; } @@ -808,18 +808,18 @@ void Session::dec_container(uint64 message_id, Query *query) { } } -void Session::cleanup_container(uint64 message_id, Query *query) { - if (query->container_id == message_id) { +void Session::cleanup_container(uint64 container_message_id, Query *query) { + if (query->container_message_id == container_message_id) { // message was sent without any container return; } // we can forget container now, since we have an answer for its part. // TODO: we can do it only for one element per container - sent_containers_.erase(query->container_id); + sent_containers_.erase(query->container_message_id); } -void Session::mark_as_known(uint64 id, Query *query) { +void Session::mark_as_known(uint64 message_id, Query *query) { { auto lock = query->query->lock(); query->query->get_data_unsafe().unknown_state_ = false; @@ -827,15 +827,15 @@ void Session::mark_as_known(uint64 id, Query *query) { if (!query->unknown) { return; } - VLOG(net_query) << "Mark as known " << tag("msg_id", id) << query->query; + VLOG(net_query) << "Mark as known " << tag("message_id", message_id) << query->query; query->unknown = false; - unknown_queries_.erase(id); + unknown_queries_.erase(message_id); if (unknown_queries_.empty()) { flush_pending_invoke_after_queries(); } } -void Session::mark_as_unknown(uint64 id, Query *query) { +void Session::mark_as_unknown(uint64 message_id, Query *query) { { auto lock = query->query->lock(); query->query->get_data_unsafe().unknown_state_ = true; @@ -843,10 +843,10 @@ void Session::mark_as_unknown(uint64 id, Query *query) { if (query->unknown) { return; } - VLOG(net_query) << "Mark as unknown " << tag("msg_id", id) << query->query; + VLOG(net_query) << "Mark as unknown " << tag("message_id", message_id) << query->query; query->unknown = true; - CHECK(id != 0); - unknown_queries_.insert(id); + CHECK(message_id != 0); + unknown_queries_.insert(message_id); } Status Session::on_update(BufferSlice packet) { @@ -862,16 +862,16 @@ Status Session::on_update(BufferSlice packet) { return Status::OK(); } -Status Session::on_message_result_ok(uint64 id, BufferSlice packet, size_t original_size) { +Status Session::on_message_result_ok(uint64 message_id, BufferSlice packet, size_t original_size) { last_success_timestamp_ = Time::now(); TlParser parser(packet.as_slice()); int32 ID = parser.fetch_int(); - auto it = sent_queries_.find(id); + auto it = sent_queries_.find(message_id); if (it == sent_queries_.end()) { - LOG(DEBUG) << "Drop result to " << tag("request_id", format::as_hex(id)) << tag("original_size", original_size) - << tag("tl", format::as_hex(ID)); + LOG(DEBUG) << "Drop result to " << tag("message_id", format::as_hex(message_id)) + << tag("original_size", original_size) << tag("tl", format::as_hex(ID)); if (original_size > 16 * 1024) { dropped_size_ += original_size; @@ -901,8 +901,8 @@ Status Session::on_message_result_ok(uint64 id, BufferSlice packet, size_t origi } } - cleanup_container(id, query_ptr); - mark_as_known(id, query_ptr); + cleanup_container(message_id, query_ptr); + mark_as_known(message_id, query_ptr); query_ptr->query->on_net_read(original_size); query_ptr->query->set_ok(std::move(packet)); query_ptr->query->set_message_id(0); @@ -913,7 +913,7 @@ Status Session::on_message_result_ok(uint64 id, BufferSlice packet, size_t origi return Status::OK(); } -void Session::on_message_result_error(uint64 id, int error_code, string message) { +void Session::on_message_result_error(uint64 message_id, int error_code, string message) { if (!check_utf8(message)) { LOG(ERROR) << "Receive invalid error message \"" << message << '"'; message = "INVALID_UTF8_ERROR_MESSAGE"; @@ -970,7 +970,7 @@ void Session::on_message_result_error(uint64 id, int error_code, string message) error_code = 500; } - if (id == 0) { + if (message_id == 0) { LOG(ERROR) << "Received an error update"; return; } @@ -982,7 +982,7 @@ void Session::on_message_result_error(uint64 id, int error_code, string message) } else { LOG(DEBUG) << "Receive error " << error_code << " : " << message; } - auto it = sent_queries_.find(id); + auto it = sent_queries_.find(message_id); if (it == sent_queries_.end()) { current_info_->connection_->force_ack(); return; @@ -991,8 +991,8 @@ void Session::on_message_result_error(uint64 id, int error_code, string message) Query *query_ptr = &it->second; VLOG(net_query) << "Return query error " << query_ptr->query; - cleanup_container(id, query_ptr); - mark_as_known(id, query_ptr); + cleanup_container(message_id, query_ptr); + mark_as_known(message_id, query_ptr); query_ptr->query->set_error(Status::Error(error_code, message), current_info_->connection_->get_name().str()); query_ptr->query->set_message_id(0); query_ptr->query->cancel_slot_.clear_event(); @@ -1001,18 +1001,18 @@ void Session::on_message_result_error(uint64 id, int error_code, string message) sent_queries_.erase(it); } -void Session::on_message_failed_inner(uint64 id, bool in_container) { - LOG(INFO) << "Message inner failed " << id; - auto it = sent_queries_.find(id); +void Session::on_message_failed_inner(uint64 message_id, bool in_container) { + LOG(INFO) << "Message inner failed " << message_id; + auto it = sent_queries_.find(message_id); if (it == sent_queries_.end()) { return; } Query *query_ptr = &it->second; if (!in_container) { - cleanup_container(id, query_ptr); + cleanup_container(message_id, query_ptr); } - mark_as_known(id, query_ptr); + mark_as_known(message_id, query_ptr); query_ptr->query->set_message_id(0); query_ptr->query->cancel_slot_.clear_event(); @@ -1021,26 +1021,26 @@ void Session::on_message_failed_inner(uint64 id, bool in_container) { sent_queries_.erase(it); } -void Session::on_message_failed(uint64 id, Status status) { - LOG(INFO) << "Message failed: " << tag("id", id) << tag("status", status); +void Session::on_message_failed(uint64 message_id, Status status) { + LOG(INFO) << "Message failed: " << tag("message_id", message_id) << tag("status", status); status.ignore(); - auto cit = sent_containers_.find(id); + auto cit = sent_containers_.find(message_id); if (cit != sent_containers_.end()) { auto container_info = std::move(cit->second); sent_containers_.erase(cit); - for (auto message_id : container_info.message_ids) { - on_message_failed_inner(message_id, true); + for (auto contained_message_id : container_info.message_ids) { + on_message_failed_inner(contained_message_id, true); } return; } - on_message_failed_inner(id, false); + on_message_failed_inner(message_id, false); } -void Session::on_message_info(uint64 id, int32 state, uint64 answer_id, int32 answer_size) { - auto it = sent_queries_.find(id); +void Session::on_message_info(uint64 message_id, int32 state, uint64 answer_message_id, int32 answer_size) { + auto it = sent_queries_.find(message_id); if (it != sent_queries_.end()) { if (it->second.query->update_is_ready()) { dec_container(it->first, &it->second); @@ -1054,7 +1054,7 @@ void Session::on_message_info(uint64 id, int32 state, uint64 answer_id, int32 an return; } } - if (id != 0) { + if (message_id != 0) { if (it == sent_queries_.end()) { return; } @@ -1063,30 +1063,31 @@ void Session::on_message_info(uint64 id, int32 state, uint64 answer_id, int32 an case 2: case 3: // message not received by server - return on_message_failed(id, Status::Error("Unknown message identifier")); + return on_message_failed(message_id, Status::Error("Unknown message identifier")); case 0: - if (answer_id == 0) { - LOG(ERROR) << "Unexpected message_info.state == 0 " << tag("id", id) << tag("state", state) - << tag("answer_id", answer_id); - return on_message_failed(id, Status::Error("Unexpected message_info.state == 0")); + if (answer_message_id == 0) { + LOG(ERROR) << "Unexpected message_info.state == 0 " << tag("message_id", message_id) << tag("state", state) + << tag("answer_message_id", answer_message_id); + return on_message_failed(message_id, Status::Error("Unexpected message_info.state == 0")); } // fallthrough case 4: - on_message_ack_impl(id, 2); + on_message_ack_impl(message_id, 2); break; default: LOG(ERROR) << "Invalid message info " << tag("state", state); } } - // ok, we are waiting for result of id. let's ask to resend it - if (answer_id != 0) { + // ok, we are waiting for result of message_id. let's ask to resend it + if (answer_message_id != 0) { if (it != sent_queries_.end()) { - VLOG_IF(net_query, id != 0) << "Resend answer " << tag("msg_id", id) << tag("answer_id", answer_id) - << tag("answer_size", answer_size) << it->second.query; + VLOG_IF(net_query, message_id != 0) + << "Resend answer " << tag("message_id", message_id) << tag("answer_message_id", answer_message_id) + << tag("answer_size", answer_size) << it->second.query; it->second.query->debug("Session: resend answer"); } - current_info_->connection_->resend_answer(answer_id); + current_info_->connection_->resend_answer(answer_message_id); } } @@ -1160,9 +1161,10 @@ void Session::connection_send_query(ConnectionInfo *info, NetQueryPtr &&net_quer message_id = auth_data_.next_message_id(Time::now_cached()); } } - VLOG(net_query) << "Send query to connection " << net_query << " [msg_id:" << format::as_hex(message_id) << "]" - << tag("invoke_after", - transform(invoke_after_ids, [](auto id) { return PSTRING() << format::as_hex(id); })); + VLOG(net_query) << "Send query to connection " << net_query << " [message_id:" << format::as_hex(message_id) << "]" + << tag("invoke_after", transform(invoke_after_ids, [](auto message_id) { + return PSTRING() << format::as_hex(message_id); + })); net_query->set_message_id(message_id); net_query->cancel_slot_.clear_event(); { @@ -1306,11 +1308,11 @@ void Session::connection_open_finish(ConnectionInfo *info, return; } if (info->ask_info_) { - for (auto &id : unknown_queries_) { - info->connection_->get_state_info(id); + for (auto &message_id : unknown_queries_) { + info->connection_->get_state_info(message_id); } - for (auto &id : to_cancel_) { - info->connection_->cancel_answer(id); + for (auto &message_id : to_cancel_) { + info->connection_->cancel_answer(message_id); } to_cancel_.clear(); } diff --git a/td/telegram/net/Session.h b/td/telegram/net/Session.h index 02d1d4dee..ee23102b8 100644 --- a/td/telegram/net/Session.h +++ b/td/telegram/net/Session.h @@ -78,7 +78,7 @@ class Session final private: struct Query final : private ListNode { - uint64 container_id; + uint64 container_message_id; NetQueryPtr query; bool ack = false; @@ -87,7 +87,7 @@ class Session final int8 connection_id; double sent_at_; Query(uint64 message_id, NetQueryPtr &&q, int8 connection_id, double sent_at) - : container_id(message_id), query(std::move(q)), connection_id(connection_id), sent_at_(sent_at) { + : container_message_id(message_id), query(std::move(q)), connection_id(connection_id), sent_at_(sent_at) { } ListNode *get_list_node() { @@ -101,7 +101,7 @@ class Session final // When connection is closed, mark all queries without ack as unknown. // Ask state of all unknown queries when new connection is created. // - // Just re-ask answer_id each time we get information about it. + // Just re-ask answer_message_id each time we get information about it. // Though mtproto::Connection must ensure delivery of such query. const int32 raw_dc_id_; // numerical datacenter ID, i.e. 2 @@ -209,33 +209,33 @@ class Session final void on_server_salt_updated() final; void on_server_time_difference_updated() final; - void on_session_created(uint64 unique_id, uint64 first_id) final; + void on_session_created(uint64 unique_id, uint64 first_message_id) final; void on_session_failed(Status status) final; - void on_container_sent(uint64 container_id, vector msg_ids) final; + void on_container_sent(uint64 container_message_id, vector message_ids) final; Status on_update(BufferSlice packet) final; - void on_message_ack(uint64 id) final; - Status on_message_result_ok(uint64 id, BufferSlice packet, size_t original_size) final; - void on_message_result_error(uint64 id, int error_code, string message) final; - void on_message_failed(uint64 id, Status status) final; + void on_message_ack(uint64 message_id) final; + Status on_message_result_ok(uint64 message_id, BufferSlice packet, size_t original_size) final; + void on_message_result_error(uint64 message_id, int error_code, string message) final; + void on_message_failed(uint64 message_id, Status status) final; - void on_message_info(uint64 id, int32 state, uint64 answer_id, int32 answer_size) final; + void on_message_info(uint64 message_id, int32 state, uint64 answer_message_id, int32 answer_size) final; Status on_destroy_auth_key() final; void flush_pending_invoke_after_queries(); bool has_queries() const; - void dec_container(uint64 message_id, Query *query); - void cleanup_container(uint64 id, Query *query); - void mark_as_known(uint64 id, Query *query); - void mark_as_unknown(uint64 id, Query *query); + void dec_container(uint64 container_message_id, Query *query); + void cleanup_container(uint64 container_message_id, Query *query); + void mark_as_known(uint64 message_id, Query *query); + void mark_as_unknown(uint64 message_id, Query *query); - void on_message_ack_impl(uint64 id, int32 type); - void on_message_ack_impl_inner(uint64 id, int32 type, bool in_container); - void on_message_failed_inner(uint64 id, bool in_container); + void on_message_ack_impl(uint64 container_message_id, int32 type); + void on_message_ack_impl_inner(uint64 message_id, int32 type, bool in_container); + void on_message_failed_inner(uint64 message_id, bool in_container); // send NetQueryPtr to parent void return_query(NetQueryPtr &&query); From 3e5f30af700145d6059dca793cc29cfab276628c Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 16 Jan 2023 12:47:37 +0300 Subject: [PATCH 23/30] Improve event_id variable names. --- tddb/td/db/BinlogKeyValue.h | 34 ++++++++-------- tddb/td/db/binlog/Binlog.cpp | 4 +- tddb/td/db/binlog/Binlog.h | 40 +++++++++---------- tddb/td/db/binlog/BinlogInterface.h | 40 +++++++++---------- tddb/td/db/binlog/ConcurrentBinlog.cpp | 12 +++--- tddb/td/db/binlog/ConcurrentBinlog.h | 12 +++--- tddb/td/db/binlog/binlog_dump.cpp | 27 +++++++------ .../binlog/detail/BinlogEventsProcessor.cpp | 38 +++++++++--------- .../db/binlog/detail/BinlogEventsProcessor.h | 19 ++++----- test/db.cpp | 8 ++-- test/secret.cpp | 16 ++++---- 11 files changed, 127 insertions(+), 123 deletions(-) diff --git a/tddb/td/db/BinlogKeyValue.h b/tddb/td/db/BinlogKeyValue.h index 32026a76e..80500f839 100644 --- a/tddb/td/db/BinlogKeyValue.h +++ b/tddb/td/db/BinlogKeyValue.h @@ -120,7 +120,7 @@ class BinlogKeyValue final : public KeyValueSyncInterface { SeqNo set(string key, string value) final { auto lock = rw_mutex_.lock_write().move_as_ok(); - uint64 old_id = 0; + uint64 old_event_id = 0; auto it_ok = map_.emplace(key, std::make_pair(value, 0)); if (!it_ok.second) { if (it_ok.first->second.first == value) { @@ -128,25 +128,25 @@ class BinlogKeyValue final : public KeyValueSyncInterface { } VLOG(binlog) << "Change value of key " << key << " from " << hex_encode(it_ok.first->second.first) << " to " << hex_encode(value); - old_id = it_ok.first->second.second; + old_event_id = it_ok.first->second.second; it_ok.first->second.first = value; } else { VLOG(binlog) << "Set value of key " << key << " to " << hex_encode(value); } bool rewrite = false; - uint64 id; - auto seq_no = binlog_->next_id(); - if (old_id != 0) { + uint64 event_id; + auto seq_no = binlog_->next_event_id(); + if (old_event_id != 0) { rewrite = true; - id = old_id; + event_id = old_event_id; } else { - id = seq_no; - it_ok.first->second.second = id; + event_id = seq_no; + it_ok.first->second.second = event_id; } lock.reset(); add_event(seq_no, - BinlogEvent::create_raw(id, magic_, rewrite ? BinlogEvent::Flags::Rewrite : 0, Event{key, value})); + BinlogEvent::create_raw(event_id, magic_, rewrite ? BinlogEvent::Flags::Rewrite : 0, Event{key, value})); return seq_no; } @@ -157,11 +157,11 @@ class BinlogKeyValue final : public KeyValueSyncInterface { return 0; } VLOG(binlog) << "Remove value of key " << key << ", which is " << hex_encode(it->second.first); - uint64 id = it->second.second; + uint64 event_id = it->second.second; map_.erase(it); - auto seq_no = binlog_->next_id(); + auto seq_no = binlog_->next_event_id(); lock.reset(); - add_event(seq_no, BinlogEvent::create_raw(id, BinlogEvent::ServiceTypes::Empty, BinlogEvent::Flags::Rewrite, + add_event(seq_no, BinlogEvent::create_raw(event_id, BinlogEvent::ServiceTypes::Empty, BinlogEvent::Flags::Rewrite, EmptyStorer())); return seq_no; } @@ -215,18 +215,18 @@ class BinlogKeyValue final : public KeyValueSyncInterface { void erase_by_prefix(Slice prefix) final { auto lock = rw_mutex_.lock_write().move_as_ok(); - vector ids; + vector event_ids; table_remove_if(map_, [&](const auto &it) { if (begins_with(it.first, prefix)) { - ids.push_back(it.second.second); + event_ids.push_back(it.second.second); return true; } return false; }); - auto seq_no = binlog_->next_id(narrow_cast(ids.size())); + auto seq_no = binlog_->next_event_id(narrow_cast(event_ids.size())); lock.reset(); - for (auto id : ids) { - add_event(seq_no, BinlogEvent::create_raw(id, BinlogEvent::ServiceTypes::Empty, BinlogEvent::Flags::Rewrite, + for (auto event_id : event_ids) { + add_event(seq_no, BinlogEvent::create_raw(event_id, BinlogEvent::ServiceTypes::Empty, BinlogEvent::Flags::Rewrite, EmptyStorer())); seq_no++; } diff --git a/tddb/td/db/binlog/Binlog.cpp b/tddb/td/db/binlog/Binlog.cpp index 1eb3eebd2..d6dd73e1f 100644 --- a/tddb/td/db/binlog/Binlog.cpp +++ b/tddb/td/db/binlog/Binlog.cpp @@ -208,8 +208,8 @@ Status Binlog::init(string path, const Callback &callback, DbKey db_key, DbKey o close().ignore(); return status; } - info_.last_id = processor_->last_id(); - last_id_ = processor_->last_id(); + info_.last_event_id = processor_->last_event_id(); + last_event_id_ = processor_->last_event_id(); if (info_.wrong_password) { close().ignore(); return Status::Error(static_cast(Error::WrongPassword), "Wrong password"); diff --git a/tddb/td/db/binlog/Binlog.h b/tddb/td/db/binlog/Binlog.h index 73e1e17e2..617173167 100644 --- a/tddb/td/db/binlog/Binlog.h +++ b/tddb/td/db/binlog/Binlog.h @@ -31,7 +31,7 @@ extern int32 VERBOSITY_NAME(binlog); struct BinlogInfo { bool was_created{false}; - uint64 last_id{0}; + uint64 last_event_id{0}; bool is_encrypted{false}; bool wrong_password{false}; bool is_opened{false}; @@ -57,16 +57,16 @@ class Binlog { Status init(string path, const Callback &callback, DbKey db_key = DbKey::empty(), DbKey old_db_key = DbKey::empty(), int32 dummy = -1, const Callback &debug_callback = Callback()) TD_WARN_UNUSED_RESULT; - uint64 next_id() { - return ++last_id_; + uint64 next_event_id() { + return ++last_event_id_; } - uint64 next_id(int32 shift) { - auto res = last_id_ + 1; - last_id_ += shift; + uint64 next_event_id(int32 shift) { + auto res = last_event_id_ + 1; + last_event_id_ += shift; return res; } - uint64 peek_next_id() const { - return last_id_ + 1; + uint64 peek_next_event_id() const { + return last_event_id_ + 1; } bool empty() const { @@ -74,22 +74,22 @@ class Binlog { } uint64 add(int32 type, const Storer &storer) { - auto log_event_id = next_id(); - add_raw_event(BinlogEvent::create_raw(log_event_id, type, 0, storer), {}); - return log_event_id; + auto event_id = next_event_id(); + add_raw_event(BinlogEvent::create_raw(event_id, type, 0, storer), {}); + return event_id; } - uint64 rewrite(uint64 log_event_id, int32 type, const Storer &storer) { - auto seq_no = next_id(); - add_raw_event(BinlogEvent::create_raw(log_event_id, type, BinlogEvent::Flags::Rewrite, storer), {}); + uint64 rewrite(uint64 event_id, int32 type, const Storer &storer) { + auto seq_no = next_event_id(); + add_raw_event(BinlogEvent::create_raw(event_id, type, BinlogEvent::Flags::Rewrite, storer), {}); return seq_no; } - uint64 erase(uint64 log_event_id) { - auto seq_no = next_id(); - add_raw_event(BinlogEvent::create_raw(log_event_id, BinlogEvent::ServiceTypes::Empty, BinlogEvent::Flags::Rewrite, - EmptyStorer()), - {}); + uint64 erase(uint64 event_id) { + auto seq_no = next_event_id(); + add_raw_event( + BinlogEvent::create_raw(event_id, BinlogEvent::ServiceTypes::Empty, BinlogEvent::Flags::Rewrite, EmptyStorer()), + {}); return seq_no; } @@ -148,7 +148,7 @@ class Binlog { unique_ptr processor_; unique_ptr events_buffer_; bool in_flush_events_buffer_{false}; - uint64 last_id_{0}; + uint64 last_event_id_{0}; double need_flush_since_ = 0; bool need_sync_{false}; enum class State { Empty, Load, Reindex, Run } state_{State::Empty}; diff --git a/tddb/td/db/binlog/BinlogInterface.h b/tddb/td/db/binlog/BinlogInterface.h index 3dcc615e1..31b1b1c1f 100644 --- a/tddb/td/db/binlog/BinlogInterface.h +++ b/tddb/td/db/binlog/BinlogInterface.h @@ -31,35 +31,35 @@ class BinlogInterface { void close_and_destroy(Promise<> promise = {}) { close_and_destroy_impl(std::move(promise)); } - void add_raw_event(BinlogDebugInfo info, uint64 id, BufferSlice &&raw_event, Promise<> promise = Promise<>()) { - add_raw_event_impl(id, std::move(raw_event), std::move(promise), info); + void add_raw_event(BinlogDebugInfo info, uint64 event_id, BufferSlice &&raw_event, Promise<> promise = Promise<>()) { + add_raw_event_impl(event_id, std::move(raw_event), std::move(promise), info); } - void add_raw_event(uint64 id, BufferSlice &&raw_event, Promise<> promise = Promise<>()) { - add_raw_event_impl(id, std::move(raw_event), std::move(promise), {}); + void add_raw_event(uint64 event_id, BufferSlice &&raw_event, Promise<> promise = Promise<>()) { + add_raw_event_impl(event_id, std::move(raw_event), std::move(promise), {}); } void lazy_sync(Promise<> promise = Promise<>()) { - add_raw_event_impl(next_id(), BufferSlice(), std::move(promise), {}); + add_raw_event_impl(next_event_id(), BufferSlice(), std::move(promise), {}); } uint64 add(int32 type, const Storer &storer, Promise<> promise = Promise<>()) { - auto log_event_id = next_id(); - add_raw_event_impl(log_event_id, BinlogEvent::create_raw(log_event_id, type, 0, storer), std::move(promise), {}); - return log_event_id; + auto event_id = next_event_id(); + add_raw_event_impl(event_id, BinlogEvent::create_raw(event_id, type, 0, storer), std::move(promise), {}); + return event_id; } - uint64 rewrite(uint64 log_event_id, int32 type, const Storer &storer, Promise<> promise = Promise<>()) { - auto seq_no = next_id(); - add_raw_event_impl(seq_no, BinlogEvent::create_raw(log_event_id, type, BinlogEvent::Flags::Rewrite, storer), + uint64 rewrite(uint64 event_id, int32 type, const Storer &storer, Promise<> promise = Promise<>()) { + auto seq_no = next_event_id(); + add_raw_event_impl(seq_no, BinlogEvent::create_raw(event_id, type, BinlogEvent::Flags::Rewrite, storer), std::move(promise), {}); return seq_no; } - uint64 erase(uint64 log_event_id, Promise<> promise = Promise<>()) { - auto seq_no = next_id(); - add_raw_event_impl(seq_no, - BinlogEvent::create_raw(log_event_id, BinlogEvent::ServiceTypes::Empty, - BinlogEvent::Flags::Rewrite, EmptyStorer()), - std::move(promise), {}); + uint64 erase(uint64 event_id, Promise<> promise = Promise<>()) { + auto seq_no = next_event_id(); + add_raw_event_impl( + seq_no, + BinlogEvent::create_raw(event_id, BinlogEvent::ServiceTypes::Empty, BinlogEvent::Flags::Rewrite, EmptyStorer()), + std::move(promise), {}); return seq_no; } @@ -67,13 +67,13 @@ class BinlogInterface { virtual void force_flush() = 0; virtual void change_key(DbKey db_key, Promise<> promise) = 0; - virtual uint64 next_id() = 0; - virtual uint64 next_id(int32 shift) = 0; + virtual uint64 next_event_id() = 0; + virtual uint64 next_event_id(int32 shift) = 0; protected: virtual void close_impl(Promise<> promise) = 0; virtual void close_and_destroy_impl(Promise<> promise) = 0; - virtual void add_raw_event_impl(uint64 id, BufferSlice &&raw_event, Promise<> promise, BinlogDebugInfo info) = 0; + virtual void add_raw_event_impl(uint64 seq_no, BufferSlice &&raw_event, Promise<> promise, BinlogDebugInfo info) = 0; }; } // namespace td diff --git a/tddb/td/db/binlog/ConcurrentBinlog.cpp b/tddb/td/db/binlog/ConcurrentBinlog.cpp index 2ee21272a..1dfe10b67 100644 --- a/tddb/td/db/binlog/ConcurrentBinlog.cpp +++ b/tddb/td/db/binlog/ConcurrentBinlog.cpp @@ -40,7 +40,7 @@ class BinlogActor final : public Actor { BinlogDebugInfo debug_info; }; void add_raw_event(uint64 seq_no, BufferSlice &&raw_event, Promise<> &&promise, BinlogDebugInfo info) { - processor_.add(seq_no, Event{std::move(raw_event), std::move(promise), info}, [&](uint64 id, Event &&event) { + processor_.add(seq_no, Event{std::move(raw_event), std::move(promise), info}, [&](uint64 event_id, Event &&event) { if (!event.raw_event.empty()) { do_add_raw_event(std::move(event.raw_event), event.debug_info); } @@ -178,9 +178,9 @@ Result ConcurrentBinlog::init(string path, const Callback &callback, void ConcurrentBinlog::init_impl(unique_ptr binlog, int32 scheduler_id) { path_ = binlog->get_path().str(); - last_id_ = binlog->peek_next_id(); + last_event_id_ = binlog->peek_next_event_id(); binlog_actor_ = create_actor_on_scheduler(PSLICE() << "Binlog " << path_, scheduler_id, - std::move(binlog), last_id_); + std::move(binlog), last_event_id_); } void ConcurrentBinlog::close_impl(Promise<> promise) { @@ -189,8 +189,10 @@ void ConcurrentBinlog::close_impl(Promise<> promise) { void ConcurrentBinlog::close_and_destroy_impl(Promise<> promise) { send_closure(std::move(binlog_actor_), &detail::BinlogActor::close_and_destroy, std::move(promise)); } -void ConcurrentBinlog::add_raw_event_impl(uint64 id, BufferSlice &&raw_event, Promise<> promise, BinlogDebugInfo info) { - send_closure(binlog_actor_, &detail::BinlogActor::add_raw_event, id, std::move(raw_event), std::move(promise), info); +void ConcurrentBinlog::add_raw_event_impl(uint64 event_id, BufferSlice &&raw_event, Promise<> promise, + BinlogDebugInfo info) { + send_closure(binlog_actor_, &detail::BinlogActor::add_raw_event, event_id, std::move(raw_event), std::move(promise), + info); } void ConcurrentBinlog::force_sync(Promise<> promise) { send_closure(binlog_actor_, &detail::BinlogActor::force_sync, std::move(promise)); diff --git a/tddb/td/db/binlog/ConcurrentBinlog.h b/tddb/td/db/binlog/ConcurrentBinlog.h index 48632afe0..51eeff280 100644 --- a/tddb/td/db/binlog/ConcurrentBinlog.h +++ b/tddb/td/db/binlog/ConcurrentBinlog.h @@ -45,11 +45,11 @@ class ConcurrentBinlog final : public BinlogInterface { void force_flush() final; void change_key(DbKey db_key, Promise<> promise) final; - uint64 next_id() final { - return last_id_.fetch_add(1, std::memory_order_relaxed); + uint64 next_event_id() final { + return last_event_id_.fetch_add(1, std::memory_order_relaxed); } - uint64 next_id(int32 shift) final { - return last_id_.fetch_add(shift, std::memory_order_relaxed); + uint64 next_event_id(int32 shift) final { + return last_event_id_.fetch_add(shift, std::memory_order_relaxed); } CSlice get_path() const { @@ -60,11 +60,11 @@ class ConcurrentBinlog final : public BinlogInterface { void init_impl(unique_ptr binlog, int scheduler_id); void close_impl(Promise<> promise) final; void close_and_destroy_impl(Promise<> promise) final; - void add_raw_event_impl(uint64 id, BufferSlice &&raw_event, Promise<> promise, BinlogDebugInfo info) final; + void add_raw_event_impl(uint64 event_id, BufferSlice &&raw_event, Promise<> promise, BinlogDebugInfo info) final; ActorOwn binlog_actor_; string path_; - std::atomic last_id_{0}; + std::atomic last_event_id_{0}; }; } // namespace td diff --git a/tddb/td/db/binlog/binlog_dump.cpp b/tddb/td/db/binlog/binlog_dump.cpp index 05f542614..7e34b6a2f 100644 --- a/tddb/td/db/binlog/binlog_dump.cpp +++ b/tddb/td/db/binlog/binlog_dump.cpp @@ -44,20 +44,20 @@ struct Trie { }; td::vector nodes_; - void do_add(int id, td::Slice value) { - nodes_[id].sum++; + void do_add(int event_id, td::Slice value) { + nodes_[event_id].sum++; if (value.empty()) { return; } auto c = static_cast(value[0]); - auto next_id = nodes_[id].next[c]; - if (next_id == 0) { - next_id = static_cast(nodes_.size()); + auto next_event_id = nodes_[event_id].next[c]; + if (next_event_id == 0) { + next_event_id = static_cast(nodes_.size()); nodes_.emplace_back(); - nodes_[id].next[c] = next_id; + nodes_[event_id].next[c] = next_event_id; } - do_add(next_id, value.substr(1)); + do_add(next_event_id, value.substr(1)); } void do_dump(td::string path, int v) { @@ -84,11 +84,11 @@ struct Trie { return; } for (int c = 0; c < 256; c++) { - auto next_id = nodes_[v].next[c]; - if (next_id == 0) { + auto next_event_id = nodes_[v].next[c]; + if (next_event_id == 0) { continue; } - do_dump(path + static_cast(c), next_id); + do_dump(path + static_cast(c), next_event_id); } } }; @@ -137,9 +137,10 @@ int main(int argc, char *argv[]) { auto key = td::TlParser(event.data_).fetch_string(); info[event.type_].trie.add(key); } - LOG(PLAIN) << "LogEvent[" << td::tag("id", td::format::as_hex(event.id_)) << td::tag("type", event.type_) - << td::tag("flags", event.flags_) << td::tag("size", event.data_.size()) - << td::tag("data", td::format::escaped(event.data_)) << "]\n"; + LOG(PLAIN) << "LogEvent[" << td::tag("event_id", td::format::as_hex(event.id_)) + << td::tag("type", event.type_) << td::tag("flags", event.flags_) + << td::tag("size", event.data_.size()) << td::tag("data", td::format::escaped(event.data_)) + << "]\n"; }) .ensure(); diff --git a/tddb/td/db/binlog/detail/BinlogEventsProcessor.cpp b/tddb/td/db/binlog/detail/BinlogEventsProcessor.cpp index 58c14a941..f1d740eaa 100644 --- a/tddb/td/db/binlog/detail/BinlogEventsProcessor.cpp +++ b/tddb/td/db/binlog/detail/BinlogEventsProcessor.cpp @@ -16,13 +16,13 @@ namespace detail { Status BinlogEventsProcessor::do_event(BinlogEvent &&event) { offset_ = event.offset_; - auto fixed_id = event.id_ * 2; - if ((event.flags_ & BinlogEvent::Flags::Rewrite) && !ids_.empty() && ids_.back() >= fixed_id) { - auto it = std::lower_bound(ids_.begin(), ids_.end(), fixed_id); - if (it == ids_.end() || *it != fixed_id) { + auto fixed_event_id = event.id_ * 2; + if ((event.flags_ & BinlogEvent::Flags::Rewrite) && !event_ids_.empty() && event_ids_.back() >= fixed_event_id) { + auto it = std::lower_bound(event_ids_.begin(), event_ids_.end(), fixed_event_id); + if (it == event_ids_.end() || *it != fixed_event_id) { return Status::Error(PSLICE() << "Ignore rewrite log event " << event.public_to_string()); } - auto pos = it - ids_.begin(); + auto pos = it - event_ids_.begin(); total_raw_events_size_ -= static_cast(events_[pos].raw_event_.size()); if (event.type_ == BinlogEvent::ServiceTypes::Empty) { *it += 1; @@ -36,15 +36,15 @@ Status BinlogEventsProcessor::do_event(BinlogEvent &&event) { } else if (event.type_ < 0) { // just skip service events } else { - if (!(ids_.empty() || ids_.back() < fixed_id)) { - return Status::Error(PSLICE() << offset_ << " " << ids_.size() << " " << ids_.back() << " " << fixed_id << " " - << event.public_to_string() << " " << total_events_ << " " + if (!(event_ids_.empty() || event_ids_.back() < fixed_event_id)) { + return Status::Error(PSLICE() << offset_ << " " << event_ids_.size() << " " << event_ids_.back() << " " + << fixed_event_id << " " << event.public_to_string() << " " << total_events_ << " " << total_raw_events_size_); } - last_id_ = event.id_; + last_event_id_ = event.id_; total_raw_events_size_ += static_cast(event.raw_event_.size()); total_events_++; - ids_.push_back(fixed_id); + event_ids_.push_back(fixed_event_id); events_.emplace_back(std::move(event)); } @@ -55,22 +55,22 @@ Status BinlogEventsProcessor::do_event(BinlogEvent &&event) { } void BinlogEventsProcessor::compactify() { - CHECK(ids_.size() == events_.size()); - auto ids_from = ids_.begin(); - auto ids_to = ids_from; + CHECK(event_ids_.size() == events_.size()); + auto event_ids_from = event_ids_.begin(); + auto event_ids_to = event_ids_from; auto events_from = events_.begin(); auto events_to = events_from; - for (; ids_from != ids_.end(); ids_from++, events_from++) { - if ((*ids_from & 1) == 0) { - *ids_to++ = *ids_from; + for (; event_ids_from != event_ids_.end(); event_ids_from++, events_from++) { + if ((*event_ids_from & 1) == 0) { + *event_ids_to++ = *event_ids_from; *events_to++ = std::move(*events_from); } } - ids_.erase(ids_to, ids_.end()); + event_ids_.erase(event_ids_to, event_ids_.end()); events_.erase(events_to, events_.end()); - total_events_ = ids_.size(); + total_events_ = event_ids_.size(); empty_events_ = 0; - CHECK(ids_.size() == events_.size()); + CHECK(event_ids_.size() == events_.size()); } } // namespace detail diff --git a/tddb/td/db/binlog/detail/BinlogEventsProcessor.h b/tddb/td/db/binlog/detail/BinlogEventsProcessor.h index 4ca70a67c..b358548a2 100644 --- a/tddb/td/db/binlog/detail/BinlogEventsProcessor.h +++ b/tddb/td/db/binlog/detail/BinlogEventsProcessor.h @@ -23,17 +23,18 @@ class BinlogEventsProcessor { template void for_each(CallbackT &&callback) { - for (size_t i = 0; i < ids_.size(); i++) { - LOG_CHECK(i == 0 || ids_[i - 1] < ids_[i]) << ids_[i - 1] << " " << events_[i - 1].public_to_string() << " " - << ids_[i] << " " << events_[i].public_to_string(); - if ((ids_[i] & 1) == 0) { + for (size_t i = 0; i < event_ids_.size(); i++) { + LOG_CHECK(i == 0 || event_ids_[i - 1] < event_ids_[i]) + << event_ids_[i - 1] << " " << events_[i - 1].public_to_string() << " " << event_ids_[i] << " " + << events_[i].public_to_string(); + if ((event_ids_[i] & 1) == 0) { callback(events_[i]); } } } - uint64 last_id() const { - return last_id_; + uint64 last_event_id() const { + return last_event_id_; } int64 offset() const { return offset_; @@ -43,12 +44,12 @@ class BinlogEventsProcessor { } private: - // holds (id * 2 + was_deleted) - std::vector ids_; + // holds (event_id * 2 + was_deleted) + std::vector event_ids_; std::vector events_; size_t total_events_{0}; size_t empty_events_{0}; - uint64 last_id_{0}; + uint64 last_event_id_{0}; int64 offset_{0}; int64 total_raw_events_size_{0}; diff --git a/test/db.cpp b/test/db.cpp index 8ed7b5eac..0b4e7d939 100644 --- a/test/db.cpp +++ b/test/db.cpp @@ -75,17 +75,17 @@ TEST(DB, binlog_encryption) { { td::Binlog binlog; binlog.init(binlog_name.str(), [](const td::BinlogEvent &x) {}).ensure(); - binlog.add_raw_event(td::BinlogEvent::create_raw(binlog.next_id(), 1, 0, td::create_storer("AAAA")), + binlog.add_raw_event(td::BinlogEvent::create_raw(binlog.next_event_id(), 1, 0, td::create_storer("AAAA")), td::BinlogDebugInfo{__FILE__, __LINE__}); - binlog.add_raw_event(td::BinlogEvent::create_raw(binlog.next_id(), 1, 0, td::create_storer("BBBB")), + binlog.add_raw_event(td::BinlogEvent::create_raw(binlog.next_event_id(), 1, 0, td::create_storer("BBBB")), td::BinlogDebugInfo{__FILE__, __LINE__}); - binlog.add_raw_event(td::BinlogEvent::create_raw(binlog.next_id(), 1, 0, td::create_storer(long_data)), + binlog.add_raw_event(td::BinlogEvent::create_raw(binlog.next_event_id(), 1, 0, td::create_storer(long_data)), td::BinlogDebugInfo{__FILE__, __LINE__}); LOG(INFO) << "SET PASSWORD"; binlog.change_key(cucumber); binlog.change_key(hello); LOG(INFO) << "OK"; - binlog.add_raw_event(td::BinlogEvent::create_raw(binlog.next_id(), 1, 0, td::create_storer("CCCC")), + binlog.add_raw_event(td::BinlogEvent::create_raw(binlog.next_event_id(), 1, 0, td::create_storer("CCCC")), td::BinlogDebugInfo{__FILE__, __LINE__}); binlog.close().ensure(); } diff --git a/test/secret.cpp b/test/secret.cpp index 01024720c..05b91af1c 100644 --- a/test/secret.cpp +++ b/test/secret.cpp @@ -382,14 +382,14 @@ class FakeBinlog final void force_flush() final { } - uint64 next_id() final { - auto res = last_id_; - last_id_++; + uint64 next_event_id() final { + auto res = last_event_id_; + last_event_id_++; return res; } - uint64 next_id(int32 shift) final { - auto res = last_id_; - last_id_ += shift; + uint64 next_event_id(int32 shift) final { + auto res = last_event_id_; + last_event_id_ += shift; return res; } template @@ -420,7 +420,7 @@ class FakeBinlog final } void close_and_destroy_impl(Promise<> promise) final { } - void add_raw_event_impl(uint64 id, BufferSlice &&raw_event, Promise<> promise, BinlogDebugInfo info) final { + void add_raw_event_impl(uint64 event_id, BufferSlice &&raw_event, Promise<> promise, BinlogDebugInfo info) final { auto event = BinlogEvent(std::move(raw_event), info); LOG(INFO) << "ADD EVENT: " << event.id_ << " " << event; pending_events_.emplace_back(); @@ -464,7 +464,7 @@ class FakeBinlog final } } bool has_request_sync = false; - uint64 last_id_ = 1; + uint64 last_event_id_ = 1; detail::BinlogEventsProcessor events_processor_; struct PendingEvent { From ad8f0c4146cb8ce77357c6f0174ff3b5508ee1da Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 16 Jan 2023 12:59:43 +0300 Subject: [PATCH 24/30] Improve some spelling. --- td/generate/scheme/td_api.tl | 4 ++-- td/generate/tl_writer_c.h | 1 - td/telegram/BackgroundManager.h | 2 +- td/telegram/Client.h | 2 +- td/telegram/ContactsManager.h | 2 +- td/telegram/MessageDb.cpp | 2 +- td/telegram/MessagesManager.cpp | 2 +- td/telegram/MessagesManager.h | 2 +- td/telegram/StickersManager.cpp | 2 +- td/telegram/StickersManager.h | 5 +++-- tdactor/td/actor/impl/ActorInfo.h | 4 ++-- tdutils/td/utils/Container.h | 2 +- tdutils/td/utils/logging.cpp | 2 +- tdutils/td/utils/port/UdpSocketFd.cpp | 6 +++--- tdutils/test/heap.cpp | 13 +++++++++---- 15 files changed, 28 insertions(+), 23 deletions(-) diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index 3980ebfdc..03b635316 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -1433,7 +1433,7 @@ replyMarkupInlineKeyboard rows:vector> = ReplyMarku //@class LoginUrlInfo @description Contains information about an inline button of type inlineKeyboardButtonTypeLoginUrl -//@description An HTTP url needs to be open @url The URL to open @skip_confirm True, if there is no need to show an ordinary open URL confirm +//@description An HTTP URL needs to be open @url The URL to open @skip_confirm True, if there is no need to show an ordinary open URL confirm loginUrlInfoOpen url:string skip_confirm:Bool = LoginUrlInfo; //@description An authorization confirmation dialog needs to be shown to the user @@ -2978,7 +2978,7 @@ groupCallStream channel_id:int32 scale:int32 time_offset:int53 = GroupCallStream //@description Represents a list of group call streams @streams A list of group call streams groupCallStreams streams:vector = GroupCallStreams; -//@description Represents an RTMP url @url The URL @stream_key Stream key +//@description Represents an RTMP URL @url The URL @stream_key Stream key rtmpUrl url:string stream_key:string = RtmpUrl; diff --git a/td/generate/tl_writer_c.h b/td/generate/tl_writer_c.h index 4311cf5e1..41bb38b5d 100644 --- a/td/generate/tl_writer_c.h +++ b/td/generate/tl_writer_c.h @@ -1019,7 +1019,6 @@ class TlWriterCCommon final : public tl::TL_writer { if (is_proxy || is_header_ != 1) { return ""; } - // return "#define CODE_" + class_name + " " + int_to_string(id) + "\n"; return ""; } diff --git a/td/telegram/BackgroundManager.h b/td/telegram/BackgroundManager.h index f623d4f91..f7d711214 100644 --- a/td/telegram/BackgroundManager.h +++ b/td/telegram/BackgroundManager.h @@ -160,7 +160,7 @@ class BackgroundManager final : public Actor { FlatHashMap, BackgroundIdHash> backgrounds_; FlatHashMap, BackgroundIdHash> - background_id_to_file_source_id_; // id -> [access_hash, file_source_id] + background_id_to_file_source_id_; // background_id -> [access_hash, file_source_id] FlatHashMap name_to_background_id_; diff --git a/td/telegram/Client.h b/td/telegram/Client.h index 86ec2b205..2f2165ebf 100644 --- a/td/telegram/Client.h +++ b/td/telegram/Client.h @@ -199,7 +199,7 @@ class ClientManager final { * if (response.id == 0) { * // process response.object as an incoming update of type td_api::Update * } else { - * // process response.object as an answer to a sent request with id response.id + * // process response.object as an answer to a sent request with identifier response.id * } * } * \endcode diff --git a/td/telegram/ContactsManager.h b/td/telegram/ContactsManager.h index fd2daca22..18c1dd36e 100644 --- a/td/telegram/ContactsManager.h +++ b/td/telegram/ContactsManager.h @@ -725,7 +725,7 @@ class ContactsManager final : public Actor { FlatHashSet photo_ids; - FlatHashMap online_member_dialogs; // id -> time + FlatHashMap online_member_dialogs; // dialog_id -> time static constexpr uint32 CACHE_VERSION = 4; uint32 cache_version = 0; diff --git a/td/telegram/MessageDb.cpp b/td/telegram/MessageDb.cpp index f1e53bb9a..7b11fcc91 100644 --- a/td/telegram/MessageDb.cpp +++ b/td/telegram/MessageDb.cpp @@ -565,7 +565,7 @@ class MessageDbImpl final : public MessageDbSyncInterface { prev_found_message_id = message_id; } - // left_message_id is always an id of suitable message, let's return it + // left_message_id is always an identifier of suitable message, let's return it return get_message({dialog_id, MessageId(left_message_id)}); } } diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index a1e740229..586c522b8 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -25826,7 +25826,7 @@ MessageId MessagesManager::get_persistent_message_id(const Dialog *d, MessageId return MessageId(); } if (message_id.is_yet_unsent()) { - // it is possible that user tries to do something with an already sent message by its temporary id + // it is possible that user tries to do something with an already sent message by its temporary identifier // we need to use real message in this case and transparently replace message_id auto it = d->yet_unsent_message_id_to_persistent_message_id.find(message_id); if (it != d->yet_unsent_message_id_to_persistent_message_id.end()) { diff --git a/td/telegram/MessagesManager.h b/td/telegram/MessagesManager.h index 99807e730..cc094862b 100644 --- a/td/telegram/MessagesManager.h +++ b/td/telegram/MessagesManager.h @@ -1599,7 +1599,7 @@ class MessagesManager final : public Actor { protected: MessagesIteratorBase() = default; - // points iterator to message with greatest id which is less or equal than message_id + // points iterator to message with greatest identifier which is less or equal than message_id MessagesIteratorBase(const Message *root, MessageId message_id) { size_t last_right_pos = 0; while (root != nullptr) { diff --git a/td/telegram/StickersManager.cpp b/td/telegram/StickersManager.cpp index cff6562a0..c404e1bb3 100644 --- a/td/telegram/StickersManager.cpp +++ b/td/telegram/StickersManager.cpp @@ -3174,7 +3174,7 @@ StickerSetId StickersManager::on_get_input_sticker_set(FileId sticker_file_id, })); } // always return empty StickerSetId, because we can't trust the set_id provided by the peer in the secret chat - // the real sticker set id will be set in on_get_sticker if and only if the sticker is really from the set + // the real sticker set identifier will be set in on_get_sticker if and only if the sticker is really from the set return StickerSetId(); } case telegram_api::inputStickerSetAnimatedEmoji::ID: diff --git a/td/telegram/StickersManager.h b/td/telegram/StickersManager.h index 908bd543a..160dcaaee 100644 --- a/td/telegram/StickersManager.h +++ b/td/telegram/StickersManager.h @@ -967,8 +967,9 @@ class StickersManager final : public Actor { bool is_inited_ = false; - WaitFreeHashMap, FileIdHash> stickers_; // file_id -> Sticker - WaitFreeHashMap, StickerSetIdHash> sticker_sets_; // id -> StickerSet + WaitFreeHashMap, FileIdHash> stickers_; // file_id -> Sticker + WaitFreeHashMap, StickerSetIdHash> + sticker_sets_; // sticker_set_id -> StickerSet WaitFreeHashMap short_name_to_sticker_set_id_; vector installed_sticker_set_ids_[MAX_STICKER_TYPE]; diff --git a/tdactor/td/actor/impl/ActorInfo.h b/tdactor/td/actor/impl/ActorInfo.h index f734f4c0b..0bb85e728 100644 --- a/tdactor/td/actor/impl/ActorInfo.h +++ b/tdactor/td/actor/impl/ActorInfo.h @@ -69,8 +69,8 @@ inline void ActorInfo::clear() { CHECK(!actor_); CHECK(!is_running()); CHECK(!is_migrating()); - // NB: must be in non migrating state - // store invalid scheduler id. + // NB: must be in non-migrating state + // store invalid scheduler identifier sched_id_.store((1 << 30) - 1, std::memory_order_relaxed); VLOG(actor) << "Clear context " << context_.get() << " for " << get_name(); context_.reset(); diff --git a/tdutils/td/utils/Container.h b/tdutils/td/utils/Container.h index a3d62c7c4..5f1e0920e 100644 --- a/tdutils/td/utils/Container.h +++ b/tdutils/td/utils/Container.h @@ -155,7 +155,7 @@ class Container { void release(int32 id) { inc_generation(id); slots_[id].data = DataT(); - if (slots_[id].generation & ~TYPE_MASK) { // generation overflow. Can't use this id anymore + if (slots_[id].generation & ~TYPE_MASK) { // generation overflow. Can't use this identifier anymore empty_slots_.push_back(id); } } diff --git a/tdutils/td/utils/logging.cpp b/tdutils/td/utils/logging.cpp index a59589b26..cf73f5dfb 100644 --- a/tdutils/td/utils/logging.cpp +++ b/tdutils/td/utils/logging.cpp @@ -80,7 +80,7 @@ Logger::Logger(LogInterface &log, const LogOptions &options, int log_level, Slic } sb_ << ']'; - // thread id + // thread identifier auto thread_id = get_thread_id(); sb_ << "[t"; if (static_cast(thread_id) < 10) { diff --git a/tdutils/td/utils/port/UdpSocketFd.cpp b/tdutils/td/utils/port/UdpSocketFd.cpp index 3c94e7021..e5db2c2fb 100644 --- a/tdutils/td/utils/port/UdpSocketFd.cpp +++ b/tdutils/td/utils/port/UdpSocketFd.cpp @@ -600,12 +600,12 @@ class UdpSocketFdImpl { case EBADF: // impossible case ENOTSOCK: // impossible - case EPIPE: // impossible for udp - case ECONNRESET: // impossible for udp + case EPIPE: // impossible for UDP + case ECONNRESET: // impossible for UDP case EDESTADDRREQ: // we checked that address is valid case ENOTCONN: // we checked that address is valid case EINTR: // we already skipped all EINTR - case EISCONN: // impossible for udp socket + case EISCONN: // impossible for UDP socket case EOPNOTSUPP: case ENOTDIR: case EFAULT: diff --git a/tdutils/test/heap.cpp b/tdutils/test/heap.cpp index bc317152d..43072fb97 100644 --- a/tdutils/test/heap.cpp +++ b/tdutils/test/heap.cpp @@ -46,10 +46,12 @@ class CheckedHeap { nodes[i].value = i; } } + static void xx(int key, const td::HeapNode *heap_node) { const Node *node = static_cast(heap_node); std::fprintf(stderr, "(%d;%d)", node->key, node->value); } + void check() const { for (auto p : set_heap) { std::fprintf(stderr, "(%d;%d)", p.first, p.second); @@ -59,13 +61,16 @@ class CheckedHeap { std::fprintf(stderr, "\n"); kheap.check(); } + int random_id() const { CHECK(!empty()); return ids[td::Random::fast(0, static_cast(ids.size() - 1))]; } + std::size_t size() const { return ids.size(); } + bool empty() const { return ids.empty(); } @@ -77,8 +82,8 @@ class CheckedHeap { ASSERT_EQ(res, kheap.top_key()); return res; } + int insert(int key) { - // std::fprintf(stderr, "insert %d\n", key); int id; if (free_ids.empty()) { UNREACHABLE(); @@ -96,15 +101,15 @@ class CheckedHeap { set_heap.emplace(key, id); return id; } + void fix_key(int new_key, int id) { - // std::fprintf(stderr, "fix key %d %d (old_key = %d)\n", new_key, id, nodes[id].key); set_heap.erase(std::make_pair(nodes[id].key, id)); nodes[id].key = new_key; kheap.fix(new_key, &nodes[id]); set_heap.emplace(new_key, id); } + void erase(int id) { - // std::fprintf(stderr, "erase %d\n", id); int pos = rev_ids[id]; CHECK(pos != -1); ids[pos] = ids.back(); @@ -116,8 +121,8 @@ class CheckedHeap { kheap.erase(&nodes[id]); set_heap.erase(std::make_pair(nodes[id].key, id)); } + void pop() { - // std::fprintf(stderr, "pop\n"); CHECK(!empty()); Node *node = static_cast(kheap.pop()); int id = node->value; From 4408af6643c25932012320ff16f05f5b2c1d8ef3 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 16 Jan 2023 13:30:03 +0300 Subject: [PATCH 25/30] Improve "id" variable names in FileManager. --- td/telegram/files/FileDb.cpp | 124 +++++++++++++------------ td/telegram/files/FileDb.h | 8 +- td/telegram/files/FileDbId.h | 4 +- td/telegram/files/FileDownloader.cpp | 6 +- td/telegram/files/FileLoadManager.cpp | 44 +++++---- td/telegram/files/FileLoadManager.h | 36 +++---- td/telegram/files/FileLoader.cpp | 18 ++-- td/telegram/files/FileManager.cpp | 129 +++++++++++++------------- td/telegram/files/FileManager.h | 8 +- td/telegram/files/PartsManager.cpp | 101 ++++++++++---------- td/telegram/files/PartsManager.h | 10 +- 11 files changed, 245 insertions(+), 243 deletions(-) diff --git a/td/telegram/files/FileDb.cpp b/td/telegram/files/FileDb.cpp index d091deb77..75f04abeb 100644 --- a/td/telegram/files/FileDb.cpp +++ b/td/telegram/files/FileDb.cpp @@ -60,8 +60,8 @@ class FileDb final : public FileDbInterface { public: class FileDbActor final : public Actor { public: - FileDbActor(FileDbId current_pmc_id, std::shared_ptr file_kv_safe) - : current_pmc_id_(current_pmc_id), file_kv_safe_(std::move(file_kv_safe)) { + FileDbActor(FileDbId max_file_db_id, std::shared_ptr file_kv_safe) + : max_file_db_id_(max_file_db_id), file_kv_safe_(std::move(file_kv_safe)) { } void close(Promise<> promise) { @@ -72,20 +72,21 @@ class FileDb final : public FileDbInterface { } void load_file_data(const string &key, Promise promise) { - promise.set_result(load_file_data_impl(actor_id(this), file_pmc(), key, current_pmc_id_)); + promise.set_result(load_file_data_impl(actor_id(this), file_pmc(), key, max_file_db_id_)); } - void clear_file_data(FileDbId id, const string &remote_key, const string &local_key, const string &generate_key) { + void clear_file_data(FileDbId file_db_id, const string &remote_key, const string &local_key, + const string &generate_key) { auto &pmc = file_pmc(); pmc.begin_write_transaction().ensure(); - if (id > current_pmc_id_) { - pmc.set("file_id", to_string(id.get())); - current_pmc_id_ = id; + if (file_db_id > max_file_db_id_) { + pmc.set("file_id", to_string(file_db_id.get())); + max_file_db_id_ = file_db_id; } - pmc.erase(PSTRING() << "file" << id.get()); - // LOG(DEBUG) << "ERASE " << format::as_hex_dump<4>(Slice(PSLICE() << "file" << id.get())); + pmc.erase(PSTRING() << "file" << file_db_id.get()); + // LOG(DEBUG) << "ERASE " << format::as_hex_dump<4>(Slice(PSLICE() << "file" << file_db_id.get())); if (!remote_key.empty()) { pmc.erase(remote_key); @@ -102,79 +103,79 @@ class FileDb final : public FileDbInterface { pmc.commit_transaction().ensure(); } - void store_file_data(FileDbId id, const string &file_data, const string &remote_key, const string &local_key, - const string &generate_key) { + void store_file_data(FileDbId file_db_id, const string &file_data, const string &remote_key, + const string &local_key, const string &generate_key) { auto &pmc = file_pmc(); pmc.begin_write_transaction().ensure(); - if (id > current_pmc_id_) { - pmc.set("file_id", to_string(id.get())); - current_pmc_id_ = id; + if (file_db_id > max_file_db_id_) { + pmc.set("file_id", to_string(file_db_id.get())); + max_file_db_id_ = file_db_id; } - pmc.set(PSTRING() << "file" << id.get(), file_data); + pmc.set(PSTRING() << "file" << file_db_id.get(), file_data); if (!remote_key.empty()) { - pmc.set(remote_key, to_string(id.get())); + pmc.set(remote_key, to_string(file_db_id.get())); } if (!local_key.empty()) { - pmc.set(local_key, to_string(id.get())); + pmc.set(local_key, to_string(file_db_id.get())); } if (!generate_key.empty()) { - pmc.set(generate_key, to_string(id.get())); + pmc.set(generate_key, to_string(file_db_id.get())); } pmc.commit_transaction().ensure(); } - void store_file_data_ref(FileDbId id, FileDbId new_id) { + void store_file_data_ref(FileDbId file_db_id, FileDbId new_file_db_id) { auto &pmc = file_pmc(); pmc.begin_write_transaction().ensure(); - if (id > current_pmc_id_) { - pmc.set("file_id", to_string(id.get())); - current_pmc_id_ = id; + if (file_db_id > max_file_db_id_) { + pmc.set("file_id", to_string(file_db_id.get())); + max_file_db_id_ = file_db_id; } - do_store_file_data_ref(id, new_id); + do_store_file_data_ref(file_db_id, new_file_db_id); pmc.commit_transaction().ensure(); } - void optimize_refs(std::vector ids, FileDbId main_id) { - LOG(INFO) << "Optimize " << ids.size() << " ids in file database to " << main_id.get(); + void optimize_refs(std::vector file_db_ids, FileDbId main_file_db_id) { + LOG(INFO) << "Optimize " << file_db_ids.size() << " file_db_ids in file database to " << main_file_db_id.get(); auto &pmc = file_pmc(); pmc.begin_write_transaction().ensure(); - for (size_t i = 0; i + 1 < ids.size(); i++) { - do_store_file_data_ref(ids[i], main_id); + for (size_t i = 0; i + 1 < file_db_ids.size(); i++) { + do_store_file_data_ref(file_db_ids[i], main_file_db_id); } pmc.commit_transaction().ensure(); } private: - FileDbId current_pmc_id_; + FileDbId max_file_db_id_; std::shared_ptr file_kv_safe_; SqliteKeyValue &file_pmc() { return file_kv_safe_->get(); } - void do_store_file_data_ref(FileDbId id, FileDbId new_id) { - file_pmc().set(PSTRING() << "file" << id.get(), PSTRING() << "@@" << new_id.get()); + void do_store_file_data_ref(FileDbId file_db_id, FileDbId new_file_db_id) { + file_pmc().set(PSTRING() << "file" << file_db_id.get(), PSTRING() << "@@" << new_file_db_id.get()); } }; explicit FileDb(std::shared_ptr kv_safe, int scheduler_id = -1) { file_kv_safe_ = std::move(kv_safe); CHECK(file_kv_safe_); - current_pmc_id_ = FileDbId(to_integer(file_kv_safe_->get().get("file_id"))); + max_file_db_id_ = FileDbId(to_integer(file_kv_safe_->get().get("file_id"))); file_db_actor_ = - create_actor_on_scheduler("FileDbActor", scheduler_id, current_pmc_id_, file_kv_safe_); + create_actor_on_scheduler("FileDbActor", scheduler_id, max_file_db_id_, file_kv_safe_); } - FileDbId create_pmc_id() final { - current_pmc_id_ = FileDbId(current_pmc_id_.get() + 1); - return current_pmc_id_; + FileDbId get_next_file_db_id() final { + max_file_db_id_ = FileDbId(max_file_db_id_.get() + 1); + return max_file_db_id_; } void close(Promise<> promise) final { @@ -186,10 +187,10 @@ class FileDb final : public FileDbInterface { } Result get_file_data_sync_impl(string key) final { - return load_file_data_impl(file_db_actor_.get(), file_kv_safe_->get(), key, current_pmc_id_); + return load_file_data_impl(file_db_actor_.get(), file_kv_safe_->get(), key, max_file_db_id_); } - void clear_file_data(FileDbId id, const FileData &file_data) final { + void clear_file_data(FileDbId file_db_id, const FileData &file_data) final { string remote_key; if (file_data.remote_.type() == RemoteFileLocation::Type::Full) { remote_key = as_key(file_data.remote_.full()); @@ -202,10 +203,11 @@ class FileDb final : public FileDbInterface { if (file_data.generate_ != nullptr) { generate_key = as_key(*file_data.generate_); } - send_closure(file_db_actor_, &FileDbActor::clear_file_data, id, remote_key, local_key, generate_key); + send_closure(file_db_actor_, &FileDbActor::clear_file_data, file_db_id, remote_key, local_key, generate_key); } - void set_file_data(FileDbId id, const FileData &file_data, bool new_remote, bool new_local, bool new_generate) final { + void set_file_data(FileDbId file_db_id, const FileData &file_data, bool new_remote, bool new_local, + bool new_generate) final { string remote_key; if (file_data.remote_.type() == RemoteFileLocation::Type::Full && new_remote) { remote_key = as_key(file_data.remote_.full()); @@ -218,16 +220,16 @@ class FileDb final : public FileDbInterface { if (file_data.generate_ != nullptr && new_generate) { generate_key = as_key(*file_data.generate_); } - // LOG(DEBUG) << "SAVE " << id.get() << " -> " << file_data << " " + // LOG(DEBUG) << "SAVE " << file_db_id.get() << " -> " << file_data << " " // << tag("remote_key", format::as_hex_dump<4>(Slice(remote_key))) // << tag("local_key", format::as_hex_dump<4>(Slice(local_key))) // << tag("generate_key", format::as_hex_dump<4>(Slice(generate_key))); - send_closure(file_db_actor_, &FileDbActor::store_file_data, id, serialize(file_data), remote_key, local_key, + send_closure(file_db_actor_, &FileDbActor::store_file_data, file_db_id, serialize(file_data), remote_key, local_key, generate_key); } - void set_file_data_ref(FileDbId id, FileDbId new_id) final { - send_closure(file_db_actor_, &FileDbActor::store_file_data_ref, id, new_id); + void set_file_data_ref(FileDbId file_db_id, FileDbId new_file_db_id) final { + send_closure(file_db_actor_, &FileDbActor::store_file_data_ref, file_db_id, new_file_db_id); } SqliteKeyValue &pmc() final { return file_kv_safe_->get(); @@ -235,39 +237,39 @@ class FileDb final : public FileDbInterface { private: ActorOwn file_db_actor_; - FileDbId current_pmc_id_; + FileDbId max_file_db_id_; std::shared_ptr file_kv_safe_; static Result load_file_data_impl(ActorId file_db_actor_id, SqliteKeyValue &pmc, - const string &key, FileDbId current_pmc_id) { + const string &key, FileDbId max_file_db_id) { // LOG(DEBUG) << "Load by key " << format::as_hex_dump<4>(Slice(key)); - TRY_RESULT(id, get_id(pmc, key)); + TRY_RESULT(file_db_id, get_file_db_id(pmc, key)); - vector ids; + vector file_db_ids; string data_str; int attempt_count = 0; while (true) { if (attempt_count > 100) { - LOG(FATAL) << "Cycle in file database? current_pmc_id=" << current_pmc_id << " key=" << key - << " links=" << format::as_array(ids); + LOG(FATAL) << "Cycle in file database? max_file_db_id=" << max_file_db_id << " key=" << key + << " links=" << format::as_array(file_db_ids); } attempt_count++; - data_str = pmc.get(PSTRING() << "file" << id.get()); + data_str = pmc.get(PSTRING() << "file" << file_db_id.get()); auto data_slice = Slice(data_str); if (data_slice.substr(0, 2) == "@@") { - ids.push_back(id); + file_db_ids.push_back(file_db_id); - id = FileDbId(to_integer(data_slice.substr(2))); + file_db_id = FileDbId(to_integer(data_slice.substr(2))); } else { break; } } - if (ids.size() > 1) { - send_closure(file_db_actor_id, &FileDbActor::optimize_refs, std::move(ids), id); + if (file_db_ids.size() > 1) { + send_closure(file_db_actor_id, &FileDbActor::optimize_refs, std::move(file_db_ids), file_db_id); } - // LOG(DEBUG) << "By ID " << id.get() << " found data " << format::as_hex_dump<4>(Slice(data_str)); + // LOG(DEBUG) << "By ID " << file_db_id.get() << " found data " << format::as_hex_dump<4>(Slice(data_str)); // LOG(INFO) << attempt_count; log_event::WithVersion parser(data_str); @@ -282,13 +284,13 @@ class FileDb final : public FileDbInterface { return std::move(data); } - static Result get_id(SqliteKeyValue &pmc, const string &key) TD_WARN_UNUSED_RESULT { - auto id_str = pmc.get(key); - // LOG(DEBUG) << "Found ID " << id_str << " by key " << format::as_hex_dump<4>(Slice(key)); - if (id_str.empty()) { - return Status::Error("There is no such a key in database"); + static Result get_file_db_id(SqliteKeyValue &pmc, const string &key) TD_WARN_UNUSED_RESULT { + auto file_db_id_str = pmc.get(key); + // LOG(DEBUG) << "Found ID " << file_db_id_str << " by key " << format::as_hex_dump<4>(Slice(key)); + if (file_db_id_str.empty()) { + return Status::Error("There is no such key in the database"); } - return FileDbId(to_integer(id_str)); + return FileDbId(to_integer(file_db_id_str)); } }; diff --git a/td/telegram/files/FileDb.h b/td/telegram/files/FileDb.h index 46483ae67..750e82220 100644 --- a/td/telegram/files/FileDb.h +++ b/td/telegram/files/FileDb.h @@ -39,7 +39,7 @@ class FileDbInterface { virtual ~FileDbInterface() = default; // non-thread-safe - virtual FileDbId create_pmc_id() = 0; + virtual FileDbId get_next_file_db_id() = 0; // thread-safe virtual void close(Promise<> promise) = 0; @@ -75,10 +75,10 @@ class FileDbInterface { return res; } - virtual void clear_file_data(FileDbId id, const FileData &file_data) = 0; - virtual void set_file_data(FileDbId id, const FileData &file_data, bool new_remote, bool new_local, + virtual void clear_file_data(FileDbId file_db_id, const FileData &file_data) = 0; + virtual void set_file_data(FileDbId file_db_id, const FileData &file_data, bool new_remote, bool new_local, bool new_generate) = 0; - virtual void set_file_data_ref(FileDbId id, FileDbId new_id) = 0; + virtual void set_file_data_ref(FileDbId file_db_id, FileDbId new_file_db_id) = 0; // For FileStatsWorker. TODO: remove it virtual SqliteKeyValue &pmc() = 0; diff --git a/td/telegram/files/FileDbId.h b/td/telegram/files/FileDbId.h index 06f04d866..e68030dd6 100644 --- a/td/telegram/files/FileDbId.h +++ b/td/telegram/files/FileDbId.h @@ -51,8 +51,8 @@ class FileDbId { } }; -inline StringBuilder &operator<<(StringBuilder &sb, const FileDbId &id) { - return sb << "FileDbId{" << id.get() << "}"; +inline StringBuilder &operator<<(StringBuilder &sb, const FileDbId &file_db_id) { + return sb << "FileDbId{" << file_db_id.get() << "}"; } } // namespace td diff --git a/td/telegram/files/FileDownloader.cpp b/td/telegram/files/FileDownloader.cpp index 4324310c0..4545ea27a 100644 --- a/td/telegram/files/FileDownloader.cpp +++ b/td/telegram/files/FileDownloader.cpp @@ -245,16 +245,16 @@ Result> FileDownloader::start_part(Part part, int32 } #endif DcId dc_id = remote_.is_web() ? G()->get_webfile_dc_id() : remote_.get_dc_id(); - auto id = UniqueId::next(UniqueId::Type::Default, static_cast(QueryType::Default)); + auto unique_id = UniqueId::next(UniqueId::Type::Default, static_cast(QueryType::Default)); net_query = remote_.is_web() ? G()->net_query_creator().create( - id, + unique_id, telegram_api::upload_getWebFile(remote_.as_input_web_file_location(), narrow_cast(part.offset), narrow_cast(size)), {}, dc_id, net_query_type, NetQuery::AuthFlag::On) : G()->net_query_creator().create( - id, + unique_id, telegram_api::upload_getFile(flags, false /*ignored*/, false /*ignored*/, remote_.as_input_file_location(), part.offset, narrow_cast(size)), {}, dc_id, net_query_type, NetQuery::AuthFlag::On); diff --git a/td/telegram/files/FileLoadManager.cpp b/td/telegram/files/FileLoadManager.cpp index 5d0d09036..3b13e60c9 100644 --- a/td/telegram/files/FileLoadManager.cpp +++ b/td/telegram/files/FileLoadManager.cpp @@ -43,7 +43,7 @@ ActorOwn &FileLoadManager::get_download_resource_manager(bool i return actor; } -void FileLoadManager::download(QueryId id, const FullRemoteFileLocation &remote_location, +void FileLoadManager::download(QueryId query_id, const FullRemoteFileLocation &remote_location, const LocalFileLocation &local, int64 size, string name, const FileEncryptionKey &encryption_key, bool search_file, int64 offset, int64 limit, int8 priority) { @@ -53,7 +53,7 @@ void FileLoadManager::download(QueryId id, const FullRemoteFileLocation &remote_ NodeId node_id = nodes_container_.create(Node()); Node *node = nodes_container_.get(node_id); CHECK(node); - node->query_id_ = id; + node->query_id_ = query_id; auto callback = make_unique(actor_shared(this, node_id)); bool is_small = size < 20 * 1024; node->loader_ = @@ -63,11 +63,11 @@ void FileLoadManager::download(QueryId id, const FullRemoteFileLocation &remote_ auto &resource_manager = get_download_resource_manager(is_small, dc_id); send_closure(resource_manager, &ResourceManager::register_worker, ActorShared(node->loader_.get(), static_cast(-1)), priority); - bool is_inserted = query_id_to_node_id_.emplace(id, node_id).second; + bool is_inserted = query_id_to_node_id_.emplace(query_id, node_id).second; CHECK(is_inserted); } -void FileLoadManager::upload(QueryId id, const LocalFileLocation &local_location, +void FileLoadManager::upload(QueryId query_id, const LocalFileLocation &local_location, const RemoteFileLocation &remote_location, int64 expected_size, const FileEncryptionKey &encryption_key, int8 priority, vector bad_parts) { if (stop_flag_) { @@ -76,17 +76,17 @@ void FileLoadManager::upload(QueryId id, const LocalFileLocation &local_location NodeId node_id = nodes_container_.create(Node()); Node *node = nodes_container_.get(node_id); CHECK(node); - node->query_id_ = id; + node->query_id_ = query_id; auto callback = make_unique(actor_shared(this, node_id)); node->loader_ = create_actor("Uploader", local_location, remote_location, expected_size, encryption_key, std::move(bad_parts), std::move(callback)); send_closure(upload_resource_manager_, &ResourceManager::register_worker, ActorShared(node->loader_.get(), static_cast(-1)), priority); - bool is_inserted = query_id_to_node_id_.emplace(id, node_id).second; + bool is_inserted = query_id_to_node_id_.emplace(query_id, node_id).second; CHECK(is_inserted); } -void FileLoadManager::upload_by_hash(QueryId id, const FullLocalFileLocation &local_location, int64 size, +void FileLoadManager::upload_by_hash(QueryId query_id, const FullLocalFileLocation &local_location, int64 size, int8 priority) { if (stop_flag_) { return; @@ -94,20 +94,20 @@ void FileLoadManager::upload_by_hash(QueryId id, const FullLocalFileLocation &lo NodeId node_id = nodes_container_.create(Node()); Node *node = nodes_container_.get(node_id); CHECK(node); - node->query_id_ = id; + node->query_id_ = query_id; auto callback = make_unique(actor_shared(this, node_id)); node->loader_ = create_actor("HashUploader", local_location, size, std::move(callback)); send_closure(upload_resource_manager_, &ResourceManager::register_worker, ActorShared(node->loader_.get(), static_cast(-1)), priority); - bool is_inserted = query_id_to_node_id_.emplace(id, node_id).second; + bool is_inserted = query_id_to_node_id_.emplace(query_id, node_id).second; CHECK(is_inserted); } -void FileLoadManager::update_priority(QueryId id, int8 priority) { +void FileLoadManager::update_priority(QueryId query_id, int8 priority) { if (stop_flag_) { return; } - auto it = query_id_to_node_id_.find(id); + auto it = query_id_to_node_id_.find(query_id); if (it == query_id_to_node_id_.end()) { return; } @@ -118,18 +118,18 @@ void FileLoadManager::update_priority(QueryId id, int8 priority) { send_closure(node->loader_, &FileLoaderActor::update_priority, priority); } -void FileLoadManager::from_bytes(QueryId id, FileType type, BufferSlice bytes, string name) { +void FileLoadManager::from_bytes(QueryId query_id, FileType type, BufferSlice bytes, string name) { if (stop_flag_) { return; } NodeId node_id = nodes_container_.create(Node()); Node *node = nodes_container_.get(node_id); CHECK(node); - node->query_id_ = id; + node->query_id_ = query_id; auto callback = make_unique(actor_shared(this, node_id)); node->loader_ = create_actor("FromBytes", type, std::move(bytes), std::move(name), std::move(callback)); - bool is_inserted = query_id_to_node_id_.emplace(id, node_id).second; + bool is_inserted = query_id_to_node_id_.emplace(query_id, node_id).second; CHECK(is_inserted); } @@ -160,23 +160,21 @@ void FileLoadManager::check_partial_local_location(PartialLocalFileLocation part } } -// void upload_reload_parts(QueryId id, vector parts); -// void upload_restart(QueryId id); -void FileLoadManager::cancel(QueryId id) { +void FileLoadManager::cancel(QueryId query_id) { if (stop_flag_) { return; } - auto it = query_id_to_node_id_.find(id); + auto it = query_id_to_node_id_.find(query_id); if (it == query_id_to_node_id_.end()) { return; } on_error_impl(it->second, Status::Error(-1, "Canceled")); } -void FileLoadManager::update_local_file_location(QueryId id, const LocalFileLocation &local) { +void FileLoadManager::update_local_file_location(QueryId query_id, const LocalFileLocation &local) { if (stop_flag_) { return; } - auto it = query_id_to_node_id_.find(id); + auto it = query_id_to_node_id_.find(query_id); if (it == query_id_to_node_id_.end()) { return; } @@ -187,11 +185,11 @@ void FileLoadManager::update_local_file_location(QueryId id, const LocalFileLoca send_closure(node->loader_, &FileLoaderActor::update_local_file_location, local); } -void FileLoadManager::update_downloaded_part(QueryId id, int64 offset, int64 limit) { +void FileLoadManager::update_downloaded_part(QueryId query_id, int64 offset, int64 limit) { if (stop_flag_) { return; } - auto it = query_id_to_node_id_.find(id); + auto it = query_id_to_node_id_.find(query_id); if (it == query_id_to_node_id_.end()) { return; } @@ -203,7 +201,7 @@ void FileLoadManager::update_downloaded_part(QueryId id, int64 offset, int64 lim } void FileLoadManager::hangup() { - nodes_container_.for_each([](auto id, auto &node) { node.loader_.reset(); }); + nodes_container_.for_each([](auto query_id, auto &node) { node.loader_.reset(); }); stop_flag_ = true; loop(); } diff --git a/td/telegram/files/FileLoadManager.h b/td/telegram/files/FileLoadManager.h index 239e701fa..865d75821 100644 --- a/td/telegram/files/FileLoadManager.h +++ b/td/telegram/files/FileLoadManager.h @@ -34,30 +34,30 @@ class FileLoadManager final : public Actor { using QueryId = uint64; class Callback : public Actor { public: - virtual void on_start_download(QueryId id) = 0; - virtual void on_partial_download(QueryId id, PartialLocalFileLocation partial_local, int64 ready_size, + virtual void on_start_download(QueryId query_id) = 0; + virtual void on_partial_download(QueryId query_id, PartialLocalFileLocation partial_local, int64 ready_size, int64 size) = 0; - virtual void on_partial_upload(QueryId id, PartialRemoteFileLocation partial_remote, int64 ready_size) = 0; - virtual void on_hash(QueryId id, string hash) = 0; - virtual void on_upload_ok(QueryId id, FileType file_type, PartialRemoteFileLocation remtoe, int64 size) = 0; - virtual void on_upload_full_ok(QueryId id, FullRemoteFileLocation remote) = 0; - virtual void on_download_ok(QueryId id, FullLocalFileLocation local, int64 size, bool is_new) = 0; - virtual void on_error(QueryId id, Status status) = 0; + virtual void on_partial_upload(QueryId query_id, PartialRemoteFileLocation partial_remote, int64 ready_size) = 0; + virtual void on_hash(QueryId query_id, string hash) = 0; + virtual void on_upload_ok(QueryId query_id, FileType file_type, PartialRemoteFileLocation remtoe, int64 size) = 0; + virtual void on_upload_full_ok(QueryId query_id, FullRemoteFileLocation remote) = 0; + virtual void on_download_ok(QueryId query_id, FullLocalFileLocation local, int64 size, bool is_new) = 0; + virtual void on_error(QueryId query_id, Status status) = 0; }; explicit FileLoadManager(ActorShared callback, ActorShared<> parent); - void download(QueryId id, const FullRemoteFileLocation &remote_location, const LocalFileLocation &local, int64 size, - string name, const FileEncryptionKey &encryption_key, bool search_file, int64 offset, int64 limit, - int8 priority); - void upload(QueryId id, const LocalFileLocation &local_location, const RemoteFileLocation &remote_location, + void download(QueryId query_id, const FullRemoteFileLocation &remote_location, const LocalFileLocation &local, + int64 size, string name, const FileEncryptionKey &encryption_key, bool search_file, int64 offset, + int64 limit, int8 priority); + void upload(QueryId query_id, const LocalFileLocation &local_location, const RemoteFileLocation &remote_location, int64 expected_size, const FileEncryptionKey &encryption_key, int8 priority, vector bad_parts); - void upload_by_hash(QueryId id, const FullLocalFileLocation &local_location, int64 size, int8 priority); - void update_priority(QueryId id, int8 priority); - void from_bytes(QueryId id, FileType type, BufferSlice bytes, string name); - void cancel(QueryId id); - void update_local_file_location(QueryId id, const LocalFileLocation &local); - void update_downloaded_part(QueryId id, int64 offset, int64 limit); + void upload_by_hash(QueryId query_id, const FullLocalFileLocation &local_location, int64 size, int8 priority); + void update_priority(QueryId query_id, int8 priority); + void from_bytes(QueryId query_id, FileType type, BufferSlice bytes, string name); + void cancel(QueryId query_id); + void update_local_file_location(QueryId query_id, const LocalFileLocation &local); + void update_downloaded_part(QueryId query_id, int64 offset, int64 limit); void get_content(string file_path, Promise promise); diff --git a/td/telegram/files/FileLoader.cpp b/td/telegram/files/FileLoader.cpp index 3793a75e0..760cb1944 100644 --- a/td/telegram/files/FileLoader.cpp +++ b/td/telegram/files/FileLoader.cpp @@ -211,15 +211,15 @@ Status FileLoader::do_loop() { NetQueryPtr query; bool is_blocking; std::tie(query, is_blocking) = std::move(query_flag); - uint64 id = UniqueId::next(); + uint64 unique_id = UniqueId::next(); if (is_blocking) { CHECK(blocking_id_ == 0); - blocking_id_ = id; + blocking_id_ = unique_id; } - part_map_[id] = std::make_pair(part, query->cancel_slot_.get_signal_new()); - // part_map_[id] = std::make_pair(part, query.get_weak()); + part_map_[unique_id] = std::make_pair(part, query->cancel_slot_.get_signal_new()); + // part_map_[unique_id] = std::make_pair(part, query.get_weak()); - auto callback = actor_shared(this, id); + auto callback = actor_shared(this, unique_id); if (delay_dispatcher_.empty()) { G()->net_query_dispatcher().dispatch_with_callback(std::move(query), std::move(callback)); } else { @@ -259,15 +259,15 @@ void FileLoader::on_result(NetQueryPtr query) { if (stop_flag_) { return; } - auto id = get_link_token(); - if (id == blocking_id_) { + auto unique_id = get_link_token(); + if (unique_id == blocking_id_) { blocking_id_ = 0; } - if (UniqueId::extract_key(id) == COMMON_QUERY_KEY) { + if (UniqueId::extract_key(unique_id) == COMMON_QUERY_KEY) { on_common_query(std::move(query)); return loop(); } - auto it = part_map_.find(id); + auto it = part_map_.find(unique_id); if (it == part_map_.end()) { LOG(WARNING) << "Got result for unknown part"; return; diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index 93203bbf0..a6a4de45e 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -1957,7 +1957,7 @@ void FileManager::flush_to_pmc(FileNodePtr node, bool new_remote, bool new_local bool create_flag = false; if (node->pmc_id_.empty()) { create_flag = true; - node->pmc_id_ = file_db_->create_pmc_id(); + node->pmc_id_ = file_db_->get_next_file_db_id(); } FileData data; @@ -2113,10 +2113,10 @@ bool FileManager::set_content(FileId file_id, BufferSlice bytes) { node->set_download_priority(FROM_BYTES_PRIORITY); - QueryId id = queries_container_.create(Query{file_id, Query::Type::SetContent}); - node->download_id_ = id; + QueryId query_id = queries_container_.create(Query{file_id, Query::Type::SetContent}); + node->download_id_ = query_id; node->is_download_started_ = true; - send_closure(file_load_manager_, &FileLoadManager::from_bytes, id, node->remote_.full.value().file_type_, + send_closure(file_load_manager_, &FileLoadManager::from_bytes, query_id, node->remote_.full.value().file_type_, std::move(bytes), node->suggested_path()); return true; } @@ -2422,10 +2422,10 @@ void FileManager::run_download(FileNodePtr node, bool force_update_priority) { if (node->need_reload_photo_ && file_view.may_reload_photo()) { LOG(INFO) << "Reload photo from file " << node->main_file_id_; - QueryId id = queries_container_.create(Query{file_id, Query::Type::DownloadReloadDialog}); - node->download_id_ = id; + QueryId query_id = queries_container_.create(Query{file_id, Query::Type::DownloadReloadDialog}); + node->download_id_ = query_id; context_->reload_photo(file_view.remote_location().get_source(), - PromiseCreator::lambda([id, actor_id = actor_id(this), file_id](Result res) { + PromiseCreator::lambda([query_id, actor_id = actor_id(this), file_id](Result res) { Status error; if (res.is_ok()) { error = Status::Error("FILE_DOWNLOAD_ID_INVALID"); @@ -2434,7 +2434,7 @@ void FileManager::run_download(FileNodePtr node, bool force_update_priority) { } VLOG(file_references) << "Got result from reload photo for file " << file_id << ": " << error; - send_closure(actor_id, &FileManager::on_error, id, std::move(error)); + send_closure(actor_id, &FileManager::on_error, query_id, std::move(error)); })); node->need_reload_photo_ = false; return; @@ -2443,16 +2443,16 @@ void FileManager::run_download(FileNodePtr node, bool force_update_priority) { // If file reference is needed if (!file_view.has_active_download_remote_location()) { VLOG(file_references) << "Do not have valid file_reference for file " << file_id; - QueryId id = queries_container_.create(Query{file_id, Query::Type::DownloadWaitFileReference}); - node->download_id_ = id; + QueryId query_id = queries_container_.create(Query{file_id, Query::Type::DownloadWaitFileReference}); + node->download_id_ = query_id; if (node->download_was_update_file_reference_) { - on_error(id, Status::Error("Can't download file: have no valid file reference")); + on_error(query_id, Status::Error("Can't download file: have no valid file reference")); return; } node->download_was_update_file_reference_ = true; context_->repair_file_reference( - file_id, PromiseCreator::lambda([id, actor_id = actor_id(this), file_id](Result res) { + file_id, PromiseCreator::lambda([query_id, actor_id = actor_id(this), file_id](Result res) { Status error; if (res.is_ok()) { error = Status::Error("FILE_DOWNLOAD_RESTART_WITH_FILE_REFERENCE"); @@ -2460,13 +2460,13 @@ void FileManager::run_download(FileNodePtr node, bool force_update_priority) { error = res.move_as_error(); } VLOG(file_references) << "Got result from FileSourceManager for file " << file_id << ": " << error; - send_closure(actor_id, &FileManager::on_error, id, std::move(error)); + send_closure(actor_id, &FileManager::on_error, query_id, std::move(error)); })); return; } - QueryId id = queries_container_.create(Query{file_id, Query::Type::Download}); - node->download_id_ = id; + QueryId query_id = queries_container_.create(Query{file_id, Query::Type::Download}); + node->download_id_ = query_id; node->is_download_started_ = false; LOG(INFO) << "Run download of file " << file_id << " of size " << node->size_ << " from " << node->remote_.full.value() << " with suggested name " << node->suggested_path() << " and encyption key " @@ -2479,7 +2479,7 @@ void FileManager::run_download(FileNodePtr node, bool force_update_priority) { download_limit += download_offset; download_offset = 0; } - send_closure(file_load_manager_, &FileLoadManager::download, id, node->remote_.full.value(), node->local_, + send_closure(file_load_manager_, &FileLoadManager::download, query_id, node->remote_.full.value(), node->local_, node->size_, node->suggested_path(), node->encryption_key_, node->can_search_locally_, download_offset, download_limit, priority); } @@ -2776,20 +2776,20 @@ void FileManager::delete_file_reference(FileId file_id, Slice file_reference) { try_flush_node_pmc(node, "delete_file_reference"); } -void FileManager::external_file_generate_write_part(int64 id, int64 offset, string data, Promise<> promise) { - send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_write_part, id, offset, - std::move(data), std::move(promise)); +void FileManager::external_file_generate_write_part(int64 generation_id, int64 offset, string data, Promise<> promise) { + send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_write_part, + static_cast(generation_id), offset, std::move(data), std::move(promise)); } -void FileManager::external_file_generate_progress(int64 id, int64 expected_size, int64 local_prefix_size, +void FileManager::external_file_generate_progress(int64 generation_id, int64 expected_size, int64 local_prefix_size, Promise<> promise) { - send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_progress, id, expected_size, - local_prefix_size, std::move(promise)); + send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_progress, + static_cast(generation_id), expected_size, local_prefix_size, std::move(promise)); } -void FileManager::external_file_generate_finish(int64 id, Status status, Promise<> promise) { - send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_finish, id, std::move(status), - std::move(promise)); +void FileManager::external_file_generate_finish(int64 generation_id, Status status, Promise<> promise) { + send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_finish, + static_cast(generation_id), std::move(status), std::move(promise)); } void FileManager::run_generate(FileNodePtr node) { @@ -2846,30 +2846,30 @@ void FileManager::run_generate(FileNodePtr node) { return; } - QueryId id = queries_container_.create(Query{file_id, Query::Type::Generate}); - node->generate_id_ = id; - send_closure(file_generate_manager_, &FileGenerateManager::generate_file, id, *node->generate_, node->local_, - node->suggested_path(), [file_manager = this, id] { - class Callback final : public FileGenerateCallback { - ActorId actor_; - uint64 query_id_; + QueryId query_id = queries_container_.create(Query{file_id, Query::Type::Generate}); + node->generate_id_ = query_id; + send_closure( + file_generate_manager_, &FileGenerateManager::generate_file, query_id, *node->generate_, node->local_, + node->suggested_path(), [file_manager = this, query_id] { + class Callback final : public FileGenerateCallback { + ActorId actor_; + uint64 query_id_; - public: - Callback(ActorId actor, QueryId id) : actor_(std::move(actor)), query_id_(id) { - } - void on_partial_generate(PartialLocalFileLocation partial_local, int64 expected_size) final { - send_closure(actor_, &FileManager::on_partial_generate, query_id_, std::move(partial_local), - expected_size); - } - void on_ok(FullLocalFileLocation local) final { - send_closure(actor_, &FileManager::on_generate_ok, query_id_, std::move(local)); - } - void on_error(Status error) final { - send_closure(actor_, &FileManager::on_error, query_id_, std::move(error)); - } - }; - return make_unique(file_manager->actor_id(file_manager), id); - }()); + public: + Callback(ActorId actor, QueryId query_id) : actor_(std::move(actor)), query_id_(query_id) { + } + void on_partial_generate(PartialLocalFileLocation partial_local, int64 expected_size) final { + send_closure(actor_, &FileManager::on_partial_generate, query_id_, std::move(partial_local), expected_size); + } + void on_ok(FullLocalFileLocation local) final { + send_closure(actor_, &FileManager::on_generate_ok, query_id_, std::move(local)); + } + void on_error(Status error) final { + send_closure(actor_, &FileManager::on_error, query_id_, std::move(error)); + } + }; + return make_unique(file_manager->actor_id(file_manager), query_id); + }()); LOG(INFO) << "File " << file_id << " generate request has sent to FileGenerateManager"; } @@ -2948,27 +2948,28 @@ void FileManager::run_upload(FileNodePtr node, vector bad_parts) { CHECK(node->upload_id_ == 0); if (file_view.has_alive_remote_location() && !file_view.has_active_upload_remote_location() && can_reuse_remote_file(file_view.get_type())) { - QueryId id = queries_container_.create(Query{file_id, Query::Type::UploadWaitFileReference}); - node->upload_id_ = id; + QueryId query_id = queries_container_.create(Query{file_id, Query::Type::UploadWaitFileReference}); + node->upload_id_ = query_id; if (node->upload_was_update_file_reference_) { - on_error(id, Status::Error("Can't upload file: have no valid file reference")); + on_error(query_id, Status::Error("Can't upload file: have no valid file reference")); return; } node->upload_was_update_file_reference_ = true; - context_->repair_file_reference( - node->main_file_id_, PromiseCreator::lambda([id, actor_id = actor_id(this)](Result res) { - send_closure(actor_id, &FileManager::on_error, id, Status::Error("FILE_UPLOAD_RESTART_WITH_FILE_REFERENCE")); - })); + context_->repair_file_reference(node->main_file_id_, + PromiseCreator::lambda([query_id, actor_id = actor_id(this)](Result res) { + send_closure(actor_id, &FileManager::on_error, query_id, + Status::Error("FILE_UPLOAD_RESTART_WITH_FILE_REFERENCE")); + })); return; } if (!node->remote_.partial && node->get_by_hash_) { LOG(INFO) << "Get file " << node->main_file_id_ << " by hash"; - QueryId id = queries_container_.create(Query{file_id, Query::Type::UploadByHash}); - node->upload_id_ = id; + QueryId query_id = queries_container_.create(Query{file_id, Query::Type::UploadByHash}); + node->upload_id_ = query_id; - send_closure(file_load_manager_, &FileLoadManager::upload_by_hash, id, node->local_.full(), node->size_, + send_closure(file_load_manager_, &FileLoadManager::upload_by_hash, query_id, node->local_.full(), node->size_, narrow_cast(-priority)); return; } @@ -2981,9 +2982,9 @@ void FileManager::run_upload(FileNodePtr node, vector bad_parts) { expected_size = 10 << 20; } - QueryId id = queries_container_.create(Query{file_id, Query::Type::Upload}); - node->upload_id_ = id; - send_closure(file_load_manager_, &FileLoadManager::upload, id, node->local_, node->remote_.partial_or_empty(), + QueryId query_id = queries_container_.create(Query{file_id, Query::Type::Upload}); + node->upload_id_ = query_id; + send_closure(file_load_manager_, &FileLoadManager::upload, query_id, node->local_, node->remote_.partial_or_empty(), expected_size, node->encryption_key_, new_priority, std::move(bad_parts)); LOG(INFO) << "File " << file_id << " upload request has sent to FileLoadManager"; @@ -4078,9 +4079,9 @@ void FileManager::hangup() { file_generate_manager_.reset(); file_load_manager_.reset(); while (!queries_container_.empty()) { - auto ids = queries_container_.ids(); - for (auto id : ids) { - on_error(id, Global::request_aborted_error()); + auto query_ids = queries_container_.ids(); + for (auto query_id : query_ids) { + on_error(query_id, Global::request_aborted_error()); } } is_closed_ = true; diff --git a/td/telegram/files/FileManager.h b/td/telegram/files/FileManager.h index f72e8deb7..e1f8d1a0d 100644 --- a/td/telegram/files/FileManager.h +++ b/td/telegram/files/FileManager.h @@ -470,9 +470,10 @@ class FileManager final : public FileLoadManager::Callback { void delete_file(FileId file_id, Promise promise, const char *source); - void external_file_generate_write_part(int64 id, int64 offset, string data, Promise<> promise); - void external_file_generate_progress(int64 id, int64 expected_size, int64 local_prefix_size, Promise<> promise); - void external_file_generate_finish(int64 id, Status status, Promise<> promise); + void external_file_generate_write_part(int64 generation_id, int64 offset, string data, Promise<> promise); + void external_file_generate_progress(int64 generation_id, int64 expected_size, int64 local_prefix_size, + Promise<> promise); + void external_file_generate_finish(int64 generation_id, Status status, Promise<> promise); Result from_persistent_id(CSlice persistent_id, FileType file_type) TD_WARN_UNUSED_RESULT; FileView get_file_view(FileId file_id) const; @@ -659,7 +660,6 @@ class FileManager final : public FileLoadManager::Callback { void on_force_reupload_success(FileId file_id); - // void release_file_node(FileNodeId id); void do_cancel_download(FileNodePtr node); void do_cancel_upload(FileNodePtr node); void do_cancel_generate(FileNodePtr node); diff --git a/td/telegram/files/PartsManager.cpp b/td/telegram/files/PartsManager.cpp index 0fdb801e3..8eaecd100 100644 --- a/td/telegram/files/PartsManager.cpp +++ b/td/telegram/files/PartsManager.cpp @@ -46,17 +46,17 @@ int32 PartsManager::set_streaming_offset(int64 offset, int64 limit) { return finish(); } - auto part_i = offset / part_size_; - if (use_part_count_limit_ && part_i >= MAX_PART_COUNT_PREMIUM) { + auto part_id = offset / part_size_; + if (use_part_count_limit_ && part_id >= MAX_PART_COUNT_PREMIUM) { streaming_offset_ = 0; - LOG(ERROR) << "Ignore streaming_offset " << offset << " in part " << part_i; + LOG(ERROR) << "Ignore streaming_offset " << offset << " in part " << part_id; return finish(); } streaming_offset_ = offset; - first_streaming_empty_part_ = narrow_cast(part_i); - first_streaming_not_ready_part_ = narrow_cast(part_i); + first_streaming_empty_part_ = narrow_cast(part_id); + first_streaming_not_ready_part_ = narrow_cast(part_id); if (part_count_ < first_streaming_empty_part_) { part_count_ = first_streaming_empty_part_; part_status_.resize(part_count_, PartStatus::Empty); @@ -75,9 +75,9 @@ void PartsManager::set_streaming_limit(int64 limit) { if (streaming_limit_ == 0) { return; } - for (int part_i = 0; part_i < part_count_; part_i++) { - if (is_part_in_streaming_limit(part_i) && part_status_[part_i] == PartStatus::Ready) { - streaming_ready_size_ += get_part(part_i).size; + for (int part_id = 0; part_id < part_count_; part_id++) { + if (is_part_in_streaming_limit(part_id) && part_status_[part_id] == PartStatus::Ready) { + streaming_ready_size_ += get_part(part_id).size; } } } @@ -231,10 +231,10 @@ string PartsManager::get_bitmask() { return bitmask_.encode(prefix_count); } -bool PartsManager::is_part_in_streaming_limit(int part_i) const { - CHECK(part_i < part_count_); - auto offset_begin = static_cast(part_i) * static_cast(get_part_size()); - auto offset_end = offset_begin + static_cast(get_part(part_i).size); +bool PartsManager::is_part_in_streaming_limit(int part_id) const { + CHECK(part_id < part_count_); + auto offset_begin = static_cast(part_id) * static_cast(get_part_size()); + auto offset_end = offset_begin + static_cast(get_part(part_id).size); if (offset_begin >= get_expected_size()) { return false; @@ -265,25 +265,25 @@ bool PartsManager::is_streaming_limit_reached() { return false; } update_first_not_ready_part(); - auto part_i = first_streaming_not_ready_part_; + auto part_id = first_streaming_not_ready_part_; // wrap - if (!unknown_size_flag_ && part_i == part_count_) { - part_i = first_not_ready_part_; + if (!unknown_size_flag_ && part_id == part_count_) { + part_id = first_not_ready_part_; } - if (part_i == part_count_) { + if (part_id == part_count_) { return false; } - return !is_part_in_streaming_limit(part_i); + return !is_part_in_streaming_limit(part_id); } Result PartsManager::start_part() { update_first_empty_part(); - auto part_i = first_streaming_empty_part_; - if (known_prefix_flag_ && part_i >= static_cast(known_prefix_size_ / part_size_)) { + auto part_id = first_streaming_empty_part_; + if (known_prefix_flag_ && part_id >= static_cast(known_prefix_size_ / part_size_)) { return Status::Error(-1, "Wait for prefix to be known"); } - if (part_i == part_count_) { + if (part_id == part_count_) { if (unknown_size_flag_) { part_count_++; if (part_count_ > MAX_PART_COUNT_PREMIUM + (use_part_count_limit_ ? 0 : 64)) { @@ -296,19 +296,19 @@ Result PartsManager::start_part() { part_status_.push_back(PartStatus::Empty); } else { if (first_empty_part_ < part_count_) { - part_i = first_empty_part_; + part_id = first_empty_part_; } else { return get_empty_part(); } } } - if (!is_part_in_streaming_limit(part_i)) { + if (!is_part_in_streaming_limit(part_id)) { return get_empty_part(); } - CHECK(part_status_[part_i] == PartStatus::Empty); - on_part_start(part_i); - return get_part(part_i); + CHECK(part_status_[part_id] == PartStatus::Empty); + on_part_start(part_id); + return get_part(part_id); } Status PartsManager::set_known_prefix(size_t size, bool is_ready) { @@ -341,22 +341,23 @@ Status PartsManager::set_known_prefix(size_t size, bool is_ready) { return Status::OK(); } -Status PartsManager::on_part_ok(int32 id, size_t part_size, size_t actual_size) { - CHECK(part_status_[id] == PartStatus::Pending); +Status PartsManager::on_part_ok(int part_id, size_t part_size, size_t actual_size) { + CHECK(part_status_[part_id] == PartStatus::Pending); pending_count_--; - part_status_[id] = PartStatus::Ready; + part_status_[part_id] = PartStatus::Ready; if (actual_size != 0) { - bitmask_.set(id); + bitmask_.set(part_id); } ready_size_ += narrow_cast(actual_size); - if (streaming_limit_ > 0 && is_part_in_streaming_limit(id)) { + if (streaming_limit_ > 0 && is_part_in_streaming_limit(part_id)) { streaming_ready_size_ += narrow_cast(actual_size); } - VLOG(file_loader) << "Transferred part " << id << " of size " << part_size << ", total ready size = " << ready_size_; + VLOG(file_loader) << "Transferred part " << part_id << " of size " << part_size + << ", total ready size = " << ready_size_; - int64 offset = narrow_cast(part_size_) * id; + int64 offset = narrow_cast(part_size_) * part_id; int64 end_offset = offset + narrow_cast(actual_size); if (unknown_size_flag_) { CHECK(part_size == part_size_); @@ -386,20 +387,20 @@ Status PartsManager::on_part_ok(int32 id, size_t part_size, size_t actual_size) return Status::OK(); } -void PartsManager::on_part_failed(int32 id) { - CHECK(part_status_[id] == PartStatus::Pending); +void PartsManager::on_part_failed(int32 part_id) { + CHECK(part_status_[part_id] == PartStatus::Pending); pending_count_--; - part_status_[id] = PartStatus::Empty; - if (id < first_empty_part_) { - first_empty_part_ = id; + part_status_[part_id] = PartStatus::Empty; + if (part_id < first_empty_part_) { + first_empty_part_ = part_id; } if (streaming_offset_ == 0) { - first_streaming_empty_part_ = id; + first_streaming_empty_part_ = part_id; return; } - auto part_i = narrow_cast(streaming_offset_ / part_size_); - if (id >= part_i && id < first_streaming_empty_part_) { - first_streaming_empty_part_ = id; + auto offset_part_id = narrow_cast(streaming_offset_ / part_size_); + if (part_id >= offset_part_id && part_id < first_streaming_empty_part_) { + first_streaming_empty_part_ = part_id; } } @@ -442,9 +443,9 @@ int64 PartsManager::get_estimated_extra() const { //TODO: delete this block if CHECK won't fail int64 sub = 0; - for (int part_i = 0; part_i < part_count_; part_i++) { - if (is_part_in_streaming_limit(part_i) && part_status_[part_i] == PartStatus::Ready) { - sub += get_part(part_i).size; + for (int part_id = 0; part_id < part_count_; part_id++) { + if (is_part_in_streaming_limit(part_id) && part_status_[part_id] == PartStatus::Ready) { + sub += get_part(part_id).size; } } CHECK(sub == streaming_ready_size_); @@ -536,25 +537,25 @@ int64 PartsManager::get_unchecked_ready_prefix_size() { return res; } -Part PartsManager::get_part(int id) const { +Part PartsManager::get_part(int part_id) const { auto size = narrow_cast(part_size_); - auto offset = size * id; + auto offset = size * part_id; auto total_size = unknown_size_flag_ ? max_size_ : get_size(); if (total_size < offset) { size = 0; } else { size = min(size, total_size - offset); } - return Part{id, offset, static_cast(size)}; + return Part{part_id, offset, static_cast(size)}; } Part PartsManager::get_empty_part() { return Part{-1, 0, 0}; } -void PartsManager::on_part_start(int32 id) { - CHECK(part_status_[id] == PartStatus::Empty); - part_status_[id] = PartStatus::Pending; +void PartsManager::on_part_start(int32 part_id) { + CHECK(part_status_[part_id] == PartStatus::Empty); + part_status_[part_id] = PartStatus::Pending; pending_count_++; } diff --git a/td/telegram/files/PartsManager.h b/td/telegram/files/PartsManager.h index 51edc7026..44734ac88 100644 --- a/td/telegram/files/PartsManager.h +++ b/td/telegram/files/PartsManager.h @@ -30,8 +30,8 @@ class PartsManager { // returns empty part if nothing to return Result start_part() TD_WARN_UNUSED_RESULT; - Status on_part_ok(int32 id, size_t part_size, size_t actual_size) TD_WARN_UNUSED_RESULT; - void on_part_failed(int32 id); + Status on_part_ok(int part_id, size_t part_size, size_t actual_size) TD_WARN_UNUSED_RESULT; + void on_part_failed(int part_id); Status set_known_prefix(size_t size, bool is_ready); void set_need_check(); void set_checked_prefix_size(int64 size); @@ -96,13 +96,13 @@ class PartsManager { static Part get_empty_part(); - Part get_part(int id) const; - void on_part_start(int32 id); + Part get_part(int part_id) const; + void on_part_start(int part_id); void update_first_empty_part(); void update_first_not_ready_part(); bool is_streaming_limit_reached(); - bool is_part_in_streaming_limit(int part_i) const; + bool is_part_in_streaming_limit(int part_id) const; }; } // namespace td From 02a4851e12d459c88c18103f9083bf6640cc4e82 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 17 Jan 2023 13:55:51 +0300 Subject: [PATCH 26/30] Fix set_option_empty. --- td/telegram/OptionManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/td/telegram/OptionManager.cpp b/td/telegram/OptionManager.cpp index 61b5837e3..a7aee453c 100644 --- a/td/telegram/OptionManager.cpp +++ b/td/telegram/OptionManager.cpp @@ -198,7 +198,7 @@ void OptionManager::set_option(Slice name, Slice value) { CHECK(!name.empty()); CHECK(Scheduler::instance()->sched_id() == current_scheduler_id_); if (value.empty()) { - if (option_pmc_->erase(name.str()) == 0) { + if (options_->erase(name.str()) == 0) { return; } option_pmc_->erase(name.str()); From 754ff0fc534260ec94f991885b9cb57b1f177c0e Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 17 Jan 2023 14:24:18 +0300 Subject: [PATCH 27/30] Improve handling of internal options with updates. --- td/telegram/MessageReaction.cpp | 8 ++---- td/telegram/MessageReaction.h | 2 +- td/telegram/OptionManager.cpp | 51 ++++++++++++++++++--------------- td/telegram/OptionManager.h | 2 ++ 4 files changed, 34 insertions(+), 29 deletions(-) diff --git a/td/telegram/MessageReaction.cpp b/td/telegram/MessageReaction.cpp index de74915d5..62ce7499a 100644 --- a/td/telegram/MessageReaction.cpp +++ b/td/telegram/MessageReaction.cpp @@ -943,13 +943,11 @@ void send_set_default_reaction_query(Td *td) { td->create_handler()->send(td->option_manager_->get_option_string("default_reaction")); } -void send_update_default_reaction_type(const string &default_reaction) { +td_api::object_ptr get_update_default_reaction_type(const string &default_reaction) { if (default_reaction.empty()) { - LOG(ERROR) << "Have no default reaction"; - return; + return nullptr; } - send_closure(G()->td(), &Td::send_update, - td_api::make_object(get_reaction_type_object(default_reaction))); + return td_api::make_object(get_reaction_type_object(default_reaction)); } void report_message_reactions(Td *td, FullMessageId full_message_id, DialogId chooser_dialog_id, diff --git a/td/telegram/MessageReaction.h b/td/telegram/MessageReaction.h index ed0f42a44..caad4b65b 100644 --- a/td/telegram/MessageReaction.h +++ b/td/telegram/MessageReaction.h @@ -224,7 +224,7 @@ void set_default_reaction(Td *td, string reaction, Promise &&promise); void send_set_default_reaction_query(Td *td); -void send_update_default_reaction_type(const string &default_reaction); +td_api::object_ptr get_update_default_reaction_type(const string &default_reaction); void report_message_reactions(Td *td, FullMessageId full_message_id, DialogId chooser_dialog_id, Promise &&promise); diff --git a/td/telegram/OptionManager.cpp b/td/telegram/OptionManager.cpp index a7aee453c..3d6efb643 100644 --- a/td/telegram/OptionManager.cpp +++ b/td/telegram/OptionManager.cpp @@ -62,19 +62,10 @@ OptionManager::OptionManager(Td *td) if (!is_internal_option(name)) { send_closure(G()->td(), &Td::send_update, td_api::make_object(name, get_option_value_object(name_value.second))); - } else if (name == "otherwise_relogin_days") { - auto days = narrow_cast(get_option_integer(name)); - if (days > 0) { - vector added_actions{SuggestedAction{SuggestedAction::Type::SetPassword, DialogId(), days}}; - send_closure(G()->td(), &Td::send_update, get_update_suggested_actions_object(added_actions, {})); - } - } else if (name == "default_reaction") { - auto value = get_option_string(name); - if (value.empty()) { - // legacy - set_option_empty(name); - } else { - send_update_default_reaction_type(value); + } else { + auto update = get_internal_option_update(name); + if (update != nullptr) { + send_closure(G()->td(), &Td::send_update, std::move(update)); } } } @@ -216,6 +207,11 @@ void OptionManager::set_option(Slice name, Slice value) { if (!is_internal_option(name)) { send_closure(G()->td(), &Td::send_update, td_api::make_object(name.str(), get_option_value_object(get_option(name)))); + } else { + auto update = get_internal_option_update(name); + if (update != nullptr) { + send_closure(G()->td(), &Td::send_update, std::move(update)); + } } } @@ -299,6 +295,20 @@ bool OptionManager::is_internal_option(Slice name) { } } +td_api::object_ptr OptionManager::get_internal_option_update(Slice name) const { + if (name == "default_reaction") { + return get_update_default_reaction_type(get_option_string(name)); + } + if (name == "otherwise_relogin_days") { + auto days = narrow_cast(get_option_integer(name)); + if (days > 0) { + vector added_actions{SuggestedAction{SuggestedAction::Type::SetPassword, DialogId(), days}}; + return get_update_suggested_actions_object(added_actions, {}); + } + } + return nullptr; +} + const vector &OptionManager::get_synchronous_options() { static const vector options{"version", "commit_hash"}; return options; @@ -334,9 +344,6 @@ void OptionManager::on_option_updated(Slice name) { } break; case 'd': - if (name == "default_reaction") { - send_update_default_reaction_type(get_option_string(name)); - } if (name == "dice_emojis") { send_closure(td_->stickers_manager_actor_, &StickersManager::on_update_dice_emojis); } @@ -414,13 +421,6 @@ void OptionManager::on_option_updated(Slice name) { if (name == "online_cloud_timeout_ms") { send_closure(td_->notification_manager_actor_, &NotificationManager::on_online_cloud_timeout_changed); } - if (name == "otherwise_relogin_days") { - auto days = narrow_cast(get_option_integer(name)); - if (days > 0) { - vector added_actions{SuggestedAction{SuggestedAction::Type::SetPassword, DialogId(), days}}; - send_closure(G()->td(), &Td::send_update, get_update_suggested_actions_object(added_actions, {})); - } - } break; case 'r': if (name == "rating_e_decay") { @@ -867,6 +867,11 @@ void OptionManager::get_current_state(vector> if (!is_internal_option(option.first)) { updates.push_back( td_api::make_object(option.first, get_option_value_object(option.second))); + } else { + auto update = get_internal_option_update(option.first); + if (update != nullptr) { + updates.push_back(std::move(update)); + } } } } diff --git a/td/telegram/OptionManager.h b/td/telegram/OptionManager.h index 13424c547..2ffe26f4b 100644 --- a/td/telegram/OptionManager.h +++ b/td/telegram/OptionManager.h @@ -72,6 +72,8 @@ class OptionManager { static bool is_internal_option(Slice name); + td_api::object_ptr get_internal_option_update(Slice name) const; + static const vector &get_synchronous_options(); static td_api::object_ptr get_unix_time_option_value_object(); From 89efc4feca1a3cfbcc516fbc02b84420f9620a53 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 17 Jan 2023 14:46:21 +0300 Subject: [PATCH 28/30] Explicitly define static constexpr fields. --- test/query_merger.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/query_merger.cpp b/test/query_merger.cpp index cdda2390f..c192f1dd4 100644 --- a/test/query_merger.cpp +++ b/test/query_merger.cpp @@ -85,6 +85,10 @@ class TestQueryMerger : public td::Actor { td::FlatHashSet pending_query_ids_; }; +constexpr std::size_t TestQueryMerger::MAX_CONCURRENT_QUERY_COUNT; +constexpr std::size_t TestQueryMerger::MAX_MERGED_QUERY_COUNT; +constexpr std::size_t TestQueryMerger::MAX_QUERY_COUNT; + TEST(QueryMerger, stress) { td::ConcurrentScheduler sched(0, 0); sched.create_actor_unsafe(0, "TestQueryMerger").release(); From 42af57ee0561ff4ea85e281c5d17512c5a649f47 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 17 Jan 2023 15:34:31 +0300 Subject: [PATCH 29/30] Support General topic in getMessageThread. --- td/telegram/MessagesManager.cpp | 35 ++++++++++++++++++++++----------- td/telegram/cli.cpp | 3 ++- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 586c522b8..824ccbca5 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -18582,19 +18582,24 @@ void MessagesManager::get_message_thread(DialogId dialog_id, MessageId message_i return promise.set_error(Status::Error(400, "Scheduled messages can't have message threads")); } - message_id = get_persistent_message_id(d, message_id); - auto m = get_message_force(d, message_id, "get_message_thread"); - if (m == nullptr) { - return promise.set_error(Status::Error(400, "Message not found")); - } + FullMessageId top_thread_full_message_id; + if (message_id == MessageId(ServerMessageId(1)) && is_forum_channel(dialog_id)) { + top_thread_full_message_id = FullMessageId{dialog_id, message_id}; + } else { + message_id = get_persistent_message_id(d, message_id); + auto m = get_message_force(d, message_id, "get_message_thread"); + if (m == nullptr) { + return promise.set_error(Status::Error(400, "Message not found")); + } - TRY_RESULT_PROMISE(promise, top_thread_full_message_id, get_top_thread_full_message_id(dialog_id, m, true)); - if ((m->reply_info.is_empty() || !m->reply_info.is_comment_) && - top_thread_full_message_id.get_message_id() != m->message_id) { - CHECK(dialog_id == top_thread_full_message_id.get_dialog_id()); - // get information about the thread from the top message - message_id = top_thread_full_message_id.get_message_id(); - CHECK(message_id.is_valid()); + TRY_RESULT_PROMISE_ASSIGN(promise, top_thread_full_message_id, get_top_thread_full_message_id(dialog_id, m, true)); + if ((m->reply_info.is_empty() || !m->reply_info.is_comment_) && + top_thread_full_message_id.get_message_id() != m->message_id) { + CHECK(dialog_id == top_thread_full_message_id.get_dialog_id()); + // get information about the thread from the top message + message_id = top_thread_full_message_id.get_message_id(); + CHECK(message_id.is_valid()); + } } auto query_promise = PromiseCreator::lambda([actor_id = actor_id(this), dialog_id, message_id, @@ -18707,6 +18712,9 @@ void MessagesManager::on_get_discussion_message(DialogId dialog_id, MessageId me return promise.set_error(Status::Error(400, "Message has no comments")); } expected_dialog_id = DialogId(m->reply_info.channel_id_); + } else if (message_id == MessageId(ServerMessageId(1)) && is_forum_channel(dialog_id)) { + // General forum topic + expected_dialog_id = dialog_id; } else { if (!m->top_thread_message_id.is_valid()) { return promise.set_error(Status::Error(400, "Message has no thread")); @@ -18760,6 +18768,9 @@ td_api::object_ptr MessagesManager::get_message_threa } if (messages.size() != 1) { is_forum_topic = false; + } else if (info.message_ids[0] == MessageId(ServerMessageId(1)) && is_forum_channel(info.dialog_id)) { + // General forum topic + is_forum_topic = true; } if (reply_info == nullptr && !is_forum_topic) { return nullptr; diff --git a/td/telegram/cli.cpp b/td/telegram/cli.cpp index ade563b9b..a721399b6 100644 --- a/td/telegram/cli.cpp +++ b/td/telegram/cli.cpp @@ -2241,10 +2241,11 @@ class CliClient final : public Actor { MessageId from_message_id; int32 offset; string limit; + get_args(args, chat_id, args); if (op == "gmth") { get_args(args, thread_message_id, args); } - get_args(args, chat_id, from_message_id, offset, limit); + get_args(args, from_message_id, offset, limit); if (op == "gmth") { send_request(td_api::make_object(chat_id, thread_message_id, from_message_id, offset, as_limit(limit))); From d912fd14311fec0d5d6b81359a4bc7d9ce2b0996 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 17 Jan 2023 16:26:20 +0300 Subject: [PATCH 30/30] Improve can_send_messages documentation. --- td/generate/scheme/td_api.tl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index 03b635316..4294e8cbd 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -562,7 +562,7 @@ inputChatPhotoAnimation animation:InputFile main_frame_timestamp:double = InputC //@description Describes actions that a user is allowed to take in a chat -//@can_send_messages True, if the user can send text messages, contacts, locations, and venues +//@can_send_messages True, if the user can send text messages, contacts, invoices, locations, and venues //@can_send_media_messages True, if the user can send audio files, documents, photos, videos, video notes, and voice notes. Implies can_send_messages permissions //@can_send_polls True, if the user can send polls. Implies can_send_messages permissions //@can_send_other_messages True, if the user can send animations, games, stickers, and dice and use inline bots. Implies can_send_messages permissions