From d229c5daed67277ae25605cef8e5989acbabe431 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 21 Aug 2021 19:57:01 +0300 Subject: [PATCH 01/91] Fix handling of empty permissions in chatMemberStatusRestricted. --- td/telegram/DialogParticipant.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/td/telegram/DialogParticipant.cpp b/td/telegram/DialogParticipant.cpp index c27000de4..bb0fde35e 100644 --- a/td/telegram/DialogParticipant.cpp +++ b/td/telegram/DialogParticipant.cpp @@ -421,6 +421,10 @@ DialogParticipantStatus get_dialog_participant_status(const tl_object_ptr(status.get()); auto permissions = st->permissions_.get(); + if (permissions == nullptr) { + return DialogParticipantStatus::Restricted(st->is_member_, st->restricted_until_date_, false, false, false, + false, false, false, false, false, false, false, false); + } bool can_send_polls = permissions->can_send_polls_; bool can_send_media = permissions->can_send_media_messages_; bool can_send_messages = permissions->can_send_messages_ || can_send_media || can_send_polls || From 43d9ba81ac677aa3d96e7b812a9e76117be687bd Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 21 Aug 2021 21:42:07 +0300 Subject: [PATCH 02/91] Prefer std::unordered_map to std::map. --- td/mtproto/RawConnection.cpp | 4 ++-- tdactor/td/actor/impl/Scheduler-decl.h | 4 ++-- tdnet/td/net/SslStream.cpp | 4 ++-- test/online.cpp | 1 + 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/td/mtproto/RawConnection.cpp b/td/mtproto/RawConnection.cpp index e4b5d6086..58462c8d0 100644 --- a/td/mtproto/RawConnection.cpp +++ b/td/mtproto/RawConnection.cpp @@ -26,8 +26,8 @@ #include "td/utils/Status.h" #include "td/utils/StorerBase.h" -#include #include +#include #include namespace td { @@ -130,7 +130,7 @@ class RawConnectionDefault final : public RawConnection { PublicFields extra_; BufferedFd socket_fd_; unique_ptr transport_; - std::map quick_ack_to_token_; + std::unordered_map quick_ack_to_token_; bool has_error_{false}; unique_ptr stats_callback_; diff --git a/tdactor/td/actor/impl/Scheduler-decl.h b/tdactor/td/actor/impl/Scheduler-decl.h index 2b17d90ed..24393a801 100644 --- a/tdactor/td/actor/impl/Scheduler-decl.h +++ b/tdactor/td/actor/impl/Scheduler-decl.h @@ -26,9 +26,9 @@ #include "td/utils/type_traits.h" #include -#include #include #include +#include #include namespace td { @@ -214,7 +214,7 @@ class Scheduler { ListNode ready_actors_list_; KHeap timeout_queue_; - std::map> pending_events_; + std::unordered_map> pending_events_; ServiceActor service_actor_; Poll poll_; diff --git a/tdnet/td/net/SslStream.cpp b/tdnet/td/net/SslStream.cpp index b9061b94d..84d9e5573 100644 --- a/tdnet/td/net/SslStream.cpp +++ b/tdnet/td/net/SslStream.cpp @@ -24,9 +24,9 @@ #include #include -#include #include #include +#include #if TD_PORT_WINDOWS #include @@ -131,7 +131,7 @@ int verify_callback(int preverify_ok, X509_STORE_CTX *ctx) { static std::mutex warning_mutex; { std::lock_guard lock(warning_mutex); - static std::map next_warning_time; + static std::unordered_map next_warning_time; double &next = next_warning_time[warning]; if (next <= now) { next = now + 300; // one warning per 5 minutes diff --git a/test/online.cpp b/test/online.cpp index 689beff9a..68b3fbe01 100644 --- a/test/online.cpp +++ b/test/online.cpp @@ -5,6 +5,7 @@ // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // #include +#include #include #include #include From d154399d705cf4bb0e8f037bcc171ae964fd4228 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 21 Aug 2021 22:41:06 +0300 Subject: [PATCH 03/91] Optimize MessageIdDuplicateChecker::check. --- td/mtproto/AuthData.cpp | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/td/mtproto/AuthData.cpp b/td/mtproto/AuthData.cpp index 365218ad1..189c74cd1 100644 --- a/td/mtproto/AuthData.cpp +++ b/td/mtproto/AuthData.cpp @@ -22,20 +22,19 @@ Status MessageIdDuplicateChecker::check(int64 message_id) { // a message comes in with msg_id lower than all or equal to any of the stored values, that message is to be // ignored. Otherwise, the new message msg_id is added to the set, and, if the number of stored msg_id values is // greater than N, the oldest (i. e. the lowest) is forgotten. - if (saved_message_ids_.size() == MAX_SAVED_MESSAGE_IDS) { - auto oldest_message_id = *saved_message_ids_.begin(); - if (message_id < oldest_message_id) { - return Status::Error(2, PSLICE() << "Ignore very old message_id " << tag("oldest message_id", oldest_message_id) - << tag("got message_id", message_id)); - } - } - if (saved_message_ids_.count(message_id) != 0) { + auto insert_result = saved_message_ids_.insert(message_id); + if (!insert_result.second) { return Status::Error(1, PSLICE() << "Ignore duplicated message_id " << tag("message_id", message_id)); } - - saved_message_ids_.insert(message_id); - if (saved_message_ids_.size() > MAX_SAVED_MESSAGE_IDS) { - saved_message_ids_.erase(saved_message_ids_.begin()); + if (saved_message_ids_.size() == MAX_SAVED_MESSAGE_IDS + 1) { + auto begin_it = saved_message_ids_.begin(); + bool is_very_old = begin_it == insert_result.first; + saved_message_ids_.erase(begin_it); + if (is_very_old) { + return Status::Error(2, PSLICE() << "Ignore very old message_id " + << tag("oldest message_id", *saved_message_ids_.begin()) + << tag("got message_id", message_id)); + } } return Status::OK(); } From 530256928996704e62244c789957e849cd780aed Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 21 Aug 2021 23:13:36 +0300 Subject: [PATCH 04/91] Fix MTProto tests. --- test/mtproto.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mtproto.cpp b/test/mtproto.cpp index 5e998ff00..c0b66e126 100644 --- a/test/mtproto.cpp +++ b/test/mtproto.cpp @@ -305,7 +305,7 @@ class HandshakeContext final : public mtproto::AuthKeyHandshakeContext { } private: - PublicRsaKeyShared public_rsa_key{DcId::empty(), false}; + PublicRsaKeyShared public_rsa_key{DcId::empty(), true}; }; class HandshakeTestActor final : public Actor { From e0f80ca008f39254b066cdb9ba0c6591321d5f4d Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 22 Aug 2021 09:39:43 +0300 Subject: [PATCH 05/91] Add DuplicateChecker benchmark. --- benchmark/bench_misc.cpp | 134 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/benchmark/bench_misc.cpp b/benchmark/bench_misc.cpp index 21aad77f7..914a6c2b0 100644 --- a/benchmark/bench_misc.cpp +++ b/benchmark/bench_misc.cpp @@ -6,6 +6,7 @@ // #include "td/utils/benchmark.h" #include "td/utils/common.h" +#include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/port/Clocks.h" #include "td/utils/port/EventFd.h" @@ -32,6 +33,7 @@ #include #include +#include class F { td::uint32 ∑ @@ -418,8 +420,140 @@ std::atomic AtomicCounterBench::counter_; #endif +class MessageIdDuplicateCheckerOld { + public: + static td::string get_description() { + return "Old"; + } + td::Status check(td::int64 message_id) { + if (saved_message_ids_.size() == MAX_SAVED_MESSAGE_IDS) { + auto oldest_message_id = *saved_message_ids_.begin(); + if (message_id < oldest_message_id) { + return td::Status::Error(2, PSLICE() << "Ignore very old message_id " + << td::tag("oldest message_id", oldest_message_id) + << td::tag("got message_id", message_id)); + } + } + if (saved_message_ids_.count(message_id) != 0) { + return td::Status::Error(1, PSLICE() << "Ignore duplicated message_id " << td::tag("message_id", message_id)); + } + + saved_message_ids_.insert(message_id); + if (saved_message_ids_.size() > MAX_SAVED_MESSAGE_IDS) { + saved_message_ids_.erase(saved_message_ids_.begin()); + } + return td::Status::OK(); + } + + private: + static constexpr size_t MAX_SAVED_MESSAGE_IDS = 1000; + std::set saved_message_ids_; +}; + +template +class MessageIdDuplicateCheckerNew { + public: + static td::string get_description() { + return PSTRING() << "New" << MAX_SAVED_MESSAGE_IDS; + } + td::Status check(td::int64 message_id) { + auto insert_result = saved_message_ids_.insert(message_id); + if (!insert_result.second) { + return td::Status::Error(1, PSLICE() << "Ignore duplicated message_id " << td::tag("message_id", message_id)); + } + if (saved_message_ids_.size() == MAX_SAVED_MESSAGE_IDS + 1) { + auto begin_it = saved_message_ids_.begin(); + bool is_very_old = begin_it == insert_result.first; + saved_message_ids_.erase(begin_it); + if (is_very_old) { + return td::Status::Error(2, PSLICE() << "Ignore very old message_id " + << td::tag("oldest message_id", *saved_message_ids_.begin()) + << td::tag("got message_id", message_id)); + } + } + return td::Status::OK(); + } + + private: + std::set saved_message_ids_; +}; + +class MessageIdDuplicateCheckerNewOther { + public: + static td::string get_description() { + return "NewOther"; + } + td::Status check(td::int64 message_id) { + if (!saved_message_ids_.insert(message_id).second) { + return td::Status::Error(1, PSLICE() << "Ignore duplicated message_id " << td::tag("message_id", message_id)); + } + if (saved_message_ids_.size() == MAX_SAVED_MESSAGE_IDS + 1) { + auto begin_it = saved_message_ids_.begin(); + bool is_very_old = *begin_it == message_id; + saved_message_ids_.erase(begin_it); + if (is_very_old) { + return td::Status::Error(2, PSLICE() << "Ignore very old message_id " + << td::tag("oldest message_id", *saved_message_ids_.begin()) + << td::tag("got message_id", message_id)); + } + } + return td::Status::OK(); + } + + private: + static constexpr size_t MAX_SAVED_MESSAGE_IDS = 1000; + std::set saved_message_ids_; +}; + +class MessageIdDuplicateCheckerNewSimple { + public: + static td::string get_description() { + return "NewSimple"; + } + td::Status check(td::int64 message_id) { + auto insert_result = saved_message_ids_.insert(message_id); + if (!insert_result.second) { + return td::Status::Error(1, "Ignore duplicated message_id"); + } + if (saved_message_ids_.size() == MAX_SAVED_MESSAGE_IDS + 1) { + auto begin_it = saved_message_ids_.begin(); + bool is_very_old = begin_it == insert_result.first; + saved_message_ids_.erase(begin_it); + if (is_very_old) { + return td::Status::Error(2, "Ignore very old message_id"); + } + } + return td::Status::OK(); + } + + private: + static constexpr size_t MAX_SAVED_MESSAGE_IDS = 1000; + std::set saved_message_ids_; +}; + +template +class DuplicateCheckerBench final : public td::Benchmark { + td::string get_description() const final { + return PSTRING() << "DuplicateCheckerBench" << T::get_description(); + } + void run(int n) final { + T checker_; + for (int i = 0; i < n; i++) { + checker_.check(i).ensure(); + } + } +}; + int main() { SET_VERBOSITY_LEVEL(VERBOSITY_NAME(DEBUG)); + + td::bench(DuplicateCheckerBench()); + td::bench(DuplicateCheckerBench>()); + td::bench(DuplicateCheckerBench()); + td::bench(DuplicateCheckerBench()); + td::bench(DuplicateCheckerBench>()); + td::bench(DuplicateCheckerBench>()); + #if !TD_THREAD_UNSUPPORTED for (int i = 1; i <= 16; i *= 2) { td::bench(ThreadSafeCounterBench(i)); From f9b71104d41205abb91d248b0eab467bc1b47ac9 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 22 Aug 2021 10:25:11 +0300 Subject: [PATCH 06/91] Add DuplicateChecker benchmark with repeated message_id. --- benchmark/bench_misc.cpp | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/benchmark/bench_misc.cpp b/benchmark/bench_misc.cpp index 914a6c2b0..8175b5768 100644 --- a/benchmark/bench_misc.cpp +++ b/benchmark/bench_misc.cpp @@ -450,7 +450,7 @@ class MessageIdDuplicateCheckerOld { std::set saved_message_ids_; }; -template +template class MessageIdDuplicateCheckerNew { public: static td::string get_description() { @@ -544,13 +544,39 @@ class DuplicateCheckerBench final : public td::Benchmark { } }; +template +class DuplicateCheckerBenchRepeat final : public td::Benchmark { + td::string get_description() const final { + return PSTRING() << "DuplicateCheckerBenchRepeat" << T::get_description(); + } + void run(int n) final { + T checker_; + for (int i = 0; i < n; i++) { + auto iter = i >> 10; + auto pos = i - (iter << 10); + if (pos < 768) { + checker_.check(iter * 768 + pos).ensure(); + } else { + checker_.check(iter * 768 + pos - 256).ensure_error(); + } + } + } +}; + int main() { SET_VERBOSITY_LEVEL(VERBOSITY_NAME(DEBUG)); + td::bench(DuplicateCheckerBenchRepeat()); + td::bench(DuplicateCheckerBenchRepeat>()); + td::bench(DuplicateCheckerBenchRepeat()); + td::bench(DuplicateCheckerBenchRepeat()); + td::bench(DuplicateCheckerBenchRepeat>()); + td::bench(DuplicateCheckerBench()); td::bench(DuplicateCheckerBench>()); td::bench(DuplicateCheckerBench()); td::bench(DuplicateCheckerBench()); + td::bench(DuplicateCheckerBench>()); td::bench(DuplicateCheckerBench>()); td::bench(DuplicateCheckerBench>()); From a2f3c4c7885f71db92baf3291a9bbfc0f1520f62 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 22 Aug 2021 10:49:40 +0300 Subject: [PATCH 07/91] Add array-based duplicate checker to benchmark. --- benchmark/bench_misc.cpp | 42 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/benchmark/bench_misc.cpp b/benchmark/bench_misc.cpp index 8175b5768..4d7627f24 100644 --- a/benchmark/bench_misc.cpp +++ b/benchmark/bench_misc.cpp @@ -31,6 +31,8 @@ #include #endif +#include +#include #include #include #include @@ -531,6 +533,42 @@ class MessageIdDuplicateCheckerNewSimple { std::set saved_message_ids_; }; +template +class MessageIdDuplicateCheckerArray { + public: + static td::string get_description() { + return PSTRING() << "Array" << MAX_SAVED_MESSAGE_IDS; + } + td::Status check(td::int64 message_id) { + if (end_pos == 2 * MAX_SAVED_MESSAGE_IDS) { + std::copy_n(&saved_message_ids_[MAX_SAVED_MESSAGE_IDS], MAX_SAVED_MESSAGE_IDS, &saved_message_ids_[0]); + end_pos = MAX_SAVED_MESSAGE_IDS; + } + if (end_pos == 0 || message_id > saved_message_ids_[end_pos - 1]) { + // fast path + saved_message_ids_[end_pos++] = message_id; + return td::Status::OK(); + } + if (end_pos >= MAX_SAVED_MESSAGE_IDS && message_id < saved_message_ids_[0]) { + return td::Status::Error(2, PSLICE() << "Ignore very old message_id " + << td::tag("oldest message_id", saved_message_ids_[0]) + << td::tag("got message_id", message_id)); + } + auto it = std::lower_bound(&saved_message_ids_[0], &saved_message_ids_[end_pos], message_id); + if (*it == message_id) { + return td::Status::Error(1, PSLICE() << "Ignore duplicated message_id " << td::tag("message_id", message_id)); + } + std::copy_backward(it, &saved_message_ids_[end_pos], &saved_message_ids_[end_pos + 1]); + *it = message_id; + ++end_pos; + return td::Status::OK(); + } + + private: + std::array saved_message_ids_; + std::size_t end_pos = 0; +}; + template class DuplicateCheckerBench final : public td::Benchmark { td::string get_description() const final { @@ -571,6 +609,8 @@ int main() { td::bench(DuplicateCheckerBenchRepeat()); td::bench(DuplicateCheckerBenchRepeat()); td::bench(DuplicateCheckerBenchRepeat>()); + td::bench(DuplicateCheckerBenchRepeat>()); + td::bench(DuplicateCheckerBenchRepeat>()); td::bench(DuplicateCheckerBench()); td::bench(DuplicateCheckerBench>()); @@ -579,6 +619,8 @@ int main() { td::bench(DuplicateCheckerBench>()); td::bench(DuplicateCheckerBench>()); td::bench(DuplicateCheckerBench>()); + td::bench(DuplicateCheckerBench>()); + td::bench(DuplicateCheckerBench>()); #if !TD_THREAD_UNSUPPORTED for (int i = 1; i <= 16; i *= 2) { From d6679a01790537b7020e49ab9813e26925401cf3 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 22 Aug 2021 10:59:50 +0300 Subject: [PATCH 08/91] Add even more DuplicateChecker benchmarks. --- benchmark/bench_misc.cpp | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/benchmark/bench_misc.cpp b/benchmark/bench_misc.cpp index 4d7627f24..2edc3659f 100644 --- a/benchmark/bench_misc.cpp +++ b/benchmark/bench_misc.cpp @@ -601,9 +601,47 @@ class DuplicateCheckerBenchRepeat final : public td::Benchmark { } }; +template +class DuplicateCheckerBenchRepeatOnly final : public td::Benchmark { + td::string get_description() const final { + return PSTRING() << "DuplicateCheckerBenchRepeatOnly" << T::get_description(); + } + void run(int n) final { + T checker_; + for (int i = 0; i < n; i++) { + auto result = checker_.check(i & 255); + CHECK(result.is_error() == i >= 256); + } + } +}; + +template +class DuplicateCheckerBenchReverse final : public td::Benchmark { + td::string get_description() const final { + return PSTRING() << "DuplicateCheckerBenchReverseAdd" << T::get_description(); + } + void run(int n) final { + T checker_; + for (int i = 0; i < n; i++) { + auto pos = i & 255; + checker_.check(i - pos + (255 - pos)).ensure(); + } + } +}; + int main() { SET_VERBOSITY_LEVEL(VERBOSITY_NAME(DEBUG)); + td::bench(DuplicateCheckerBenchReverse>()); + td::bench(DuplicateCheckerBenchReverse>()); + td::bench(DuplicateCheckerBenchReverse>()); + td::bench(DuplicateCheckerBenchReverse>()); + + td::bench(DuplicateCheckerBenchRepeatOnly>()); + td::bench(DuplicateCheckerBenchRepeatOnly>()); + td::bench(DuplicateCheckerBenchRepeatOnly>()); + td::bench(DuplicateCheckerBenchRepeatOnly>()); + td::bench(DuplicateCheckerBenchRepeat()); td::bench(DuplicateCheckerBenchRepeat>()); td::bench(DuplicateCheckerBenchRepeat()); From 7b3f60ea85647e891cec0d063f35efcacee8bf26 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 22 Aug 2021 11:01:49 +0300 Subject: [PATCH 09/91] Fix warning. --- benchmark/bench_misc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/bench_misc.cpp b/benchmark/bench_misc.cpp index 2edc3659f..8b5db4329 100644 --- a/benchmark/bench_misc.cpp +++ b/benchmark/bench_misc.cpp @@ -610,7 +610,7 @@ class DuplicateCheckerBenchRepeatOnly final : public td::Benchmark { T checker_; for (int i = 0; i < n; i++) { auto result = checker_.check(i & 255); - CHECK(result.is_error() == i >= 256); + CHECK(result.is_error() == (i >= 256)); } } }; From e66ee3eeae2313a26b3e3a2104e8b95ccffb4629 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 22 Aug 2021 11:47:57 +0300 Subject: [PATCH 10/91] Add check for code 2. --- benchmark/bench_misc.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/benchmark/bench_misc.cpp b/benchmark/bench_misc.cpp index 8b5db4329..e3671a3e7 100644 --- a/benchmark/bench_misc.cpp +++ b/benchmark/bench_misc.cpp @@ -103,7 +103,7 @@ BENCH(NewObj, "new struct, then delete") { } #if !TD_THREAD_UNSUPPORTED -BENCH(ThreadNew, "new struct, then delete in several threads") { +BENCH(ThreadNew, "new struct, then delete in 2 threads") { NewObjBench a, b; td::thread ta([&] { a.run(n / 2); }); td::thread tb([&] { b.run(n - n / 2); }); @@ -593,6 +593,10 @@ class DuplicateCheckerBenchRepeat final : public td::Benchmark { auto iter = i >> 10; auto pos = i - (iter << 10); if (pos < 768) { + if (iter >= 3 && pos == 0) { + auto error = checker_.check((iter - 3) * 768 + pos); + CHECK(error.error().code() == 2); + } checker_.check(iter * 768 + pos).ensure(); } else { checker_.check(iter * 768 + pos - 256).ensure_error(); From 7faf7aeba7a1298612ce0b5099b0eb04fa567752 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 22 Aug 2021 13:16:10 +0300 Subject: [PATCH 11/91] Improve logging on too old update. --- td/mtproto/SessionConnection.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/td/mtproto/SessionConnection.cpp b/td/mtproto/SessionConnection.cpp index 47305b0af..5721d7186 100644 --- a/td/mtproto/SessionConnection.cpp +++ b/td/mtproto/SessionConnection.cpp @@ -304,8 +304,7 @@ Status SessionConnection::on_packet(const MsgInfo &info, uint64 req_msg_id, cons if (req_msg_id != 0) { callback_->on_message_result_error(req_msg_id, rpc_error.error_code_, rpc_error.error_message_.str()); } else { - LOG(ERROR) << "Receive rpc_error as update: [" << rpc_error.error_code_ << "][" << rpc_error.error_message_ - << "]"; + LOG(ERROR) << "Receive rpc_error as update: [" << rpc_error.error_code_ << "][" << rpc_error.error_message_ << "]"; } return Status::OK(); } @@ -499,12 +498,15 @@ Status SessionConnection::on_slice_packet(const MsgInfo &info, Slice packet) { auto status = auth_data_->check_update(info.message_id); if (status.is_error()) { if (status.code() == 2) { - LOG(WARNING) << "Receive too old update: " << status; + LOG(WARNING) << "Receive too old update from " << get_name() << " created in " << (Time::now() - created_at_) + << " in container " << container_id_ << " from session " << auth_data_->get_session_id() + << " with message_id " << info.message_id << ", main_message_id = " << main_message_id_ + << ", seq_no = " << info.seq_no << " and original size " << info.size << ": " << status; callback_->on_session_failed(Status::Error("Receive too old update")); return status; } - VLOG(mtproto) << "Skip update " << info.message_id << " from " << get_name() << " created in " - << (Time::now() - created_at_) << ": " << status; + VLOG(mtproto) << "Skip update " << info.message_id << " of size " << info.size << " with seq_no " << info.seq_no + << " from " << get_name() << " created in " << (Time::now() - created_at_) << ": " << status; return Status::OK(); } else { VLOG(mtproto) << "Got update from " << get_name() << " created in " << (Time::now() - created_at_) From 4a3f56e6c1097fecf0fc45367dd07f6efc2f2e82 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 22 Aug 2021 13:23:21 +0300 Subject: [PATCH 12/91] Check that updates aren't received in rpc_result. --- td/mtproto/SessionConnection.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/td/mtproto/SessionConnection.cpp b/td/mtproto/SessionConnection.cpp index 5721d7186..fa77defaf 100644 --- a/td/mtproto/SessionConnection.cpp +++ b/td/mtproto/SessionConnection.cpp @@ -250,6 +250,10 @@ Status SessionConnection::on_packet_rpc_result(const MsgInfo &info, Slice packet if (parser.get_error()) { return Status::Error(PSLICE() << "Failed to parse mtproto_api::rpc_result: " << parser.get_error()); } + if (req_msg_id == 0) { + LOG(ERROR) << "Receive an update in rpc_result: message_id = " << info.message_id << ", seq_no = " << info.seq_no; + return Status::Error("Receive an update in rpc_result"); + } auto object_begin_pos = packet.size() - parser.get_left_len(); int32 id = parser.fetch_int(); From b3aa31d39800c5f88e23c3975b64723814c7d523 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 22 Aug 2021 13:36:47 +0300 Subject: [PATCH 13/91] Minor cut_tail improvements. --- td/mtproto/SessionConnection.cpp | 19 ++++++++++--------- td/mtproto/SessionConnection.h | 12 ++++++------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/td/mtproto/SessionConnection.cpp b/td/mtproto/SessionConnection.cpp index fa77defaf..d85bc387d 100644 --- a/td/mtproto/SessionConnection.cpp +++ b/td/mtproto/SessionConnection.cpp @@ -192,7 +192,6 @@ unique_ptr SessionConnection::move_as_raw_connection() { return std::move(raw_connection_); } -/*** SessionConnection ***/ BufferSlice SessionConnection::as_buffer_slice(Slice packet) { return current_buffer_slice_->from_slice(packet); } @@ -429,7 +428,7 @@ Status SessionConnection::on_packet(const MsgInfo &info, const mtproto_api::pong } Status SessionConnection::on_packet(const MsgInfo &info, const mtproto_api::future_salts &salts) { VLOG(mtproto) << "FUTURE_SALTS"; - std::vector new_salts; + vector new_salts; for (auto &it : salts.salts_) { new_salts.push_back( ServerSalt{it->salt_, static_cast(it->valid_since_), static_cast(it->valid_until_)}); @@ -440,7 +439,7 @@ Status SessionConnection::on_packet(const MsgInfo &info, const mtproto_api::futu return Status::OK(); } -Status SessionConnection::on_msgs_state_info(const std::vector &ids, Slice info) { +Status SessionConnection::on_msgs_state_info(const vector &ids, Slice info) { if (ids.size() != info.size()) { return Status::Error(PSLICE() << tag("ids.size()", ids.size()) << " != " << tag("info.size()", info.size())); } @@ -897,7 +896,7 @@ void SessionConnection::flush_packet() { send_till++; } } - std::vector queries; + vector queries; if (send_till == to_send_.size()) { queries = std::move(to_send_); } else if (send_till != 0) { @@ -922,14 +921,16 @@ void SessionConnection::flush_packet() { << tag("resend", to_resend_answer_.size()) << tag("cancel", to_cancel_answer_.size()) << tag("destroy_key", destroy_auth_key) << tag("auth_id", auth_data_->get_auth_key().id()); - auto cut_tail = [](auto &v, size_t size, Slice name) { + auto cut_tail = [](vector &v, size_t size, Slice name) { if (size >= v.size()) { - return std::move(v); + auto result = std::move(v); + v.clear(); + return result; } - LOG(WARNING) << "Too much ids in container: " << v.size() << " " << name; - std::decay_t res(std::make_move_iterator(v.end() - size), std::make_move_iterator(v.end())); + LOG(WARNING) << "Too much message identifiers in container " << name << ": " << v.size() << " instead of " << size; + vector result(v.end() - size, v.end()); v.resize(v.size() - size); - return res; + return result; }; // no more than 8192 ids per container.. diff --git a/td/mtproto/SessionConnection.h b/td/mtproto/SessionConnection.h index 24bc82103..69db54bb4 100644 --- a/td/mtproto/SessionConnection.h +++ b/td/mtproto/SessionConnection.h @@ -164,15 +164,15 @@ class SessionConnection final struct ServiceQuery { enum Type { GetStateInfo, ResendAnswer } type; - std::vector message_ids; + vector message_ids; }; - std::vector to_resend_answer_; - std::vector to_cancel_answer_; - std::vector to_get_state_info_; + vector to_resend_answer_; + vector to_cancel_answer_; + vector to_get_state_info_; std::unordered_map service_queries_; // nobody cleans up this map. But it should be really small. - std::unordered_map> container_to_service_msg_; + std::unordered_map> container_to_service_msg_; double last_read_at_ = 0; double last_ping_at_ = 0; @@ -233,7 +233,7 @@ class SessionConnection final Status on_packet(const MsgInfo &info, const mtproto_api::pong &pong) TD_WARN_UNUSED_RESULT; Status on_packet(const MsgInfo &info, const mtproto_api::future_salts &salts) TD_WARN_UNUSED_RESULT; - Status on_msgs_state_info(const std::vector &ids, Slice info) TD_WARN_UNUSED_RESULT; + Status on_msgs_state_info(const vector &ids, Slice info) TD_WARN_UNUSED_RESULT; Status on_packet(const MsgInfo &info, const mtproto_api::msgs_state_info &msgs_state_info) TD_WARN_UNUSED_RESULT; Status on_packet(const MsgInfo &info, const mtproto_api::msgs_all_info &msgs_all_info) TD_WARN_UNUSED_RESULT; Status on_packet(const MsgInfo &info, const mtproto_api::msg_detailed_info &msg_detailed_info) TD_WARN_UNUSED_RESULT; From 6507fb7602e4c25210788abbaa4e18e5d2abc506 Mon Sep 17 00:00:00 2001 From: levlam Date: Sun, 22 Aug 2021 22:08:46 +0300 Subject: [PATCH 14/91] Use array-based MessageIdDuplicateChecker. --- benchmark/bench_misc.cpp | 26 +++++++++++++------------- td/mtproto/AuthData.cpp | 32 +++++++++++++++++++------------- td/mtproto/AuthData.h | 17 +++++++++++------ 3 files changed, 43 insertions(+), 32 deletions(-) diff --git a/benchmark/bench_misc.cpp b/benchmark/bench_misc.cpp index e3671a3e7..aeec2232a 100644 --- a/benchmark/bench_misc.cpp +++ b/benchmark/bench_misc.cpp @@ -533,40 +533,40 @@ class MessageIdDuplicateCheckerNewSimple { std::set saved_message_ids_; }; -template +template class MessageIdDuplicateCheckerArray { public: static td::string get_description() { - return PSTRING() << "Array" << MAX_SAVED_MESSAGE_IDS; + return PSTRING() << "Array" << max_size; } td::Status check(td::int64 message_id) { - if (end_pos == 2 * MAX_SAVED_MESSAGE_IDS) { - std::copy_n(&saved_message_ids_[MAX_SAVED_MESSAGE_IDS], MAX_SAVED_MESSAGE_IDS, &saved_message_ids_[0]); - end_pos = MAX_SAVED_MESSAGE_IDS; + if (end_pos_ == 2 * max_size) { + std::copy_n(&saved_message_ids_[max_size], max_size, &saved_message_ids_[0]); + end_pos_ = max_size; } - if (end_pos == 0 || message_id > saved_message_ids_[end_pos - 1]) { + if (end_pos_ == 0 || message_id > saved_message_ids_[end_pos_ - 1]) { // fast path - saved_message_ids_[end_pos++] = message_id; + saved_message_ids_[end_pos_++] = message_id; return td::Status::OK(); } - if (end_pos >= MAX_SAVED_MESSAGE_IDS && message_id < saved_message_ids_[0]) { + if (end_pos_ >= max_size && message_id < saved_message_ids_[0]) { return td::Status::Error(2, PSLICE() << "Ignore very old message_id " << td::tag("oldest message_id", saved_message_ids_[0]) << td::tag("got message_id", message_id)); } - auto it = std::lower_bound(&saved_message_ids_[0], &saved_message_ids_[end_pos], message_id); + auto it = std::lower_bound(&saved_message_ids_[0], &saved_message_ids_[end_pos_], message_id); if (*it == message_id) { return td::Status::Error(1, PSLICE() << "Ignore duplicated message_id " << td::tag("message_id", message_id)); } - std::copy_backward(it, &saved_message_ids_[end_pos], &saved_message_ids_[end_pos + 1]); + std::copy_backward(it, &saved_message_ids_[end_pos_], &saved_message_ids_[end_pos_ + 1]); *it = message_id; - ++end_pos; + ++end_pos_; return td::Status::OK(); } private: - std::array saved_message_ids_; - std::size_t end_pos = 0; + std::array saved_message_ids_; + std::size_t end_pos_ = 0; }; template diff --git a/td/mtproto/AuthData.cpp b/td/mtproto/AuthData.cpp index 189c74cd1..2dfff4bcd 100644 --- a/td/mtproto/AuthData.cpp +++ b/td/mtproto/AuthData.cpp @@ -17,25 +17,31 @@ namespace td { namespace mtproto { -Status MessageIdDuplicateChecker::check(int64 message_id) { +Status check_message_id_duplicates(int64 *saved_message_ids, size_t max_size, size_t &end_pos, int64 message_id) { // In addition, the identifiers (msg_id) of the last N messages received from the other side must be stored, and if // a message comes in with msg_id lower than all or equal to any of the stored values, that message is to be // ignored. Otherwise, the new message msg_id is added to the set, and, if the number of stored msg_id values is // greater than N, the oldest (i. e. the lowest) is forgotten. - auto insert_result = saved_message_ids_.insert(message_id); - if (!insert_result.second) { + if (end_pos == 2 * max_size) { + std::copy_n(&saved_message_ids[max_size], max_size, &saved_message_ids[0]); + end_pos = max_size; + } + if (end_pos == 0 || message_id > saved_message_ids[end_pos - 1]) { + // fast path + saved_message_ids[end_pos++] = message_id; + return Status::OK(); + } + if (end_pos >= max_size && message_id < saved_message_ids[0]) { + return Status::Error(2, PSLICE() << "Ignore very old message_id " << tag("oldest message_id", saved_message_ids[0]) + << tag("got message_id", message_id)); + } + auto it = std::lower_bound(&saved_message_ids[0], &saved_message_ids[end_pos], message_id); + if (*it == message_id) { return Status::Error(1, PSLICE() << "Ignore duplicated message_id " << tag("message_id", message_id)); } - if (saved_message_ids_.size() == MAX_SAVED_MESSAGE_IDS + 1) { - auto begin_it = saved_message_ids_.begin(); - bool is_very_old = begin_it == insert_result.first; - saved_message_ids_.erase(begin_it); - if (is_very_old) { - return Status::Error(2, PSLICE() << "Ignore very old message_id " - << tag("oldest message_id", *saved_message_ids_.begin()) - << tag("got message_id", message_id)); - } - } + std::copy_backward(it, &saved_message_ids[end_pos], &saved_message_ids[end_pos + 1]); + *it = message_id; + ++end_pos; return Status::OK(); } diff --git a/td/mtproto/AuthData.h b/td/mtproto/AuthData.h index 4a9717048..89d34f02b 100644 --- a/td/mtproto/AuthData.h +++ b/td/mtproto/AuthData.h @@ -12,7 +12,7 @@ #include "td/utils/Slice.h" #include "td/utils/Status.h" -#include +#include namespace td { namespace mtproto { @@ -37,13 +37,18 @@ void parse(ServerSalt &salt, ParserT &parser) { salt.valid_until = parser.fetch_double(); } +Status check_message_id_duplicates(int64 *saved_message_ids, size_t max_size, size_t &end_pos, int64 message_id); + +template class MessageIdDuplicateChecker { public: - Status check(int64 message_id); + Status check(int64 message_id) { + return check_message_id_duplicates(&saved_message_ids_[0], max_size, end_pos_, message_id); + } private: - static constexpr size_t MAX_SAVED_MESSAGE_IDS = 1000; - std::set saved_message_ids_; + std::array saved_message_ids_; + size_t end_pos_ = 0; }; class AuthData { @@ -274,8 +279,8 @@ class AuthData { std::vector future_salts_; - MessageIdDuplicateChecker duplicate_checker_; - MessageIdDuplicateChecker updates_duplicate_checker_; + MessageIdDuplicateChecker<1000> duplicate_checker_; + MessageIdDuplicateChecker<1000> updates_duplicate_checker_; void update_salt(double now); }; From 3749e1f672172ba547fc2d11636d756bee27a79c Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 23 Aug 2021 00:08:05 +0300 Subject: [PATCH 15/91] Add warning for very old updates. --- td/mtproto/AuthData.h | 5 +++++ td/mtproto/SessionConnection.cpp | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/td/mtproto/AuthData.h b/td/mtproto/AuthData.h index 89d34f02b..ef44aca82 100644 --- a/td/mtproto/AuthData.h +++ b/td/mtproto/AuthData.h @@ -245,6 +245,10 @@ class AuthData { return updates_duplicate_checker_.check(message_id); } + Status recheck_update(int64 message_id) { + return updates_duplicate_rechecker_.check(message_id); + } + int32 next_seq_no(bool is_content_related) { int32 res = seq_no_; if (is_content_related) { @@ -281,6 +285,7 @@ class AuthData { MessageIdDuplicateChecker<1000> duplicate_checker_; MessageIdDuplicateChecker<1000> updates_duplicate_checker_; + MessageIdDuplicateChecker<100> updates_duplicate_rechecker_; void update_salt(double now); }; diff --git a/td/mtproto/SessionConnection.cpp b/td/mtproto/SessionConnection.cpp index d85bc387d..6ac30a068 100644 --- a/td/mtproto/SessionConnection.cpp +++ b/td/mtproto/SessionConnection.cpp @@ -499,6 +499,14 @@ Status SessionConnection::on_slice_packet(const MsgInfo &info, Slice packet) { // It is an update... I hope. auto status = auth_data_->check_update(info.message_id); + auto recheck_status = auth_data_->recheck_update(info.message_id); + if (recheck_status.is_error() && recheck_status.code() == 2) { + LOG(WARNING) << "Receive very old update from " << get_name() << " created in " << (Time::now() - created_at_) + << " in container " << container_id_ << " from session " << auth_data_->get_session_id() + << " with message_id " << info.message_id << ", main_message_id = " << main_message_id_ + << ", seq_no = " << info.seq_no << " and original size " << info.size << ": " << status << ' ' + << recheck_status; + } if (status.is_error()) { if (status.code() == 2) { LOG(WARNING) << "Receive too old update from " << get_name() << " created in " << (Time::now() - created_at_) From 828e1e22a0531e84abd65a6d0533083ea0a137f6 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 23 Aug 2021 11:10:46 +0300 Subject: [PATCH 16/91] Skip unneeded getFullChannel requests. --- td/telegram/ContactsManager.cpp | 93 +++++++++++++++++---------------- td/telegram/ContactsManager.h | 4 +- 2 files changed, 49 insertions(+), 48 deletions(-) diff --git a/td/telegram/ContactsManager.cpp b/td/telegram/ContactsManager.cpp index cab0b7bf2..14ae36920 100644 --- a/td/telegram/ContactsManager.cpp +++ b/td/telegram/ContactsManager.cpp @@ -4854,7 +4854,7 @@ string ContactsManager::get_dialog_about(DialogId dialog_id) { break; } case DialogType::Channel: { - auto channel_full = get_channel_full_force(dialog_id.get_channel_id(), "get_dialog_about"); + auto channel_full = get_channel_full_force(dialog_id.get_channel_id(), false, "get_dialog_about"); if (channel_full != nullptr) { return channel_full->description; } @@ -6106,7 +6106,7 @@ void ContactsManager::on_update_bot_commands(DialogId dialog_id, UserId bot_user } case DialogType::Channel: { ChannelId channel_id(dialog_id.get_channel_id()); - auto channel_full = get_channel_full(channel_id, "on_update_bot_commands"); + auto channel_full = get_channel_full(channel_id, true, "on_update_bot_commands"); if (channel_full != nullptr) { if (bot_commands.empty()) { if (td::remove_if(channel_full->bot_commands, is_from_bot)) { @@ -6337,7 +6337,7 @@ void ContactsManager::set_channel_username(ChannelId channel_id, const string &u } if (!username.empty() && c->username.empty()) { - auto channel_full = get_channel_full(channel_id, "set_channel_username"); + auto channel_full = get_channel_full(channel_id, false, "set_channel_username"); if (channel_full != nullptr && !channel_full->can_set_username) { return promise.set_error(Status::Error(3, "Can't set supergroup username")); } @@ -6369,7 +6369,7 @@ void ContactsManager::set_channel_sticker_set(ChannelId channel_id, StickerSetId } } - auto channel_full = get_channel_full(channel_id, "set_channel_sticker_set"); + auto channel_full = get_channel_full(channel_id, false, "set_channel_sticker_set"); if (channel_full != nullptr && !channel_full->can_set_sticker_set) { return promise.set_error(Status::Error(3, "Can't set supergroup sticker set")); } @@ -6597,7 +6597,7 @@ void ContactsManager::get_channel_statistics_dc_id(DialogId dialog_id, bool for_ return promise.set_error(Status::Error(400, "Chat info not found")); } - auto channel_full = get_channel_full_force(channel_id, "get_channel_statistics_dc_id"); + auto channel_full = get_channel_full_force(channel_id, true, "get_channel_statistics_dc_id"); if (channel_full == nullptr || !channel_full->stats_dc_id.is_exact() || (for_full_statistics && !channel_full->can_view_statistics)) { auto query_promise = PromiseCreator::lambda([actor_id = actor_id(this), channel_id, for_full_statistics, @@ -6618,7 +6618,7 @@ void ContactsManager::get_channel_statistics_dc_id_impl(ChannelId channel_id, bo return promise.set_error(Status::Error(500, "Request aborted")); } - auto channel_full = get_channel_full(channel_id, "get_channel_statistics_dc_id_impl"); + auto channel_full = get_channel_full(channel_id, false, "get_channel_statistics_dc_id_impl"); if (channel_full == nullptr) { return promise.set_error(Status::Error(400, "Chat full info not found")); } @@ -9517,7 +9517,7 @@ void ContactsManager::on_load_channel_full_from_database(ChannelId channel_id, s // G()->td_db()->get_sqlite_pmc()->erase(get_channel_full_database_key(channel_id), Auto()); // return; - if (get_channel_full(channel_id, "on_load_channel_full_from_database") != nullptr || value.empty()) { + if (get_channel_full(channel_id, true, "on_load_channel_full_from_database") != nullptr || value.empty()) { return; } @@ -9606,12 +9606,13 @@ void ContactsManager::on_load_channel_full_from_database(ChannelId channel_id, s } } -ContactsManager::ChannelFull *ContactsManager::get_channel_full_force(ChannelId channel_id, const char *source) { +ContactsManager::ChannelFull *ContactsManager::get_channel_full_force(ChannelId channel_id, bool only_local, + const char *source) { if (!have_channel_force(channel_id)) { return nullptr; } - ChannelFull *channel_full = get_channel_full(channel_id, source); + ChannelFull *channel_full = get_channel_full(channel_id, only_local, source); if (channel_full != nullptr) { return channel_full; } @@ -9625,7 +9626,7 @@ ContactsManager::ChannelFull *ContactsManager::get_channel_full_force(ChannelId LOG(INFO) << "Trying to load full " << channel_id << " from database from " << source; on_load_channel_full_from_database( channel_id, G()->td_db()->get_sqlite_sync_pmc()->get(get_channel_full_database_key(channel_id)), source); - return get_channel_full(channel_id, source); + return get_channel_full(channel_id, only_local, source); } void ContactsManager::for_each_secret_chat_with_user(UserId user_id, std::function f) { @@ -10431,7 +10432,7 @@ void ContactsManager::on_get_chat_full(tl_object_ptr &&c invalidated_channels_full_.erase(channel_id); if (!G()->close_flag()) { - auto channel_full = get_channel_full(channel_id, "on_get_channel_full"); + auto channel_full = get_channel_full(channel_id, true, "on_get_channel_full"); if (channel_full != nullptr) { if (channel_full->repair_request_version != 0 && channel_full->repair_request_version < channel_full->speculative_version) { @@ -10666,7 +10667,7 @@ void ContactsManager::on_get_chat_full(tl_object_ptr &&c update_channel_full(channel_full, channel_id); if (linked_channel_id.is_valid()) { - auto linked_channel_full = get_channel_full_force(linked_channel_id, "on_get_chat_full"); + auto linked_channel_full = get_channel_full_force(linked_channel_id, true, "on_get_chat_full"); on_update_channel_full_linked_channel_id(linked_channel_full, linked_channel_id, channel_id); if (linked_channel_full != nullptr) { update_channel_full(linked_channel_full, linked_channel_id); @@ -10716,7 +10717,7 @@ void ContactsManager::on_get_channel_full_failed(ChannelId channel_id) { } LOG(INFO) << "Failed to get " << channel_id; - auto channel_full = get_channel_full(channel_id, "on_get_channel_full"); + auto channel_full = get_channel_full(channel_id, true, "on_get_channel_full"); if (channel_full != nullptr) { channel_full->repair_request_version = 0; } @@ -11738,7 +11739,7 @@ void ContactsManager::on_get_channel_participants( } if (participant_count != -1 || administrator_count != -1) { - auto channel_full = get_channel_full_force(channel_id, "on_get_channel_participants_success"); + auto channel_full = get_channel_full_force(channel_id, true, "on_get_channel_participants_success"); if (channel_full != nullptr) { if (administrator_count == -1) { administrator_count = channel_full->administrator_count; @@ -11847,7 +11848,7 @@ bool ContactsManager::speculative_add_count(int32 &count, int32 delta_count, int void ContactsManager::speculative_add_channel_participants(ChannelId channel_id, const vector &added_user_ids, UserId inviter_user_id, int32 date, bool by_me) { auto it = cached_channel_participants_.find(channel_id); - auto channel_full = get_channel_full_force(channel_id, "speculative_add_channel_participants"); + auto channel_full = get_channel_full_force(channel_id, true, "speculative_add_channel_participants"); bool is_participants_cache_changed = false; int32 delta_participant_count = 0; @@ -11913,7 +11914,7 @@ void ContactsManager::speculative_delete_channel_participant(ChannelId channel_i } if (is_user_bot(deleted_user_id)) { - auto channel_full = get_channel_full_force(channel_id, "speculative_delete_channel_participant"); + auto channel_full = get_channel_full_force(channel_id, true, "speculative_delete_channel_participant"); if (channel_full != nullptr && td::remove(channel_full->bot_user_ids, deleted_user_id)) { channel_full->need_save_to_database = true; update_channel_full(channel_full, channel_id); @@ -11934,7 +11935,7 @@ void ContactsManager::speculative_add_channel_participant_count(ChannelId channe return; } - auto channel_full = get_channel_full_force(channel_id, "speculative_add_channel_participant_count"); + auto channel_full = get_channel_full_force(channel_id, true, "speculative_add_channel_participant_count"); auto min_count = channel_full == nullptr ? 0 : channel_full->administrator_count; auto c = get_channel_force(channel_id); @@ -11964,7 +11965,7 @@ void ContactsManager::speculative_add_channel_user(ChannelId channel_id, UserId auto c = get_channel_force(channel_id); // channel full must be loaded before c->participant_count is updated, because on_load_channel_full_from_database // must copy the initial c->participant_count before it is speculatibely updated - auto channel_full = get_channel_full_force(channel_id, "speculative_add_channel_user"); + auto channel_full = get_channel_full_force(channel_id, true, "speculative_add_channel_user"); int32 min_count = 0; if (channel_full != nullptr) { channel_full->is_changed |= speculative_add_count(channel_full->administrator_count, @@ -12077,7 +12078,7 @@ void ContactsManager::speculative_add_channel_user(ChannelId channel_id, UserId void ContactsManager::drop_channel_photos(ChannelId channel_id, bool is_empty, bool drop_channel_full_photo, const char *source) { if (drop_channel_full_photo) { - auto channel_full = get_channel_full(channel_id, "drop_channel_photos"); // must not load ChannelFull + auto channel_full = get_channel_full(channel_id, true, "drop_channel_photos"); // must not load ChannelFull if (channel_full == nullptr) { return; } @@ -12096,7 +12097,7 @@ void ContactsManager::drop_channel_photos(ChannelId channel_id, bool is_empty, b void ContactsManager::invalidate_channel_full(ChannelId channel_id, bool need_drop_slow_mode_delay) { LOG(INFO) << "Invalidate supergroup full for " << channel_id; - auto channel_full = get_channel_full(channel_id, "invalidate_channel_full"); // must not load ChannelFull + auto channel_full = get_channel_full(channel_id, true, "invalidate_channel_full"); // must not load ChannelFull if (channel_full != nullptr) { do_invalidate_channel_full(channel_full, need_drop_slow_mode_delay); update_channel_full(channel_full, channel_id); @@ -12206,7 +12207,7 @@ void ContactsManager::on_get_permanent_dialog_invite_link(DialogId dialog_id, co } case DialogType::Channel: { auto channel_id = dialog_id.get_channel_id(); - auto channel_full = get_channel_full_force(channel_id, "on_get_permanent_dialog_invite_link"); + auto channel_full = get_channel_full_force(channel_id, true, "on_get_permanent_dialog_invite_link"); if (channel_full != nullptr && update_permanent_invite_link(channel_full->invite_link, invite_link)) { channel_full->is_changed = true; update_channel_full(channel_full, channel_id); @@ -12273,7 +12274,7 @@ void ContactsManager::on_update_channel_full_linked_channel_id(ChannelFull *chan if (channel_full != nullptr && channel_full->linked_channel_id != linked_channel_id && channel_full->linked_channel_id.is_valid()) { get_channel_force(channel_full->linked_channel_id); - get_channel_full_force(channel_full->linked_channel_id, "on_update_channel_full_linked_channel_id 0"); + get_channel_full_force(channel_full->linked_channel_id, true, "on_update_channel_full_linked_channel_id 0"); } auto old_linked_linked_channel_id = get_linked_channel_id(linked_channel_id); @@ -12295,7 +12296,7 @@ void ContactsManager::on_update_channel_full_linked_channel_id(ChannelFull *chan reload_channel(channel_full->linked_channel_id, Auto()); } auto linked_channel_full = - get_channel_full_force(channel_full->linked_channel_id, "on_update_channel_full_linked_channel_id 1"); + get_channel_full_force(channel_full->linked_channel_id, true, "on_update_channel_full_linked_channel_id 1"); if (linked_channel_full != nullptr && linked_channel_full->linked_channel_id == channel_id) { linked_channel_full->linked_channel_id = ChannelId(); linked_channel_full->is_changed = true; @@ -12316,7 +12317,7 @@ void ContactsManager::on_update_channel_full_linked_channel_id(ChannelFull *chan reload_channel(channel_full->linked_channel_id, Auto()); } auto linked_channel_full = - get_channel_full_force(channel_full->linked_channel_id, "on_update_channel_full_linked_channel_id 2"); + get_channel_full_force(channel_full->linked_channel_id, true, "on_update_channel_full_linked_channel_id 2"); if (linked_channel_full != nullptr && linked_channel_full->linked_channel_id != channel_id) { linked_channel_full->linked_channel_id = channel_id; linked_channel_full->is_changed = true; @@ -13114,7 +13115,7 @@ void ContactsManager::on_channel_status_changed(const Channel *c, ChannelId chan bool need_reload_group_call = old_status.can_manage_calls() != new_status.can_manage_calls(); if (old_status.can_manage_invite_links() && !new_status.can_manage_invite_links()) { - auto channel_full = get_channel_full(channel_id, "on_channel_status_changed"); + auto channel_full = get_channel_full(channel_id, true, "on_channel_status_changed"); if (channel_full != nullptr) { // otherwise invite_link will be dropped when the channel is loaded on_update_channel_full_invite_link(channel_full, nullptr); do_invalidate_channel_full(channel_full, !c->is_slow_mode_enabled); @@ -13212,7 +13213,7 @@ void ContactsManager::on_channel_username_changed(const Channel *c, ChannelId ch void ContactsManager::on_update_channel_description(ChannelId channel_id, string &&description) { CHECK(channel_id.is_valid()); - auto channel_full = get_channel_full_force(channel_id, "on_update_channel_description"); + auto channel_full = get_channel_full_force(channel_id, true, "on_update_channel_description"); if (channel_full == nullptr) { return; } @@ -13226,7 +13227,7 @@ void ContactsManager::on_update_channel_description(ChannelId channel_id, string void ContactsManager::on_update_channel_sticker_set(ChannelId channel_id, StickerSetId sticker_set_id) { CHECK(channel_id.is_valid()); - auto channel_full = get_channel_full_force(channel_id, "on_update_channel_sticker_set"); + auto channel_full = get_channel_full_force(channel_id, true, "on_update_channel_sticker_set"); if (channel_full == nullptr) { return; } @@ -13239,14 +13240,14 @@ void ContactsManager::on_update_channel_sticker_set(ChannelId channel_id, Sticke void ContactsManager::on_update_channel_linked_channel_id(ChannelId channel_id, ChannelId group_channel_id) { if (channel_id.is_valid()) { - auto channel_full = get_channel_full_force(channel_id, "on_update_channel_linked_channel_id 1"); + auto channel_full = get_channel_full_force(channel_id, true, "on_update_channel_linked_channel_id 1"); on_update_channel_full_linked_channel_id(channel_full, channel_id, group_channel_id); if (channel_full != nullptr) { update_channel_full(channel_full, channel_id); } } if (group_channel_id.is_valid()) { - auto channel_full = get_channel_full_force(group_channel_id, "on_update_channel_linked_channel_id 2"); + auto channel_full = get_channel_full_force(group_channel_id, true, "on_update_channel_linked_channel_id 2"); on_update_channel_full_linked_channel_id(channel_full, group_channel_id, channel_id); if (channel_full != nullptr) { update_channel_full(channel_full, group_channel_id); @@ -13255,7 +13256,7 @@ void ContactsManager::on_update_channel_linked_channel_id(ChannelId channel_id, } void ContactsManager::on_update_channel_location(ChannelId channel_id, const DialogLocation &location) { - auto channel_full = get_channel_full_force(channel_id, "on_update_channel_location"); + auto channel_full = get_channel_full_force(channel_id, true, "on_update_channel_location"); if (channel_full != nullptr) { on_update_channel_full_location(channel_full, channel_id, location); update_channel_full(channel_full, channel_id); @@ -13267,7 +13268,7 @@ void ContactsManager::on_update_channel_slow_mode_delay(ChannelId channel_id, in if (G()->close_flag()) { return promise.set_error(Status::Error(500, "Request aborted")); } - auto channel_full = get_channel_full_force(channel_id, "on_update_channel_slow_mode_delay"); + auto channel_full = get_channel_full_force(channel_id, true, "on_update_channel_slow_mode_delay"); if (channel_full != nullptr) { on_update_channel_full_slow_mode_delay(channel_full, channel_id, slow_mode_delay, 0); update_channel_full(channel_full, channel_id); @@ -13276,7 +13277,7 @@ void ContactsManager::on_update_channel_slow_mode_delay(ChannelId channel_id, in } void ContactsManager::on_update_channel_slow_mode_next_send_date(ChannelId channel_id, int32 slow_mode_next_send_date) { - auto channel_full = get_channel_full_force(channel_id, "on_update_channel_slow_mode_next_send_date"); + auto channel_full = get_channel_full_force(channel_id, true, "on_update_channel_slow_mode_next_send_date"); if (channel_full != nullptr) { on_update_channel_full_slow_mode_next_send_date(channel_full, slow_mode_next_send_date); update_channel_full(channel_full, channel_id); @@ -13290,7 +13291,7 @@ void ContactsManager::on_update_channel_bot_user_ids(ChannelId channel_id, vecto return; } - auto channel_full = get_channel_full_force(channel_id, "on_update_channel_bot_user_ids"); + auto channel_full = get_channel_full_force(channel_id, true, "on_update_channel_bot_user_ids"); if (channel_full == nullptr) { send_closure_later(G()->messages_manager(), &MessagesManager::on_dialog_bots_updated, DialogId(channel_id), std::move(bot_user_ids), false); @@ -13317,7 +13318,7 @@ void ContactsManager::on_update_channel_is_all_history_available(ChannelId chann return promise.set_error(Status::Error(500, "Request aborted")); } CHECK(channel_id.is_valid()); - auto channel_full = get_channel_full_force(channel_id, "on_update_channel_is_all_history_available"); + auto channel_full = get_channel_full_force(channel_id, true, "on_update_channel_is_all_history_available"); if (channel_full != nullptr && channel_full->is_all_history_available != is_all_history_available) { channel_full->is_all_history_available = is_all_history_available; channel_full->is_changed = true; @@ -14214,7 +14215,7 @@ bool ContactsManager::get_channel_has_linked_channel(const Channel *c) { ChannelId ContactsManager::get_channel_linked_channel_id(ChannelId channel_id) { auto channel_full = get_channel_full_const(channel_id); if (channel_full == nullptr) { - channel_full = get_channel_full_force(channel_id, "get_channel_linked_channel_id"); + channel_full = get_channel_full_force(channel_id, false, "get_channel_linked_channel_id"); if (channel_full == nullptr) { return ChannelId(); } @@ -14225,7 +14226,7 @@ ChannelId ContactsManager::get_channel_linked_channel_id(ChannelId channel_id) { int32 ContactsManager::get_channel_slow_mode_delay(ChannelId channel_id) { auto channel_full = get_channel_full_const(channel_id); if (channel_full == nullptr) { - channel_full = get_channel_full_force(channel_id, "get_channel_slow_mode_delay"); + channel_full = get_channel_full_force(channel_id, false, "get_channel_slow_mode_delay"); if (channel_full == nullptr) { return 0; } @@ -14323,14 +14324,15 @@ const ContactsManager::ChannelFull *ContactsManager::get_channel_full(ChannelId return get_channel_full_const(channel_id); } -ContactsManager::ChannelFull *ContactsManager::get_channel_full(ChannelId channel_id, const char *source) { +ContactsManager::ChannelFull *ContactsManager::get_channel_full(ChannelId channel_id, bool only_local, + const char *source) { auto p = channels_full_.find(channel_id); if (p == channels_full_.end()) { return nullptr; } auto channel_full = p->second.get(); - if (channel_full->is_expired() && !td_->auth_manager_->is_bot()) { + if (!only_local && channel_full->is_expired() && !td_->auth_manager_->is_bot()) { send_get_channel_full_query(channel_full, channel_id, Auto(), source); } @@ -14347,7 +14349,7 @@ ContactsManager::ChannelFull *ContactsManager::add_channel_full(ChannelId channe } bool ContactsManager::load_channel_full(ChannelId channel_id, bool force, Promise &&promise, const char *source) { - auto channel_full = get_channel_full_force(channel_id, source); + auto channel_full = get_channel_full_force(channel_id, true, source); if (channel_full == nullptr) { send_get_channel_full_query(channel_full, channel_id, std::move(promise), source); return false; @@ -14357,8 +14359,7 @@ bool ContactsManager::load_channel_full(ChannelId channel_id, bool force, Promis send_get_channel_full_query(channel_full, channel_id, std::move(promise), "load expired channel_full"); return false; } else { - // request has already been sent in get_channel_full_force - // send_get_channel_full_query(channel_full, channel_id, Auto(), "load expired channel_full"); + send_get_channel_full_query(channel_full, channel_id, Auto(), "load expired channel_full"); } } @@ -14367,7 +14368,7 @@ bool ContactsManager::load_channel_full(ChannelId channel_id, bool force, Promis } void ContactsManager::reload_channel_full(ChannelId channel_id, Promise &&promise, const char *source) { - send_get_channel_full_query(get_channel_full(channel_id, "reload_channel_full"), channel_id, std::move(promise), + send_get_channel_full_query(get_channel_full(channel_id, true, "reload_channel_full"), channel_id, std::move(promise), source); } @@ -15045,7 +15046,7 @@ void ContactsManager::get_channel_participants(ChannelId channel_id, return promise.set_error(Status::Error(400, "Parameter offset must be non-negative")); } - auto channel_full = get_channel_full_force(channel_id, "do_get_channel_participants"); + auto channel_full = get_channel_full_force(channel_id, false, "get_channel_participants"); if (channel_full != nullptr && !channel_full->is_expired() && !channel_full->can_get_participants) { return promise.set_error(Status::Error(400, "Member list is inaccessible")); } @@ -15172,7 +15173,7 @@ void ContactsManager::on_load_administrator_users_finished(DialogId dialog_id, } void ContactsManager::on_update_channel_administrator_count(ChannelId channel_id, int32 administrator_count) { - auto channel_full = get_channel_full_force(channel_id, "on_update_channel_administrator_count"); + auto channel_full = get_channel_full_force(channel_id, true, "on_update_channel_administrator_count"); if (channel_full != nullptr && channel_full->administrator_count != administrator_count) { channel_full->administrator_count = administrator_count; channel_full->is_changed = true; @@ -15568,7 +15569,7 @@ void ContactsManager::on_chat_update(telegram_api::channel &channel, const char update_channel(c, channel_id); if (need_update_participant_count) { - auto channel_full = get_channel_full(channel_id, "on_chat_update"); + auto channel_full = get_channel_full(channel_id, true, "on_chat_update"); if (channel_full != nullptr && channel_full->participant_count != participant_count) { channel_full->participant_count = participant_count; channel_full->is_changed = true; @@ -15677,7 +15678,7 @@ void ContactsManager::on_chat_update(telegram_api::channelForbidden &channel, co update_channel(c, channel_id); if (need_drop_participant_count) { - auto channel_full = get_channel_full(channel_id, "on_chat_update"); + auto channel_full = get_channel_full(channel_id, true, "on_chat_update"); if (channel_full != nullptr && channel_full->participant_count != 0) { channel_full->participant_count = 0; channel_full->administrator_count = 0; diff --git a/td/telegram/ContactsManager.h b/td/telegram/ContactsManager.h index cf39630ab..1972dfda1 100644 --- a/td/telegram/ContactsManager.h +++ b/td/telegram/ContactsManager.h @@ -1129,8 +1129,8 @@ class ContactsManager final : public Actor { const ChannelFull *get_channel_full(ChannelId channel_id) const; const ChannelFull *get_channel_full_const(ChannelId channel_id) const; - ChannelFull *get_channel_full(ChannelId channel_id, const char *source); - ChannelFull *get_channel_full_force(ChannelId channel_id, const char *source); + ChannelFull *get_channel_full(ChannelId channel_id, bool only_local, const char *source); + ChannelFull *get_channel_full_force(ChannelId channel_id, bool only_local, const char *source); ChannelFull *add_channel_full(ChannelId channel_id); From ddb3c56ddb118d4ac71237d516cf6e05c0a73157 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 23 Aug 2021 11:17:25 +0300 Subject: [PATCH 17/91] Add one more MessageIdDuplicateChecker benchmark. --- benchmark/bench_misc.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/benchmark/bench_misc.cpp b/benchmark/bench_misc.cpp index aeec2232a..f0a4bea06 100644 --- a/benchmark/bench_misc.cpp +++ b/benchmark/bench_misc.cpp @@ -633,9 +633,28 @@ class DuplicateCheckerBenchReverse final : public td::Benchmark { } }; +template +class DuplicateCheckerBenchEvenOdd final : public td::Benchmark { + td::string get_description() const final { + return PSTRING() << "DuplicateCheckerBenchEvenOdd" << T::get_description(); + } + void run(int n) final { + T checker_; + for (int i = 0; i < n; i++) { + auto pos = i & 255; + checker_.check(i - pos + (pos * 2) % 256 + (pos * 2) / 256).ensure(); + } + } +}; + int main() { SET_VERBOSITY_LEVEL(VERBOSITY_NAME(DEBUG)); + td::bench(DuplicateCheckerBenchEvenOdd>()); + td::bench(DuplicateCheckerBenchEvenOdd>()); + td::bench(DuplicateCheckerBenchEvenOdd>()); + td::bench(DuplicateCheckerBenchEvenOdd>()); + td::bench(DuplicateCheckerBenchReverse>()); td::bench(DuplicateCheckerBenchReverse>()); td::bench(DuplicateCheckerBenchReverse>()); From 8e7eba4f8cf2c71d72287cfb5fb3b9a2eca88dda Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 23 Aug 2021 11:59:37 +0300 Subject: [PATCH 18/91] Reload ChannelFull in get_channel_statistics_dc_id. --- td/telegram/ContactsManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/td/telegram/ContactsManager.cpp b/td/telegram/ContactsManager.cpp index 14ae36920..62b13ca38 100644 --- a/td/telegram/ContactsManager.cpp +++ b/td/telegram/ContactsManager.cpp @@ -6597,7 +6597,7 @@ void ContactsManager::get_channel_statistics_dc_id(DialogId dialog_id, bool for_ return promise.set_error(Status::Error(400, "Chat info not found")); } - auto channel_full = get_channel_full_force(channel_id, true, "get_channel_statistics_dc_id"); + auto channel_full = get_channel_full_force(channel_id, false, "get_channel_statistics_dc_id"); if (channel_full == nullptr || !channel_full->stats_dc_id.is_exact() || (for_full_statistics && !channel_full->can_view_statistics)) { auto query_promise = PromiseCreator::lambda([actor_id = actor_id(this), channel_id, for_full_statistics, From bba085318e3a4a2b3174d0dfcd3982f981aabed3 Mon Sep 17 00:00:00 2001 From: levlam Date: Tue, 24 Aug 2021 17:13:51 +0300 Subject: [PATCH 19/91] Return BackgroundType from on_get_background. --- td/telegram/BackgroundManager.cpp | 78 +++++++++++++++++++------------ td/telegram/BackgroundManager.h | 9 ++-- td/telegram/BackgroundType.h | 4 ++ td/telegram/Td.cpp | 2 +- 4 files changed, 59 insertions(+), 34 deletions(-) diff --git a/td/telegram/BackgroundManager.cpp b/td/telegram/BackgroundManager.cpp index 0d8859d96..32970552e 100644 --- a/td/telegram/BackgroundManager.cpp +++ b/td/telegram/BackgroundManager.cpp @@ -53,7 +53,7 @@ class GetBackgroundQuery final : public Td::ResultHandler { telegram_api::object_ptr &&input_wallpaper) { background_id_ = background_id; background_name_ = background_name; - LOG(INFO) << "Load " << background_id_ << "/" << background_name_ << " from server: " << to_string(input_wallpaper); + LOG(INFO) << "Load " << background_id_ << "/" << background_name_ << " from server"; send_query(G()->net_query_creator().create(telegram_api::account_getWallPaper(std::move(input_wallpaper)))); } @@ -145,7 +145,7 @@ class UploadBackgroundQuery final : public Td::ResultHandler { type_ = type; for_dark_theme_ = for_dark_theme; send_query(G()->net_query_creator().create(telegram_api::account_uploadWallPaper( - std::move(input_file), type_.get_mime_type(), type.get_input_wallpaper_settings()))); + std::move(input_file), type_.get_mime_type(), type_.get_input_wallpaper_settings()))); } void on_result(uint64 id, BufferSlice packet) final { @@ -549,7 +549,8 @@ void BackgroundManager::on_load_background_from_database(string name, string val td_api::object_ptr BackgroundManager::get_update_selected_background_object( bool for_dark_theme) const { return td_api::make_object( - for_dark_theme, get_background_object(set_background_id_[for_dark_theme], for_dark_theme)); + for_dark_theme, + get_background_object(set_background_id_[for_dark_theme], for_dark_theme, &set_background_type_[for_dark_theme])); } void BackgroundManager::send_update_selected_background(bool for_dark_theme) const { @@ -714,8 +715,15 @@ void BackgroundManager::on_installed_background(BackgroundId background_id, Back return promise.set_error(result.move_as_error()); } - if (!td::contains(installed_background_ids_, background_id)) { - installed_background_ids_.insert(installed_background_ids_.begin(), background_id); + size_t i; + for (i = 0; i < installed_backgrounds_.size(); i++) { + if (installed_backgrounds_[i].first == background_id) { + installed_backgrounds_[i].second = type; + break; + } + } + if (i == installed_backgrounds_.size()) { + installed_backgrounds_.insert(installed_backgrounds_.begin(), {background_id, type}); } set_background_id(background_id, type, for_dark_theme); promise.set_value(Unit()); @@ -841,11 +849,14 @@ void BackgroundManager::on_uploaded_background_file(FileId file_id, const Backgr Promise &&promise) { CHECK(wallpaper != nullptr); - BackgroundId background_id = on_get_background(BackgroundId(), string(), std::move(wallpaper)); + auto added_background = on_get_background(BackgroundId(), string(), std::move(wallpaper)); + auto background_id = added_background.first; if (!background_id.is_valid()) { td_->file_manager_->cancel_upload(file_id); return promise.set_error(Status::Error(500, "Receive wrong uploaded background")); } + LOG_IF(ERROR, added_background.second != type) + << "Type of uploaded background has changed from " << type << " to " << added_background.second; const auto *background = get_background(background_id); CHECK(background != nullptr); @@ -888,7 +899,8 @@ void BackgroundManager::on_removed_background(BackgroundId background_id, Result if (result.is_error()) { return promise.set_error(result.move_as_error()); } - td::remove(installed_background_ids_, background_id); + td::remove_if(installed_backgrounds_, + [background_id](const auto &background) { return background.first == background_id; }); if (background_id == set_background_id_[0]) { set_background_id(BackgroundId(), BackgroundType(), false); } @@ -919,7 +931,7 @@ void BackgroundManager::on_reset_background(Result &&result, Promise if (result.is_error()) { return promise.set_error(result.move_as_error()); } - installed_background_ids_.clear(); + installed_backgrounds_.clear(); set_background_id(BackgroundId(), BackgroundType(), false); set_background_id(BackgroundId(), BackgroundType(), true); if (!local_background_ids_[0].empty()) { @@ -1031,9 +1043,9 @@ string BackgroundManager::get_background_name_database_key(const string &name) { return PSTRING() << "bgn" << name; } -BackgroundId BackgroundManager::on_get_background(BackgroundId expected_background_id, - const string &expected_background_name, - telegram_api::object_ptr wallpaper_ptr) { +std::pair BackgroundManager::on_get_background( + BackgroundId expected_background_id, const string &expected_background_name, + telegram_api::object_ptr wallpaper_ptr) { CHECK(wallpaper_ptr != nullptr); if (wallpaper_ptr->get_id() == telegram_api::wallPaperNoFile::ID) { @@ -1041,13 +1053,13 @@ BackgroundId BackgroundManager::on_get_background(BackgroundId expected_backgrou if (wallpaper->settings_ == nullptr) { LOG(ERROR) << "Receive wallPaperNoFile without settings: " << to_string(wallpaper); - return BackgroundId(); + return {}; } auto background_id = BackgroundId(wallpaper->id_); if (!background_id.is_valid() || background_id.is_local()) { LOG(ERROR) << "Receive " << to_string(wallpaper); - return BackgroundId(); + return {}; } Background background; @@ -1059,14 +1071,14 @@ BackgroundId BackgroundManager::on_get_background(BackgroundId expected_backgrou background.name = background.type.get_link(); add_background(background); - return background_id; + return {background_id, background.type}; } auto wallpaper = move_tl_object_as(wallpaper_ptr); auto background_id = BackgroundId(wallpaper->id_); if (!background_id.is_valid() || background_id.is_local() || is_background_name_local(wallpaper->slug_)) { LOG(ERROR) << "Receive " << to_string(wallpaper); - return BackgroundId(); + return {}; } if (expected_background_id.is_valid() && background_id != expected_background_id) { LOG(ERROR) << "Expected " << expected_background_id << ", but receive " << to_string(wallpaper); @@ -1075,7 +1087,7 @@ BackgroundId BackgroundManager::on_get_background(BackgroundId expected_backgrou int32 document_id = wallpaper->document_->get_id(); if (document_id == telegram_api::documentEmpty::ID) { LOG(ERROR) << "Receive " << to_string(wallpaper); - return BackgroundId(); + return {}; } CHECK(document_id == telegram_api::document::ID); @@ -1087,7 +1099,7 @@ BackgroundId BackgroundManager::on_get_background(BackgroundId expected_backgrou Document::Type::General, true, is_pattern); if (!document.file_id.is_valid()) { LOG(ERROR) << "Receive wrong document in " << to_string(wallpaper); - return BackgroundId(); + return {}; } CHECK(document.type == Document::Type::General); // guaranteed by is_background parameter to on_get_document @@ -1114,7 +1126,7 @@ BackgroundId BackgroundManager::on_get_background(BackgroundId expected_backgrou log_event_store(background).as_slice().str(), Auto()); } - return background_id; + return {background_id, background.type}; } void BackgroundManager::on_get_backgrounds(Result> result) { @@ -1123,7 +1135,7 @@ void BackgroundManager::on_get_backgrounds(Result(wallpapers_ptr); for (auto &wallpaper : wallpapers->wallpapers_) { - auto background_id = on_get_background(BackgroundId(), string(), std::move(wallpaper)); - if (background_id.is_valid()) { - installed_background_ids_.push_back(background_id); + auto background = on_get_background(BackgroundId(), string(), std::move(wallpaper)); + if (background.first.is_valid()) { + installed_backgrounds_.push_back(std::move(background)); } } @@ -1179,16 +1191,24 @@ td_api::object_ptr BackgroundManager::get_background_object( } td_api::object_ptr BackgroundManager::get_backgrounds_object(bool for_dark_theme) const { - auto backgrounds = transform(installed_background_ids_, [this, for_dark_theme](BackgroundId background_id) { - return get_background_object(background_id, for_dark_theme); - }); + auto backgrounds = transform(installed_backgrounds_, + [this, for_dark_theme](const std::pair &background) { + return get_background_object(background.first, for_dark_theme, &background.second); + }); auto background_id = set_background_id_[for_dark_theme]; - if (background_id.is_valid() && !td::contains(installed_background_ids_, background_id)) { - backgrounds.push_back(get_background_object(background_id, for_dark_theme)); + bool have_background = false; + for (const auto &background : installed_backgrounds_) { + if (background_id == background.first) { + have_background = true; + break; + } + } + if (background_id.is_valid() && !have_background) { + backgrounds.push_back(get_background_object(background_id, for_dark_theme, nullptr)); } for (auto local_background_id : local_background_ids_[for_dark_theme]) { if (local_background_id != background_id) { - backgrounds.push_back(get_background_object(local_background_id, for_dark_theme)); + backgrounds.push_back(get_background_object(local_background_id, for_dark_theme, nullptr)); } } std::stable_sort(backgrounds.begin(), backgrounds.end(), diff --git a/td/telegram/BackgroundManager.h b/td/telegram/BackgroundManager.h index 7df9486b4..daec4b2a3 100644 --- a/td/telegram/BackgroundManager.h +++ b/td/telegram/BackgroundManager.h @@ -51,12 +51,13 @@ class BackgroundManager final : public Actor { void reset_backgrounds(Promise &&promise); td_api::object_ptr get_background_object(BackgroundId background_id, bool for_dark_theme, - const BackgroundType *type = nullptr) const; + const BackgroundType *type) const; td_api::object_ptr get_backgrounds_object(bool for_dark_theme) const; - BackgroundId on_get_background(BackgroundId expected_background_id, const string &expected_background_name, - telegram_api::object_ptr wallpaper_ptr); + std::pair on_get_background( + BackgroundId expected_background_id, const string &expected_background_name, + telegram_api::object_ptr wallpaper_ptr); FileSourceId get_background_file_source_id(BackgroundId background_id, int64 access_hash); @@ -167,7 +168,7 @@ class BackgroundManager final : public Actor { BackgroundId set_background_id_[2]; BackgroundType set_background_type_[2]; - vector installed_background_ids_; + vector> installed_backgrounds_; vector> pending_get_backgrounds_queries_; diff --git a/td/telegram/BackgroundType.h b/td/telegram/BackgroundType.h index b30087dd2..e84fd84ef 100644 --- a/td/telegram/BackgroundType.h +++ b/td/telegram/BackgroundType.h @@ -125,6 +125,10 @@ class BackgroundType { bool operator==(const BackgroundType &lhs, const BackgroundType &rhs); +inline bool operator!=(const BackgroundType &lhs, const BackgroundType &rhs) { + return !(lhs == rhs); +} + StringBuilder &operator<<(StringBuilder &string_builder, const BackgroundType &type); } // namespace td diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index b599b0500..99a738b19 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -2901,7 +2901,7 @@ class SetBackgroundRequest final : public RequestActor<> { } void do_send_result() final { - send_result(td->background_manager_->get_background_object(background_id_, for_dark_theme_)); + send_result(td->background_manager_->get_background_object(background_id_, for_dark_theme_, nullptr)); } public: From dbdf69d0763d759a6f709976b568c5823b04c914 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 25 Aug 2021 11:03:07 +0300 Subject: [PATCH 20/91] Simplify on_update_dialog_last_pinned_message_id. --- td/telegram/MessagesManager.cpp | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index d29be6f8a..a7688941a 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -29634,19 +29634,6 @@ void MessagesManager::on_update_dialog_last_pinned_message_id(DialogId dialog_id return; } - if (d->last_pinned_message_id == pinned_message_id) { - LOG(INFO) << "Pinned message in " << d->dialog_id << " is still " << pinned_message_id; - if (!d->is_last_pinned_message_id_inited) { - d->is_last_pinned_message_id_inited = true; - on_dialog_updated(dialog_id, "on_update_dialog_last_pinned_message_id"); - } - Message *m = get_message_force(d, pinned_message_id, "on_update_dialog_last_pinned_message_id"); - if (m != nullptr && update_message_is_pinned(d, m, true, "on_update_dialog_last_pinned_message_id")) { - on_message_changed(d, m, true, "on_update_dialog_last_pinned_message_id"); - } - return; - } - set_dialog_last_pinned_message_id(d, pinned_message_id); } @@ -29657,7 +29644,7 @@ void MessagesManager::set_dialog_last_pinned_message_id(Dialog *d, MessageId pin on_message_changed(d, m, true, "set_dialog_last_pinned_message_id"); } - if (d->last_pinned_message_id == pinned_message_id) { + if (d->is_last_pinned_message_id_inited && d->last_pinned_message_id == pinned_message_id) { return; } d->last_pinned_message_id = pinned_message_id; From 628648cd6cfe81f1f7619072703abae586552df9 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 25 Aug 2021 11:09:24 +0300 Subject: [PATCH 21/91] Use set_dialog_last_pinned_message_id instead of on_update_dialog_last_pinned_message_id. --- td/telegram/MessagesManager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index a7688941a..0cb8bda0d 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -10097,7 +10097,7 @@ bool MessagesManager::update_message_is_pinned(Dialog *d, Message *m, bool is_pi make_tl_object(d->dialog_id.get(), m->message_id.get(), is_pinned)); if (is_pinned) { if (d->is_last_pinned_message_id_inited && m->message_id > d->last_pinned_message_id) { - on_update_dialog_last_pinned_message_id(d->dialog_id, m->message_id); + set_dialog_last_pinned_message_id(d, m->message_id); } } else { if (d->is_last_pinned_message_id_inited && m->message_id == d->last_pinned_message_id) { @@ -32699,7 +32699,7 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, uniq if (*need_update && m->message_id.is_server() && message_content_type == MessageContentType::PinMessage) { auto pinned_message_id = get_message_content_pinned_message_id(m->content.get()); if (d->is_last_pinned_message_id_inited && pinned_message_id > d->last_pinned_message_id) { - on_update_dialog_last_pinned_message_id(dialog_id, pinned_message_id); + set_dialog_last_pinned_message_id(d, pinned_message_id); } } From 8559392f6f7600c720fa0958d40a4ff8135a5743 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 25 Aug 2021 11:21:17 +0300 Subject: [PATCH 22/91] Fix Dialog pts in run_after_channel_difference. --- td/telegram/MessagesManager.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 0cb8bda0d..9352e6b0f 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -16973,7 +16973,7 @@ void MessagesManager::process_discussion_message_impl( MessageId top_message_id; for (auto &message : result->messages_) { auto full_message_id = - on_get_message(std::move(message), false, true, false, false, false, "process_discussion_message"); + on_get_message(std::move(message), false, true, false, false, false, "process_discussion_message_impl"); if (full_message_id.get_message_id().is_valid()) { CHECK(full_message_id.get_dialog_id() == expected_dialog_id); message_thread_info.message_ids.push_back(full_message_id.get_message_id()); @@ -35610,12 +35610,11 @@ void MessagesManager::run_after_channel_difference(DialogId dialog_id, Promisepts, true, "run_after_channel_difference"); + const Dialog *d = get_dialog(dialog_id); + get_channel_difference(dialog_id, d == nullptr ? load_channel_pts(dialog_id) : d->pts, true, + "run_after_channel_difference"); } bool MessagesManager::running_get_channel_difference(DialogId dialog_id) const { From fcfbb38f637c0e2ed9b1741794e34427872c27e5 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 25 Aug 2021 11:29:57 +0300 Subject: [PATCH 23/91] Fix MessageId::is_valid_scheduled. --- td/telegram/MessageId.cpp | 4 ++-- td/telegram/MessagesManager.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/td/telegram/MessageId.cpp b/td/telegram/MessageId.cpp index ab5b85065..0c96b8e2b 100644 --- a/td/telegram/MessageId.cpp +++ b/td/telegram/MessageId.cpp @@ -35,7 +35,7 @@ bool MessageId::is_valid() const { } bool MessageId::is_valid_scheduled() const { - if (id <= 0 || id > max().get()) { + if (id <= 0 || id > (static_cast(1) << 51)) { return false; } int32 type = (id & TYPE_MASK); @@ -43,7 +43,7 @@ bool MessageId::is_valid_scheduled() const { } MessageType MessageId::get_type() const { - if (id <= 0 || id > max().get()) { + if (id <= 0 || id > (static_cast(1) << 51)) { return MessageType::None; } diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index 9352e6b0f..8f7542b43 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -15273,7 +15273,7 @@ unique_ptr MessagesManager::do_delete_scheduled_messag bool is_permanently_deleted, const char *source) { CHECK(d != nullptr); - CHECK(message_id.is_valid_scheduled()); + LOG_CHECK(message_id.is_valid_scheduled()) << d->dialog_id << ' ' << message_id << ' ' << source; unique_ptr *v = treap_find_message(&d->scheduled_messages, message_id); if (*v == nullptr) { From e31a48998d1bd57c9f4d6009418d381c08f4133a Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 26 Aug 2021 17:23:18 +0300 Subject: [PATCH 24/91] Remove "Filling gap" warnings during getDifference. --- td/telegram/UpdatesManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/td/telegram/UpdatesManager.cpp b/td/telegram/UpdatesManager.cpp index 95f327dc8..5f87db20d 100644 --- a/td/telegram/UpdatesManager.cpp +++ b/td/telegram/UpdatesManager.cpp @@ -242,7 +242,7 @@ void UpdatesManager::fill_gap(void *td, const char *source) { } auto updates_manager = static_cast(td)->updates_manager_.get(); - if (source != nullptr) { + if (source != nullptr && !updates_manager->running_get_difference_) { LOG(WARNING) << "Filling gap in " << source << " by running getDifference"; } From ca03135f02094b66d0a2a827753eb11b1cf949b5 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 26 Aug 2021 18:05:27 +0300 Subject: [PATCH 25/91] Use (1 << 30) as pts overflow mark. At least 1e9 must be used, because pts can jump from 1 to 1e9 when switching to tmessages. --- td/telegram/UpdatesManager.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/td/telegram/UpdatesManager.cpp b/td/telegram/UpdatesManager.cpp index 5f87db20d..ef1c3c71e 100644 --- a/td/telegram/UpdatesManager.cpp +++ b/td/telegram/UpdatesManager.cpp @@ -1749,7 +1749,7 @@ void UpdatesManager::on_pending_updates(vector= 1000000000 && seq_begin < seq_ - 1000000000) { + if (seq_ >= (1 << 30) && seq_begin < seq_ - (1 << 30)) { set_seq_gap_timeout(0.001); } if (seq_end > seq_) { @@ -1975,7 +1975,7 @@ void UpdatesManager::add_pending_pts_update(tl_object_ptr } } - if (new_pts <= old_pts || (old_pts >= 1 && new_pts - 500000000 > old_pts)) { + if (new_pts <= old_pts || (old_pts >= 1 && new_pts - (1 << 30) > old_pts)) { td_->messages_manager_->skip_old_pending_pts_update(std::move(update), new_pts, old_pts, pts_count, source); return promise.set_value(Unit()); } @@ -2166,7 +2166,7 @@ void UpdatesManager::process_postponed_pts_updates() { while (update_it != postponed_pts_updates_.end()) { auto new_pts = update_it->second.pts; auto pts_count = update_it->second.pts_count; - if (new_pts <= old_pts || (old_pts >= 1 && new_pts - 500000000 > old_pts)) { + if (new_pts <= old_pts || (old_pts >= 1 && new_pts - (1 << 30) > old_pts)) { skipped_update_count++; td_->messages_manager_->skip_old_pending_pts_update(std::move(update_it->second.update), new_pts, old_pts, pts_count, "process_postponed_pts_updates"); @@ -2281,7 +2281,7 @@ void UpdatesManager::process_pending_seq_updates() { auto update_it = pending_seq_updates_.begin(); auto &update = update_it->second; auto seq_begin = update.seq_begin; - if (seq_begin - 1 > seq_ && seq_begin - 500000000 <= seq_) { + if (seq_begin - 1 > seq_ && seq_begin - (1 << 30) <= seq_) { // the updates will be applied later break; } @@ -2330,7 +2330,7 @@ void UpdatesManager::process_pending_qts_updates() { auto update_it = pending_qts_updates_.begin(); auto qts = update_it->first; auto old_qts = get_qts(); - if (qts - 1 > old_qts && qts - 500000000 <= old_qts) { + if (qts - 1 > old_qts && qts - (1 << 30) <= old_qts) { // the update will be applied later break; } From cf7c14fb592499578f2f2eea5ba53e641fa178e9 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 26 Aug 2021 18:50:28 +0300 Subject: [PATCH 26/91] Remove unneeded document.is_changed. --- td/telegram/AnimationsManager.cpp | 25 +++--------------- td/telegram/AnimationsManager.h | 4 +-- td/telegram/AudiosManager.cpp | 20 +++------------ td/telegram/AudiosManager.h | 4 +-- td/telegram/DocumentsManager.cpp | 20 +++------------ td/telegram/DocumentsManager.h | 4 +-- td/telegram/MessageContent.cpp | 42 ++++++++++++++++++++----------- td/telegram/StickersManager.cpp | 39 ++++++++-------------------- td/telegram/StickersManager.h | 4 +-- td/telegram/StickersManager.hpp | 1 - td/telegram/VideoNotesManager.cpp | 19 +++----------- td/telegram/VideoNotesManager.h | 4 +-- td/telegram/VideosManager.cpp | 24 +++--------------- td/telegram/VideosManager.h | 4 +-- td/telegram/VoiceNotesManager.cpp | 18 +++---------- td/telegram/VoiceNotesManager.h | 4 +-- 16 files changed, 63 insertions(+), 173 deletions(-) diff --git a/td/telegram/AnimationsManager.cpp b/td/telegram/AnimationsManager.cpp index f00230b44..f5ba25b51 100644 --- a/td/telegram/AnimationsManager.cpp +++ b/td/telegram/AnimationsManager.cpp @@ -158,8 +158,6 @@ tl_object_ptr AnimationsManager::get_animation_object(FileId auto &animation = animations_[file_id]; LOG_CHECK(animation != nullptr) << source << " " << file_id << " " << static_cast(td_->file_manager_->get_file_view(file_id).get_type()); - // TODO can we make that function const? - animation->is_changed = false; auto thumbnail = animation->animated_thumbnail.file_id.is_valid() ? get_thumbnail_object(td_->file_manager_.get(), animation->animated_thumbnail, PhotoFormat::Mpeg4) @@ -183,26 +181,21 @@ FileId AnimationsManager::on_get_animation(unique_ptr new_animation, if (a->mime_type != new_animation->mime_type) { LOG(DEBUG) << "Animation " << file_id << " info has changed"; a->mime_type = new_animation->mime_type; - a->is_changed = true; } if (a->file_name != new_animation->file_name) { LOG(DEBUG) << "Animation " << file_id << " file name has changed"; a->file_name = std::move(new_animation->file_name); - a->is_changed = true; } if (a->dimensions != new_animation->dimensions) { LOG(DEBUG) << "Animation " << file_id << " dimensions has changed"; a->dimensions = new_animation->dimensions; - a->is_changed = true; } if (a->duration != new_animation->duration) { LOG(DEBUG) << "Animation " << file_id << " duration has changed"; a->duration = new_animation->duration; - a->is_changed = true; } if (a->minithumbnail != new_animation->minithumbnail) { a->minithumbnail = std::move(new_animation->minithumbnail); - a->is_changed = true; } if (a->thumbnail != new_animation->thumbnail) { if (!a->thumbnail.file_id.is_valid()) { @@ -212,7 +205,6 @@ FileId AnimationsManager::on_get_animation(unique_ptr new_animation, << new_animation->thumbnail; } a->thumbnail = new_animation->thumbnail; - a->is_changed = true; } if (a->animated_thumbnail != new_animation->animated_thumbnail) { if (!a->animated_thumbnail.file_id.is_valid()) { @@ -222,15 +214,12 @@ FileId AnimationsManager::on_get_animation(unique_ptr new_animation, << " to " << new_animation->animated_thumbnail; } a->animated_thumbnail = new_animation->animated_thumbnail; - a->is_changed = true; } if (a->has_stickers != new_animation->has_stickers && new_animation->has_stickers) { a->has_stickers = new_animation->has_stickers; - a->is_changed = true; } if (a->sticker_file_ids != new_animation->sticker_file_ids && !new_animation->sticker_file_ids.empty()) { a->sticker_file_ids = std::move(new_animation->sticker_file_ids); - a->is_changed = true; } } @@ -280,24 +269,18 @@ FileId AnimationsManager::dup_animation(FileId new_id, FileId old_id) { return new_id; } -bool AnimationsManager::merge_animations(FileId new_id, FileId old_id, bool can_delete_old) { - if (!old_id.is_valid()) { - LOG(ERROR) << "Old file identifier is invalid"; - return true; - } +void AnimationsManager::merge_animations(FileId new_id, FileId old_id, bool can_delete_old) { + CHECK(old_id.is_valid() && new_id.is_valid()); + CHECK(new_id != old_id); LOG(INFO) << "Merge animations " << new_id << " and " << old_id; const Animation *old_ = get_animation(old_id); CHECK(old_ != nullptr); - if (old_id == new_id) { - return old_->is_changed; - } bool need_merge = true; auto new_it = animations_.find(new_id); if (new_it == animations_.end()) { auto &old = animations_[old_id]; - old->is_changed = true; if (!can_delete_old) { dup_animation(new_id, old_id); } else { @@ -308,7 +291,6 @@ bool AnimationsManager::merge_animations(FileId new_id, FileId old_id, bool can_ Animation *new_ = new_it->second.get(); CHECK(new_ != nullptr); - new_->is_changed = true; if (old_->thumbnail != new_->thumbnail) { // LOG_STATUS(td_->file_manager_->merge(new_->thumbnail.file_id, old_->thumbnail.file_id)); } @@ -322,7 +304,6 @@ bool AnimationsManager::merge_animations(FileId new_id, FileId old_id, bool can_ if (can_delete_old) { animations_.erase(old_id); } - return true; } void AnimationsManager::create_animation(FileId file_id, string minithumbnail, PhotoSize thumbnail, diff --git a/td/telegram/AnimationsManager.h b/td/telegram/AnimationsManager.h index ee0ea546d..56e444040 100644 --- a/td/telegram/AnimationsManager.h +++ b/td/telegram/AnimationsManager.h @@ -55,7 +55,7 @@ class AnimationsManager final : public Actor { FileId dup_animation(FileId new_id, FileId old_id); - bool merge_animations(FileId new_id, FileId old_id, bool can_delete_old); + void merge_animations(FileId new_id, FileId old_id, bool can_delete_old); void on_update_animation_search_emojis(string animation_search_emojis); @@ -110,8 +110,6 @@ class AnimationsManager final : public Actor { vector sticker_file_ids; FileId file_id; - - bool is_changed = true; }; const Animation *get_animation(FileId file_id) const; diff --git a/td/telegram/AudiosManager.cpp b/td/telegram/AudiosManager.cpp index 6021b4d85..8912eeea2 100644 --- a/td/telegram/AudiosManager.cpp +++ b/td/telegram/AudiosManager.cpp @@ -36,7 +36,6 @@ tl_object_ptr AudiosManager::get_audio_object(FileId file_id) { auto &audio = audios_[file_id]; CHECK(audio != nullptr); - audio->is_changed = false; return make_tl_object( audio->duration, audio->title, audio->performer, audio->file_name, audio->mime_type, get_minithumbnail_object(audio->minithumbnail), @@ -56,23 +55,19 @@ FileId AudiosManager::on_get_audio(unique_ptr