From 06135cccf92dfcde4da0b120a506cacf419d5c16 Mon Sep 17 00:00:00 2001 From: levlam Date: Sat, 18 Apr 2020 01:55:54 +0300 Subject: [PATCH] Add parameters offset and limit to getTrendingStickerSets. GitOrigin-RevId: 360c14f4cd357d23c3537ab26ee55a5b5ed29e81 --- td/generate/scheme/td_api.tl | 8 +- td/generate/scheme/td_api.tlo | Bin 171028 -> 171092 bytes td/telegram/PollManager.cpp | 2 +- td/telegram/StickersManager.cpp | 350 ++++++++++++++++++++++++++--- td/telegram/StickersManager.h | 34 ++- td/telegram/Td.cpp | 23 +- td/telegram/cli.cpp | 30 ++- tddb/td/db/SqliteKeyValueAsync.cpp | 8 + tddb/td/db/SqliteKeyValueAsync.h | 1 + 9 files changed, 410 insertions(+), 46 deletions(-) diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index e470ab98..e1c5bbda 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -3016,7 +3016,7 @@ updateStickerSet sticker_set:stickerSet = Update; //@description The list of installed sticker sets was updated @is_masks True, if the list of installed mask sticker sets was updated @sticker_set_ids The new list of installed ordinary sticker sets updateInstalledStickerSets is_masks:Bool sticker_set_ids:vector = Update; -//@description The list of trending sticker sets was updated or some of them were viewed @sticker_sets The new list of trending sticker sets +//@description The list of trending sticker sets was updated or some of them were viewed @sticker_sets The prefix of the list of trending sticker sets updateTrendingStickerSets sticker_sets:stickerSets = Update; //@description The list of recently used stickers was updated @is_attached True, if the list of stickers attached to photo or video files was updated, otherwise the list of sent stickers is updated @sticker_ids The new list of file identifiers of recently used stickers @@ -3893,8 +3893,10 @@ getInstalledStickerSets is_masks:Bool = StickerSets; //@description Returns a list of archived sticker sets @is_masks Pass true to return mask stickers sets; pass false to return ordinary sticker sets @offset_sticker_set_id Identifier of the sticker set from which to return the result @limit The maximum number of sticker sets to return getArchivedStickerSets is_masks:Bool offset_sticker_set_id:int64 limit:int32 = StickerSets; -//@description Returns a list of trending sticker sets -getTrendingStickerSets = StickerSets; +//@description Returns a list of trending sticker sets. For the optimal performance the number of returned users is chosen by the library +//@offset The offset from which to return the sticker sets; must be non-negative +//@limit The maximum number of sticker sets to be returned; must be non-negative. Fewer sticker sets may be returned than specified by the limit, even if the end of the list has not been reached +getTrendingStickerSets offset:int32 limit:int32 = StickerSets; //@description Returns a list of sticker sets attached to a file. Currently only photos and videos can have attached sticker sets @file_id File identifier getAttachedStickerSets file_id:int32 = StickerSets; diff --git a/td/generate/scheme/td_api.tlo b/td/generate/scheme/td_api.tlo index aa7d5b40deccfd0585693a2abdb95e0219b1112c..c257192d790b84ca26515e05763694a8683bf394 100644 GIT binary patch delta 41 xcmbQTgX_u;u7(!IEld@zEH1ULmQAm5Wl~^dn%-!|vW diff --git a/td/telegram/PollManager.cpp b/td/telegram/PollManager.cpp index c0221dcd..de4dfedf 100644 --- a/td/telegram/PollManager.cpp +++ b/td/telegram/PollManager.cpp @@ -934,7 +934,7 @@ void PollManager::get_poll_voters(PollId poll_id, FullMessageId full_message_id, auto cur_offset = narrow_cast(voters.voter_user_ids.size()); if (offset > cur_offset) { - return promise.set_error(Status::Error(400, "Too big offset specified, voters can be received only consequently")); + return promise.set_error(Status::Error(400, "Too big offset specified; voters can be received only consequently")); } if (offset < cur_offset) { vector result; diff --git a/td/telegram/StickersManager.cpp b/td/telegram/StickersManager.cpp index 5885a7a7..491b686d 100644 --- a/td/telegram/StickersManager.cpp +++ b/td/telegram/StickersManager.cpp @@ -271,7 +271,7 @@ class GetArchivedStickerSetsQuery : public Td::ResultHandler { class GetFeaturedStickerSetsQuery : public Td::ResultHandler { public: void send(int32 hash) { - LOG(INFO) << "Get featured sticker sets with hash " << hash; + LOG(INFO) << "Get trending sticker sets with hash " << hash; send_query(G()->net_query_creator().create(telegram_api::messages_getFeaturedStickers(hash))); } @@ -283,11 +283,41 @@ class GetFeaturedStickerSetsQuery : public Td::ResultHandler { auto ptr = result_ptr.move_as_ok(); LOG(DEBUG) << "Receive result for GetFeaturedStickerSetsQuery " << to_string(ptr); - td->stickers_manager_->on_get_featured_sticker_sets(std::move(ptr)); + td->stickers_manager_->on_get_featured_sticker_sets(-1, -1, 0, std::move(ptr)); } void on_error(uint64 id, Status status) override { - td->stickers_manager_->on_get_featured_sticker_sets_failed(std::move(status)); + td->stickers_manager_->on_get_featured_sticker_sets_failed(-1, -1, 0, std::move(status)); + } +}; + +class GetOldFeaturedStickerSetsQuery : public Td::ResultHandler { + int32 offset_; + int32 limit_; + uint32 generation_; + + public: + void send(int32 offset, int32 limit, uint32 generation) { + offset_ = offset; + limit_ = limit; + generation_ = generation; + LOG(INFO) << "Get old trending sticker sets with offset = " << offset << " and limit = " << limit; + send_query(G()->net_query_creator().create(telegram_api::messages_getOldFeaturedStickers(offset, limit, 0))); + } + + void on_result(uint64 id, BufferSlice packet) override { + auto result_ptr = fetch_result(packet); + if (result_ptr.is_error()) { + return on_error(id, result_ptr.move_as_error()); + } + + auto ptr = result_ptr.move_as_ok(); + LOG(DEBUG) << "Receive result for GetOldFeaturedStickerSetsQuery " << to_string(ptr); + td->stickers_manager_->on_get_featured_sticker_sets(offset_, limit_, generation_, std::move(ptr)); + } + + void on_error(uint64 id, Status status) override { + td->stickers_manager_->on_get_featured_sticker_sets_failed(offset_, limit_, generation_, std::move(status)); } }; @@ -800,7 +830,7 @@ class UninstallStickerSetQuery : public Td::ResultHandler { class ReadFeaturedStickerSetsQuery : public Td::ResultHandler { public: void send(vector sticker_set_ids) { - LOG(INFO) << "Read featured sticker sets " << format::as_array(sticker_set_ids); + LOG(INFO) << "Read trending sticker sets " << format::as_array(sticker_set_ids); send_query(G()->net_query_creator().create( telegram_api::messages_readFeaturedStickers(StickersManager::convert_sticker_set_ids(sticker_set_ids)))); } @@ -1135,6 +1165,19 @@ void StickersManager::init() { } load_special_sticker_set(special_sticker_sets_[SpecialStickerSetType::AnimatedEmoji]); load_special_sticker_set(special_sticker_sets_[SpecialStickerSetType::AnimatedDice]); + + if (G()->parameters().use_file_db) { + auto old_featured_sticker_set_count_str = G()->td_db()->get_binlog_pmc()->get("old_featured_sticker_set_count"); + if (!old_featured_sticker_set_count_str.empty()) { + old_featured_sticker_set_count_ = to_integer(old_featured_sticker_set_count_str); + } + if (!G()->td_db()->get_binlog_pmc()->get("invalidate_old_featured_sticker_sets").empty()) { + invalidate_old_featured_sticker_sets(); + } + } else { + G()->td_db()->get_binlog_pmc()->erase("old_featured_sticker_set_count"); + G()->td_db()->get_binlog_pmc()->erase("invalidate_old_featured_sticker_sets"); + } } void StickersManager::init_special_sticker_set(SpecialStickerSet &sticker_set, int64 sticker_set_id, int64 access_hash, @@ -1649,12 +1692,21 @@ void StickersManager::reload_featured_sticker_sets(bool force) { if (!td_->auth_manager_->is_bot() && next_featured_sticker_sets_load_time_ >= 0 && (next_featured_sticker_sets_load_time_ < Time::now() || force)) { - LOG_IF(INFO, force) << "Reload featured sticker sets"; + LOG_IF(INFO, force) << "Reload trending sticker sets"; next_featured_sticker_sets_load_time_ = -1; td_->create_handler()->send(featured_sticker_sets_hash_); } } +void StickersManager::reload_old_featured_sticker_sets(uint32 generation) { + if (generation != 0 && generation != old_featured_sticker_set_generation_) { + return; + } + td_->create_handler()->send(static_cast(old_featured_sticker_set_ids_.size()), + OLD_FEATURED_STICKER_SET_SLICE_SIZE, + old_featured_sticker_set_generation_); +} + StickerSetId StickersManager::on_get_input_sticker_set(FileId sticker_file_id, tl_object_ptr &&set_ptr, MultiPromiseActor *load_data_multipromise_ptr) { @@ -2883,14 +2935,14 @@ void StickersManager::on_load_installed_sticker_sets_from_database(bool is_masks return; } - LOG(INFO) << "Successfully loaded installed " << (is_masks ? "mask " : "") << "sticker sets list of size " + LOG(INFO) << "Successfully loaded installed " << (is_masks ? "mask " : "") << "sticker set list of size " << value.size() << " from database"; StickerSetListLogEvent log_event; auto status = log_event_parse(log_event, value); if (status.is_error()) { // can't happen unless database is broken - LOG(ERROR) << "Can't load installed sticker sets list: " << status << ' ' << format::as_hex_dump<4>(Slice(value)); + LOG(ERROR) << "Can't load installed sticker set list: " << status << ' ' << format::as_hex_dump<4>(Slice(value)); return reload_installed_sticker_sets(is_masks, true); } @@ -2911,6 +2963,8 @@ void StickersManager::on_load_installed_sticker_sets_from_database(bool is_masks if (result.is_ok()) { send_closure(G()->stickers_manager(), &StickersManager::on_load_installed_sticker_sets_finished, is_masks, std::move(sticker_set_ids), true); + } else { + send_closure(G()->stickers_manager(), &StickersManager::reload_installed_sticker_sets, is_masks, true); } })); } @@ -2986,7 +3040,7 @@ string StickersManager::get_sticker_set_database_value(const StickerSet *s, bool void StickersManager::update_sticker_set(StickerSet *sticker_set) { CHECK(sticker_set != nullptr); if (sticker_set->is_changed || sticker_set->need_save_to_database) { - if (G()->parameters().use_file_db && sticker_set->need_save_to_database) { + if (G()->parameters().use_file_db) { LOG(INFO) << "Save " << sticker_set->id << " to database"; if (sticker_set->is_inited) { G()->td_db()->get_sqlite_pmc()->set(get_sticker_set_database_key(sticker_set->id), @@ -3100,6 +3154,7 @@ void StickersManager::on_load_sticker_set_from_database(StickerSetId sticker_set } if (value.empty()) { + LOG(INFO) << "Failed to find in the database " << sticker_set_id; return do_reload_sticker_set(sticker_set_id, get_input_sticker_set(sticker_set), Auto()); } @@ -3205,7 +3260,9 @@ void StickersManager::view_featured_sticker_sets(const vector &sti for (auto sticker_set_id : sticker_set_ids) { auto set = get_sticker_set(sticker_set_id); if (set != nullptr && !set->is_viewed) { - need_update_featured_sticker_sets_ = true; + if (td::contains(featured_sticker_set_ids_, sticker_set_id)) { + need_update_featured_sticker_sets_ = true; + } set->is_viewed = true; pending_viewed_featured_sticker_set_ids_.insert(sticker_set_id); update_sticker_set(set); @@ -3215,7 +3272,7 @@ void StickersManager::view_featured_sticker_sets(const vector &sti send_update_featured_sticker_sets(); if (!pending_viewed_featured_sticker_set_ids_.empty() && !pending_featured_sticker_set_views_timeout_.has_timeout()) { - LOG(INFO) << "Have pending viewed featured sticker sets"; + LOG(INFO) << "Have pending viewed trending sticker sets"; pending_featured_sticker_set_views_timeout_.set_callback(read_featured_sticker_sets); pending_featured_sticker_set_views_timeout_.set_callback_data(static_cast(td_)); pending_featured_sticker_set_views_timeout_.set_timeout_in(MAX_FEATURED_STICKER_SET_VIEW_DELAY); @@ -3316,29 +3373,154 @@ void StickersManager::on_get_archived_sticker_sets( send_update_installed_sticker_sets(); } -vector StickersManager::get_featured_sticker_sets(Promise &&promise) { +std::pair> StickersManager::get_featured_sticker_sets(int32 offset, int32 limit, + Promise &&promise) { + if (offset < 0) { + promise.set_error(Status::Error(3, "Parameter offset must be non-negative")); + return {}; + } + + if (limit < 0) { + promise.set_error(Status::Error(3, "Parameter limit must be non-negative")); + return {}; + } + if (limit == 0) { + offset = 0; + } + if (!are_featured_sticker_sets_loaded_) { load_featured_sticker_sets(std::move(promise)); return {}; } reload_featured_sticker_sets(false); + auto set_count = static_cast(featured_sticker_set_ids_.size()); + auto total_count = set_count + (old_featured_sticker_set_count_ == -1 ? 1 : old_featured_sticker_set_count_); + if (offset < set_count) { + if (limit > set_count - offset) { + limit = set_count - offset; + } + promise.set_value(Unit()); + auto begin = featured_sticker_set_ids_.begin() + offset; + return {total_count, {begin, begin + limit}}; + } + + if (offset == set_count && are_old_featured_sticker_sets_invalidated_) { + invalidate_old_featured_sticker_sets(); + } + + if (offset < total_count || old_featured_sticker_set_count_ == -1) { + offset -= set_count; + set_count = static_cast(old_featured_sticker_set_ids_.size()); + if (offset < set_count) { + if (limit > set_count - offset) { + limit = set_count - offset; + } + promise.set_value(Unit()); + auto begin = old_featured_sticker_set_ids_.begin() + offset; + return {total_count, {begin, begin + limit}}; + } + if (offset > set_count) { + promise.set_error( + Status::Error(400, "Too big offset specified; trending sticker sets can be received only consequently")); + return {}; + } + + load_old_featured_sticker_sets(std::move(promise)); + return {}; + } + promise.set_value(Unit()); - return featured_sticker_set_ids_; + return {total_count, Auto()}; +} + +void StickersManager::on_old_featured_sticker_sets_invalidated() { + LOG(INFO) << "Invalidate old trending sticker sets"; + are_old_featured_sticker_sets_invalidated_ = true; + + if (!G()->parameters().use_file_db) { + return; + } + + G()->td_db()->get_binlog_pmc()->set("invalidate_old_featured_sticker_sets", "1"); +} + +void StickersManager::invalidate_old_featured_sticker_sets() { + LOG(INFO) << "Invalidate old featured sticker sets"; + if (G()->parameters().use_file_db) { + G()->td_db()->get_binlog_pmc()->erase("invalidate_old_featured_sticker_sets"); + G()->td_db()->get_sqlite_pmc()->erase_by_prefix("sssoldfeatured", Auto()); + } + are_old_featured_sticker_sets_invalidated_ = false; + old_featured_sticker_set_ids_.clear(); + + old_featured_sticker_set_generation_++; + auto promises = std::move(load_old_featured_sticker_sets_queries_); + load_old_featured_sticker_sets_queries_.clear(); + for (auto &promise : promises) { + promise.set_error(Status::Error(400, "Trending sticker sets was updated")); + } +} + +void StickersManager::set_old_featured_sticker_set_count(int32 count) { + if (old_featured_sticker_set_count_ == count) { + return; + } + + on_old_featured_sticker_sets_invalidated(); + + old_featured_sticker_set_count_ = count; + need_update_featured_sticker_sets_ = true; + + if (!G()->parameters().use_file_db) { + return; + } + + LOG(INFO) << "Save old trending sticker set count " << count << " to binlog"; + G()->td_db()->get_binlog_pmc()->set("old_featured_sticker_set_count", to_string(count)); +} + +void StickersManager::fix_old_featured_sticker_set_count() { + auto known_count = static_cast(old_featured_sticker_set_ids_.size()); + if (old_featured_sticker_set_count_ < known_count) { + if (old_featured_sticker_set_count_ >= 0) { + LOG(ERROR) << "Have old trending sticker set count " << old_featured_sticker_set_count_ << ", but have " + << known_count << " old trending sticker sets"; + } + set_old_featured_sticker_set_count(known_count); + } + if (old_featured_sticker_set_count_ > known_count && known_count % OLD_FEATURED_STICKER_SET_SLICE_SIZE != 0) { + LOG(ERROR) << "Have " << known_count << " old sticker sets out of " << old_featured_sticker_set_count_; + set_old_featured_sticker_set_count(known_count); + } } void StickersManager::on_get_featured_sticker_sets( + int32 offset, int32 limit, uint32 generation, tl_object_ptr &&sticker_sets_ptr) { - next_featured_sticker_sets_load_time_ = Time::now_cached() + Random::fast(30 * 60, 50 * 60); + if (offset < 0) { + next_featured_sticker_sets_load_time_ = Time::now_cached() + Random::fast(30 * 60, 50 * 60); + } int32 constructor_id = sticker_sets_ptr->get_id(); if (constructor_id == telegram_api::messages_featuredStickersNotModified::ID) { - LOG(INFO) << "Featured stickers are not modified"; + LOG(INFO) << "Trending sticker sets are not modified"; + auto *stickers = static_cast(sticker_sets_ptr.get()); + if (offset >= 0 && generation == old_featured_sticker_set_generation_) { + set_old_featured_sticker_set_count(stickers->count_); + fix_old_featured_sticker_set_count(); + } + send_update_featured_sticker_sets(); return; } CHECK(constructor_id == telegram_api::messages_featuredStickers::ID); auto featured_stickers = move_tl_object_as(sticker_sets_ptr); + if (offset >= 0 && generation == old_featured_sticker_set_generation_) { + set_old_featured_sticker_set_count(featured_stickers->count_); + // the count will be fixed in on_load_old_featured_sticker_sets_finished + } + std::unordered_set unread_sticker_set_ids; for (auto &unread_sticker_set_id : featured_stickers->unread_) { unread_sticker_set_ids.insert(StickerSetId(unread_sticker_set_id)); @@ -3366,24 +3548,50 @@ void StickersManager::on_get_featured_sticker_sets( send_update_installed_sticker_sets(); + if (offset >= 0) { + if (generation == old_featured_sticker_set_generation_) { + if (G()->parameters().use_file_db) { + LOG(INFO) << "Save old trending sticker sets to database with offset " << old_featured_sticker_set_ids_.size(); + CHECK(old_featured_sticker_set_ids_.size() % OLD_FEATURED_STICKER_SET_SLICE_SIZE == 0); + StickerSetListLogEvent log_event(featured_sticker_set_ids); + G()->td_db()->get_sqlite_pmc()->set(PSTRING() << "sssoldfeatured" << old_featured_sticker_set_ids_.size(), + log_event_store(log_event).as_slice().str(), Auto()); + } + on_load_old_featured_sticker_sets_finished(generation, std::move(featured_sticker_set_ids)); + } + + send_update_featured_sticker_sets(); // because of changed count + return; + } + on_load_featured_sticker_sets_finished(std::move(featured_sticker_set_ids)); - LOG_IF(ERROR, featured_sticker_sets_hash_ != featured_stickers->hash_) << "Featured sticker sets hash mismatch"; + LOG_IF(ERROR, featured_sticker_sets_hash_ != featured_stickers->hash_) << "Trending sticker sets hash mismatch"; if (!G()->parameters().use_file_db) { return; } - LOG(INFO) << "Save featured sticker sets to database"; + LOG(INFO) << "Save trending sticker sets to database"; StickerSetListLogEvent log_event(featured_sticker_set_ids_); G()->td_db()->get_sqlite_pmc()->set("sssfeatured", log_event_store(log_event).as_slice().str(), Auto()); } -void StickersManager::on_get_featured_sticker_sets_failed(Status error) { +void StickersManager::on_get_featured_sticker_sets_failed(int32 offset, int32 limit, uint32 generation, Status error) { CHECK(error.is_error()); - next_featured_sticker_sets_load_time_ = Time::now_cached() + Random::fast(5, 10); - auto promises = std::move(load_featured_sticker_sets_queries_); - load_featured_sticker_sets_queries_.clear(); + vector> promises; + if (offset >= 0) { + if (generation != old_featured_sticker_set_generation_) { + return; + } + promises = std::move(load_old_featured_sticker_sets_queries_); + load_old_featured_sticker_sets_queries_.clear(); + } else { + next_featured_sticker_sets_load_time_ = Time::now_cached() + Random::fast(5, 10); + promises = std::move(load_featured_sticker_sets_queries_); + load_featured_sticker_sets_queries_.clear(); + } + for (auto &promise : promises) { promise.set_error(error.clone()); } @@ -3392,6 +3600,7 @@ void StickersManager::on_get_featured_sticker_sets_failed(Status error) { void StickersManager::load_featured_sticker_sets(Promise &&promise) { if (td_->auth_manager_->is_bot()) { are_featured_sticker_sets_loaded_ = true; + old_featured_sticker_set_count_ = 0; } if (are_featured_sticker_sets_loaded_) { promise.set_value(Unit()); @@ -3400,14 +3609,14 @@ void StickersManager::load_featured_sticker_sets(Promise &&promise) { load_featured_sticker_sets_queries_.push_back(std::move(promise)); if (load_featured_sticker_sets_queries_.size() == 1u) { if (G()->parameters().use_file_db) { - LOG(INFO) << "Trying to load featured sticker sets from database"; + LOG(INFO) << "Trying to load trending sticker sets from database"; G()->td_db()->get_sqlite_pmc()->get("sssfeatured", PromiseCreator::lambda([](string value) { send_closure(G()->stickers_manager(), &StickersManager::on_load_featured_sticker_sets_from_database, std::move(value)); })); } else { - LOG(INFO) << "Trying to load featured sticker sets from server"; + LOG(INFO) << "Trying to load trending sticker sets from server"; reload_featured_sticker_sets(true); } } @@ -3415,18 +3624,18 @@ void StickersManager::load_featured_sticker_sets(Promise &&promise) { void StickersManager::on_load_featured_sticker_sets_from_database(string value) { if (value.empty()) { - LOG(INFO) << "Featured sticker sets aren't found in database"; + LOG(INFO) << "Trending sticker sets aren't found in database"; reload_featured_sticker_sets(true); return; } - LOG(INFO) << "Successfully loaded featured sticker sets list of size " << value.size() << " from database"; + LOG(INFO) << "Successfully loaded trending sticker set list of size " << value.size() << " from database"; StickerSetListLogEvent log_event; auto status = log_event_parse(log_event, value); if (status.is_error()) { // can't happen unless database is broken - LOG(ERROR) << "Can't load featured sticker sets list: " << status << ' ' << format::as_hex_dump<4>(Slice(value)); + LOG(ERROR) << "Can't load trending sticker set list: " << status << ' ' << format::as_hex_dump<4>(Slice(value)); return reload_featured_sticker_sets(true); } @@ -3445,11 +3654,17 @@ void StickersManager::on_load_featured_sticker_sets_from_database(string value) if (result.is_ok()) { send_closure(G()->stickers_manager(), &StickersManager::on_load_featured_sticker_sets_finished, std::move(sticker_set_ids)); + } else { + send_closure(G()->stickers_manager(), &StickersManager::reload_featured_sticker_sets, true); } })); } void StickersManager::on_load_featured_sticker_sets_finished(vector &&featured_sticker_set_ids) { + if (!featured_sticker_set_ids_.empty() && featured_sticker_set_ids != featured_sticker_set_ids_) { + // always invalidate old featured sticker sets when current featured sticker sets change + on_old_featured_sticker_sets_invalidated(); + } featured_sticker_set_ids_ = std::move(featured_sticker_set_ids); are_featured_sticker_sets_loaded_ = true; need_update_featured_sticker_sets_ = true; @@ -3461,6 +3676,85 @@ void StickersManager::on_load_featured_sticker_sets_finished(vector &&promise) { + CHECK(!td_->auth_manager_->is_bot()); + CHECK(old_featured_sticker_set_ids_.size() % OLD_FEATURED_STICKER_SET_SLICE_SIZE == 0); + load_old_featured_sticker_sets_queries_.push_back(std::move(promise)); + if (load_old_featured_sticker_sets_queries_.size() == 1u) { + if (G()->parameters().use_file_db) { + LOG(INFO) << "Trying to load old trending sticker sets from database with offset " + << old_featured_sticker_set_ids_.size(); + G()->td_db()->get_sqlite_pmc()->get( + PSTRING() << "sssoldfeatured" << old_featured_sticker_set_ids_.size(), + PromiseCreator::lambda([generation = old_featured_sticker_set_generation_](string value) { + send_closure(G()->stickers_manager(), &StickersManager::on_load_old_featured_sticker_sets_from_database, + generation, std::move(value)); + })); + } else { + LOG(INFO) << "Trying to load old trending sticker sets from server with offset " + << old_featured_sticker_set_ids_.size(); + reload_old_featured_sticker_sets(); + } + } +} + +void StickersManager::on_load_old_featured_sticker_sets_from_database(uint32 generation, string value) { + if (generation != old_featured_sticker_set_generation_) { + return; + } + if (value.empty()) { + LOG(INFO) << "Old trending sticker sets aren't found in database"; + return reload_old_featured_sticker_sets(); + } + + LOG(INFO) << "Successfully loaded old trending sticker set list of size " << value.size() + << " from database with offset " << old_featured_sticker_set_ids_.size(); + + StickerSetListLogEvent log_event; + auto status = log_event_parse(log_event, value); + if (status.is_error()) { + // can't happen unless database is broken + LOG(ERROR) << "Can't load old trending sticker set list: " << status << ' ' << format::as_hex_dump<4>(Slice(value)); + return reload_old_featured_sticker_sets(); + } + + vector sets_to_load; + for (auto sticker_set_id : log_event.sticker_set_ids) { + StickerSet *sticker_set = get_sticker_set(sticker_set_id); + CHECK(sticker_set != nullptr); + if (!sticker_set->is_inited) { + sets_to_load.push_back(sticker_set_id); + } + } + + load_sticker_sets_without_stickers( + std::move(sets_to_load), + PromiseCreator::lambda( + [generation, sticker_set_ids = std::move(log_event.sticker_set_ids)](Result<> result) mutable { + if (result.is_ok()) { + send_closure(G()->stickers_manager(), &StickersManager::on_load_old_featured_sticker_sets_finished, + generation, std::move(sticker_set_ids)); + } else { + send_closure(G()->stickers_manager(), &StickersManager::reload_old_featured_sticker_sets, generation); + } + })); +} + +void StickersManager::on_load_old_featured_sticker_sets_finished(uint32 generation, + vector &&featured_sticker_set_ids) { + if (generation != old_featured_sticker_set_generation_) { + fix_old_featured_sticker_set_count(); // must never be needed + return; + } + append(old_featured_sticker_set_ids_, std::move(featured_sticker_set_ids)); + fix_old_featured_sticker_set_count(); + auto promises = std::move(load_old_featured_sticker_sets_queries_); + load_old_featured_sticker_sets_queries_.clear(); + for (auto &promise : promises) { + promise.set_value(Unit()); + } +} + vector StickersManager::get_attached_sticker_sets(FileId file_id, Promise &&promise) { if (!file_id.is_valid()) { promise.set_error(Status::Error(5, "Wrong file_id specified")); @@ -4316,8 +4610,10 @@ void StickersManager::send_update_installed_sticker_sets(bool from_database) { } td_api::object_ptr StickersManager::get_update_trending_sticker_sets_object() const { + auto total_count = static_cast(featured_sticker_set_ids_.size()) + + (old_featured_sticker_set_count_ == -1 ? 1 : old_featured_sticker_set_count_); return td_api::make_object( - get_sticker_sets_object(-1, featured_sticker_set_ids_, 5)); + get_sticker_sets_object(total_count, featured_sticker_set_ids_, 5)); } void StickersManager::send_update_featured_sticker_sets() { @@ -5604,7 +5900,7 @@ void StickersManager::after_get_difference() { if (td_->is_online()) { get_installed_sticker_sets(false, Auto()); get_installed_sticker_sets(true, Auto()); - get_featured_sticker_sets(Auto()); + get_featured_sticker_sets(0, 1000, Auto()); get_recent_stickers(false, Auto()); get_recent_stickers(true, Auto()); get_favorite_stickers(Auto()); diff --git a/td/telegram/StickersManager.h b/td/telegram/StickersManager.h index be573475..316f5c8e 100644 --- a/td/telegram/StickersManager.h +++ b/td/telegram/StickersManager.h @@ -132,11 +132,12 @@ class StickersManager : public Actor { vector> &&sticker_sets, int32 total_count); - vector get_featured_sticker_sets(Promise &&promise); + std::pair> get_featured_sticker_sets(int32 offset, int32 limit, Promise &&promise); - void on_get_featured_sticker_sets(tl_object_ptr &&sticker_sets_ptr); + void on_get_featured_sticker_sets(int32 offset, int32 limit, uint32 generation, + tl_object_ptr &&sticker_sets_ptr); - void on_get_featured_sticker_sets_failed(Status error); + void on_get_featured_sticker_sets_failed(int32 offset, int32 limit, uint32 generation, Status error); vector get_attached_sticker_sets(FileId file_id, Promise &&promise); @@ -271,6 +272,7 @@ class StickersManager : public Actor { private: static constexpr int32 MAX_FEATURED_STICKER_SET_VIEW_DELAY = 5; + static constexpr int32 OLD_FEATURED_STICKER_SET_SLICE_SIZE = 20; static constexpr int32 MAX_FOUND_STICKERS = 100; // server side limit static constexpr int64 MAX_STICKER_FILE_SIZE = 1 << 19; // server side limit @@ -435,6 +437,8 @@ class StickersManager : public Actor { void load_featured_sticker_sets(Promise &&promise); + void load_old_featured_sticker_sets(Promise &&promise); + void load_recent_stickers(bool is_attached, Promise &&promise); void on_load_installed_sticker_sets_from_database(bool is_masks, string value); @@ -446,6 +450,11 @@ class StickersManager : public Actor { void on_load_featured_sticker_sets_finished(vector &&featured_sticker_set_ids); + void on_load_old_featured_sticker_sets_from_database(uint32 generation, string value); + + void on_load_old_featured_sticker_sets_finished(uint32 generation, + vector &&old_featured_sticker_set_ids); + void on_load_recent_stickers_from_database(bool is_attached, string value); void on_load_recent_stickers_finished(bool is_attached, vector &&recent_sticker_ids, @@ -455,6 +464,18 @@ class StickersManager : public Actor { void send_update_installed_sticker_sets(bool from_database = false); + void reload_old_featured_sticker_sets(uint32 generation = 0); + + void on_old_featured_sticker_sets_invalidated(); + + void invalidate_old_featured_sticker_sets(); + + void set_old_featured_sticker_set_count(int32 count); + + // must be called after every call to set_old_featured_sticker_set_count or + // any change of old_featured_sticker_set_ids_ size + void fix_old_featured_sticker_set_count(); + td_api::object_ptr get_update_trending_sticker_sets_object() const; void send_update_featured_sticker_sets(); @@ -576,6 +597,7 @@ class StickersManager : public Actor { vector installed_sticker_set_ids_[2]; vector featured_sticker_set_ids_; + vector old_featured_sticker_set_ids_; vector recent_sticker_ids_[2]; vector favorite_sticker_ids_; @@ -588,6 +610,9 @@ class StickersManager : public Actor { int32 featured_sticker_sets_hash_ = 0; int32 recent_stickers_hash_[2] = {0, 0}; + int32 old_featured_sticker_set_count_ = -1; + uint32 old_featured_sticker_set_generation_ = 1; + bool need_update_installed_sticker_sets_[2] = {false, false}; bool need_update_featured_sticker_sets_ = false; bool need_update_recent_stickers_[2] = {false, false}; @@ -597,8 +622,11 @@ class StickersManager : public Actor { bool are_recent_stickers_loaded_[2] = {false, false}; bool are_favorite_stickers_loaded_ = false; + bool are_old_featured_sticker_sets_invalidated_ = false; + vector> load_installed_sticker_sets_queries_[2]; vector> load_featured_sticker_sets_queries_; + vector> load_old_featured_sticker_sets_queries_; vector> load_recent_stickers_queries_[2]; vector> repair_recent_stickers_queries_[2]; vector> load_favorite_stickers_queries_; diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index 0b648b1d..0299aa71 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -2337,18 +2337,22 @@ class GetArchivedStickerSetsRequest : public RequestActor<> { }; class GetTrendingStickerSetsRequest : public RequestActor<> { - vector sticker_set_ids_; + std::pair> sticker_set_ids_; + int32 offset_; + int32 limit_; void do_run(Promise &&promise) override { - sticker_set_ids_ = td->stickers_manager_->get_featured_sticker_sets(std::move(promise)); + sticker_set_ids_ = td->stickers_manager_->get_featured_sticker_sets(offset_, limit_, std::move(promise)); } void do_send_result() override { - send_result(td->stickers_manager_->get_sticker_sets_object(-1, sticker_set_ids_, 5)); + send_result(td->stickers_manager_->get_sticker_sets_object(sticker_set_ids_.first, sticker_set_ids_.second, 5)); } public: - GetTrendingStickerSetsRequest(ActorShared td, uint64 request_id) : RequestActor(std::move(td), request_id) { + GetTrendingStickerSetsRequest(ActorShared td, uint64 request_id, int32 offset, int32 limit) + : RequestActor(std::move(td), request_id), offset_(offset), limit_(limit) { + set_tries(3); } }; @@ -4335,9 +4339,12 @@ void Td::send_update(tl_object_ptr &&object) { case td_api::updateUserStatus::ID: VLOG(td_requests) << "Sending update: " << oneline(to_string(object)); break; - case td_api::updateTrendingStickerSets::ID: - VLOG(td_requests) << "Sending update: updateTrendingStickerSets { ... }"; + case td_api::updateTrendingStickerSets::ID: { + auto sticker_sets = static_cast(object.get())->sticker_sets_.get(); + VLOG(td_requests) << "Sending update: updateTrendingStickerSets { total_count = " << sticker_sets->total_count_ + << ", count = " << sticker_sets->sets_.size() << " }"; break; + } case td_api::updateOption::ID / 2: case td_api::updateChatReadInbox::ID / 2: case td_api::updateUnreadMessageCount::ID / 2: @@ -4351,7 +4358,7 @@ void Td::send_update(tl_object_ptr &&object) { } callback_->on_result(0, std::move(object)); -} +} // namespace td void Td::send_result(uint64 id, tl_object_ptr object) { if (id == 0) { @@ -6300,7 +6307,7 @@ void Td::on_request(uint64 id, const td_api::getArchivedStickerSets &request) { void Td::on_request(uint64 id, const td_api::getTrendingStickerSets &request) { CHECK_IS_USER(); - CREATE_NO_ARGS_REQUEST(GetTrendingStickerSetsRequest); + CREATE_REQUEST(GetTrendingStickerSetsRequest, request.offset_, request.limit_); } void Td::on_request(uint64 id, const td_api::getAttachedStickerSets &request) { diff --git a/td/telegram/cli.cpp b/td/telegram/cli.cpp index 453954cf..cf61cdba 100644 --- a/td/telegram/cli.cpp +++ b/td/telegram/cli.cpp @@ -626,8 +626,22 @@ class CliClient final : public Actor { } void on_result(uint64 generation, uint64 id, td_api::object_ptr result) { + auto result_str = to_string(result); + if (result != nullptr) { + switch (result->get_id()) { + case td_api::stickerSets::ID: { + auto sticker_sets = static_cast(result.get()); + result_str = PSTRING() << "StickerSets { total_count = " << sticker_sets->total_count_ + << ", count = " << sticker_sets->sets_.size() << "}"; + break; + } + default: + break; + } + } + if (id > 0 && GET_VERBOSITY_LEVEL() < VERBOSITY_NAME(td_requests)) { - LOG(ERROR) << "Receive result [" << generation << "][id=" << id << "] " << to_string(result); + LOG(ERROR) << "Receive result [" << generation << "][id=" << id << "] " << result_str; } auto as_json_str = json_encode(ToJson(result)); @@ -642,7 +656,7 @@ class CliClient final : public Actor { // LOG(INFO) << "Receive result [" << generation << "][id=" << id << "] " << as_json_str; if (generation != generation_) { - LOG(INFO) << "Drop received from previous Client " << to_string(result); + LOG(INFO) << "Drop received from previous Client " << result_str; return; } @@ -728,7 +742,7 @@ class CliClient final : public Actor { on_get_file(*static_cast(result.get())->file_); break; case td_api::updateConnectionState::ID: - LOG(WARNING) << to_string(result); + LOG(WARNING) << result_str; break; } } @@ -2243,7 +2257,15 @@ class CliClient final : public Actor { send_request(td_api::make_object( as_bool(is_masks), to_integer(offset_sticker_set_id), to_integer(limit))); } else if (op == "gtss") { - send_request(td_api::make_object()); + string offset; + string limit; + + std::tie(offset, limit) = split(args); + if (limit.empty()) { + limit = "1000"; + } + send_request( + td_api::make_object(to_integer(offset), to_integer(limit))); } else if (op == "gatss") { send_request(td_api::make_object(as_file_id(args))); } else if (op == "storage") { diff --git a/tddb/td/db/SqliteKeyValueAsync.cpp b/tddb/td/db/SqliteKeyValueAsync.cpp index 15af7201..2ac46f32 100644 --- a/tddb/td/db/SqliteKeyValueAsync.cpp +++ b/tddb/td/db/SqliteKeyValueAsync.cpp @@ -29,6 +29,9 @@ class SqliteKeyValueAsync : public SqliteKeyValueAsyncInterface { void erase(string key, Promise<> promise) override { send_closure_later(impl_, &Impl::erase, std::move(key), std::move(promise)); } + void erase_by_prefix(string key_prefix, Promise<> promise) override { + send_closure_later(impl_, &Impl::erase_by_prefix, std::move(key_prefix), std::move(promise)); + } void get(string key, Promise promise) override { send_closure_later(impl_, &Impl::get, std::move(key), std::move(promise)); } @@ -67,6 +70,11 @@ class SqliteKeyValueAsync : public SqliteKeyValueAsyncInterface { cnt_++; do_flush(false /*force*/); } + void erase_by_prefix(string key_prefix, Promise<> promise) { + do_flush(true /*force*/); + kv_->erase_by_prefix(key_prefix); + promise.set_value(Unit()); + } void get(const string &key, Promise promise) { auto it = buffer_.find(key); diff --git a/tddb/td/db/SqliteKeyValueAsync.h b/tddb/td/db/SqliteKeyValueAsync.h index 62293c82..7dfa8c98 100644 --- a/tddb/td/db/SqliteKeyValueAsync.h +++ b/tddb/td/db/SqliteKeyValueAsync.h @@ -20,6 +20,7 @@ class SqliteKeyValueAsyncInterface { virtual void set(string key, string value, Promise<> promise) = 0; virtual void erase(string key, Promise<> promise) = 0; + virtual void erase_by_prefix(string key_prefix, Promise<> promise) = 0; virtual void get(string key, Promise promise) = 0; virtual void close(Promise<> promise) = 0;