diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index ae8ba480..e7171ec4 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -1551,18 +1551,18 @@ userStatusLastMonth = UserStatus; //@description Represents a list of stickers @stickers List of stickers stickers stickers:vector = Stickers; -//@description Represents a list of all emoji corresponding to a sticker in a sticker set. The list is only for informational purposes, because a sticker is always sent with a fixed emoji from the corresponding Sticker object @emojis List of emojis -stickerEmojis emojis:vector = StickerEmojis; +//@description Represents a list of emoji @emojis List of emojis +emojis emojis:vector = Emojis; //@description Represents a sticker set -//@id Identifier of the sticker set @title Title of the sticker set @name Name of the sticker set @thumbnail Sticker set thumbnail in WEBP format; may be null +//@id Identifier of the sticker set @title Title of the sticker set @name Name of the sticker set @thumbnail Sticker set thumbnail in WEBP format with width and height 100; may be null //@is_installed True, if the sticker set has been installed by the current user @is_archived True, if the sticker set has been archived. A sticker set can't be installed and archived simultaneously //@is_official True, if the sticker set is official @is_masks True, if the stickers in the set are masks @is_viewed True for already viewed trending sticker sets -//@stickers List of stickers in this set @emojis A list of emoji corresponding to the stickers in the same order -stickerSet id:int64 title:string name:string thumbnail:photoSize is_installed:Bool is_archived:Bool is_official:Bool is_masks:Bool is_viewed:Bool stickers:vector emojis:vector = StickerSet; +//@stickers List of stickers in this set @emojis A list of emoji corresponding to the stickers in the same order. The list is only for informational purposes, because a sticker is always sent with a fixed emoji from the corresponding Sticker object +stickerSet id:int64 title:string name:string thumbnail:photoSize is_installed:Bool is_archived:Bool is_official:Bool is_masks:Bool is_viewed:Bool stickers:vector emojis:vector = StickerSet; //@description Represents short information about a sticker set -//@id Identifier of the sticker set @title Title of the sticker set @name Name of the sticker set @thumbnail Sticker set thumbnail in WEBP format; may be null +//@id Identifier of the sticker set @title Title of the sticker set @name Name of the sticker set @thumbnail Sticker set thumbnail in WEBP format with width and height 100; may be null //@is_installed True, if the sticker set has been installed by current user @is_archived True, if the sticker set has been archived. A sticker set can't be installed and archived simultaneously //@is_official True, if the sticker set is official @is_masks True, if the stickers in the set are masks @is_viewed True for already viewed trending sticker sets //@size Total number of stickers in the set @covers Contains up to the first 5 stickers from the set, depending on the context. If the client needs more stickers the full set should be requested @@ -3506,8 +3506,11 @@ addFavoriteSticker sticker:InputFile = Ok; //@description Removes a sticker from the list of favorite stickers @sticker Sticker file to delete from the list removeFavoriteSticker sticker:InputFile = Ok; -//@description Returns emoji corresponding to a sticker @sticker Sticker file identifier -getStickerEmojis sticker:InputFile = StickerEmojis; +//@description Returns emoji corresponding to a sticker. The list is only for informational purposes, because a sticker is always sent with a fixed emoji from the corresponding Sticker object @sticker Sticker file identifier +getStickerEmojis sticker:InputFile = Emojis; + +//@description Searches for emojis by a short text. Supported only if the file database is enabled @text Text to search for @exact_match True, if only emojis, which exactly match text needs to be returned +searchEmojis text:string exact_match:Bool = Emojis; //@description Returns saved animations diff --git a/td/generate/scheme/td_api.tlo b/td/generate/scheme/td_api.tlo index 258e3cdf..eeda68bc 100644 Binary files a/td/generate/scheme/td_api.tlo and b/td/generate/scheme/td_api.tlo differ diff --git a/td/telegram/LanguagePackManager.cpp b/td/telegram/LanguagePackManager.cpp index 19d01ff4..3fd320fd 100644 --- a/td/telegram/LanguagePackManager.cpp +++ b/td/telegram/LanguagePackManager.cpp @@ -237,6 +237,45 @@ void LanguagePackManager::tear_down() { } } +vector LanguagePackManager::get_used_language_codes() const { + if (language_pack_.empty() || language_code_.empty()) { + return {}; + } + + std::lock_guard packs_lock(database_->mutex_); + auto pack_it = database_->language_packs_.find(language_pack_); + CHECK(pack_it != database_->language_packs_.end()); + + LanguageInfo *info = nullptr; + LanguagePack *pack = pack_it->second.get(); + std::lock_guard languages_lock(pack->mutex_); + if (is_custom_language_code(language_code_)) { + auto custom_it = pack->custom_language_pack_infos_.find(language_code_); + if (custom_it != pack->custom_language_pack_infos_.end()) { + info = &custom_it->second; + } + } else { + for (auto &server_info : pack->server_language_pack_infos_) { + if (server_info.first == language_code_) { + info = &server_info.second; + } + } + } + CHECK(info != nullptr); + + vector result; + if (language_code_.size() <= 2) { + result.push_back(language_code_); + } + if (!info->base_language_code_.empty()) { + result.push_back(info->base_language_code_); + } + if (!info->plural_code_.empty()) { + result.push_back(info->plural_code_); + } + return result; +} + void LanguagePackManager::on_language_pack_changed() { auto new_language_pack = G()->shared_config().get_option_string("localization_target"); if (new_language_pack == language_pack_) { diff --git a/td/telegram/LanguagePackManager.h b/td/telegram/LanguagePackManager.h index 169388fe..8219feeb 100644 --- a/td/telegram/LanguagePackManager.h +++ b/td/telegram/LanguagePackManager.h @@ -43,6 +43,8 @@ class LanguagePackManager : public NetQueryCallback { static bool is_custom_language_code(Slice language_code); + vector get_used_language_codes() const; + void on_language_pack_changed(); void on_language_code_changed(); diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index b36c9f72..66a51913 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -25576,6 +25576,7 @@ MessagesManager::Message *MessagesManager::continue_send_message(DialogId dialog void MessagesManager::on_binlog_events(vector &&events) { for (auto &event : events) { + CHECK(event.id_ != 0); switch (event.type_) { case LogEvent::HandlerType::SendMessage: { if (!G()->parameters().use_message_db) { diff --git a/td/telegram/StickersManager.cpp b/td/telegram/StickersManager.cpp index d2c94c73..e6b618f8 100644 --- a/td/telegram/StickersManager.cpp +++ b/td/telegram/StickersManager.cpp @@ -23,6 +23,7 @@ #include "td/telegram/files/FileType.h" #include "td/telegram/Global.h" #include "td/telegram/logevent/LogEvent.h" +#include "td/telegram/LanguagePackManager.h" #include "td/telegram/MessagesManager.h" #include "td/telegram/misc.h" #include "td/telegram/net/DcId.h" @@ -112,6 +113,59 @@ class SearchStickersQuery : public Td::ResultHandler { } }; +class GetEmojiKeywordsLanguageQuery : public Td::ResultHandler { + Promise> promise_; + + public: + explicit GetEmojiKeywordsLanguageQuery(Promise> &&promise) : promise_(std::move(promise)) { + } + + void send(vector &&language_codes) { + send_query(G()->net_query_creator().create( + create_storer(telegram_api::messages_getEmojiKeywordsLanguages(std::move(language_codes))))); + } + void on_result(uint64 id, BufferSlice packet) override { + auto result_ptr = fetch_result(packet); + if (result_ptr.is_error()) { + return on_error(id, result_ptr.move_as_error()); + } + + auto result = + transform(result_ptr.move_as_ok(), [](auto &&emoji_language) { return std::move(emoji_language->lang_code_); }); + promise_.set_value(std::move(result)); + } + + void on_error(uint64 id, Status status) override { + promise_.set_error(std::move(status)); + } +}; + +class GetEmojiKeywordsQuery : public Td::ResultHandler { + Promise> promise_; + + public: + explicit GetEmojiKeywordsQuery(Promise> &&promise) + : promise_(std::move(promise)) { + } + + void send(const string &language_code) { + send_query(G()->net_query_creator().create(create_storer(telegram_api::messages_getEmojiKeywords(language_code)))); + } + + void on_result(uint64 id, BufferSlice packet) override { + auto result_ptr = fetch_result(packet); + if (result_ptr.is_error()) { + return on_error(id, result_ptr.move_as_error()); + } + + promise_.set_value(result_ptr.move_as_ok()); + } + + void on_error(uint64 id, Status status) override { + promise_.set_error(std::move(status)); + } +}; + class GetArchivedStickerSetsQuery : public Td::ResultHandler { Promise promise_; int64 offset_sticker_set_id_; @@ -993,7 +1047,7 @@ tl_object_ptr StickersManager::get_sticker_set_object(int64 CHECK(sticker_set->was_loaded); std::vector> stickers; - std::vector> emojis; + std::vector> emojis; for (auto sticker_id : sticker_set->sticker_ids) { stickers.push_back(get_sticker_object(sticker_id)); @@ -1001,7 +1055,7 @@ tl_object_ptr StickersManager::get_sticker_set_object(int64 if (it == sticker_set->sticker_emojis_map_.end()) { emojis.push_back(Auto()); } else { - emojis.push_back(make_tl_object(vector(it->second))); + emojis.push_back(make_tl_object(vector(it->second))); } } return make_tl_object(sticker_set->id, sticker_set->title, sticker_set->short_name, @@ -4600,6 +4654,267 @@ vector StickersManager::get_sticker_emojis(const tl_object_ptrsecond; } +string StickersManager::get_emoji_language_code_version_database_key(const string &language_code) { + return PSTRING() << "emojiv$" << language_code; +} + +int32 StickersManager::get_emoji_language_code_version(const string &language_code) { + auto it = emoji_language_code_versions_.find(language_code); + if (it != emoji_language_code_versions_.end()) { + return it->second; + } + auto &result = emoji_language_code_versions_[language_code]; + result = to_integer( + G()->td_db()->get_sqlite_sync_pmc()->get(get_emoji_language_code_version_database_key(language_code))); + return result; +} + +string StickersManager::get_language_emojis_database_key(const string &language_code, const string &text) { + return PSTRING() << "emoji$" << language_code << '$' << text; +} + +vector StickersManager::search_language_emojis(const string &language_code, const string &text, + bool exact_match) const { + LOG(INFO) << "Search for \"" << text << "\" in language " << language_code; + auto key = get_language_emojis_database_key(language_code, text); + if (exact_match) { + string emojis = G()->td_db()->get_sqlite_sync_pmc()->get(key); + return full_split(emojis, '$'); + } else { + vector result; + G()->td_db()->get_sqlite_sync_pmc()->get_by_prefix(key, [&result](Slice key, Slice value) { + for (auto &emoji : full_split(value, '$')) { + result.push_back(emoji.str()); + } + return true; + }); + return result; + } +} + +string StickersManager::get_emoji_language_codes_database_key(const vector &language_codes) { + return PSTRING() << "emojilc$" << implode(language_codes, '$'); +} + +void StickersManager::load_language_codes(vector language_codes, string key, Promise &&promise) { + auto &promises = load_language_codes_queries_[key]; + promises.push_back(std::move(promise)); + if (promises.size() != 1) { + // query has already been sent, just wait for the result + return; + } + + auto query_promise = + PromiseCreator::lambda([actor_id = actor_id(this), key = std::move(key)](Result> &&result) { + send_closure(actor_id, &StickersManager::on_get_language_codes, key, std::move(result)); + }); + td_->create_handler(std::move(query_promise))->send(std::move(language_codes)); +} + +void StickersManager::on_get_language_codes(const string &key, Result> &&result) { + auto queries_it = load_language_codes_queries_.find(key); + CHECK(queries_it != load_language_codes_queries_.end()); + CHECK(!queries_it->second.empty()); + auto promises = std::move(queries_it->second); + load_language_codes_queries_.erase(queries_it); + + if (result.is_error()) { + if (!G()->close_flag()) { + LOG(ERROR) << "Reseive " << result.error() << " from GetEmojiKeywordsLanguageQuery"; + } + for (auto &promise : promises) { + promise.set_error(result.error().clone()); + } + return; + } + + auto language_codes = result.move_as_ok(); + LOG(INFO) << "Receive language codes " << language_codes << " for emojis search"; + language_codes.erase(std::remove_if(language_codes.begin(), language_codes.end(), + [](const auto &language_code) { + if (language_code.empty() || language_code.find('$') != string::npos) { + LOG(ERROR) << "Receive language_code \"" << language_code << '"'; + return true; + } + return false; + }), + language_codes.end()); + if (language_codes.empty()) { + LOG(ERROR) << "Language codes list is empty"; + language_codes.emplace_back("en"); + } + + G()->td_db()->get_sqlite_pmc()->set(key, implode(language_codes, '$'), Auto()); + auto it = emoji_language_codes_.find(key); + CHECK(it != emoji_language_codes_.end()); + it->second = std::move(language_codes); + + for (auto &promise : promises) { + promise.set_value(Unit()); + } +} + +vector StickersManager::get_emoji_language_codes(Promise &promise) { + vector language_codes = td_->language_pack_manager_->get_actor_unsafe()->get_used_language_codes(); + auto system_language_code = G()->mtproto_header().get_system_language_code(); + if (!system_language_code.empty() && system_language_code.find('$') == string::npos) { + language_codes.push_back(system_language_code); + } + + if (language_codes.empty()) { + LOG(ERROR) << "List of language codes is empty"; + language_codes.push_back("en"); + } + std::sort(language_codes.begin(), language_codes.end()); + language_codes.erase(std::unique(language_codes.begin(), language_codes.end()), language_codes.end()); + + auto key = get_emoji_language_codes_database_key(language_codes); + auto it = emoji_language_codes_.find(key); + if (it == emoji_language_codes_.end()) { + it = emoji_language_codes_.emplace(key, full_split(G()->td_db()->get_sqlite_sync_pmc()->get(key), '$')).first; + } + if (it->second.empty()) { + load_language_codes(std::move(language_codes), std::move(key), std::move(promise)); + } + return it->second; +} + +void StickersManager::load_emoji_keywords(const string &language_code, Promise &&promise) { + auto &promises = load_emoji_keywords_queries_[language_code]; + promises.push_back(std::move(promise)); + if (promises.size() != 1) { + // query has already been sent, just wait for the result + return; + } + + auto query_promise = PromiseCreator::lambda( + [actor_id = actor_id(this), + language_code](Result> &&result) mutable { + send_closure(actor_id, &StickersManager::on_get_emoji_keywords, language_code, std::move(result)); + }); + td_->create_handler(std::move(query_promise))->send(language_code); +} + +void StickersManager::on_get_emoji_keywords( + const string &language_code, Result> &&result) { + auto it = load_emoji_keywords_queries_.find(language_code); + CHECK(it != load_emoji_keywords_queries_.end()); + CHECK(!it->second.empty()); + auto promises = std::move(it->second); + load_emoji_keywords_queries_.erase(it); + + if (result.is_error()) { + if (!G()->close_flag()) { + LOG(ERROR) << "Reseive " << result.error() << " from GetEmojiKeywordsQuery"; + } + for (auto &promise : promises) { + promise.set_error(result.error().clone()); + } + return; + } + + auto version = get_emoji_language_code_version(language_code); + CHECK(version == 0); + + MultiPromiseActorSafe mpas{"SaveEmojiKeywordsMultiPromiseActor"}; + for (auto &promise : promises) { + mpas.add_promise(std::move(promise)); + } + + auto lock = mpas.get_promise(); + + auto keywords = result.move_as_ok(); + LOG(INFO) << "Receive " << keywords->keywords_.size() << " emoji keywords for language " << language_code; + LOG_IF(ERROR, language_code != keywords->lang_code_) + << "Receive keywords for " << keywords->lang_code_ << " instead of " << language_code; + LOG_IF(ERROR, keywords->from_version_ != 0) << "Receive keywords from version " << keywords->from_version_; + version = keywords->version_; + if (version <= 0) { + LOG(ERROR) << "Receive keywords to version " << version; + version = 1; + } + for (auto &keyword_ptr : keywords->keywords_) { + switch (keyword_ptr->get_id()) { + case telegram_api::emojiKeyword::ID: { + auto keyword = telegram_api::move_object_as(keyword_ptr); + auto text = utf8_to_lower(keyword->keyword_); + bool is_good = true; + for (auto &emoji : keyword->emoticons_) { + if (emoji.find('$') != string::npos) { + LOG(ERROR) << "Receive emoji \"" << emoji << "\" from server for " << text; + is_good = false; + } + } + if (is_good) { + G()->td_db()->get_sqlite_pmc()->set(get_language_emojis_database_key(language_code, text), + implode(keyword->emoticons_, '$'), mpas.get_promise()); + } + break; + } + case telegram_api::emojiKeywordDeleted::ID: + LOG(ERROR) << "Receive emojiKeywordDeleted in keywords for " << language_code; + break; + default: + UNREACHABLE(); + } + } + G()->td_db()->get_sqlite_pmc()->set(get_emoji_language_code_version_database_key(language_code), to_string(version), + mpas.get_promise()); + emoji_language_code_versions_[language_code] = version; + + lock.set_value(Unit()); +} + +vector StickersManager::search_emojis(const string &text, bool exact_match, bool force, + Promise &&promise) { + if (text.empty() || !G()->parameters().use_file_db /* have SQLite PMC */) { + promise.set_value(Unit()); + return {}; + } + + auto language_codes = get_emoji_language_codes(promise); + if (language_codes.empty()) { + // promise was consumed + return {}; + } + + vector languages_to_load; + for (auto &language_code : language_codes) { + auto version = get_emoji_language_code_version(language_code); + if (version == 0) { + languages_to_load.push_back(language_code); + } + } + + if (!languages_to_load.empty()) { + if (!force) { + MultiPromiseActorSafe mpas{"LoadEmojiLanguagesMultiPromiseActor"}; + mpas.add_promise(std::move(promise)); + + auto lock = mpas.get_promise(); + for (auto &language_code : languages_to_load) { + load_emoji_keywords(language_code, mpas.get_promise()); + } + lock.set_value(Unit()); + return {}; + } else { + LOG(ERROR) << "Have no " << languages_to_load << " emoji keywords"; + } + } + + auto text_lowered = utf8_to_lower(text); + vector result; + for (auto &language_code : language_codes) { + combine(result, search_language_emojis(language_code, text_lowered, exact_match)); + } + + std::sort(result.begin(), result.end()); + result.erase(std::unique(result.begin(), result.end()), result.end()); + + promise.set_value(Unit()); + return result; +} + string StickersManager::remove_emoji_modifiers(string emoji) { static const Slice modifiers[] = {u8"\uFE0E" /* variation selector-15 */, u8"\uFE0F" /* variation selector-16 */, diff --git a/td/telegram/StickersManager.h b/td/telegram/StickersManager.h index f870ccc4..6dd535e5 100644 --- a/td/telegram/StickersManager.h +++ b/td/telegram/StickersManager.h @@ -195,6 +195,8 @@ class StickersManager : public Actor { vector get_sticker_emojis(const tl_object_ptr &input_file, Promise &&promise); + vector search_emojis(const string &text, bool exact_match, bool force, Promise &&promise); + void reload_installed_sticker_sets(bool is_masks, bool force); void reload_featured_sticker_sets(bool force); @@ -467,6 +469,27 @@ class StickersManager : public Actor { void tear_down() override; + static string get_emoji_language_code_version_database_key(const string &language_code); + + static string get_language_emojis_database_key(const string &language_code, const string &text); + + static string get_emoji_language_codes_database_key(const vector &language_codes); + + int32 get_emoji_language_code_version(const string &language_code); + + vector get_emoji_language_codes(Promise &promise); + + void load_language_codes(vector language_codes, string key, Promise &&promise); + + void on_get_language_codes(const string &key, Result> &&result); + + vector search_language_emojis(const string &language_code, const string &text, bool exact_match) const; + + void load_emoji_keywords(const string &language_code, Promise &&promise); + + void on_get_emoji_keywords(const string &language_code, + Result> &&result); + static string remove_emoji_modifiers(string emoji); Td *td_; @@ -545,6 +568,11 @@ class StickersManager : public Actor { std::shared_ptr upload_sticker_file_callback_; std::unordered_map>, FileIdHash> being_uploaded_files_; + + std::unordered_map> emoji_language_codes_; + std::unordered_map emoji_language_code_versions_; + std::unordered_map>> load_emoji_keywords_queries_; + std::unordered_map>> load_language_codes_queries_; }; } // namespace td diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index 105590b1..814716a0 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -2597,7 +2597,7 @@ class GetStickerEmojisRequest : public RequestActor<> { } void do_send_result() override { - send_result(make_tl_object(std::move(emojis_))); + send_result(td_api::make_object(std::move(emojis_))); } public: @@ -2607,6 +2607,27 @@ class GetStickerEmojisRequest : public RequestActor<> { } }; +class SearchEmojisRequest : public RequestActor<> { + string text_; + bool exact_match_; + + vector emojis_; + + void do_run(Promise &&promise) override { + emojis_ = td->stickers_manager_->search_emojis(text_, exact_match_, get_tries() < 2, std::move(promise)); + } + + void do_send_result() override { + send_result(td_api::make_object(std::move(emojis_))); + } + + public: + SearchEmojisRequest(ActorShared td, uint64 request_id, string &&text, bool exact_match) + : RequestActor(std::move(td), request_id), text_(std::move(text)), exact_match_(exact_match) { + set_tries(3); + } +}; + class GetSavedAnimationsRequest : public RequestActor<> { vector animation_ids_; @@ -6338,6 +6359,12 @@ void Td::on_request(uint64 id, td_api::getStickerEmojis &request) { CREATE_REQUEST(GetStickerEmojisRequest, std::move(request.sticker_)); } +void Td::on_request(uint64 id, td_api::searchEmojis &request) { + CHECK_IS_USER(); + CLEAN_INPUT_STRING(request.text_); + CREATE_REQUEST(SearchEmojisRequest, std::move(request.text_), request.exact_match_); +} + void Td::on_request(uint64 id, const td_api::getSavedAnimations &request) { CHECK_IS_USER(); CREATE_NO_ARGS_REQUEST(GetSavedAnimationsRequest); diff --git a/td/telegram/Td.h b/td/telegram/Td.h index d5ea4794..4fd0dcd1 100644 --- a/td/telegram/Td.h +++ b/td/telegram/Td.h @@ -810,6 +810,8 @@ class Td final : public NetQueryCallback { void on_request(uint64 id, td_api::getStickerEmojis &request); + void on_request(uint64 id, td_api::searchEmojis &request); + void on_request(uint64 id, const td_api::getFavoriteStickers &request); void on_request(uint64 id, td_api::addFavoriteSticker &request); diff --git a/td/telegram/cli.cpp b/td/telegram/cli.cpp index c70b2af0..f194697b 100644 --- a/td/telegram/cli.cpp +++ b/td/telegram/cli.cpp @@ -2210,6 +2210,10 @@ class CliClient final : public Actor { send_request(td_api::make_object(as_bool(args))); } else if (op == "gse") { send_request(td_api::make_object(as_input_file_id(args))); + } else if (op == "se") { + send_request(td_api::make_object(args, false)); + } else if (op == "see") { + send_request(td_api::make_object(args, true)); } else { op_not_found_count++; } diff --git a/td/telegram/net/MtprotoHeader.h b/td/telegram/net/MtprotoHeader.h index 4094298c..36aa819e 100644 --- a/td/telegram/net/MtprotoHeader.h +++ b/td/telegram/net/MtprotoHeader.h @@ -73,6 +73,10 @@ class MtprotoHeader { return anonymous_header_; } + string get_system_language_code() const { + return options_.system_language_code; + } + private: Options options_; string default_header_;