From 08db51cdce777815b14e8dda4ad88533223a1e47 Mon Sep 17 00:00:00 2001 From: levlam Date: Thu, 5 Jul 2018 21:28:11 +0300 Subject: [PATCH] In-memory thread-safe cache for language pack strings. GitOrigin-RevId: 71dc10a925f1000590501c4ddfc307806e242e51 --- td/generate/scheme/td_api.tl | 2 +- td/telegram/LanguagePackManager.cpp | 202 +++++++++++++++++++++++----- td/telegram/LanguagePackManager.h | 50 ++++++- tdutils/td/utils/port/RwMutex.h | 1 + 4 files changed, 222 insertions(+), 33 deletions(-) diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index d78933ae..726645df 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -1661,7 +1661,7 @@ languagePackStringValue key:string value:string = LanguagePackString; //@few_value Value for few objects @many_value Value for many objects @other_value Default value languagePackStringPluralized key:string zero_value:string one_value:string two_value:string few_value:string many_value:string other_value:string = LanguagePackString; -//@description A deleted language pack string @key String key +//@description A deleted language pack string, the value should be taken from a built-in english language pack @key String key languagePackStringDeleted key:string = LanguagePackString; diff --git a/td/telegram/LanguagePackManager.cpp b/td/telegram/LanguagePackManager.cpp index 7b7946c6..024ba14d 100644 --- a/td/telegram/LanguagePackManager.cpp +++ b/td/telegram/LanguagePackManager.cpp @@ -66,6 +66,117 @@ void LanguagePackManager::inc_generation() { language_pack_version_ = -1; } +LanguagePackManager::Language *LanguagePackManager::get_language(const string &language_pack, + const string &language_code) { + std::unique_lock lock(language_packs_mutex_); + auto it = language_packs_.find(language_pack); + if (it == language_packs_.end()) { + return nullptr; + } + LanguagePack *pack = it->second.get(); + lock.unlock(); + return get_language(pack, language_code); +} + +LanguagePackManager::Language *LanguagePackManager::get_language(LanguagePack *language_pack, + const string &language_code) { + CHECK(language_pack != nullptr); + std::lock_guard lock(language_pack->mutex_); + auto it = language_pack->languages_.find(language_code); + if (it == language_pack->languages_.end()) { + return nullptr; + } + return it->second.get(); +} + +LanguagePackManager::Language *LanguagePackManager::add_language(const string &language_pack, + const string &language_code) { + std::lock_guard packs_lock(language_packs_mutex_); + auto pack_it = language_packs_.find(language_pack); + if (pack_it == language_packs_.end()) { + pack_it = language_packs_.emplace(language_pack, make_unique()).first; + } + LanguagePack *pack = pack_it->second.get(); + + std::lock_guard languages_lock(pack->mutex_); + auto code_it = pack->languages_.find(language_code); + if (code_it == pack->languages_.end()) { + code_it = pack->languages_.emplace(language_code, make_unique()).first; + } + return code_it->second.get(); +} + +bool LanguagePackManager::language_has_string_unsafe(Language *language, const string &key) { + return language->ordinary_strings_.count(key) != 0 || language->pluralized_strings_.count(key) != 0 || + language->deleted_strings_.count(key) != 0; +} + +bool LanguagePackManager::language_has_strings(Language *language, const vector &keys) { + if (language == nullptr) { + return false; + } + + std::lock_guard lock(language->mutex_); + if (keys.empty()) { + return language->version_ != -1; + } + for (auto &key : keys) { + if (!language_has_string_unsafe(language, key)) { + return false; + } + } + return true; +} + +td_api::object_ptr LanguagePackManager::get_language_pack_string_object( + const std::pair &str) { + return td_api::make_object(str.first, str.second); +} + +td_api::object_ptr LanguagePackManager::get_language_pack_string_object( + const std::pair &str) { + return td_api::make_object( + str.first, str.second.zero_value_, str.second.one_value_, str.second.two_value_, str.second.few_value_, + str.second.many_value_, str.second.other_value_); +} + +td_api::object_ptr LanguagePackManager::get_language_pack_string_object(const string &str) { + return td_api::make_object(str); +} + +td_api::object_ptr LanguagePackManager::get_language_pack_strings_object( + Language *language, const vector &keys) { + CHECK(language != nullptr); + + std::lock_guard lock(language->mutex_); + vector> strings; + if (keys.empty()) { + for (auto &str : language->ordinary_strings_) { + strings.push_back(get_language_pack_string_object(str)); + } + for (auto &str : language->pluralized_strings_) { + strings.push_back(get_language_pack_string_object(str)); + } + } else { + for (auto &key : keys) { + auto ordinary_it = language->ordinary_strings_.find(key); + if (ordinary_it != language->ordinary_strings_.end()) { + strings.push_back(get_language_pack_string_object(*ordinary_it)); + continue; + } + auto pluralized_it = language->pluralized_strings_.find(key); + if (pluralized_it != language->pluralized_strings_.end()) { + strings.push_back(get_language_pack_string_object(*pluralized_it)); + continue; + } + LOG_IF(ERROR, language->deleted_strings_.count(key) == 0) << "Have no string for key " << key; + strings.push_back(get_language_pack_string_object(key)); + } + } + + return td_api::make_object(std::move(strings)); +} + void LanguagePackManager::get_languages(Promise> promise) { auto request_promise = PromiseCreator::lambda([promise = std::move(promise)](Result r_query) mutable { auto r_result = fetch_result(std::move(r_query)); @@ -88,17 +199,15 @@ void LanguagePackManager::get_languages(Promise keys, Promise> promise) { - bool is_all = keys.empty(); - auto result_promise = - PromiseCreator::lambda([actor_id = actor_id(this), is_all, promise = std::move(promise)]( - Result>> r_result) mutable { - send_closure(actor_id, &LanguagePackManager::on_get_language_pack_strings, std::move(r_result), is_all, - std::move(promise)); - }); + Language *language = get_language(language_pack_, language_code); + if (language_has_strings(language, keys)) { + return promise.set_value(get_language_pack_strings_object(language, keys)); + } - if (is_all) { + if (keys.empty()) { auto request_promise = - PromiseCreator::lambda([promise = std::move(result_promise)](Result r_query) mutable { + PromiseCreator::lambda([actor_id = actor_id(this), language_pack = language_pack_, language_code, + promise = std::move(promise)](Result r_query) mutable { auto r_result = fetch_result(std::move(r_query)); if (r_result.is_error()) { return promise.set_error(r_result.move_as_error()); @@ -108,19 +217,26 @@ void LanguagePackManager::get_language_pack_strings(string language_code, vector LOG(INFO) << "Receive language pack for language " << result->lang_code_ << " from version " << result->from_version_ << " with version " << result->version_ << " of size " << result->strings_.size(); - promise.set_value(std::move(result->strings_)); + LOG_IF(ERROR, result->lang_code_ != language_code) + << "Receive strings for " << result->lang_code_ << " instead of " << language_code; + LOG_IF(ERROR, result->from_version_ != 0) << "Receive lang pack from version " << result->from_version_; + send_closure(actor_id, &LanguagePackManager::on_get_language_pack_strings, std::move(language_pack), + std::move(language_code), result->version_, vector(), std::move(result->strings_), + std::move(promise)); }); send_with_promise(G()->net_query_creator().create(create_storer(telegram_api::langpack_getLangPack(language_code))), std::move(request_promise)); } else { auto request_promise = - PromiseCreator::lambda([promise = std::move(result_promise)](Result r_query) mutable { + PromiseCreator::lambda([actor_id = actor_id(this), language_pack = language_pack_, language_code, keys, + promise = std::move(promise)](Result r_query) mutable { auto r_result = fetch_result(std::move(r_query)); if (r_result.is_error()) { return promise.set_error(r_result.move_as_error()); } - promise.set_value(r_result.move_as_ok()); + send_closure(actor_id, &LanguagePackManager::on_get_language_pack_strings, std::move(language_pack), + std::move(language_code), -1, std::move(keys), r_result.move_as_ok(), std::move(promise)); }); send_with_promise(G()->net_query_creator().create( create_storer(telegram_api::langpack_getStrings(language_code, std::move(keys)))), @@ -129,36 +245,57 @@ void LanguagePackManager::get_language_pack_strings(string language_code, vector } void LanguagePackManager::on_get_language_pack_strings( - Result>> r_result, bool ia_all, + string language_pack, string language_code, int32 version, vector keys, + vector> results, Promise> promise) { - if (r_result.is_error()) { - return promise.set_error(r_result.move_as_error()); - } - auto result = - transform(r_result.move_as_ok(), [](const auto &string_ptr) -> tl_object_ptr { - CHECK(string_ptr != nullptr); - switch (string_ptr->get_id()) { + Language *language = get_language(language_pack, language_code); + if (language == nullptr || (language->version_ < version || !keys.empty())) { + if (language == nullptr) { + language = add_language(language_pack, language_code); + CHECK(language != nullptr); + } + std::lock_guard lock(language->mutex_); + if (language->version_ < version || !keys.empty()) { + if (language->version_ < version) { + language->version_ = version; + } + for (auto &result : results) { + CHECK(result != nullptr); + switch (result->get_id()) { case telegram_api::langPackString::ID: { - auto str = static_cast(string_ptr.get()); - return make_tl_object(str->key_, str->value_); + auto str = static_cast(result.get()); + language->ordinary_strings_[str->key_] = std::move(str->value_); + break; } case telegram_api::langPackStringPluralized::ID: { - auto str = static_cast(string_ptr.get()); - return make_tl_object(str->key_, str->zero_value_, str->one_value_, - str->two_value_, str->few_value_, - str->many_value_, str->other_value_); + auto str = static_cast(result.get()); + language->pluralized_strings_[str->key_] = PluralizedString{ + std::move(str->zero_value_), std::move(str->one_value_), std::move(str->two_value_), + std::move(str->few_value_), std::move(str->many_value_), std::move(str->other_value_)}; + break; } case telegram_api::langPackStringDeleted::ID: { - auto str = static_cast(string_ptr.get()); - return make_tl_object(str->key_); + auto str = static_cast(result.get()); + language->deleted_strings_.insert(std::move(str->key_)); + break; } default: UNREACHABLE(); - return nullptr; + break; } - }); + } + for (auto &key : keys) { + if (!language_has_string_unsafe(language, key)) { + LOG(ERROR) << "Doesn't receive key " << key << " from server"; + language->deleted_strings_.insert(key); + } + } - promise.set_value(make_tl_object(std::move(result))); + // TODO save language + } + } + + promise.set_value(get_language_pack_strings_object(language, keys)); } void LanguagePackManager::on_result(NetQueryPtr query) { @@ -177,4 +314,7 @@ void LanguagePackManager::hangup() { stop(); } +std::mutex LanguagePackManager::language_packs_mutex_; +std::unordered_map> LanguagePackManager::language_packs_; + } // namespace td diff --git a/td/telegram/LanguagePackManager.h b/td/telegram/LanguagePackManager.h index e0b6cd4f..745c1526 100644 --- a/td/telegram/LanguagePackManager.h +++ b/td/telegram/LanguagePackManager.h @@ -17,6 +17,11 @@ #include "td/utils/Container.h" #include "td/utils/Status.h" +#include +#include +#include +#include + namespace td { class LanguagePackManager : public NetQueryCallback { @@ -36,6 +41,28 @@ class LanguagePackManager : public NetQueryCallback { Promise> promise); private: + struct PluralizedString { + string zero_value_; + string one_value_; + string two_value_; + string few_value_; + string many_value_; + string other_value_; + }; + + struct Language { + std::mutex mutex_; // TODO RwMutex + std::atomic version_ = -1; + std::unordered_map ordinary_strings_; + std::unordered_map pluralized_strings_; + std::unordered_set deleted_strings_; + }; + + struct LanguagePack { + std::mutex mutex_; + std::unordered_map> languages_; + }; + ActorShared<> parent_; string language_pack_; @@ -44,9 +71,30 @@ class LanguagePackManager : public NetQueryCallback { int32 language_pack_version_ = -1; + static std::mutex language_packs_mutex_; + static std::unordered_map> language_packs_; + + static Language *get_language(const string &language_pack, const string &language_code); + static Language *get_language(LanguagePack *language_pack, const string &language_code); + + static Language *add_language(const string &language_pack, const string &language_code); + + static bool language_has_string_unsafe(Language *language, const string &key); + static bool language_has_strings(Language *language, const vector &keys); + + static td_api::object_ptr get_language_pack_string_object( + const std::pair &str); + static td_api::object_ptr get_language_pack_string_object( + const std::pair &str); + static td_api::object_ptr get_language_pack_string_object(const string &str); + + static td_api::object_ptr get_language_pack_strings_object(Language *language, + const vector &keys); + void inc_generation(); - void on_get_language_pack_strings(Result>> r_result, bool ia_all, + void on_get_language_pack_strings(string language_pack, string language_code, int32 version, vector keys, + vector> results, Promise> promise); void on_result(NetQueryPtr query) override; diff --git a/tdutils/td/utils/port/RwMutex.h b/tdutils/td/utils/port/RwMutex.h index eee5f3dc..2eb97e9b 100644 --- a/tdutils/td/utils/port/RwMutex.h +++ b/tdutils/td/utils/port/RwMutex.h @@ -17,6 +17,7 @@ #endif namespace td { + class RwMutex { public: RwMutex() {