diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index 370fb8293..29939bd7b 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -1722,8 +1722,8 @@ languagePackString key:string value:LanguagePackStringValue = LanguagePackString //@description Contains a list of language pack strings @strings A list of language pack strings languagePackStrings strings:vector = LanguagePackStrings; -//@description Contains information about a language @code Language code @name Language name @native_name Language native name -languageInfo code:string name:string native_name:string = LanguageInfo; +//@description Contains information about a language @code Language code @name Language name @native_name Language native name @local_key_count Total number of locally available non-deleted keys from the language +languageInfo code:string name:string native_name:string key_count:int32 = LanguageInfo; //@description Contains information about a language pack @languages List of available languages languagePack languages:vector = LanguagePack; @@ -3136,7 +3136,7 @@ getLanguagePackInfo = LanguagePack; //@description Returns strings from a language in the used language pack by their keys @language_code Language code of strings to return @keys Language pack keys of strings to return; may be empty to get all available strings getLanguagePackStrings language_code:string keys:vector = LanguagePackStrings; -//@description Adds or changes a custom language to the used language pack @info Information about the language. Language code must start with 'X' @strings New language pack strings +//@description Adds or changes a custom language to the used language pack @info Information about the language. Language code must start with 'X', consist only of English letters, digits and hyphens and be not longer than 64 characters @strings New language pack strings setCustomLanguage info:languageInfo strings:vector = Ok; //@description Sets new value for a string in a custom language in the used language pack @language_code The code of previously added custom language, must start with 'X' @new_string New language pack string diff --git a/td/generate/scheme/td_api.tlo b/td/generate/scheme/td_api.tlo index 529a16a4f..f714146f9 100644 Binary files a/td/generate/scheme/td_api.tlo and b/td/generate/scheme/td_api.tlo differ diff --git a/td/telegram/LanguagePackManager.cpp b/td/telegram/LanguagePackManager.cpp index 0a7735bbe..3caeadd65 100644 --- a/td/telegram/LanguagePackManager.cpp +++ b/td/telegram/LanguagePackManager.cpp @@ -42,7 +42,9 @@ struct LanguagePackManager::PluralizedString { struct LanguagePackManager::Language { std::mutex mutex_; std::atomic version_{-1}; + std::atomic key_count_{0}; bool is_full_ = false; + bool was_loaded_full_ = false; bool has_get_difference_query_ = false; std::unordered_map ordinary_strings_; std::unordered_map pluralized_strings_; @@ -77,16 +79,16 @@ bool LanguagePackManager::check_language_pack_name(Slice name) { return false; } } - return true; + return name.size() <= 64; } bool LanguagePackManager::check_language_code_name(Slice name) { for (auto c : name) { - if (c != '-' && !is_alpha(c)) { + if (c != '-' && !is_alpha(c) && !is_digit(c)) { return false; } } - return true; + return name.size() <= 64 && (is_custom_language_code(name) || name.empty() || name.size() >= 2); } bool LanguagePackManager::is_custom_language_code(Slice language_code) { @@ -115,6 +117,26 @@ static int32 load_database_language_version(SqliteKeyValue *kv) { return to_integer(str_version); } +static int32 load_database_language_key_count(SqliteKeyValue *kv) { + CHECK(kv != nullptr); + if (kv->empty()) { + return 0; + } + string str_key_count = kv->get("!key_count"); + if (str_key_count.empty()) { + // calculate key count once for the database and cache it + int key_count = 0; + for (auto &str : kv->get_all()) { + key_count += str.first[0] != '!' && (str.second[0] == '1' || str.second[0] == '2'); + } + LOG(INFO) << "Set language key count in database to " << key_count; + kv->set("!key_count", to_string(key_count)); + return key_count; + } + + return to_integer(str_key_count); +} + LanguagePackManager::LanguageDatabase *LanguagePackManager::add_language_database(const string &path) { auto it = language_databases_.find(path); if (it != language_databases_.end()) { @@ -321,13 +343,14 @@ LanguagePackManager::Language *LanguagePackManager::add_language(LanguageDatabas .init_with_connection(database->database_.clone(), get_database_table_name(language_pack, language_code)) .ensure(); language->version_ = load_database_language_version(&language->kv_); + language->key_count_ = load_database_language_key_count(&language->kv_); } code_it = pack->languages_.emplace(language_code, std::move(language)).first; } return code_it->second.get(); } -bool LanguagePackManager::language_has_string_unsafe(Language *language, const string &key) { +bool LanguagePackManager::language_has_string_unsafe(const Language *language, const string &key) { return language->ordinary_strings_.count(key) != 0 || language->pluralized_strings_.count(key) != 0 || language->deleted_strings_.count(key) != 0; } @@ -379,27 +402,26 @@ bool LanguagePackManager::load_language_strings(LanguageDatabase *database, Lang return false; } - LOG(DEBUG) << "Begin to load a language from database"; std::lock_guard database_lock(database->mutex_); std::lock_guard language_lock(language->mutex_); if (language->is_full_) { - LOG(DEBUG) << "The language was already loaded"; + LOG(DEBUG) << "The language is full in memory"; return true; } if (language->kv_.empty()) { LOG(DEBUG) << "The language has no database"; return false; } + LOG(DEBUG) << "Begin to load a language from database"; if (keys.empty()) { - if (language->version_ == -1) { - LOG(DEBUG) << "There is nothing to load"; + if (language->version_ == -1 && language->was_loaded_full_) { + LOG(DEBUG) << "The language has already been loaded"; return false; } auto all_strings = language->kv_.get_all(); for (auto &str : all_strings) { - if (str.first == "!version") { - CHECK(to_integer(str.second) == language->version_); + if (str.first[0] == '!') { continue; } @@ -408,17 +430,26 @@ bool LanguagePackManager::load_language_strings(LanguageDatabase *database, Lang load_language_string_unsafe(language, str.first, str.second); } } + language->was_loaded_full_ = true; + + if (language->version_ == -1) { + return false; + } + language->is_full_ = true; language->deleted_strings_.clear(); return true; } + + bool have_all = true; for (auto &key : keys) { if (!language_has_string_unsafe(language, key)) { auto value = language->kv_.get(key); if (value.empty()) { if (language->version_ == -1) { LOG(DEBUG) << "Have no string with key " << key << " in the database"; - return false; + have_all = false; + continue; } // have full language in the database, so this string is just deleted @@ -427,7 +458,7 @@ bool LanguagePackManager::load_language_strings(LanguageDatabase *database, Lang load_language_string_unsafe(language, key, value); } } - return true; + return have_all; } td_api::object_ptr LanguagePackManager::get_language_pack_string_value_object( @@ -460,7 +491,7 @@ td_api::object_ptr LanguagePackManager::get_language } td_api::object_ptr LanguagePackManager::get_language_pack_string_value_object( - Language *language, const string &key) { + const Language *language, const string &key) { CHECK(language != nullptr); auto ordinary_it = language->ordinary_strings_.find(key); if (ordinary_it != language->ordinary_strings_.end()) { @@ -474,8 +505,8 @@ td_api::object_ptr LanguagePackManager::get_lan return get_language_pack_string_value_object(key); } -td_api::object_ptr LanguagePackManager::get_language_pack_string_object(Language *language, - const string &key) { +td_api::object_ptr LanguagePackManager::get_language_pack_string_object( + const Language *language, const string &key) { return td_api::make_object(key, get_language_pack_string_value_object(language, key)); } @@ -502,34 +533,48 @@ td_api::object_ptr LanguagePackManager::get_languag } void LanguagePackManager::get_languages(Promise> promise) { + auto request_promise = PromiseCreator::lambda([actor_id = actor_id(this), language_pack = language_pack_, + promise = std::move(promise)](Result r_query) mutable { + auto r_result = fetch_result(std::move(r_query)); + if (r_result.is_error()) { + return promise.set_error(r_result.move_as_error()); + } + + send_closure(actor_id, &LanguagePackManager::on_get_languages, r_result.move_as_ok(), std::move(language_pack), + std::move(promise)); + }); + send_with_promise(G()->net_query_creator().create(create_storer(telegram_api::langpack_getLanguages(language_pack_))), + std::move(request_promise)); +} + +void LanguagePackManager::on_get_languages(vector> languages, + string language_pack, + Promise> promise) { auto results = make_tl_object(); - std::lock_guard packs_lock(database_->mutex_); - auto pack_it = database_->language_packs_.find(language_pack_); - if (pack_it != database_->language_packs_.end()) { - LanguagePack *pack = pack_it->second.get(); - for (auto &info : pack->language_infos_) { - results->languages_.push_back( - make_tl_object(info.first, info.second.name, info.second.native_name)); + { + std::lock_guard packs_lock(database_->mutex_); + auto pack_it = database_->language_packs_.find(language_pack); + if (pack_it != database_->language_packs_.end()) { + LanguagePack *pack = pack_it->second.get(); + for (auto &info : pack->language_infos_) { + results->languages_.push_back( + make_tl_object(info.first, info.second.name, info.second.native_name, 0)); + } } } - auto request_promise = PromiseCreator::lambda( - [results = std::move(results), promise = std::move(promise)](Result r_query) mutable { - auto r_result = fetch_result(std::move(r_query)); - if (r_result.is_error()) { - return promise.set_error(r_result.move_as_error()); - } + for (auto &language : languages) { + results->languages_.push_back( + make_tl_object(language->lang_code_, language->name_, language->native_name_, 0)); + } - auto languages = r_result.move_as_ok(); - for (auto &language : languages) { - results->languages_.push_back( - make_tl_object(language->lang_code_, language->name_, language->native_name_)); - } - promise.set_value(std::move(results)); - }); - send_with_promise(G()->net_query_creator().create(create_storer(telegram_api::langpack_getLanguages(language_pack_))), - std::move(request_promise)); + for (auto &language_info : results->languages_) { + auto language = add_language(database_, language_pack, language_info->code_); + language_info->key_count_ = language->key_count_; + } + + promise.set_value(std::move(results)); } void LanguagePackManager::get_language_pack_strings(string language_code, vector keys, @@ -540,15 +585,12 @@ void LanguagePackManager::get_language_pack_strings(string language_code, vector } } - Language *language = get_language(database_, language_pack_, language_code); + Language *language = add_language(database_, language_pack_, language_code); if (language_has_strings(language, keys)) { return promise.set_value(get_language_pack_strings_object(language, keys)); } - if (!database_->database_.empty()) { - language = add_language(database_, language_pack_, language_code); - if (load_language_strings(database_, language, keys)) { - return promise.set_value(get_language_pack_strings_object(language, keys)); - } + if (load_language_strings(database_, language, keys)) { + return promise.set_value(get_language_pack_strings_object(language, keys)); } if (keys.empty()) { @@ -617,7 +659,8 @@ td_api::object_ptr LanguagePackManager::get_language_pack_string std::lock_guard lock(language->mutex_); return get_language_pack_string_value_object(language, key); } - if (!database->database_.empty() && load_language_strings(database, language, keys)) { + if (load_language_strings(database, language, keys)) { + std::lock_guard lock(language->mutex_); return get_language_pack_string_value_object(language, key); } return td_api::make_object(404, "Not Found"); @@ -632,15 +675,16 @@ bool LanguagePackManager::is_valid_key(Slice key) { return !key.empty(); } -void LanguagePackManager::save_strings_to_database(Language *language, int32 new_version, - vector> strings) { - LOG(DEBUG) << "Save to database a language with new version " << new_version; +void LanguagePackManager::save_strings_to_database(SqliteKeyValue *kv, int32 new_version, bool new_is_full, + int32 new_key_count, vector> strings) { + LOG(DEBUG) << "Save to database a language with new version " << new_version << " and " << strings.size() + << " new strings"; if (new_version == -1 && strings.empty()) { return; } std::lock_guard lock(database_->mutex_); - auto kv = &language->kv_; + CHECK(kv != nullptr); if (kv->empty()) { LOG(DEBUG) << "There is no associated database key-value"; return; @@ -658,7 +702,7 @@ void LanguagePackManager::save_strings_to_database(Language *language, int32 new continue; } - if (language->is_full_ && str.second == "3") { + if (new_is_full && str.second == "3") { kv->erase(str.first); } else { kv->set(str.first, str.second); @@ -669,6 +713,10 @@ void LanguagePackManager::save_strings_to_database(Language *language, int32 new LOG(DEBUG) << "Set language version in database to " << new_version; kv->set("!version", to_string(new_version)); } + if (new_key_count != -1) { + LOG(DEBUG) << "Set language key count in database to " << new_key_count; + kv->set("!key_count", to_string(new_key_count)); + } kv->commit_transaction().ensure(); } @@ -679,13 +727,18 @@ void LanguagePackManager::on_get_language_pack_strings( Language *language = get_language(database_, language_pack, language_code); bool is_version_changed = false; int32 new_database_version = -1; + int32 new_key_count = -1; + bool new_is_full = false; vector> database_strings; if (language == nullptr || language->version_ < version || !keys.empty()) { if (language == nullptr) { language = add_language(database_, language_pack, language_code); CHECK(language != nullptr); } + load_language_strings(database_, language, keys); + std::lock_guard lock(language->mutex_); + int32 key_count_delta = 0; if (language->version_ < version || !keys.empty()) { vector> strings; if (language->version_ < version && !(is_diff && language->version_ == -1)) { @@ -702,11 +755,12 @@ void LanguagePackManager::on_get_language_pack_strings( auto str = static_cast(result.get()); auto it = language->ordinary_strings_.find(str->key_); if (it == language->ordinary_strings_.end()) { + key_count_delta++; it = language->ordinary_strings_.emplace(str->key_, std::move(str->value_)).first; } else { it->second = std::move(str->value_); } - language->pluralized_strings_.erase(str->key_); + key_count_delta -= language->pluralized_strings_.erase(str->key_); language->deleted_strings_.erase(str->key_); if (is_diff) { strings.push_back(get_language_pack_string_object(*it)); @@ -721,11 +775,12 @@ void LanguagePackManager::on_get_language_pack_strings( std::move(str->many_value_), std::move(str->other_value_)}; auto it = language->pluralized_strings_.find(str->key_); if (it == language->pluralized_strings_.end()) { + key_count_delta++; it = language->pluralized_strings_.emplace(str->key_, std::move(value)).first; } else { it->second = std::move(value); } - language->ordinary_strings_.erase(str->key_); + key_count_delta -= language->ordinary_strings_.erase(str->key_); language->deleted_strings_.erase(str->key_); if (is_diff) { strings.push_back(get_language_pack_string_object(*it)); @@ -738,8 +793,8 @@ void LanguagePackManager::on_get_language_pack_strings( } case telegram_api::langPackStringDeleted::ID: { auto str = static_cast(result.get()); - language->ordinary_strings_.erase(str->key_); - language->pluralized_strings_.erase(str->key_); + key_count_delta -= language->ordinary_strings_.erase(str->key_); + key_count_delta -= language->pluralized_strings_.erase(str->key_); language->deleted_strings_.insert(std::move(str->key_)); if (is_diff) { strings.push_back(get_language_pack_string_object(str->key_)); @@ -765,6 +820,11 @@ void LanguagePackManager::on_get_language_pack_strings( } } + if (key_count_delta != 0) { + new_key_count = language->key_count_ + key_count_delta; + language->key_count_ = new_key_count; + } + if (is_diff) { send_closure( G()->td(), &Td::send_update, @@ -776,10 +836,15 @@ void LanguagePackManager::on_get_language_pack_strings( language->is_full_ = true; language->deleted_strings_.clear(); } + new_is_full = language->is_full_; } } + if (is_custom_language_code(language_code) && new_database_version == -1) { + new_database_version = 1; + } - save_strings_to_database(language, new_database_version, std::move(database_strings)); + save_strings_to_database(&language->kv_, new_database_version, new_is_full, new_key_count, + std::move(database_strings)); if (is_diff) { CHECK(language != nullptr); @@ -965,6 +1030,7 @@ Status LanguagePackManager::do_delete_language(string language_code) { } std::lock_guard language_lock(language->mutex_); language->version_ = -1; + language->key_count_ = load_database_language_key_count(&language->kv_); language->is_full_ = false; language->ordinary_strings_.clear(); language->pluralized_strings_.clear(); diff --git a/td/telegram/LanguagePackManager.h b/td/telegram/LanguagePackManager.h index 51a5eac42..b70a765f8 100644 --- a/td/telegram/LanguagePackManager.h +++ b/td/telegram/LanguagePackManager.h @@ -24,6 +24,8 @@ namespace td { +class SqliteKeyValue; + class LanguagePackManager : public NetQueryCallback { public: explicit LanguagePackManager(ActorShared<> parent) : parent_(std::move(parent)) { @@ -90,7 +92,7 @@ class LanguagePackManager : public NetQueryCallback { static Language *add_language(LanguageDatabase *database, const string &language_pack, const string &language_code); - static bool language_has_string_unsafe(Language *language, const string &key); + static bool language_has_string_unsafe(const Language *language, const string &key); static bool language_has_strings(Language *language, const vector &keys); static void load_language_string_unsafe(Language *language, const string &key, string &value); @@ -107,10 +109,10 @@ class LanguagePackManager : public NetQueryCallback { const std::pair &str); static td_api::object_ptr get_language_pack_string_object(const string &str); - static td_api::object_ptr get_language_pack_string_value_object(Language *language, - const string &key); + static td_api::object_ptr get_language_pack_string_value_object( + const Language *language, const string &key); - static td_api::object_ptr get_language_pack_string_object(Language *language, + static td_api::object_ptr get_language_pack_string_object(const Language *language, const string &key); static td_api::object_ptr get_language_pack_strings_object(Language *language, @@ -123,7 +125,8 @@ class LanguagePackManager : public NetQueryCallback { static bool is_valid_key(Slice key); - void save_strings_to_database(Language *language, int32 new_version, vector> strings); + void save_strings_to_database(SqliteKeyValue *kv, int32 new_version, bool new_is_full, int32 new_key_count, + vector> strings); void on_get_language_pack_strings(string language_pack, string language_code, int32 version, bool is_diff, vector keys, vector> results, @@ -131,6 +134,9 @@ class LanguagePackManager : public NetQueryCallback { void on_failed_get_difference(string language_pack, string language_code); + void on_get_languages(vector> languages, string language_pack, + Promise> promise); + Status do_delete_language(string language_code); void on_result(NetQueryPtr query) override; diff --git a/td/telegram/cli.cpp b/td/telegram/cli.cpp index 866975e02..7083c9eb0 100644 --- a/td/telegram/cli.cpp +++ b/td/telegram/cli.cpp @@ -1788,7 +1788,7 @@ class CliClient final : public Actor { "DELETED", make_tl_object())); send_request(make_tl_object( - make_tl_object(language_code, name, native_name), std::move(strings))); + make_tl_object(language_code, name, native_name, 3), std::move(strings))); } else if (op == "sclsv" || op == "sclsp" || op == "sclsd") { string language_code; string key;