Custom language packs support.

GitOrigin-RevId: e7f76319dae5be3e20f81b41a0226e5f96f91ba1

td/generate/scheme/td_api.tl

@@ -2235,7 +2235,7 @@ updateFavoriteStickers sticker_ids:vector<int32> = Update;
 updateSavedAnimations animation_ids:vector<int32> = Update;
 
 //@description Some language pack strings was updated @language_pack Changed language pack @language_code Code of the language, which was updated @strings List of changed language pack strings
-updateLanguagePack language_pack:string language_code:string strings:vector<LanguagePackString> = Update;
+updateLanguagePackStrings language_pack:string language_code:string strings:vector<LanguagePackString> = Update;
 
 //@description The connection state has changed @state The new connection state
 updateConnectionState state:ConnectionState = Update;
@@ -3085,12 +3085,18 @@ getSupportUser = User;
 getWallpapers = Wallpapers;
 
 
-//@description Returns information about used language pack
-getLanguagePack = LanguagePack;
+//@description Returns information about the used language pack
+getLanguagePackInfo = LanguagePack;
 
-//@description Returns strings from used language pack on specified language by their keys @language_code Language code of strings to return @keys Language pack keys of strings to return; may be empty to get all available strings
+//@description Returns strings from a language in the used language pack by their keys @language_code Language code of strings to return @keys Language pack keys of strings to return; may be empty to get all available strings
 getLanguagePackStrings language_code:string keys:vector<string> = LanguagePackStrings;
 
+//@description Adds or changes a custom language to the used language pack @info Information about the language. Language code should start with 'X' @strings New language pack strings
+setCustomLanguage info:languageInfo strings:vector<LanguagePackString> = Ok;
+
+//@description Deletes all information about a language in the used language pack. Currently used language can't be deleted @language_code Language code of a language to delete
+deleteLanguage language_code:string = Ok;
+
 
 //@description Registers the currently used device for receiving push notifications @device_token Device token @other_user_ids List of at most 100 user identifiers of other users currently using the client
 registerDevice device_token:DeviceToken other_user_ids:vector<int32> = Ok;

td/telegram/LanguagePackManager.cpp

@@ -9,10 +9,12 @@
 #include "td/telegram/ConfigShared.h"
 #include "td/telegram/Global.h"
 #include "td/telegram/logevent/LogEvent.h"
+#include "td/telegram/misc.h"
 #include "td/telegram/net/NetQueryDispatcher.h"
 #include "td/telegram/Td.h"
 
 #include "td/telegram/td_api.h"
+#include "td/telegram/td_api.hpp"
 #include "td/telegram/telegram_api.h"
 
 #include "td/db/SqliteDb.h"
@@ -77,7 +79,11 @@ bool LanguagePackManager::check_language_code_name(Slice name) {
       return false;
     }
   }
-  return name.size() >= 2 || name.empty();
+  return true;
+}
+
+bool LanguagePackManager::is_custom_language_code(Slice language_code) {
+  return !language_code.empty() && language_code[0] == 'X';
 }
 
 static Result<SqliteDb> open_database(const string &path) {
@@ -173,6 +179,10 @@ void LanguagePackManager::on_language_code_changed() {
 }
 
 void LanguagePackManager::on_language_pack_version_changed(int32 new_version) {
+  if (is_custom_language_code(language_code_)) {
+    return;
+  }
+
   Language *language = get_language(database_, language_pack_, language_code_);
   int32 version = language == nullptr ? static_cast<int32>(-1) : language->version_.load();
   if (version == -1) {
@@ -698,9 +708,9 @@ void LanguagePackManager::on_get_language_pack_strings(
       }
 
       if (is_diff) {
-        // do we need to send update to all client instances?
-        send_closure(G()->td(), &Td::send_update,
-                     td_api::make_object<td_api::updateLanguagePack>(language_pack, language_code, std::move(strings)));
+        send_closure(
+            G()->td(), &Td::send_update,
+            td_api::make_object<td_api::updateLanguagePackStrings>(language_pack, language_code, std::move(strings)));
       }
 
       if (keys.empty() && !is_diff) {
@@ -742,6 +752,102 @@ void LanguagePackManager::on_failed_get_difference(string language_pack, string
   }
 }
 
+void LanguagePackManager::set_custom_language(string language_code, string language_name, string language_native_name,
+                                              vector<tl_object_ptr<td_api::LanguagePackString>> strings,
+                                              Promise<Unit> &&promise) {
+  if (!is_custom_language_code(language_code)) {
+    return promise.set_error(Status::Error(400, "Custom language code must begin with 'X'"));
+  }
+  if (!check_language_code_name(language_code)) {
+    return promise.set_error(Status::Error(400, "Language code name must contain only letters and hyphen"));
+  }
+
+  vector<tl_object_ptr<telegram_api::LangPackString>> server_strings;
+  for (auto &str : strings) {
+    if (str == nullptr) {
+      return promise.set_error(Status::Error(400, "Language strings must not be null"));
+    }
+    string key;
+    downcast_call(*str, [&key](auto &value) { key = std::move(value.key_); });
+    if (!is_valid_key(key)) {
+      return promise.set_error(Status::Error(400, "Key is invalid"));
+    }
+    switch (str->get_id()) {
+      case td_api::languagePackStringValue::ID: {
+        auto value = static_cast<td_api::languagePackStringValue *>(str.get());
+        if (!clean_input_string(value->value_)) {
+          return promise.set_error(Status::Error(400, "Strings must be encoded in UTF-8"));
+        }
+        server_strings.push_back(
+            make_tl_object<telegram_api::langPackString>(std::move(key), std::move(value->value_)));
+        break;
+      }
+      case td_api::languagePackStringPluralized::ID: {
+        auto value = static_cast<td_api::languagePackStringPluralized *>(str.get());
+        if (!clean_input_string(value->zero_value_) || !clean_input_string(value->one_value_) ||
+            !clean_input_string(value->two_value_) || !clean_input_string(value->few_value_) ||
+            !clean_input_string(value->many_value_) || !clean_input_string(value->other_value_)) {
+          return promise.set_error(Status::Error(400, "Strings must be encoded in UTF-8"));
+        }
+        server_strings.push_back(make_tl_object<telegram_api::langPackStringPluralized>(
+            31, std::move(key), std::move(value->zero_value_), std::move(value->one_value_),
+            std::move(value->two_value_), std::move(value->few_value_), std::move(value->many_value_),
+            std::move(value->other_value_)));
+        break;
+      }
+      case td_api::languagePackStringDeleted::ID:
+        // there is no reason to save deleted strings in a custom language pack to database
+        break;
+      default:
+        UNREACHABLE();
+        break;
+    }
+  }
+
+  // TODO atomic replace using "ALTER TABLE new_X RENAME TO X";
+  do_delete_language(language_code).ensure();
+  on_get_language_pack_strings(language_pack_, language_code, 1, false, vector<string>(), std::move(server_strings),
+                               Auto());
+  promise.set_value(Unit());
+}
+
+void LanguagePackManager::delete_language(string language_code, Promise<Unit> &&promise) {
+  if (language_code_ == language_code) {
+    return promise.set_error(Status::Error(400, "Currently used language can't be deleted"));
+  }
+
+  auto status = do_delete_language(language_code);
+  if (status.is_error()) {
+    promise.set_error(std::move(status));
+  } else {
+    promise.set_value(Unit());
+  }
+}
+
+Status LanguagePackManager::do_delete_language(string language_code) {
+  std::lock_guard<std::mutex> packs_lock(database_->mutex_);
+  auto pack_it = database_->language_packs_.find(language_pack_);
+  if (pack_it == database_->language_packs_.end()) {
+    return Status::OK();
+  }
+  LanguagePack *pack = pack_it->second.get();
+
+  std::lock_guard<std::mutex> languages_lock(pack->mutex_);
+  auto code_it = pack->languages_.find(language_code);
+  if (code_it == pack->languages_.end()) {
+    return Status::OK();
+  }
+  auto language = code_it->second.get();
+  if (language->has_get_difference_query_) {
+    return Status::Error(400, "Language can't be deleted now, try again later");
+  }
+  if (!language->kv_.empty()) {
+    language->kv_.drop().ignore();
+  }
+  pack->languages_.erase(language_code);
+  return Status::OK();
+}
+
 void LanguagePackManager::on_result(NetQueryPtr query) {
   auto token = get_link_token();
   container_.extract(token).set_value(std::move(query));

td/telegram/LanguagePackManager.h

@@ -37,6 +37,8 @@ class LanguagePackManager : public NetQueryCallback {
 
   static bool check_language_code_name(Slice name);
 
+  static bool is_custom_language_code(Slice language_code);
+
   void on_language_pack_changed();
 
   void on_language_code_changed();
@@ -54,6 +56,11 @@ class LanguagePackManager : public NetQueryCallback {
 
   void on_update_language_pack(tl_object_ptr<telegram_api::langPackDifference> difference);
 
+  void set_custom_language(string language_code, string language_name, string language_native_name,
+                           vector<tl_object_ptr<td_api::LanguagePackString>> strings, Promise<Unit> &&promise);
+
+  void delete_language(string language_code, Promise<Unit> &&promise);
+
  private:
   struct PluralizedString;
   struct Language;
@@ -108,6 +115,8 @@ class LanguagePackManager : public NetQueryCallback {
 
   void on_failed_get_difference(string language_pack, string language_code);
 
+  Status do_delete_language(string language_code);
+
   void on_result(NetQueryPtr query) override;
 
   void start_up() override;

td/telegram/Td.cpp

@@ -3265,8 +3265,10 @@ bool Td::is_preinitialization_request(int32 id) {
 bool Td::is_preauthentication_request(int32 id) {
   switch (id) {
     case td_api::processDcUpdate::ID:
-    case td_api::getLanguagePack::ID:
+    case td_api::getLanguagePackInfo::ID:
     case td_api::getLanguagePackStrings::ID:
+    case td_api::setCustomLanguage::ID:
+    case td_api::deleteLanguage::ID:
     case td_api::getOption::ID:
     case td_api::setOption::ID:
     case td_api::setNetworkType::ID:
@@ -5986,7 +5988,7 @@ void Td::on_request(uint64 id, const td_api::resetAllNotificationSettings &reque
   send_closure(actor_id(this), &Td::send_result, id, make_tl_object<td_api::ok>());
 }
 
-void Td::on_request(uint64 id, const td_api::getLanguagePack &request) {
+void Td::on_request(uint64 id, const td_api::getLanguagePackInfo &request) {
   CHECK_IS_USER();
   CREATE_REQUEST_PROMISE();
   send_closure(language_pack_manager_, &LanguagePackManager::get_languages, std::move(promise));
@@ -6003,6 +6005,28 @@ void Td::on_request(uint64 id, td_api::getLanguagePackStrings &request) {
                std::move(request.language_code_), std::move(request.keys_), std::move(promise));
 }
 
+void Td::on_request(uint64 id, td_api::setCustomLanguage &request) {
+  CHECK_IS_USER();
+  if (request.info_ == nullptr) {
+    return send_error_raw(id, 400, "Language info must not be empty");
+  }
+  CLEAN_INPUT_STRING(request.info_->code_);
+  CLEAN_INPUT_STRING(request.info_->name_);
+  CLEAN_INPUT_STRING(request.info_->native_name_);
+  CREATE_OK_REQUEST_PROMISE();
+  send_closure(language_pack_manager_, &LanguagePackManager::set_custom_language, std::move(request.info_->code_),
+               std::move(request.info_->name_), std::move(request.info_->native_name_), std::move(request.strings_),
+               std::move(promise));
+}
+
+void Td::on_request(uint64 id, td_api::deleteLanguage &request) {
+  CHECK_IS_USER();
+  CLEAN_INPUT_STRING(request.language_code_);
+  CREATE_OK_REQUEST_PROMISE();
+  send_closure(language_pack_manager_, &LanguagePackManager::delete_language, std::move(request.language_code_),
+               std::move(promise));
+}
+
 void Td::on_request(uint64 id, td_api::getOption &request) {
   CLEAN_INPUT_STRING(request.name_);
 

td/telegram/Td.h

@@ -755,10 +755,14 @@ class Td final : public NetQueryCallback {
 
   void on_request(uint64 id, td_api::reportChat &request);
 
-  void on_request(uint64 id, const td_api::getLanguagePack &request);
+  void on_request(uint64 id, const td_api::getLanguagePackInfo &request);
 
   void on_request(uint64 id, td_api::getLanguagePackStrings &request);
 
+  void on_request(uint64 id, td_api::setCustomLanguage &request);
+
+  void on_request(uint64 id, td_api::deleteLanguage &request);
+
   void on_request(uint64 id, td_api::getOption &request);
 
   void on_request(uint64 id, td_api::setOption &request);

td/telegram/cli.cpp

@@ -1735,8 +1735,8 @@ class CliClient final : public Actor {
 
       std::tie(chat_id, message_id) = split(args);
       send_request(make_tl_object<td_api::deleteChatReplyMarkup>(as_chat_id(chat_id), as_message_id(message_id)));
-    } else if (op == "glp") {
-      send_request(make_tl_object<td_api::getLanguagePack>());
+    } else if (op == "glpi") {
+      send_request(make_tl_object<td_api::getLanguagePackInfo>());
     } else if (op == "glps") {
       string language_code;
       string keys;
@@ -1754,6 +1754,26 @@ class CliClient final : public Actor {
       std::tie(language_code, key) = split(args);
       send_request(
           make_tl_object<td_api::getLanguagePackString>(language_database_path, language_pack, language_code, key));
+    } else if (op == "scl") {
+      string language_code;
+      string name;
+      string native_name;
+      string key;
+
+      std::tie(language_code, args) = split(args);
+      std::tie(name, args) = split(args);
+      std::tie(native_name, key) = split(args);
+
+      vector<tl_object_ptr<td_api::LanguagePackString>> strings;
+      strings.push_back(make_tl_object<td_api::languagePackStringValue>(key, "Ordinary value"));
+      strings.push_back(make_tl_object<td_api::languagePackStringPluralized>("Plu", "Zero", string("One\0One", 7), "Two", "Few",
+                                                                             "Many", "Other"));
+      strings.push_back(make_tl_object<td_api::languagePackStringDeleted>("DELETED"));
+
+      send_request(make_tl_object<td_api::setCustomLanguage>(
+          make_tl_object<td_api::languageInfo>(language_code, name, native_name), std::move(strings)));
+    } else if (op == "dl") {
+      send_request(make_tl_object<td_api::deleteLanguage>(args));
     } else if (op == "go") {
       send_request(make_tl_object<td_api::getOption>(args));
     } else if (op == "sob") {

td/telegram/net/MtprotoHeader.cpp

@@ -6,6 +6,8 @@
 //
 #include "td/telegram/net/MtprotoHeader.h"
 
+#include "td/telegram/LanguagePackManager.h"
+
 #include "td/utils/tl_helpers.h"
 
 namespace td {
@@ -47,7 +49,8 @@ class HeaderStorer {
     }
     store(options.application_version, storer);
     store(options.system_language_code, storer);
-    if (is_anonymous || options.language_pack.empty()) {
+    if (is_anonymous || options.language_pack.empty() ||
+        LanguagePackManager::is_custom_language_code(options.language_code)) {
       store(Slice(), storer);
       store(Slice(), storer);
     } else {

tddb/td/db/SqliteKeyValue.cpp

@@ -24,9 +24,7 @@ Result<bool> SqliteKeyValue::init(string path) {
 }
 
 Status SqliteKeyValue::init_with_connection(SqliteDb connection, string table_name) {
-  auto init_guard = ScopeExit() + [&]() {
-    clear();
-  };
+  auto init_guard = ScopeExit() + [&]() { close(); };
   db_ = std::move(connection);
   table_name_ = std::move(table_name);
   TRY_STATUS(init(db_, table_name_));
@@ -60,13 +58,14 @@ Status SqliteKeyValue::init_with_connection(SqliteDb connection, string table_na
   return Status::OK();
 }
 
-void SqliteKeyValue::close_and_destroy() {
-  drop(db_, table_name_).ensure();
-  auto path = std::move(path_);
-  clear();
-  if (!path.empty()) {
-    destroy(path).ignore();
+Status SqliteKeyValue::drop() {
+  if (empty()) {
+    return Status::OK();
   }
+
+  auto result = drop(db_, table_name_);
+  close();
+  return result;
 }
 
 SqliteKeyValue::SeqNo SqliteKeyValue::set(Slice key, Slice value) {

tddb/td/db/SqliteKeyValue.h

@@ -42,14 +42,10 @@ class SqliteKeyValue {
   }
 
   void close() {
-    clear();
+    *this = SqliteKeyValue();
   }
 
-  static Status destroy(Slice path) TD_WARN_UNUSED_RESULT {
-    return SqliteDb::destroy(path);
-  }
-
-  void close_and_destroy();
+  Status drop();
 
   SeqNo set(Slice key, Slice value);
 
@@ -104,10 +100,6 @@ class SqliteKeyValue {
     }
   }
 
-  void clear() {
-    *this = SqliteKeyValue();
-  }
-
  private:
   string path_;
   string table_name_;