2018-12-31 20:04:05 +01:00
|
|
|
//
|
2022-01-01 01:35:39 +01:00
|
|
|
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022
|
2018-12-31 20:04:05 +01:00
|
|
|
//
|
|
|
|
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
|
|
|
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
|
|
|
//
|
|
|
|
#include "td/telegram/TdDb.h"
|
|
|
|
|
|
|
|
#include "td/telegram/DialogDb.h"
|
|
|
|
#include "td/telegram/files/FileDb.h"
|
2019-04-03 01:22:34 +02:00
|
|
|
#include "td/telegram/Global.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
#include "td/telegram/logevent/LogEvent.h"
|
|
|
|
#include "td/telegram/MessagesDb.h"
|
2019-04-11 22:24:39 +02:00
|
|
|
#include "td/telegram/Td.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
#include "td/telegram/TdParameters.h"
|
2018-04-09 20:06:37 +02:00
|
|
|
#include "td/telegram/Version.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
|
2019-01-07 00:49:49 +01:00
|
|
|
#include "td/db/binlog/Binlog.h"
|
2019-01-07 00:44:29 +01:00
|
|
|
#include "td/db/binlog/ConcurrentBinlog.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
#include "td/db/BinlogKeyValue.h"
|
2019-01-07 01:17:11 +01:00
|
|
|
#include "td/db/SqliteConnectionSafe.h"
|
2019-04-26 00:47:25 +02:00
|
|
|
#include "td/db/SqliteDb.h"
|
2018-07-18 03:11:48 +02:00
|
|
|
#include "td/db/SqliteKeyValue.h"
|
|
|
|
#include "td/db/SqliteKeyValueAsync.h"
|
|
|
|
#include "td/db/SqliteKeyValueSafe.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
|
2022-06-14 15:11:38 +02:00
|
|
|
#include "td/actor/actor.h"
|
2021-09-18 23:47:05 +02:00
|
|
|
#include "td/actor/MultiPromise.h"
|
|
|
|
|
2019-02-12 22:26:36 +01:00
|
|
|
#include "td/utils/common.h"
|
2019-04-22 02:46:51 +02:00
|
|
|
#include "td/utils/format.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
#include "td/utils/logging.h"
|
2020-06-07 17:14:52 +02:00
|
|
|
#include "td/utils/misc.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
#include "td/utils/port/path.h"
|
|
|
|
#include "td/utils/Random.h"
|
2021-05-17 14:21:11 +02:00
|
|
|
#include "td/utils/SliceBuilder.h"
|
2019-04-22 02:46:51 +02:00
|
|
|
#include "td/utils/StringBuilder.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
|
2020-06-07 17:14:52 +02:00
|
|
|
#include <algorithm>
|
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
namespace td {
|
2018-07-20 19:50:38 +02:00
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
namespace {
|
2018-07-20 19:50:38 +02:00
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
std::string get_binlog_path(const TdParameters ¶meters) {
|
|
|
|
return PSTRING() << parameters.database_directory << "td" << (parameters.use_test_dc ? "_test" : "") << ".binlog";
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string get_sqlite_path(const TdParameters ¶meters) {
|
|
|
|
const string db_name = "db" + (parameters.use_test_dc ? string("_test") : string());
|
|
|
|
return parameters.database_directory + db_name + ".sqlite";
|
|
|
|
}
|
|
|
|
|
|
|
|
Status init_binlog(Binlog &binlog, string path, BinlogKeyValue<Binlog> &binlog_pmc, BinlogKeyValue<Binlog> &config_pmc,
|
2022-06-10 15:44:53 +02:00
|
|
|
TdDb::OpenedDatabase &events, DbKey key) {
|
2018-12-31 20:04:05 +01:00
|
|
|
auto callback = [&](const BinlogEvent &event) {
|
|
|
|
switch (event.type_) {
|
|
|
|
case LogEvent::HandlerType::SecretChats:
|
|
|
|
events.to_secret_chats_manager.push_back(event.clone());
|
|
|
|
break;
|
|
|
|
case LogEvent::HandlerType::Users:
|
|
|
|
events.user_events.push_back(event.clone());
|
|
|
|
break;
|
|
|
|
case LogEvent::HandlerType::Chats:
|
|
|
|
events.chat_events.push_back(event.clone());
|
|
|
|
break;
|
|
|
|
case LogEvent::HandlerType::Channels:
|
|
|
|
events.channel_events.push_back(event.clone());
|
|
|
|
break;
|
|
|
|
case LogEvent::HandlerType::SecretChatInfos:
|
|
|
|
events.secret_chat_events.push_back(event.clone());
|
|
|
|
break;
|
|
|
|
case LogEvent::HandlerType::WebPages:
|
|
|
|
events.web_page_events.push_back(event.clone());
|
|
|
|
break;
|
2019-02-21 15:40:37 +01:00
|
|
|
case LogEvent::HandlerType::SetPollAnswer:
|
2019-02-22 16:09:55 +01:00
|
|
|
case LogEvent::HandlerType::StopPoll:
|
2019-02-21 15:40:37 +01:00
|
|
|
events.to_poll_manager.push_back(event.clone());
|
|
|
|
break;
|
2018-12-31 20:04:05 +01:00
|
|
|
case LogEvent::HandlerType::SendMessage:
|
|
|
|
case LogEvent::HandlerType::DeleteMessage:
|
2021-10-27 14:38:09 +02:00
|
|
|
case LogEvent::HandlerType::DeleteMessagesOnServer:
|
2018-12-31 20:04:05 +01:00
|
|
|
case LogEvent::HandlerType::ReadHistoryOnServer:
|
|
|
|
case LogEvent::HandlerType::ReadMessageContentsOnServer:
|
|
|
|
case LogEvent::HandlerType::ForwardMessages:
|
|
|
|
case LogEvent::HandlerType::SendBotStartMessage:
|
|
|
|
case LogEvent::HandlerType::SendScreenshotTakenNotificationMessage:
|
|
|
|
case LogEvent::HandlerType::SendInlineQueryResultMessage:
|
2021-10-27 14:38:09 +02:00
|
|
|
case LogEvent::HandlerType::DeleteDialogHistoryOnServer:
|
2018-12-31 20:04:05 +01:00
|
|
|
case LogEvent::HandlerType::ReadAllDialogMentionsOnServer:
|
2021-11-19 14:00:21 +01:00
|
|
|
case LogEvent::HandlerType::DeleteAllChannelMessagesFromSenderOnServer:
|
2018-12-31 20:04:05 +01:00
|
|
|
case LogEvent::HandlerType::ToggleDialogIsPinnedOnServer:
|
|
|
|
case LogEvent::HandlerType::ReorderPinnedDialogsOnServer:
|
|
|
|
case LogEvent::HandlerType::SaveDialogDraftMessageOnServer:
|
2018-04-28 21:31:42 +02:00
|
|
|
case LogEvent::HandlerType::UpdateDialogNotificationSettingsOnServer:
|
2018-04-28 21:50:12 +02:00
|
|
|
case LogEvent::HandlerType::ResetAllNotificationSettingsOnServer:
|
2021-09-24 09:42:39 +02:00
|
|
|
case LogEvent::HandlerType::ToggleDialogReportSpamStateOnServer:
|
2021-10-27 14:27:09 +02:00
|
|
|
case LogEvent::HandlerType::RegetDialog:
|
2018-12-31 20:04:05 +01:00
|
|
|
case LogEvent::HandlerType::GetChannelDifference:
|
2018-06-20 03:02:02 +02:00
|
|
|
case LogEvent::HandlerType::ReadHistoryInSecretChat:
|
2018-06-27 23:08:44 +02:00
|
|
|
case LogEvent::HandlerType::ToggleDialogIsMarkedAsUnreadOnServer:
|
2019-08-27 16:23:01 +02:00
|
|
|
case LogEvent::HandlerType::SetDialogFolderIdOnServer:
|
2021-10-27 14:38:09 +02:00
|
|
|
case LogEvent::HandlerType::DeleteScheduledMessagesOnServer:
|
2020-09-20 02:00:01 +02:00
|
|
|
case LogEvent::HandlerType::ToggleDialogIsBlockedOnServer:
|
2020-09-22 00:13:06 +02:00
|
|
|
case LogEvent::HandlerType::ReadMessageThreadHistoryOnServer:
|
2020-10-18 00:26:36 +02:00
|
|
|
case LogEvent::HandlerType::BlockMessageSenderFromRepliesOnServer:
|
2020-10-23 00:56:06 +02:00
|
|
|
case LogEvent::HandlerType::UnpinAllDialogMessagesOnServer:
|
2021-10-27 14:38:09 +02:00
|
|
|
case LogEvent::HandlerType::DeleteAllCallMessagesOnServer:
|
2021-10-25 19:39:22 +02:00
|
|
|
case LogEvent::HandlerType::DeleteDialogMessagesByDateOnServer:
|
2022-01-30 10:37:24 +01:00
|
|
|
case LogEvent::HandlerType::ReadAllDialogReactionsOnServer:
|
2018-12-31 20:04:05 +01:00
|
|
|
events.to_messages_manager.push_back(event.clone());
|
|
|
|
break;
|
2022-04-11 11:45:52 +02:00
|
|
|
case LogEvent::HandlerType::UpdateScopeNotificationSettingsOnServer:
|
|
|
|
events.to_notification_settings_manager.push_back(event.clone());
|
|
|
|
break;
|
2019-03-31 03:30:25 +02:00
|
|
|
case LogEvent::HandlerType::AddMessagePushNotification:
|
2019-04-07 22:57:54 +02:00
|
|
|
case LogEvent::HandlerType::EditMessagePushNotification:
|
2019-03-31 03:30:25 +02:00
|
|
|
events.to_notification_manager.push_back(event.clone());
|
|
|
|
break;
|
2022-07-18 18:21:47 +02:00
|
|
|
case LogEvent::HandlerType::SaveAppLog:
|
|
|
|
events.save_app_log_events.push_back(event.clone());
|
2019-03-31 03:30:25 +02:00
|
|
|
break;
|
2018-12-31 20:04:05 +01:00
|
|
|
case LogEvent::HandlerType::BinlogPmcMagic:
|
|
|
|
binlog_pmc.external_init_handle(event);
|
|
|
|
break;
|
|
|
|
case LogEvent::HandlerType::ConfigPmcMagic:
|
|
|
|
config_pmc.external_init_handle(event);
|
|
|
|
break;
|
|
|
|
default:
|
2020-09-22 01:15:09 +02:00
|
|
|
LOG(FATAL) << "Unsupported log event type " << event.type_;
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
auto binlog_info = binlog.init(std::move(path), callback, std::move(key));
|
|
|
|
if (binlog_info.is_error()) {
|
|
|
|
return binlog_info.move_as_error();
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
} // namespace
|
|
|
|
|
|
|
|
std::shared_ptr<FileDbInterface> TdDb::get_file_db_shared() {
|
|
|
|
return file_db_;
|
|
|
|
}
|
2019-01-06 22:06:52 +01:00
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
std::shared_ptr<SqliteConnectionSafe> &TdDb::get_sqlite_connection_safe() {
|
|
|
|
return sql_connection_;
|
|
|
|
}
|
2019-01-06 22:06:52 +01:00
|
|
|
|
2019-04-03 11:26:20 +02:00
|
|
|
BinlogInterface *TdDb::get_binlog_impl(const char *file, int line) {
|
|
|
|
LOG_CHECK(binlog_) << G()->close_flag() << " " << file << " " << line;
|
2018-12-31 20:04:05 +01:00
|
|
|
return binlog_.get();
|
|
|
|
}
|
2019-01-06 22:06:52 +01:00
|
|
|
|
|
|
|
std::shared_ptr<KeyValueSyncInterface> TdDb::get_binlog_pmc_shared() {
|
2019-01-07 00:44:29 +01:00
|
|
|
CHECK(binlog_pmc_);
|
2018-12-31 20:04:05 +01:00
|
|
|
return binlog_pmc_;
|
|
|
|
}
|
2019-01-06 22:06:52 +01:00
|
|
|
|
2019-01-06 23:58:09 +01:00
|
|
|
std::shared_ptr<KeyValueSyncInterface> TdDb::get_config_pmc_shared() {
|
2019-01-07 00:44:29 +01:00
|
|
|
CHECK(config_pmc_);
|
2019-01-06 23:20:38 +01:00
|
|
|
return config_pmc_;
|
|
|
|
}
|
|
|
|
|
2019-01-07 00:44:29 +01:00
|
|
|
KeyValueSyncInterface *TdDb::get_binlog_pmc() {
|
2018-12-31 20:04:05 +01:00
|
|
|
CHECK(binlog_pmc_);
|
|
|
|
return binlog_pmc_.get();
|
|
|
|
}
|
2019-01-06 22:06:52 +01:00
|
|
|
|
2019-01-07 00:44:29 +01:00
|
|
|
KeyValueSyncInterface *TdDb::get_config_pmc() {
|
2018-12-31 20:04:05 +01:00
|
|
|
CHECK(config_pmc_);
|
|
|
|
return config_pmc_.get();
|
|
|
|
}
|
|
|
|
|
2018-07-18 03:11:48 +02:00
|
|
|
SqliteKeyValue *TdDb::get_sqlite_sync_pmc() {
|
2018-12-31 20:04:05 +01:00
|
|
|
CHECK(common_kv_safe_);
|
|
|
|
return &common_kv_safe_->get();
|
|
|
|
}
|
|
|
|
|
|
|
|
SqliteKeyValueAsyncInterface *TdDb::get_sqlite_pmc() {
|
|
|
|
CHECK(common_kv_async_);
|
|
|
|
return common_kv_async_.get();
|
|
|
|
}
|
|
|
|
|
|
|
|
MessagesDbSyncInterface *TdDb::get_messages_db_sync() {
|
|
|
|
return &messages_db_sync_safe_->get();
|
|
|
|
}
|
|
|
|
MessagesDbAsyncInterface *TdDb::get_messages_db_async() {
|
|
|
|
return messages_db_async_.get();
|
|
|
|
}
|
|
|
|
DialogDbSyncInterface *TdDb::get_dialog_db_sync() {
|
|
|
|
return &dialog_db_sync_safe_->get();
|
|
|
|
}
|
|
|
|
DialogDbAsyncInterface *TdDb::get_dialog_db_async() {
|
|
|
|
return dialog_db_async_.get();
|
|
|
|
}
|
|
|
|
|
|
|
|
CSlice TdDb::binlog_path() const {
|
|
|
|
return binlog_->get_path();
|
|
|
|
}
|
|
|
|
CSlice TdDb::sqlite_path() const {
|
|
|
|
return sqlite_path_;
|
|
|
|
}
|
|
|
|
|
|
|
|
void TdDb::flush_all() {
|
2019-01-06 23:20:38 +01:00
|
|
|
LOG(INFO) << "Flush all databases";
|
2018-12-31 20:04:05 +01:00
|
|
|
if (messages_db_async_) {
|
|
|
|
messages_db_async_->force_flush();
|
|
|
|
}
|
|
|
|
binlog_->force_flush();
|
|
|
|
}
|
2019-01-06 23:20:38 +01:00
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
void TdDb::close_all(Promise<> on_finished) {
|
2019-01-06 23:20:38 +01:00
|
|
|
LOG(INFO) << "Close all databases";
|
2018-12-31 20:04:05 +01:00
|
|
|
do_close(std::move(on_finished), false /*destroy_flag*/);
|
|
|
|
}
|
|
|
|
|
|
|
|
void TdDb::close_and_destroy_all(Promise<> on_finished) {
|
2019-01-06 23:20:38 +01:00
|
|
|
LOG(INFO) << "Destroy all databases";
|
2018-12-31 20:04:05 +01:00
|
|
|
do_close(std::move(on_finished), true /*destroy_flag*/);
|
|
|
|
}
|
2019-01-06 23:20:38 +01:00
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
void TdDb::do_close(Promise<> on_finished, bool destroy_flag) {
|
2018-12-12 00:48:56 +01:00
|
|
|
MultiPromiseActorSafe mpas{"TdDbCloseMultiPromiseActor"};
|
2018-12-31 20:04:05 +01:00
|
|
|
mpas.add_promise(PromiseCreator::lambda(
|
|
|
|
[promise = std::move(on_finished), sql_connection = std::move(sql_connection_), destroy_flag](Unit) mutable {
|
|
|
|
if (sql_connection) {
|
2019-02-12 17:17:20 +01:00
|
|
|
LOG_CHECK(sql_connection.unique()) << sql_connection.use_count();
|
2018-12-31 20:04:05 +01:00
|
|
|
if (destroy_flag) {
|
|
|
|
sql_connection->close_and_destroy();
|
|
|
|
} else {
|
|
|
|
sql_connection->close();
|
|
|
|
}
|
|
|
|
sql_connection.reset();
|
|
|
|
}
|
|
|
|
promise.set_value(Unit());
|
|
|
|
}));
|
|
|
|
auto lock = mpas.get_promise();
|
|
|
|
|
|
|
|
if (file_db_) {
|
|
|
|
file_db_->close(mpas.get_promise());
|
|
|
|
file_db_.reset();
|
|
|
|
}
|
|
|
|
|
|
|
|
common_kv_safe_.reset();
|
|
|
|
if (common_kv_async_) {
|
|
|
|
common_kv_async_->close(mpas.get_promise());
|
|
|
|
}
|
|
|
|
|
|
|
|
messages_db_sync_safe_.reset();
|
|
|
|
if (messages_db_async_) {
|
|
|
|
messages_db_async_->close(mpas.get_promise());
|
|
|
|
}
|
|
|
|
|
|
|
|
dialog_db_sync_safe_.reset();
|
|
|
|
if (dialog_db_async_) {
|
|
|
|
dialog_db_async_->close(mpas.get_promise());
|
|
|
|
}
|
|
|
|
|
|
|
|
// binlog_pmc is dependent on binlog_ and anyway it doesn't support close_and_destroy
|
|
|
|
CHECK(binlog_pmc_.unique());
|
|
|
|
binlog_pmc_.reset();
|
|
|
|
CHECK(config_pmc_.unique());
|
|
|
|
config_pmc_.reset();
|
|
|
|
|
|
|
|
if (binlog_) {
|
|
|
|
if (destroy_flag) {
|
|
|
|
binlog_->close_and_destroy(mpas.get_promise());
|
|
|
|
} else {
|
|
|
|
binlog_->close(mpas.get_promise());
|
|
|
|
}
|
|
|
|
binlog_.reset();
|
|
|
|
}
|
2021-10-07 17:14:49 +02:00
|
|
|
|
|
|
|
lock.set_value(Unit());
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
|
2022-06-29 23:46:02 +02:00
|
|
|
Status TdDb::init_sqlite(const TdParameters ¶meters, const DbKey &key, const DbKey &old_key,
|
2018-12-31 20:04:05 +01:00
|
|
|
BinlogKeyValue<Binlog> &binlog_pmc) {
|
|
|
|
CHECK(!parameters.use_message_db || parameters.use_chat_info_db);
|
|
|
|
CHECK(!parameters.use_chat_info_db || parameters.use_file_db);
|
|
|
|
|
2019-10-03 19:38:47 +02:00
|
|
|
const string sql_database_path = get_sqlite_path(parameters);
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
bool use_sqlite = parameters.use_file_db;
|
|
|
|
bool use_file_db = parameters.use_file_db;
|
|
|
|
bool use_dialog_db = parameters.use_message_db;
|
|
|
|
bool use_message_db = parameters.use_message_db;
|
|
|
|
if (!use_sqlite) {
|
2019-10-03 19:38:47 +02:00
|
|
|
unlink(sql_database_path).ignore();
|
2018-12-31 20:04:05 +01:00
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2019-10-03 19:38:47 +02:00
|
|
|
sqlite_path_ = sql_database_path;
|
2021-09-22 18:04:56 +02:00
|
|
|
TRY_RESULT(db_instance, SqliteDb::change_key(sqlite_path_, true, key, old_key));
|
2020-08-14 16:48:43 +02:00
|
|
|
sql_connection_ = std::make_shared<SqliteConnectionSafe>(sql_database_path, key, db_instance.get_cipher_version());
|
2020-08-14 16:11:58 +02:00
|
|
|
sql_connection_->set(std::move(db_instance));
|
2018-12-31 20:04:05 +01:00
|
|
|
auto &db = sql_connection_->get();
|
2022-01-21 14:52:49 +01:00
|
|
|
if (parameters.use_custom_db_format) {
|
|
|
|
TRY_STATUS(db.exec("PRAGMA journal_mode=OFF"));
|
|
|
|
TRY_STATUS(db.exec("PRAGMA synchronous=OFF"))
|
|
|
|
TRY_STATUS(db.exec("PRAGMA temp_store=MEMORY"));
|
|
|
|
TRY_STATUS(db.exec("PRAGMA secure_delete=0"));
|
|
|
|
TRY_STATUS(db.exec("PRAGMA mmap_size=30000000000"));
|
|
|
|
} else {
|
|
|
|
TRY_STATUS(db.exec("PRAGMA journal_mode=WAL"));
|
|
|
|
TRY_STATUS(db.exec("PRAGMA secure_delete=1"));
|
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
// Init databases
|
|
|
|
// Do initialization once and before everything else to avoid "database is locked" error.
|
|
|
|
// Must be in a transaction
|
|
|
|
|
|
|
|
// NB: when database is dropped we should also drop corresponding binlog events
|
|
|
|
TRY_STATUS(db.exec("BEGIN TRANSACTION"));
|
|
|
|
|
|
|
|
// Get 'PRAGMA user_version'
|
|
|
|
TRY_RESULT(user_version, db.user_version());
|
2021-11-17 07:09:01 +01:00
|
|
|
LOG(INFO) << "Got PRAGMA user_version = " << user_version;
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
// init DialogDb
|
|
|
|
bool dialog_db_was_created = false;
|
|
|
|
if (use_dialog_db) {
|
2020-05-03 00:10:54 +02:00
|
|
|
TRY_STATUS(init_dialog_db(db, user_version, binlog_pmc, dialog_db_was_created));
|
2018-12-31 20:04:05 +01:00
|
|
|
} else {
|
|
|
|
TRY_STATUS(drop_dialog_db(db, user_version));
|
|
|
|
}
|
|
|
|
|
|
|
|
// init MessagesDb
|
|
|
|
if (use_message_db) {
|
|
|
|
TRY_STATUS(init_messages_db(db, user_version));
|
|
|
|
} else {
|
|
|
|
TRY_STATUS(drop_messages_db(db, user_version));
|
|
|
|
}
|
|
|
|
|
|
|
|
// init filesDb
|
|
|
|
if (use_file_db) {
|
|
|
|
TRY_STATUS(init_file_db(db, user_version));
|
|
|
|
} else {
|
|
|
|
TRY_STATUS(drop_file_db(db, user_version));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Update 'PRAGMA user_version'
|
|
|
|
auto db_version = current_db_version();
|
|
|
|
if (db_version != user_version) {
|
2018-10-08 15:29:44 +02:00
|
|
|
LOG(WARNING) << "Set PRAGMA user_version = " << db_version;
|
2018-12-31 20:04:05 +01:00
|
|
|
TRY_STATUS(db.set_user_version(db_version));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (dialog_db_was_created) {
|
2020-05-03 00:10:54 +02:00
|
|
|
binlog_pmc.erase_by_prefix("pinned_dialog_ids");
|
2019-08-22 17:24:02 +02:00
|
|
|
binlog_pmc.erase_by_prefix("last_server_dialog_date");
|
|
|
|
binlog_pmc.erase_by_prefix("unread_message_count");
|
|
|
|
binlog_pmc.erase_by_prefix("unread_dialog_count");
|
2018-06-13 23:07:24 +02:00
|
|
|
binlog_pmc.erase("sponsored_dialog_id");
|
2018-12-31 20:04:05 +01:00
|
|
|
binlog_pmc.erase_by_prefix("top_dialogs");
|
2022-03-02 14:36:23 +01:00
|
|
|
binlog_pmc.erase("dlds_counter");
|
|
|
|
binlog_pmc.erase_by_prefix("dlds#");
|
2018-10-08 15:29:44 +02:00
|
|
|
}
|
|
|
|
if (user_version == 0) {
|
2018-12-31 20:04:05 +01:00
|
|
|
binlog_pmc.erase("next_contacts_sync_date");
|
2021-09-26 20:15:31 +02:00
|
|
|
binlog_pmc.erase("saved_contact_count");
|
|
|
|
binlog_pmc.erase("old_featured_sticker_set_count");
|
|
|
|
binlog_pmc.erase("invalidate_old_featured_sticker_sets");
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
binlog_pmc.force_sync({});
|
|
|
|
|
|
|
|
TRY_STATUS(db.exec("COMMIT TRANSACTION"));
|
|
|
|
|
2022-06-29 23:46:02 +02:00
|
|
|
file_db_ = create_file_db(sql_connection_);
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
common_kv_safe_ = std::make_shared<SqliteKeyValueSafe>("common", sql_connection_);
|
2022-06-29 23:46:02 +02:00
|
|
|
common_kv_async_ = create_sqlite_key_value_async(common_kv_safe_);
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
if (use_dialog_db) {
|
|
|
|
dialog_db_sync_safe_ = create_dialog_db_sync(sql_connection_);
|
2022-06-29 23:46:02 +02:00
|
|
|
dialog_db_async_ = create_dialog_db_async(dialog_db_sync_safe_);
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
|
2022-06-30 19:51:59 +02:00
|
|
|
if (use_message_db) {
|
2018-12-31 20:04:05 +01:00
|
|
|
messages_db_sync_safe_ = create_messages_db_sync(sql_connection_);
|
2022-06-29 23:46:02 +02:00
|
|
|
messages_db_async_ = create_messages_db_async(messages_db_sync_safe_);
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2022-06-10 15:44:53 +02:00
|
|
|
void TdDb::open(int32 scheduler_id, TdParameters parameters, DbKey key, Promise<OpenedDatabase> &&promise) {
|
2022-06-29 23:46:02 +02:00
|
|
|
Scheduler::instance()->run_on_scheduler(
|
|
|
|
scheduler_id, [parameters = std::move(parameters), key = std::move(key), promise = std::move(promise)](
|
|
|
|
Unit) mutable { TdDb::open_impl(std::move(parameters), std::move(key), std::move(promise)); });
|
2022-06-29 23:28:25 +02:00
|
|
|
}
|
|
|
|
|
2022-06-29 23:46:02 +02:00
|
|
|
void TdDb::open_impl(TdParameters parameters, DbKey key, Promise<OpenedDatabase> &&promise) {
|
2022-06-10 15:44:53 +02:00
|
|
|
OpenedDatabase result;
|
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
// Init pmc
|
|
|
|
Binlog *binlog_ptr = nullptr;
|
|
|
|
auto binlog = std::shared_ptr<Binlog>(new Binlog, [&](Binlog *ptr) { binlog_ptr = ptr; });
|
|
|
|
|
2018-09-27 03:19:03 +02:00
|
|
|
auto binlog_pmc = make_unique<BinlogKeyValue<Binlog>>();
|
|
|
|
auto config_pmc = make_unique<BinlogKeyValue<Binlog>>();
|
2018-12-31 20:04:05 +01:00
|
|
|
binlog_pmc->external_init_begin(static_cast<int32>(LogEvent::HandlerType::BinlogPmcMagic));
|
|
|
|
config_pmc->external_init_begin(static_cast<int32>(LogEvent::HandlerType::ConfigPmcMagic));
|
|
|
|
|
|
|
|
bool encrypt_binlog = !key.is_empty();
|
2019-04-11 22:24:39 +02:00
|
|
|
VLOG(td_init) << "Start binlog loading";
|
2022-06-10 15:44:53 +02:00
|
|
|
TRY_STATUS_PROMISE(
|
|
|
|
promise, init_binlog(*binlog, get_binlog_path(parameters), *binlog_pmc, *config_pmc, result, std::move(key)));
|
2019-04-11 22:24:39 +02:00
|
|
|
VLOG(td_init) << "Finish binlog loading";
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
binlog_pmc->external_init_finish(binlog);
|
2019-04-11 22:24:39 +02:00
|
|
|
VLOG(td_init) << "Finish initialization of binlog PMC";
|
2018-12-31 20:04:05 +01:00
|
|
|
config_pmc->external_init_finish(binlog);
|
2019-04-11 22:24:39 +02:00
|
|
|
VLOG(td_init) << "Finish initialization of config PMC";
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
DbKey new_sqlite_key;
|
|
|
|
DbKey old_sqlite_key;
|
|
|
|
bool encrypt_sqlite = encrypt_binlog;
|
|
|
|
bool drop_sqlite_key = false;
|
|
|
|
auto sqlite_key = binlog_pmc->get("sqlite_key");
|
|
|
|
if (encrypt_sqlite) {
|
|
|
|
if (sqlite_key.empty()) {
|
|
|
|
sqlite_key = string(32, ' ');
|
|
|
|
Random::secure_bytes(sqlite_key);
|
|
|
|
binlog_pmc->set("sqlite_key", sqlite_key);
|
|
|
|
binlog_pmc->force_sync(Auto());
|
|
|
|
}
|
|
|
|
new_sqlite_key = DbKey::raw_key(std::move(sqlite_key));
|
|
|
|
} else {
|
|
|
|
if (!sqlite_key.empty()) {
|
|
|
|
old_sqlite_key = DbKey::raw_key(std::move(sqlite_key));
|
|
|
|
drop_sqlite_key = true;
|
|
|
|
}
|
|
|
|
}
|
2019-04-11 22:24:39 +02:00
|
|
|
VLOG(td_init) << "Start to init database";
|
2022-06-10 15:44:53 +02:00
|
|
|
auto db = make_unique<TdDb>();
|
2022-06-29 23:46:02 +02:00
|
|
|
auto init_sqlite_status = db->init_sqlite(parameters, new_sqlite_key, old_sqlite_key, *binlog_pmc);
|
2019-04-11 22:24:39 +02:00
|
|
|
VLOG(td_init) << "Finish to init database";
|
2018-12-31 20:04:05 +01:00
|
|
|
if (init_sqlite_status.is_error()) {
|
2018-12-13 23:48:36 +01:00
|
|
|
LOG(ERROR) << "Destroy bad SQLite database because of " << init_sqlite_status;
|
2022-06-10 15:44:53 +02:00
|
|
|
if (db->sql_connection_ != nullptr) {
|
|
|
|
db->sql_connection_->get().close();
|
2019-09-16 21:01:55 +02:00
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
SqliteDb::destroy(get_sqlite_path(parameters)).ignore();
|
2022-06-29 23:46:02 +02:00
|
|
|
TRY_STATUS_PROMISE(promise, db->init_sqlite(parameters, new_sqlite_key, old_sqlite_key, *binlog_pmc));
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
if (drop_sqlite_key) {
|
|
|
|
binlog_pmc->erase("sqlite_key");
|
|
|
|
binlog_pmc->force_sync(Auto());
|
|
|
|
}
|
|
|
|
|
2019-04-11 22:24:39 +02:00
|
|
|
VLOG(td_init) << "Create concurrent_binlog_pmc";
|
2018-12-31 20:04:05 +01:00
|
|
|
auto concurrent_binlog_pmc = std::make_shared<BinlogKeyValue<ConcurrentBinlog>>();
|
|
|
|
concurrent_binlog_pmc->external_init_begin(binlog_pmc->get_magic());
|
|
|
|
concurrent_binlog_pmc->external_init_handle(std::move(*binlog_pmc));
|
|
|
|
|
2019-04-11 22:24:39 +02:00
|
|
|
VLOG(td_init) << "Create concurrent_config_pmc";
|
2018-12-31 20:04:05 +01:00
|
|
|
auto concurrent_config_pmc = std::make_shared<BinlogKeyValue<ConcurrentBinlog>>();
|
|
|
|
concurrent_config_pmc->external_init_begin(config_pmc->get_magic());
|
|
|
|
concurrent_config_pmc->external_init_handle(std::move(*config_pmc));
|
|
|
|
|
|
|
|
binlog.reset();
|
|
|
|
binlog_pmc.reset();
|
|
|
|
config_pmc.reset();
|
|
|
|
|
|
|
|
CHECK(binlog_ptr != nullptr);
|
2019-04-11 22:24:39 +02:00
|
|
|
VLOG(td_init) << "Create concurrent_binlog";
|
2022-06-29 23:46:02 +02:00
|
|
|
auto concurrent_binlog = std::make_shared<ConcurrentBinlog>(unique_ptr<Binlog>(binlog_ptr));
|
2018-12-31 20:04:05 +01:00
|
|
|
|
2019-04-11 22:24:39 +02:00
|
|
|
VLOG(td_init) << "Init concurrent_binlog_pmc";
|
2018-12-31 20:04:05 +01:00
|
|
|
concurrent_binlog_pmc->external_init_finish(concurrent_binlog);
|
2019-04-11 22:24:39 +02:00
|
|
|
VLOG(td_init) << "Init concurrent_config_pmc";
|
2018-12-31 20:04:05 +01:00
|
|
|
concurrent_config_pmc->external_init_finish(concurrent_binlog);
|
|
|
|
|
2022-06-10 15:44:53 +02:00
|
|
|
db->binlog_pmc_ = std::move(concurrent_binlog_pmc);
|
|
|
|
db->config_pmc_ = std::move(concurrent_config_pmc);
|
|
|
|
db->binlog_ = std::move(concurrent_binlog);
|
2018-12-31 20:04:05 +01:00
|
|
|
|
2022-06-10 15:44:53 +02:00
|
|
|
result.database = std::move(db);
|
|
|
|
|
|
|
|
promise.set_value(std::move(result));
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
TdDb::TdDb() = default;
|
|
|
|
TdDb::~TdDb() = default;
|
|
|
|
|
2022-06-21 01:02:48 +02:00
|
|
|
void TdDb::check_parameters(int32 scheduler_id, TdParameters parameters, Promise<CheckedParameters> promise) {
|
2022-06-29 23:28:25 +02:00
|
|
|
Scheduler::instance()->run_on_scheduler(
|
|
|
|
scheduler_id, [parameters = std::move(parameters), promise = std::move(promise)](Unit) mutable {
|
|
|
|
TdDb::check_parameters_impl(std::move(parameters), std::move(promise));
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
void TdDb::check_parameters_impl(TdParameters parameters, Promise<CheckedParameters> promise) {
|
2022-06-21 01:02:48 +02:00
|
|
|
CheckedParameters result;
|
|
|
|
|
|
|
|
auto prepare_dir = [](string dir) -> Result<string> {
|
|
|
|
CHECK(!dir.empty());
|
|
|
|
if (dir.back() != TD_DIR_SLASH) {
|
|
|
|
dir += TD_DIR_SLASH;
|
|
|
|
}
|
|
|
|
TRY_STATUS(mkpath(dir, 0750));
|
|
|
|
TRY_RESULT(real_dir, realpath(dir, true));
|
|
|
|
if (dir.back() != TD_DIR_SLASH) {
|
|
|
|
dir += TD_DIR_SLASH;
|
|
|
|
}
|
|
|
|
return real_dir;
|
|
|
|
};
|
|
|
|
|
|
|
|
auto r_database_directory = prepare_dir(parameters.database_directory);
|
|
|
|
if (r_database_directory.is_error()) {
|
|
|
|
VLOG(td_init) << "Invalid database_directory";
|
|
|
|
return promise.set_error(Status::Error(PSLICE()
|
|
|
|
<< "Can't init database in the directory \"" << parameters.database_directory
|
|
|
|
<< "\": " << r_database_directory.error()));
|
|
|
|
}
|
|
|
|
result.database_directory = r_database_directory.move_as_ok();
|
|
|
|
parameters.database_directory = result.database_directory;
|
|
|
|
|
|
|
|
auto r_files_directory = prepare_dir(parameters.files_directory);
|
|
|
|
if (r_files_directory.is_error()) {
|
|
|
|
VLOG(td_init) << "Invalid files_directory";
|
|
|
|
return promise.set_error(Status::Error(PSLICE() << "Can't init files directory \"" << parameters.files_directory
|
|
|
|
<< "\": " << r_files_directory.error()));
|
|
|
|
}
|
|
|
|
result.files_directory = r_files_directory.move_as_ok();
|
|
|
|
|
|
|
|
Binlog binlog;
|
|
|
|
auto status = binlog.init(get_binlog_path(parameters), Binlog::Callback());
|
|
|
|
if (status.is_error() && status.code() != Binlog::Error::WrongPassword) {
|
|
|
|
LOG(WARNING) << "Failed to check binlog: " << status;
|
|
|
|
return promise.set_error(std::move(status));
|
|
|
|
}
|
|
|
|
result.is_database_encrypted = binlog.get_info().wrong_password;
|
|
|
|
binlog.close(false /*need_sync*/).ensure();
|
|
|
|
|
|
|
|
promise.set_value(std::move(result));
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
2019-04-11 22:24:39 +02:00
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
void TdDb::change_key(DbKey key, Promise<> promise) {
|
|
|
|
get_binlog()->change_key(std::move(key), std::move(promise));
|
|
|
|
}
|
2019-04-11 22:24:39 +02:00
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
Status TdDb::destroy(const TdParameters ¶meters) {
|
|
|
|
SqliteDb::destroy(get_sqlite_path(parameters)).ignore();
|
|
|
|
Binlog::destroy(get_binlog_path(parameters)).ignore();
|
|
|
|
return Status::OK();
|
|
|
|
}
|
2019-04-11 22:24:39 +02:00
|
|
|
|
2021-10-19 17:11:16 +02:00
|
|
|
void TdDb::with_db_path(const std::function<void(CSlice)> &callback) {
|
2018-12-31 20:04:05 +01:00
|
|
|
SqliteDb::with_db_path(sqlite_path(), callback);
|
|
|
|
callback(binlog_path());
|
|
|
|
}
|
2018-07-20 19:50:38 +02:00
|
|
|
|
2019-04-17 11:17:51 +02:00
|
|
|
Result<string> TdDb::get_stats() {
|
2019-09-15 05:19:46 +02:00
|
|
|
auto sb = StringBuilder({}, true);
|
2019-04-17 11:17:51 +02:00
|
|
|
auto &sql = sql_connection_->get();
|
|
|
|
auto run_query = [&](CSlice query, Slice desc) -> Status {
|
|
|
|
TRY_RESULT(stmt, sql.get_statement(query));
|
|
|
|
TRY_STATUS(stmt.step());
|
|
|
|
CHECK(stmt.has_row());
|
|
|
|
auto key_size = stmt.view_int64(0);
|
|
|
|
auto value_size = stmt.view_int64(1);
|
|
|
|
auto count = stmt.view_int64(2);
|
|
|
|
sb << query << "\n";
|
|
|
|
sb << desc << ":\n";
|
|
|
|
sb << format::as_size(key_size + value_size) << "\t";
|
|
|
|
sb << format::as_size(key_size) << "\t";
|
|
|
|
sb << format::as_size(value_size) << "\t";
|
|
|
|
sb << format::as_size((key_size + value_size) / (count ? count : 1)) << "\t";
|
|
|
|
sb << "\n";
|
|
|
|
return Status::OK();
|
|
|
|
};
|
2019-04-26 04:10:02 +02:00
|
|
|
auto run_kv_query = [&](Slice mask, Slice table = Slice("common")) {
|
2019-04-17 11:17:51 +02:00
|
|
|
return run_query(PSLICE() << "SELECT SUM(length(k)), SUM(length(v)), COUNT(*) FROM " << table << " WHERE k like '"
|
|
|
|
<< mask << "'",
|
|
|
|
PSLICE() << table << ":" << mask);
|
|
|
|
};
|
|
|
|
TRY_STATUS(run_query("SELECT 0, SUM(length(data)), COUNT(*) FROM messages WHERE 1", "messages"));
|
|
|
|
TRY_STATUS(run_query("SELECT 0, SUM(length(data)), COUNT(*) FROM dialogs WHERE 1", "dialogs"));
|
|
|
|
TRY_STATUS(run_kv_query("%", "common"));
|
|
|
|
TRY_STATUS(run_kv_query("%", "files"));
|
2019-04-19 00:11:58 +02:00
|
|
|
TRY_STATUS(run_kv_query("wp%"));
|
2019-04-17 11:17:51 +02:00
|
|
|
TRY_STATUS(run_kv_query("wpurl%"));
|
|
|
|
TRY_STATUS(run_kv_query("wpiv%"));
|
|
|
|
TRY_STATUS(run_kv_query("us%"));
|
|
|
|
TRY_STATUS(run_kv_query("ch%"));
|
|
|
|
TRY_STATUS(run_kv_query("ss%"));
|
|
|
|
TRY_STATUS(run_kv_query("gr%"));
|
2020-06-02 16:38:26 +02:00
|
|
|
|
2020-06-07 17:14:52 +02:00
|
|
|
vector<int32> prev(1);
|
2020-06-02 17:19:44 +02:00
|
|
|
size_t count = 0;
|
|
|
|
int32 max_bad_to = 0;
|
|
|
|
size_t bad_count = 0;
|
2020-06-02 16:38:26 +02:00
|
|
|
file_db_->pmc().get_by_range("file0", "file:", [&](Slice key, Slice value) {
|
|
|
|
if (value.substr(0, 2) != "@@") {
|
|
|
|
return true;
|
|
|
|
}
|
2020-06-02 17:19:44 +02:00
|
|
|
count++;
|
2020-06-07 17:14:52 +02:00
|
|
|
auto from = to_integer<int32>(key.substr(4));
|
|
|
|
auto to = to_integer<int32>(value.substr(2));
|
2020-06-02 17:19:44 +02:00
|
|
|
if (from <= to) {
|
|
|
|
LOG(DEBUG) << "Have forward reference from " << from << " to " << to;
|
|
|
|
if (to > max_bad_to) {
|
|
|
|
max_bad_to = to;
|
|
|
|
}
|
|
|
|
bad_count++;
|
|
|
|
return true;
|
|
|
|
}
|
2020-06-02 16:38:26 +02:00
|
|
|
if (static_cast<size_t>(from) >= prev.size()) {
|
|
|
|
prev.resize(from + 1);
|
|
|
|
}
|
2020-06-02 17:19:44 +02:00
|
|
|
if (static_cast<size_t>(to) >= prev.size()) {
|
|
|
|
prev.resize(to + 1);
|
|
|
|
}
|
2020-06-02 16:38:26 +02:00
|
|
|
prev[from] = to;
|
|
|
|
return true;
|
|
|
|
});
|
|
|
|
for (size_t i = 1; i < prev.size(); i++) {
|
|
|
|
if (!prev[i]) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
prev[i] = prev[prev[i]] + 1;
|
|
|
|
}
|
2020-06-02 17:19:44 +02:00
|
|
|
sb << "Max file database depth out of " << prev.size() << '/' << count
|
|
|
|
<< " elements: " << *std::max_element(prev.begin(), prev.end()) << "\n";
|
|
|
|
sb << "Have " << bad_count << " forward references with maximum reference to " << max_bad_to;
|
2020-06-02 16:38:26 +02:00
|
|
|
|
2019-04-17 11:17:51 +02:00
|
|
|
return sb.as_cslice().str();
|
|
|
|
}
|
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
} // namespace td
|