tdlight/td/telegram/files/FileDb.cpp
2022-01-01 03:35:39 +03:00

298 lines
10 KiB
C++

//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/telegram/files/FileDb.h"
#include "td/telegram/files/FileData.h"
#include "td/telegram/files/FileData.hpp"
#include "td/telegram/files/FileLocation.h"
#include "td/telegram/files/FileLocation.hpp"
#include "td/telegram/logevent/LogEvent.h"
#include "td/telegram/Version.h"
#include "td/db/SqliteConnectionSafe.h"
#include "td/db/SqliteDb.h"
#include "td/db/SqliteKeyValue.h"
#include "td/db/SqliteKeyValueSafe.h"
#include "td/actor/actor.h"
#include "td/utils/format.h"
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include "td/utils/Slice.h"
#include "td/utils/SliceBuilder.h"
#include "td/utils/Status.h"
#include "td/utils/tl_helpers.h"
#include "td/utils/tl_parsers.h"
namespace td {
Status drop_file_db(SqliteDb &db, int32 version) {
LOG(WARNING) << "Drop file_db " << tag("version", version) << tag("current_db_version", current_db_version());
TRY_STATUS(SqliteKeyValue::drop(db, "files"));
return Status::OK();
}
Status init_file_db(SqliteDb &db, int32 version) {
LOG(INFO) << "Init file database " << tag("version", version);
// Check if database exists
TRY_RESULT(has_table, db.has_table("files"));
if (!has_table) {
version = 0;
} else if (version < static_cast<int32>(DbVersion::FixFileRemoteLocationKeyBug)) {
TRY_STATUS(drop_file_db(db, version));
version = 0;
}
if (version == 0) {
TRY_STATUS(SqliteKeyValue::init(db, "files"));
}
return Status::OK();
}
class FileDb final : public FileDbInterface {
public:
class FileDbActor final : public Actor {
public:
FileDbActor(FileDbId current_pmc_id, std::shared_ptr<SqliteKeyValueSafe> file_kv_safe)
: current_pmc_id_(current_pmc_id), file_kv_safe_(std::move(file_kv_safe)) {
}
void close(Promise<> promise) {
file_kv_safe_.reset();
LOG(INFO) << "FileDb is closed";
promise.set_value(Unit());
stop();
}
void load_file_data(const string &key, Promise<FileData> promise) {
promise.set_result(load_file_data_impl(actor_id(this), file_pmc(), key, current_pmc_id_));
}
void clear_file_data(FileDbId id, const string &remote_key, const string &local_key, const string &generate_key) {
auto &pmc = file_pmc();
pmc.begin_write_transaction().ensure();
if (id > current_pmc_id_) {
pmc.set("file_id", to_string(id.get()));
current_pmc_id_ = id;
}
pmc.erase(PSTRING() << "file" << id.get());
// LOG(DEBUG) << "ERASE " << format::as_hex_dump<4>(Slice(PSLICE() << "file" << id.get()));
if (!remote_key.empty()) {
pmc.erase(remote_key);
// LOG(DEBUG) << "ERASE remote " << format::as_hex_dump<4>(Slice(remote_key));
}
if (!local_key.empty()) {
pmc.erase(local_key);
// LOG(DEBUG) << "ERASE local " << format::as_hex_dump<4>(Slice(local_key));
}
if (!generate_key.empty()) {
pmc.erase(generate_key);
}
pmc.commit_transaction().ensure();
}
void store_file_data(FileDbId id, const string &file_data, const string &remote_key, const string &local_key,
const string &generate_key) {
auto &pmc = file_pmc();
pmc.begin_write_transaction().ensure();
if (id > current_pmc_id_) {
pmc.set("file_id", to_string(id.get()));
current_pmc_id_ = id;
}
pmc.set(PSTRING() << "file" << id.get(), file_data);
if (!remote_key.empty()) {
pmc.set(remote_key, to_string(id.get()));
}
if (!local_key.empty()) {
pmc.set(local_key, to_string(id.get()));
}
if (!generate_key.empty()) {
pmc.set(generate_key, to_string(id.get()));
}
pmc.commit_transaction().ensure();
}
void store_file_data_ref(FileDbId id, FileDbId new_id) {
auto &pmc = file_pmc();
pmc.begin_write_transaction().ensure();
if (id > current_pmc_id_) {
pmc.set("file_id", to_string(id.get()));
current_pmc_id_ = id;
}
do_store_file_data_ref(id, new_id);
pmc.commit_transaction().ensure();
}
void optimize_refs(std::vector<FileDbId> ids, FileDbId main_id) {
LOG(INFO) << "Optimize " << ids.size() << " ids in file database to " << main_id.get();
auto &pmc = file_pmc();
pmc.begin_write_transaction().ensure();
for (size_t i = 0; i + 1 < ids.size(); i++) {
do_store_file_data_ref(ids[i], main_id);
}
pmc.commit_transaction().ensure();
}
private:
FileDbId current_pmc_id_;
std::shared_ptr<SqliteKeyValueSafe> file_kv_safe_;
SqliteKeyValue &file_pmc() {
return file_kv_safe_->get();
}
void do_store_file_data_ref(FileDbId id, FileDbId new_id) {
file_pmc().set(PSTRING() << "file" << id.get(), PSTRING() << "@@" << new_id.get());
}
};
explicit FileDb(std::shared_ptr<SqliteKeyValueSafe> kv_safe, int scheduler_id = -1) {
file_kv_safe_ = std::move(kv_safe);
CHECK(file_kv_safe_);
current_pmc_id_ = FileDbId(to_integer<uint64>(file_kv_safe_->get().get("file_id")));
file_db_actor_ =
create_actor_on_scheduler<FileDbActor>("FileDbActor", scheduler_id, current_pmc_id_, file_kv_safe_);
}
FileDbId create_pmc_id() final {
current_pmc_id_ = FileDbId(current_pmc_id_.get() + 1);
return current_pmc_id_;
}
void close(Promise<> promise) final {
send_closure(std::move(file_db_actor_), &FileDbActor::close, std::move(promise));
}
void get_file_data_impl(string key, Promise<FileData> promise) final {
send_closure(file_db_actor_, &FileDbActor::load_file_data, std::move(key), std::move(promise));
}
Result<FileData> get_file_data_sync_impl(string key) final {
return load_file_data_impl(file_db_actor_.get(), file_kv_safe_->get(), key, current_pmc_id_);
}
void clear_file_data(FileDbId id, const FileData &file_data) final {
string remote_key;
if (file_data.remote_.type() == RemoteFileLocation::Type::Full) {
remote_key = as_key(file_data.remote_.full());
}
string local_key;
if (file_data.local_.type() == LocalFileLocation::Type::Full) {
local_key = as_key(file_data.local_.full());
}
string generate_key;
if (file_data.generate_ != nullptr) {
generate_key = as_key(*file_data.generate_);
}
send_closure(file_db_actor_, &FileDbActor::clear_file_data, id, remote_key, local_key, generate_key);
}
void set_file_data(FileDbId id, const FileData &file_data, bool new_remote, bool new_local, bool new_generate) final {
string remote_key;
if (file_data.remote_.type() == RemoteFileLocation::Type::Full && new_remote) {
remote_key = as_key(file_data.remote_.full());
}
string local_key;
if (file_data.local_.type() == LocalFileLocation::Type::Full && new_local) {
local_key = as_key(file_data.local_.full());
}
string generate_key;
if (file_data.generate_ != nullptr && new_generate) {
generate_key = as_key(*file_data.generate_);
}
// LOG(DEBUG) << "SAVE " << id.get() << " -> " << file_data << " "
// << tag("remote_key", format::as_hex_dump<4>(Slice(remote_key)))
// << tag("local_key", format::as_hex_dump<4>(Slice(local_key)))
// << tag("generate_key", format::as_hex_dump<4>(Slice(generate_key)));
send_closure(file_db_actor_, &FileDbActor::store_file_data, id, serialize(file_data), remote_key, local_key,
generate_key);
}
void set_file_data_ref(FileDbId id, FileDbId new_id) final {
send_closure(file_db_actor_, &FileDbActor::store_file_data_ref, id, new_id);
}
SqliteKeyValue &pmc() final {
return file_kv_safe_->get();
}
private:
ActorOwn<FileDbActor> file_db_actor_;
FileDbId current_pmc_id_;
std::shared_ptr<SqliteKeyValueSafe> file_kv_safe_;
static Result<FileData> load_file_data_impl(ActorId<FileDbActor> file_db_actor_id, SqliteKeyValue &pmc,
const string &key, FileDbId current_pmc_id) {
// LOG(DEBUG) << "Load by key " << format::as_hex_dump<4>(Slice(key));
TRY_RESULT(id, get_id(pmc, key));
vector<FileDbId> ids;
string data_str;
int attempt_count = 0;
while (true) {
if (attempt_count > 100) {
LOG(FATAL) << "Cycle in file database? current_pmc_id=" << current_pmc_id << " key=" << key
<< " links=" << format::as_array(ids);
}
attempt_count++;
data_str = pmc.get(PSTRING() << "file" << id.get());
auto data_slice = Slice(data_str);
if (data_slice.substr(0, 2) == "@@") {
ids.push_back(id);
id = FileDbId(to_integer<uint64>(data_slice.substr(2)));
} else {
break;
}
}
if (ids.size() > 1) {
send_closure(file_db_actor_id, &FileDbActor::optimize_refs, std::move(ids), id);
}
// LOG(DEBUG) << "By ID " << id.get() << " found data " << format::as_hex_dump<4>(Slice(data_str));
// LOG(INFO) << attempt_count;
log_event::WithVersion<TlParser> parser(data_str);
parser.set_version(static_cast<int32>(Version::Initial));
FileData data;
data.parse(parser, true);
parser.fetch_end();
auto status = parser.get_status();
if (status.is_error()) {
return std::move(status);
}
return std::move(data);
}
static Result<FileDbId> get_id(SqliteKeyValue &pmc, const string &key) TD_WARN_UNUSED_RESULT {
auto id_str = pmc.get(key);
// LOG(DEBUG) << "Found ID " << id_str << " by key " << format::as_hex_dump<4>(Slice(key));
if (id_str.empty()) {
return Status::Error("There is no such a key in database");
}
return FileDbId(to_integer<uint64>(id_str));
}
};
std::shared_ptr<FileDbInterface> create_file_db(std::shared_ptr<SqliteConnectionSafe> connection, int scheduler_id) {
auto kv = std::make_shared<SqliteKeyValueSafe>("files", std::move(connection));
return std::make_shared<FileDb>(std::move(kv), scheduler_id);
}
} // namespace td