tdlight-team/tdlight
tdlight-team
/
tdlight
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add strongly typed FileDbId. 

GitOrigin-RevId: 234c9ec0ecda3f2ba48e0fd00fa06a8ec0188ca4
This commit is contained in:
levlam 2019-01-06 23:39:10 +03:00
parent 5438119bb4
commit 60e4d27011
7 changed files with 112 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -482,6 +482,7 @@ set(TDLIB_SOURCE
td/telegram/DraftMessage.h
td/telegram/files/FileBitmask.h
td/telegram/files/FileDb.h
td/telegram/files/FileDbId.h
td/telegram/files/FileDownloader.h
td/telegram/files/FileEncryptionKey.h
td/telegram/files/FileFromBytes.h

67
td/telegram/files/FileDb.cpp
View File

@ -59,8 +59,7 @@ class FileDb : public FileDbInterface {
public:
class FileDbActor : public Actor {
public:
using Id = FileDbInterface::Id;
FileDbActor(Id current_pmc_id, std::shared_ptr<SqliteKeyValueSafe> file_kv_safe)
FileDbActor(FileDbId current_pmc_id, std::shared_ptr<SqliteKeyValueSafe> file_kv_safe)
: current_pmc_id_(current_pmc_id), file_kv_safe_(std::move(file_kv_safe)) {
}
@ -75,7 +74,7 @@ class FileDb : public FileDbInterface {
promise.set_result(load_file_data_impl(actor_id(this), file_pmc(), key));
}
void clear_file_data(Id id, const string &remote_key, const string &local_key, const string &generate_key) {
void clear_file_data(FileDbId id, const string &remote_key, const string &local_key, const string &generate_key) {
auto &pmc = file_pmc();
pmc.begin_transaction().ensure();
SCOPE_EXIT {
@ -83,12 +82,12 @@ class FileDb : public FileDbInterface {
};
if (id > current_pmc_id_) {
pmc.set("file_id", to_string(id));
pmc.set("file_id", to_string(id.get()));
current_pmc_id_ = id;
}
pmc.erase(PSTRING() << "file" << id);
LOG(DEBUG) << "ERASE " << format::as_hex_dump<4>(Slice(PSLICE() << "file" << to_string(id)));
pmc.erase(PSTRING() << "file" << id.get());
LOG(DEBUG) << "ERASE " << format::as_hex_dump<4>(Slice(PSLICE() << "file" << id.get()));
if (!remote_key.empty()) {
pmc.erase(remote_key);
@ -102,7 +101,7 @@ class FileDb : public FileDbInterface {
pmc.erase(generate_key);
}
}
void store_file_data(Id id, const string &file_data, const string &remote_key, const string &local_key,
void store_file_data(FileDbId id, const string &file_data, const string &remote_key, const string &local_key,
const string &generate_key) {
auto &pmc = file_pmc();
pmc.begin_transaction().ensure();
@ -111,23 +110,23 @@ class FileDb : public FileDbInterface {
};
if (id > current_pmc_id_) {
pmc.set("file_id", to_string(id));
pmc.set("file_id", to_string(id.get()));
current_pmc_id_ = id;
}
pmc.set(PSTRING() << "file" << id, file_data);
pmc.set(PSTRING() << "file" << id.get(), file_data);
if (!remote_key.empty()) {
pmc.set(remote_key, to_string(id));
pmc.set(remote_key, to_string(id.get()));
}
if (!local_key.empty()) {
pmc.set(local_key, to_string(id));
pmc.set(local_key, to_string(id.get()));
}
if (!generate_key.empty()) {
pmc.set(generate_key, to_string(id));
pmc.set(generate_key, to_string(id.get()));
}
}
void store_file_data_ref(Id id, Id new_id) {
void store_file_data_ref(FileDbId id, FileDbId new_id) {
auto &pmc = file_pmc();
pmc.begin_transaction().ensure();
SCOPE_EXIT {
@ -135,15 +134,15 @@ class FileDb : public FileDbInterface {
};
if (id > current_pmc_id_) {
pmc.set("file_id", to_string(id));
pmc.set("file_id", to_string(id.get()));
current_pmc_id_ = id;
}
do_store_file_data_ref(id, new_id);
}
void optimize_refs(const std::vector<Id> ids, Id main_id) {
LOG(INFO) << "Optimize ids in file database" << format::as_array(ids) << " " << main_id;
void optimize_refs(const std::vector<FileDbId> ids, FileDbId main_id) {
LOG(INFO) << "Optimize " << ids.size() << " ids in file database to " << main_id.get();
auto &pmc = file_pmc();
pmc.begin_transaction().ensure();
SCOPE_EXIT {
@ -155,28 +154,29 @@ class FileDb : public FileDbInterface {
}
private:
Id current_pmc_id_;
FileDbId current_pmc_id_;
std::shared_ptr<SqliteKeyValueSafe> file_kv_safe_;
SqliteKeyValue &file_pmc() {
return file_kv_safe_->get();
}
void do_store_file_data_ref(Id id, Id new_id) {
file_pmc().set(PSTRING() << "file" << id, PSTRING() << "@@" << new_id);
void do_store_file_data_ref(FileDbId id, FileDbId new_id) {
file_pmc().set(PSTRING() << "file" << id.get(), PSTRING() << "@@" << new_id.get());
}
};
explicit FileDb(std::shared_ptr<SqliteKeyValueSafe> kv_safe, int scheduler_id = -1) {
file_kv_safe_ = std::move(kv_safe);
CHECK(file_kv_safe_);
current_pmc_id_ = to_integer<int32>(file_kv_safe_->get().get("file_id"));
current_pmc_id_ = FileDbId(to_integer<uint64>(file_kv_safe_->get().get("file_id")));
file_db_actor_ =
create_actor_on_scheduler<FileDbActor>("FileDbActor", scheduler_id, current_pmc_id_, file_kv_safe_);
}
Id create_pmc_id() override {
return ++current_pmc_id_;
FileDbId create_pmc_id() override {
current_pmc_id_ = FileDbId(current_pmc_id_.get() + 1);
return current_pmc_id_;
}
void close(Promise<> promise) override {
@ -191,7 +191,7 @@ class FileDb : public FileDbInterface {
return load_file_data_impl(file_db_actor_.get(), file_kv_safe_->get(), key);
}
void clear_file_data(Id id, const FileData &file_data) override {
void clear_file_data(FileDbId id, const FileData &file_data) override {
string remote_key;
if (file_data.remote_.type() == RemoteFileLocation::Type::Full) {
remote_key = as_key(file_data.remote_.full());
@ -206,7 +206,8 @@ class FileDb : public FileDbInterface {
}
send_closure(file_db_actor_, &FileDbActor::clear_file_data, id, remote_key, local_key, generate_key);
}
void set_file_data(Id id, const FileData &file_data, bool new_remote, bool new_local, bool new_generate) override {
void set_file_data(FileDbId id, const FileData &file_data, bool new_remote, bool new_local,
bool new_generate) override {
string remote_key;
if (file_data.remote_.type() == RemoteFileLocation::Type::Full && new_remote) {
remote_key = as_key(file_data.remote_.full());
@ -219,14 +220,14 @@ class FileDb : public FileDbInterface {
if (file_data.generate_ != nullptr && new_generate) {
generate_key = as_key(*file_data.generate_);
}
LOG(DEBUG) << "SAVE " << id << " -> " << file_data << " "
LOG(DEBUG) << "SAVE " << id.get() << " -> " << file_data << " "
<< tag("remote", format::as_hex_dump<4>(Slice(remote_key)))
<< tag("local", format::as_hex_dump<4>(Slice(local_key)));
send_closure(file_db_actor_, &FileDbActor::store_file_data, id, serialize(file_data), remote_key, local_key,
generate_key);
}
void set_file_data_ref(Id id, Id new_id) override {
void set_file_data_ref(FileDbId id, FileDbId new_id) override {
send_closure(file_db_actor_, &FileDbActor::store_file_data_ref, id, new_id);
}
SqliteKeyValue &pmc() override {
@ -235,7 +236,7 @@ class FileDb : public FileDbInterface {
private:
ActorOwn<FileDbActor> file_db_actor_;
Id current_pmc_id_;
FileDbId current_pmc_id_;
std::shared_ptr<SqliteKeyValueSafe> file_kv_safe_;
static Result<FileData> load_file_data_impl(ActorId<FileDbActor> file_db_actor_id, SqliteKeyValue &pmc,
@ -243,7 +244,7 @@ class FileDb : public FileDbInterface {
//LOG(DEBUG) << "Load by key " << format::as_hex_dump<4>(Slice(key));
TRY_RESULT(id, get_id(pmc, key));
vector<Id> ids;
vector<FileDbId> ids;
string data_str;
int attempt_count = 0;
while (true) {
@ -252,13 +253,13 @@ class FileDb : public FileDbInterface {
}
attempt_count++;
data_str = pmc.get(PSTRING() << "file" << id);
data_str = pmc.get(PSTRING() << "file" << id.get());
auto data_slice = Slice(data_str);
if (data_slice.substr(0, 2) == "@@") {
ids.push_back(id);
id = to_integer<Id>(data_slice.substr(2));
id = FileDbId(to_integer<uint64>(data_slice.substr(2)));
} else {
break;
}
@ -266,7 +267,7 @@ class FileDb : public FileDbInterface {
if (ids.size() > 1) {
send_closure(file_db_actor_id, &FileDbActor::optimize_refs, std::move(ids), id);
}
//LOG(DEBUG) << "By id " << id << " found data " << format::as_hex_dump<4>(Slice(data_str));
//LOG(DEBUG) << "By id " << id.get() << " found data " << format::as_hex_dump<4>(Slice(data_str));
//LOG(INFO) << attempt_count;
FileData data;
@ -277,13 +278,13 @@ class FileDb : public FileDbInterface {
return std::move(data);
}
static Result<Id> get_id(SqliteKeyValue &pmc, const string &key) TD_WARN_UNUSED_RESULT {
static Result<FileDbId> get_id(SqliteKeyValue &pmc, const string &key) TD_WARN_UNUSED_RESULT {
auto id_str = pmc.get(key);
//LOG(DEBUG) << "Found id " << id_str << " by key " << format::as_hex_dump<4>(Slice(key));
if (id_str.empty()) {
return Status::Error("There is no such a key in database");
}
return to_integer<Id>(id_str);
return FileDbId(to_integer<uint64>(id_str));
}
};

18
td/telegram/files/FileDb.h
View File

@ -6,6 +6,7 @@
//
#pragma once
#include "td/telegram/files/FileDbId.h"
#include "td/telegram/files/FileLocation.h"
#include "td/actor/PromiseFuture.h"
@ -16,12 +17,11 @@
#include <memory>
namespace td {
class SqliteDb;
class SqliteConnectionSafe;
class SqliteKeyValue;
} // namespace td
namespace td {
Status drop_file_db(SqliteDb &db, int32 version) TD_WARN_UNUSED_RESULT;
Status init_file_db(SqliteDb &db, int32 version) TD_WARN_UNUSED_RESULT;
@ -29,18 +29,15 @@ class FileDbInterface;
std::shared_ptr<FileDbInterface> create_file_db(std::shared_ptr<SqliteConnectionSafe> connection,
int32 scheduler_id = -1) TD_WARN_UNUSED_RESULT;
using FileDbId = uint64;
class FileDbInterface {
public:
using Id = FileDbId;
FileDbInterface() = default;
FileDbInterface(const FileDbInterface &) = delete;
FileDbInterface &operator=(const FileDbInterface &) = delete;
virtual ~FileDbInterface() = default;
// non thread safe
virtual Id create_pmc_id() = 0;
virtual FileDbId create_pmc_id() = 0;
// thread safe
virtual void close(Promise<> promise) = 0;
@ -60,9 +57,10 @@ class FileDbInterface {
return res;
}
virtual void clear_file_data(Id id, const FileData &file_data) = 0;
virtual void set_file_data(Id id, const FileData &file_data, bool new_remote, bool new_local, bool new_generate) = 0;
virtual void set_file_data_ref(Id id, Id new_id) = 0;
virtual void clear_file_data(FileDbId id, const FileData &file_data) = 0;
virtual void set_file_data(FileDbId id, const FileData &file_data, bool new_remote, bool new_local,
bool new_generate) = 0;
virtual void set_file_data_ref(FileDbId id, FileDbId new_id) = 0;
// For FileStatsWorker. TODO: remove it
virtual SqliteKeyValue &pmc() = 0;
@ -71,5 +69,5 @@ class FileDbInterface {
virtual void get_file_data_impl(string key, Promise<FileData> promise) = 0;
virtual Result<FileData> get_file_data_sync_impl(string key) = 0;
};
;
} // namespace td

54
td/telegram/files/FileDbId.h Normal file
View File

@ -0,0 +1,54 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/StringBuilder.h"
#include <type_traits>
namespace td {
class FileDbId {
uint64 id = 0;
public:
FileDbId() = default;
FileDbId(uint64 file_db_id) : id(file_db_id) {
}
template <class T1, typename = std::enable_if_t<std::is_convertible<T1, uint64>::value>>
FileDbId(T1 file_db_id) = delete;
bool empty() const {
return id == 0;
}
bool is_valid() const {
return id > 0;
}
uint64 get() const {
return id;
}
bool operator<(const FileDbId &other) const {
return id < other.id;
}
bool operator>(const FileDbId &other) const {
return id > other.id;
}
bool operator==(const FileDbId &other) const {
return id == other.id;
}
bool operator!=(const FileDbId &other) const {
return id != other.id;
}
};
} // namespace td

1
td/telegram/files/FileId.h
View File

@ -64,4 +64,5 @@ struct FileIdHash {
inline StringBuilder &operator<<(StringBuilder &string_builder, FileId file_id) {
return string_builder << file_id.get() << "(" << file_id.get_remote() << ")";
}
} // namespace td

23
td/telegram/files/FileManager.cpp
View File

@ -258,7 +258,7 @@ bool FileNode::need_pmc_flush() const {
}
// already in pmc
if (pmc_id_ != 0) {
if (pmc_id_.is_valid()) {
return true;
}
@ -848,7 +848,7 @@ Result<FileId> FileManager::register_file(FileData data, FileLocationSource file
data.owner_dialog_id_, std::move(data.encryption_key_), file_id,
static_cast<int8>(has_remote));
node->remote_source_ = file_location_source;
node->pmc_id_ = data.pmc_id_;
node->pmc_id_ = FileDbId(data.pmc_id_);
get_file_id_info(file_id)->node_id_ = file_node_id;
node->file_ids_.push_back(file_id);
@ -1125,15 +1125,16 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
}
}
int node_i = std::make_tuple(y_node->pmc_id_ != 0, x_node->pmc_id_, y_node->file_ids_.size(), main_file_id_i == 1) >
std::make_tuple(x_node->pmc_id_ != 0, y_node->pmc_id_, x_node->file_ids_.size(), main_file_id_i == 0);
int node_i =
std::make_tuple(y_node->pmc_id_.is_valid(), x_node->pmc_id_, y_node->file_ids_.size(), main_file_id_i == 1) >
std::make_tuple(x_node->pmc_id_.is_valid(), y_node->pmc_id_, x_node->file_ids_.size(), main_file_id_i == 0);
auto other_node_i = 1 - node_i;
FileNodePtr node = nodes[node_i];
FileNodePtr other_node = nodes[other_node_i];
auto file_view = FileView(node);
LOG(DEBUG) << "x_node->pmc_id_ = " << x_node->pmc_id_ << ", y_node->pmc_id_ = " << y_node->pmc_id_
LOG(DEBUG) << "x_node->pmc_id_ = " << x_node->pmc_id_.get() << ", y_node->pmc_id_ = " << y_node->pmc_id_.get()
<< ", x_node_size = " << x_node->file_ids_.size() << ", y_node_size = " << y_node->file_ids_.size()
<< ", node_i = " << node_i << ", local_i = " << local_i << ", remote_i = " << remote_i
<< ", generate_i = " << generate_i << ", size_i = " << size_i << ", remote_name_i = " << remote_name_i
@ -1269,7 +1270,7 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
run_download(node);
run_upload(node, {});
if (other_pmc_id != 0) {
if (other_pmc_id.is_valid()) {
// node might not changed, but we need to merge nodes in pmc anyway
node->on_pmc_changed();
}
@ -1284,7 +1285,7 @@ void FileManager::try_flush_node_full(FileNodePtr node, bool new_remote, bool ne
if (file_db_) {
load_from_pmc(node, true, true, true);
flush_to_pmc(node, new_remote, new_local, new_generate);
if (other_pmc_id != 0 && node->pmc_id_ != other_pmc_id) {
if (other_pmc_id.is_valid() && node->pmc_id_ != other_pmc_id) {
file_db_->set_file_data_ref(other_pmc_id, node->pmc_id_);
}
}
@ -1323,7 +1324,7 @@ void FileManager::clear_from_pmc(FileNodePtr node) {
if (!file_db_) {
return;
}
if (node->pmc_id_ == 0) {
if (node->pmc_id_.empty()) {
return;
}
@ -1340,7 +1341,7 @@ void FileManager::clear_from_pmc(FileNodePtr node) {
data.generate_ = make_unique<FullGenerateFileLocation>(*node->generate_);
}
file_db_->clear_file_data(node->pmc_id_, data);
node->pmc_id_ = 0;
node->pmc_id_ = FileDbId();
}
void FileManager::flush_to_pmc(FileNodePtr node, bool new_remote, bool new_local, bool new_generate) {
@ -1349,13 +1350,13 @@ void FileManager::flush_to_pmc(FileNodePtr node, bool new_remote, bool new_local
}
FileView view(node);
bool create_flag = false;
if (node->pmc_id_ == 0) {
if (node->pmc_id_.empty()) {
create_flag = true;
node->pmc_id_ = file_db_->create_pmc_id();
}
FileData data;
data.pmc_id_ = node->pmc_id_;
data.pmc_id_ = node->pmc_id_.get();
data.local_ = node->local_;
if (data.local_.type() == LocalFileLocation::Type::Full) {
prepare_path_for_pmc(data.local_.full().file_type_, data.local_.full().path_);

3
td/telegram/files/FileManager.h
View File

@ -11,6 +11,7 @@
#include "td/telegram/DialogId.h"
#include "td/telegram/files/FileDb.h"
#include "td/telegram/files/FileDbId.h"
#include "td/telegram/files/FileEncryptionKey.h"
#include "td/telegram/files/FileGenerateManager.h"
#include "td/telegram/files/FileId.h"
@ -112,7 +113,7 @@ class FileNode {
string url_;
DialogId owner_dialog_id_;
FileEncryptionKey encryption_key_;
FileDbId pmc_id_ = 0;
FileDbId pmc_id_;
std::vector<FileId> file_ids_;
FileId main_file_id_;