tdlight-team/tdlight
tdlight-team/tdlight
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Increased files TTL to 120 seconds, fixed rehash buckets count, lock other methods when cleaning, unique_ptr for nodes

Browse Source
This commit is contained in:
Andrea Cavalli 2020-08-02 00:28:01 +02:00
parent 22192cda3c
commit 000375e21b
2 changed files with 149 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

175
td/telegram/files/FileManager.cpp
View File

@ -49,7 +49,7 @@
#include <unordered_set> #include <unordered_set>
#include <utility> #include <utility>
#define FILE_TTL 20 #define FILE_TTL 120
namespace td { namespace td {
namespace { namespace {
@ -300,7 +300,7 @@ void FileNode::set_partial_remote_location(const PartialRemoteFileLocation &remo
on_changed(); on_changed();
} }
bool FileNode::delete_file_reference(Slice file_reference) { bool FileNode::delete_file_reference_internal(Slice file_reference) {
if (!remote_.full) { if (!remote_.full) {
VLOG(file_references) << "Can't delete file reference, because there is no remote location"; VLOG(file_references) << "Can't delete file reference, because there is no remote location";
return false; return false;
@ -1038,6 +1038,11 @@ FileManager::FileIdInfo *FileManager::get_file_id_info(FileId file_id) {
} }
FileId FileManager::dup_file_id(FileId file_id) { FileId FileManager::dup_file_id(FileId file_id) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
return dup_file_id_internal(file_id);
}
FileId FileManager::dup_file_id_internal(FileId file_id) {
int32 file_node_id; int32 file_node_id;
auto *file_node = get_file_node_raw(file_id, &file_node_id); auto *file_node = get_file_node_raw(file_id, &file_node_id);
if (!file_node) { if (!file_node) {
@ -1073,7 +1078,7 @@ void FileManager::try_forget_file_id(FileId file_id) {
} }
FileId FileManager::register_empty(FileType type) { FileId FileManager::register_empty(FileType type) {
return register_local(FullLocalFileLocation(type, "", 0), DialogId(), 0, false, true).ok(); return register_local_internal(FullLocalFileLocation(type, "", 0), DialogId(), 0, false, true).ok();
} }
void FileManager::on_file_unlink(const FullLocalFileLocation &location) { void FileManager::on_file_unlink(const FullLocalFileLocation &location) {
@ -1093,6 +1098,12 @@ void FileManager::on_file_unlink(const FullLocalFileLocation &location) {
Result<FileId> FileManager::register_local(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size, Result<FileId> FileManager::register_local(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size,
bool get_by_hash, bool force, bool skip_file_size_checks) { bool get_by_hash, bool force, bool skip_file_size_checks) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
return register_local_internal(location, owner_dialog_id, size, get_by_hash, force, skip_file_size_checks);
}
Result<FileId> FileManager::register_local_internal(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size,
bool get_by_hash, bool force, bool skip_file_size_checks) {
// TODO: use get_by_hash // TODO: use get_by_hash
FileData data; FileData data;
data.local_ = LocalFileLocation(std::move(location)); data.local_ = LocalFileLocation(std::move(location));
@ -1103,6 +1114,12 @@ Result<FileId> FileManager::register_local(FullLocalFileLocation location, Dialo
} }
FileId FileManager::register_remote(const FullRemoteFileLocation &location, FileLocationSource file_location_source, FileId FileManager::register_remote(const FullRemoteFileLocation &location, FileLocationSource file_location_source,
DialogId owner_dialog_id, int64 size, int64 expected_size, string remote_name) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
return register_remote_internal(location, file_location_source, owner_dialog_id, size, expected_size, std::move(remote_name));
}
FileId FileManager::register_remote_internal(const FullRemoteFileLocation &location, FileLocationSource file_location_source,
DialogId owner_dialog_id, int64 size, int64 expected_size, string remote_name) { DialogId owner_dialog_id, int64 size, int64 expected_size, string remote_name) {
FileData data; FileData data;
data.remote_ = RemoteFileLocation(location); data.remote_ = RemoteFileLocation(location);
@ -1123,7 +1140,7 @@ FileId FileManager::register_remote(const FullRemoteFileLocation &location, File
FileId FileManager::register_url(string url, FileType file_type, FileLocationSource file_location_source, FileId FileManager::register_url(string url, FileType file_type, FileLocationSource file_location_source,
DialogId owner_dialog_id) { DialogId owner_dialog_id) {
auto file_id = register_generate(file_type, file_location_source, url, "#url#", owner_dialog_id, 0).ok(); auto file_id = register_generate_internal(file_type, file_location_source, url, "#url#", owner_dialog_id, 0).ok();
auto file_node = get_file_node(file_id); auto file_node = get_file_node(file_id);
CHECK(file_node); CHECK(file_node);
file_node->set_url(url); file_node->set_url(url);
@ -1133,6 +1150,13 @@ FileId FileManager::register_url(string url, FileType file_type, FileLocationSou
Result<FileId> FileManager::register_generate(FileType file_type, FileLocationSource file_location_source, Result<FileId> FileManager::register_generate(FileType file_type, FileLocationSource file_location_source,
string original_path, string conversion, DialogId owner_dialog_id, string original_path, string conversion, DialogId owner_dialog_id,
int64 expected_size) { int64 expected_size) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
return register_generate_internal(file_type, file_location_source, std::move(original_path), std::move(conversion), owner_dialog_id, expected_size);
}
Result<FileId> FileManager::register_generate_internal(FileType file_type, FileLocationSource file_location_source,
string original_path, string conversion, DialogId owner_dialog_id,
int64 expected_size) {
// add #mtime# into conversion // add #mtime# into conversion
if (!original_path.empty() && conversion[0] != '#' && PathView(original_path).is_absolute()) { if (!original_path.empty() && conversion[0] != '#' && PathView(original_path).is_absolute()) {
auto file_paths = log_interface->get_file_paths(); auto file_paths = log_interface->get_file_paths();
@ -1190,13 +1214,13 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi
// create FileNode // create FileNode
auto file_node_id = next_file_node_id(); auto file_node_id = next_file_node_id();
auto &node = file_nodes_[file_node_id]; auto &node = file_nodes_[file_node_id];
node = FileNode(std::move(data.local_), NewRemoteFileLocation(data.remote_, file_location_source), node = td::make_unique<FileNode>(std::move(data.local_), NewRemoteFileLocation(data.remote_, file_location_source),
std::move(data.generate_), data.size_, data.expected_size_, std::move(data.generate_), data.size_, data.expected_size_,
std::move(data.remote_name_), std::move(data.url_), data.owner_dialog_id_, std::move(data.remote_name_), std::move(data.url_), data.owner_dialog_id_,
std::move(data.encryption_key_), file_id, static_cast<int8>(has_remote)); std::move(data.encryption_key_), file_id, static_cast<int8>(has_remote));
node.pmc_id_ = FileDbId(data.pmc_id_); node->pmc_id_ = FileDbId(data.pmc_id_);
get_file_id_info(file_id)->node_id_ = file_node_id; get_file_id_info(file_id)->node_id_ = file_node_id;
node.file_ids_.push_back(file_id); node->file_ids_.push_back(file_id);
FileView file_view(get_file_node(file_id)); FileView file_view(get_file_node(file_id));
@ -1243,12 +1267,12 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi
int new_cnt = new_remote + new_local + new_generate; int new_cnt = new_remote + new_local + new_generate;
if (data.pmc_id_ == 0 && file_db_ && new_cnt > 0) { if (data.pmc_id_ == 0 && file_db_ && new_cnt > 0) {
node.need_load_from_pmc_ = true; node->need_load_from_pmc_ = true;
} }
bool no_sync_merge = to_merge.size() == 1 && new_cnt == 0; bool no_sync_merge = to_merge.size() == 1 && new_cnt == 0;
for (auto id : to_merge) { for (auto id : to_merge) {
// may invalidate node // may invalidate node
merge(file_id, id, no_sync_merge).ignore(); merge_internal(file_id, id, no_sync_merge).ignore();
} }
try_flush_node(get_file_node(file_id), "register_file"); try_flush_node(get_file_node(file_id), "register_file");
@ -1422,6 +1446,11 @@ void FileManager::do_cancel_generate(FileNodePtr node) {
} }
Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) { Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
return merge_internal(x_file_id, y_file_id, no_sync);
}
Result<FileId> FileManager::merge_internal(FileId x_file_id, FileId y_file_id, bool no_sync) {
LOG(DEBUG) << "Merge new file " << x_file_id << " and old file " << y_file_id; LOG(DEBUG) << "Merge new file " << x_file_id << " and old file " << y_file_id;
if (!x_file_id.is_valid()) { if (!x_file_id.is_valid()) {
@ -1636,7 +1665,7 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
file_id_info->node_id_ = node_ids[node_i]; file_id_info->node_id_ = node_ids[node_i];
send_updates_flag |= file_id_info->send_updates_flag_; send_updates_flag |= file_id_info->send_updates_flag_;
} }
other_node = {this}; other_node = {};
if (send_updates_flag) { if (send_updates_flag) {
// node might not changed, but other_node might changed, so we need to send update anyway // node might not changed, but other_node might changed, so we need to send update anyway
@ -1679,6 +1708,11 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
} }
void FileManager::add_file_source(FileId file_id, FileSourceId file_source_id) { void FileManager::add_file_source(FileId file_id, FileSourceId file_source_id) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
return add_file_source_internal(file_id, file_source_id);
}
void FileManager::add_file_source_internal(FileId file_id, FileSourceId file_source_id) {
auto node = get_file_node(file_id); auto node = get_file_node(file_id);
if (!node) { if (!node) {
return; return;
@ -1689,6 +1723,7 @@ void FileManager::add_file_source(FileId file_id, FileSourceId file_source_id) {
node->on_pmc_changed(); node->on_pmc_changed();
try_flush_node_pmc(node, "add_file_source"); try_flush_node_pmc(node, "add_file_source");
} }
} }
void FileManager::remove_file_source(FileId file_id, FileSourceId file_source_id) { void FileManager::remove_file_source(FileId file_id, FileSourceId file_source_id) {
@ -1706,6 +1741,7 @@ void FileManager::remove_file_source(FileId file_id, FileSourceId file_source_id
void FileManager::change_files_source(FileSourceId file_source_id, const vector<FileId> &old_file_ids, void FileManager::change_files_source(FileSourceId file_source_id, const vector<FileId> &old_file_ids,
const vector<FileId> &new_file_ids) { const vector<FileId> &new_file_ids) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
if (old_file_ids == new_file_ids) { if (old_file_ids == new_file_ids) {
return; return;
} }
@ -1722,7 +1758,7 @@ void FileManager::change_files_source(FileSourceId file_source_id, const vector<
} }
} }
for (auto file_id : new_main_file_ids) { for (auto file_id : new_main_file_ids) {
add_file_source(file_id, file_source_id); add_file_source_internal(file_id, file_source_id);
} }
} }
@ -1881,7 +1917,7 @@ FileNode *FileManager::get_file_node_raw(FileId file_id, FileNodeId *file_node_i
if (file_node_id != nullptr) { if (file_node_id != nullptr) {
*file_node_id = node_id; *file_node_id = node_id;
} }
return &file_nodes_[node_id]; return file_nodes_[node_id].get();
} }
FileNodePtr FileManager::get_sync_file_node(FileId file_id) { FileNodePtr FileManager::get_sync_file_node(FileId file_id) {
@ -1927,7 +1963,7 @@ void FileManager::load_from_pmc(FileNodePtr node, bool new_remote, bool new_loca
TRY_RESULT(file_data, file_db_->get_file_data_sync(location)); TRY_RESULT(file_data, file_db_->get_file_data_sync(location));
TRY_RESULT(new_file_id, TRY_RESULT(new_file_id,
register_file(std::move(file_data), FileLocationSource::FromDatabase, "load_from_pmc", false)); register_file(std::move(file_data), FileLocationSource::FromDatabase, "load_from_pmc", false));
TRY_RESULT(main_file_id, merge(file_id, new_file_id)); TRY_RESULT(main_file_id, merge_internal(file_id, new_file_id));
file_id = main_file_id; file_id = main_file_id;
return Status::OK(); return Status::OK();
}; };
@ -1960,6 +1996,7 @@ bool FileManager::set_encryption_key(FileId file_id, FileEncryptionKey key) {
} }
bool FileManager::set_content(FileId file_id, BufferSlice bytes) { bool FileManager::set_content(FileId file_id, BufferSlice bytes) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
if (G()->shared_config().get_option_boolean("ignore_inline_thumbnails")) { if (G()->shared_config().get_option_boolean("ignore_inline_thumbnails")) {
return false; return false;
} }
@ -1994,6 +2031,7 @@ bool FileManager::set_content(FileId file_id, BufferSlice bytes) {
} }
void FileManager::get_content(FileId file_id, Promise<BufferSlice> promise) { void FileManager::get_content(FileId file_id, Promise<BufferSlice> promise) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
auto node = get_sync_file_node(file_id); auto node = get_sync_file_node(file_id);
if (!node) { if (!node) {
return promise.set_error(Status::Error("Unknown file_id")); return promise.set_error(Status::Error("Unknown file_id"));
@ -2011,6 +2049,7 @@ void FileManager::get_content(FileId file_id, Promise<BufferSlice> promise) {
void FileManager::read_file_part(FileId file_id, int32 offset, int32 count, int left_tries, void FileManager::read_file_part(FileId file_id, int32 offset, int32 count, int left_tries,
Promise<td_api::object_ptr<td_api::filePart>> promise) { Promise<td_api::object_ptr<td_api::filePart>> promise) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
if (G()->close_flag()) { if (G()->close_flag()) {
return promise.set_error(Status::Error(500, "Request aborted")); return promise.set_error(Status::Error(500, "Request aborted"));
} }
@ -2089,6 +2128,7 @@ void FileManager::read_file_part(FileId file_id, int32 offset, int32 count, int
} }
void FileManager::delete_file(FileId file_id, Promise<Unit> promise, const char *source) { void FileManager::delete_file(FileId file_id, Promise<Unit> promise, const char *source) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
LOG(INFO) << "Trying to delete file " << file_id << " from " << source; LOG(INFO) << "Trying to delete file " << file_id << " from " << source;
auto node = get_sync_file_node(file_id); auto node = get_sync_file_node(file_id);
if (!node) { if (!node) {
@ -2125,6 +2165,7 @@ void FileManager::delete_file(FileId file_id, Promise<Unit> promise, const char
void FileManager::download(FileId file_id, std::shared_ptr<DownloadCallback> callback, int32 new_priority, int64 offset, void FileManager::download(FileId file_id, std::shared_ptr<DownloadCallback> callback, int32 new_priority, int64 offset,
int64 limit) { int64 limit) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
LOG(INFO) << "Download file " << file_id << " with priority " << new_priority; LOG(INFO) << "Download file " << file_id << " with priority " << new_priority;
auto node = get_sync_file_node(file_id); auto node = get_sync_file_node(file_id);
if (!node) { if (!node) {
@ -2516,6 +2557,7 @@ void FileManager::resume_upload(FileId file_id, std::vector<int> bad_parts, std:
} }
bool FileManager::delete_partial_remote_location(FileId file_id) { bool FileManager::delete_partial_remote_location(FileId file_id) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
auto node = get_sync_file_node(file_id); auto node = get_sync_file_node(file_id);
if (!node) { if (!node) {
LOG(INFO) << "Wrong file identifier " << file_id; LOG(INFO) << "Wrong file identifier " << file_id;
@ -2550,6 +2592,11 @@ bool FileManager::delete_partial_remote_location(FileId file_id) {
} }
void FileManager::delete_file_reference(FileId file_id, string file_reference) { void FileManager::delete_file_reference(FileId file_id, string file_reference) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
return delete_file_reference_internal(file_id, file_reference);
}
void FileManager::delete_file_reference_internal(FileId file_id, string file_reference) {
VLOG(file_references) << "Delete file reference of file " << file_id << " " VLOG(file_references) << "Delete file reference of file " << file_id << " "
<< tag("reference_base64", base64_encode(file_reference)); << tag("reference_base64", base64_encode(file_reference));
auto node = get_sync_file_node(file_id); auto node = get_sync_file_node(file_id);
@ -2557,7 +2604,7 @@ void FileManager::delete_file_reference(FileId file_id, string file_reference) {
LOG(ERROR) << "Wrong file identifier " << file_id; LOG(ERROR) << "Wrong file identifier " << file_id;
return; return;
} }
node->delete_file_reference(file_reference); node->delete_file_reference_internal(file_reference);
auto remote = get_remote(file_id.get_remote()); auto remote = get_remote(file_id.get_remote());
if (remote != nullptr) { if (remote != nullptr) {
VLOG(file_references) << "Do delete file reference of remote file " << file_id; VLOG(file_references) << "Do delete file reference of remote file " << file_id;
@ -2572,17 +2619,20 @@ void FileManager::delete_file_reference(FileId file_id, string file_reference) {
} }
void FileManager::external_file_generate_write_part(int64 id, int32 offset, string data, Promise<> promise) { void FileManager::external_file_generate_write_part(int64 id, int32 offset, string data, Promise<> promise) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_write_part, id, offset, send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_write_part, id, offset,
std::move(data), std::move(promise)); std::move(data), std::move(promise));
} }
void FileManager::external_file_generate_progress(int64 id, int32 expected_size, int32 local_prefix_size, void FileManager::external_file_generate_progress(int64 id, int32 expected_size, int32 local_prefix_size,
Promise<> promise) { Promise<> promise) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_progress, id, expected_size, send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_progress, id, expected_size,
local_prefix_size, std::move(promise)); local_prefix_size, std::move(promise));
} }
void FileManager::external_file_generate_finish(int64 id, Status status, Promise<> promise) { void FileManager::external_file_generate_finish(int64 id, Status status, Promise<> promise) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_finish, id, std::move(status), send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_finish, id, std::move(status),
std::move(promise)); std::move(promise));
} }
@ -2784,10 +2834,12 @@ void FileManager::run_upload(FileNodePtr node, std::vector<int> bad_parts) {
void FileManager::upload(FileId file_id, std::shared_ptr<UploadCallback> callback, int32 new_priority, void FileManager::upload(FileId file_id, std::shared_ptr<UploadCallback> callback, int32 new_priority,
uint64 upload_order) { uint64 upload_order) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
return resume_upload(file_id, std::vector<int>(), std::move(callback), new_priority, upload_order); return resume_upload(file_id, std::vector<int>(), std::move(callback), new_priority, upload_order);
} }
void FileManager::cancel_upload(FileId file_id) { void FileManager::cancel_upload(FileId file_id) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
return resume_upload(file_id, std::vector<int>(), nullptr, 0, 0); return resume_upload(file_id, std::vector<int>(), nullptr, 0, 0);
} }
@ -2912,6 +2964,7 @@ FileView FileManager::get_sync_file_view(FileId file_id) {
} }
td_api::object_ptr<td_api::file> FileManager::get_file_object(FileId file_id, bool with_main_file_id) { td_api::object_ptr<td_api::file> FileManager::get_file_object(FileId file_id, bool with_main_file_id) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
auto file_view = get_sync_file_view(file_id); auto file_view = get_sync_file_view(file_id);
if (file_view.empty()) { if (file_view.empty()) {
@ -2955,6 +3008,7 @@ td_api::object_ptr<td_api::file> FileManager::get_file_object(FileId file_id, bo
vector<int32> FileManager::get_file_ids_object(const vector<FileId> &file_ids, bool with_main_file_id) { vector<int32> FileManager::get_file_ids_object(const vector<FileId> &file_ids, bool with_main_file_id) {
return transform(file_ids, [this, with_main_file_id](FileId file_id) { return transform(file_ids, [this, with_main_file_id](FileId file_id) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
auto file_view = get_sync_file_view(file_id); auto file_view = get_sync_file_view(file_id);
auto result_file_id = file_id; auto result_file_id = file_id;
auto *file_info = get_file_id_info(result_file_id); auto *file_info = get_file_id_info(result_file_id);
@ -2996,7 +3050,7 @@ Result<FileId> FileManager::check_input_file_id(FileType type, Result<FileId> re
if (!file_view.has_remote_location()) { if (!file_view.has_remote_location()) {
// TODO why not return file_id here? We will dup it anyway // TODO why not return file_id here? We will dup it anyway
// But it will not be duped if has_input_media(), so for now we can't return main_file_id // But it will not be duped if has_input_media(), so for now we can't return main_file_id
return dup_file_id(file_id); return dup_file_id_internal(file_id);
} }
int32 remote_id = file_id.get_remote(); int32 remote_id = file_id.get_remote();
@ -3012,6 +3066,7 @@ Result<FileId> FileManager::check_input_file_id(FileType type, Result<FileId> re
Result<FileId> FileManager::get_input_thumbnail_file_id(const tl_object_ptr<td_api::InputFile> &thumbnail_input_file, Result<FileId> FileManager::get_input_thumbnail_file_id(const tl_object_ptr<td_api::InputFile> &thumbnail_input_file,
DialogId owner_dialog_id, bool is_encrypted) { DialogId owner_dialog_id, bool is_encrypted) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
if (thumbnail_input_file == nullptr) { if (thumbnail_input_file == nullptr) {
return Status::Error(6, "inputThumbnail not specified"); return Status::Error(6, "inputThumbnail not specified");
} }
@ -3019,7 +3074,7 @@ Result<FileId> FileManager::get_input_thumbnail_file_id(const tl_object_ptr<td_a
switch (thumbnail_input_file->get_id()) { switch (thumbnail_input_file->get_id()) {
case td_api::inputFileLocal::ID: { case td_api::inputFileLocal::ID: {
const string &path = static_cast<const td_api::inputFileLocal *>(thumbnail_input_file.get())->path_; const string &path = static_cast<const td_api::inputFileLocal *>(thumbnail_input_file.get())->path_;
return register_local( return register_local_internal(
FullLocalFileLocation(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail, path, 0), FullLocalFileLocation(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail, path, 0),
owner_dialog_id, 0, false); owner_dialog_id, 0, false);
} }
@ -3029,7 +3084,7 @@ Result<FileId> FileManager::get_input_thumbnail_file_id(const tl_object_ptr<td_a
return Status::Error(6, "InputFileRemote is not supported for thumbnails"); return Status::Error(6, "InputFileRemote is not supported for thumbnails");
case td_api::inputFileGenerated::ID: { case td_api::inputFileGenerated::ID: {
auto *generated_thumbnail = static_cast<const td_api::inputFileGenerated *>(thumbnail_input_file.get()); auto *generated_thumbnail = static_cast<const td_api::inputFileGenerated *>(thumbnail_input_file.get());
return register_generate(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail, return register_generate_internal(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail,
FileLocationSource::FromUser, generated_thumbnail->original_path_, FileLocationSource::FromUser, generated_thumbnail->original_path_,
generated_thumbnail->conversion_, owner_dialog_id, generated_thumbnail->expected_size_); generated_thumbnail->conversion_, owner_dialog_id, generated_thumbnail->expected_size_);
} }
@ -3042,6 +3097,7 @@ Result<FileId> FileManager::get_input_thumbnail_file_id(const tl_object_ptr<td_a
Result<FileId> FileManager::get_input_file_id(FileType type, const tl_object_ptr<td_api::InputFile> &file, Result<FileId> FileManager::get_input_file_id(FileType type, const tl_object_ptr<td_api::InputFile> &file,
DialogId owner_dialog_id, bool allow_zero, bool is_encrypted, DialogId owner_dialog_id, bool allow_zero, bool is_encrypted,
bool get_by_hash, bool is_secure) { bool get_by_hash, bool is_secure) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
if (file == nullptr) { if (file == nullptr) {
if (allow_zero) { if (allow_zero) {
return FileId(); return FileId();
@ -3079,7 +3135,7 @@ Result<FileId> FileManager::get_input_file_id(FileType type, const tl_object_ptr
} }
} }
} }
TRY_RESULT(file_id, register_local(FullLocalFileLocation(new_type, path, 0), owner_dialog_id, 0, get_by_hash)); TRY_RESULT(file_id, register_local_internal(FullLocalFileLocation(new_type, path, 0), owner_dialog_id, 0, get_by_hash));
if (!hash.empty()) { if (!hash.empty()) {
file_hash_to_file_id_[hash] = file_id; file_hash_to_file_id_[hash] = file_id;
} }
@ -3101,7 +3157,7 @@ Result<FileId> FileManager::get_input_file_id(FileType type, const tl_object_ptr
} }
case td_api::inputFileGenerated::ID: { case td_api::inputFileGenerated::ID: {
auto *generated_file = static_cast<const td_api::inputFileGenerated *>(file.get()); auto *generated_file = static_cast<const td_api::inputFileGenerated *>(file.get());
return register_generate(new_type, FileLocationSource::FromUser, generated_file->original_path_, return register_generate_internal(new_type, FileLocationSource::FromUser, generated_file->original_path_,
generated_file->conversion_, owner_dialog_id, generated_file->expected_size_); generated_file->conversion_, owner_dialog_id, generated_file->expected_size_);
} }
default: default:
@ -3115,6 +3171,7 @@ Result<FileId> FileManager::get_input_file_id(FileType type, const tl_object_ptr
Result<FileId> FileManager::get_map_thumbnail_file_id(Location location, int32 zoom, int32 width, int32 height, Result<FileId> FileManager::get_map_thumbnail_file_id(Location location, int32 zoom, int32 width, int32 height,
int32 scale, DialogId owner_dialog_id) { int32 scale, DialogId owner_dialog_id) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
if (!location.is_valid_map_point()) { if (!location.is_valid_map_point()) {
return Status::Error(6, "Invalid location specified"); return Status::Error(6, "Invalid location specified");
} }
@ -3141,12 +3198,13 @@ Result<FileId> FileManager::get_map_thumbnail_file_id(Location location, int32 z
string conversion = PSTRING() << "#map#" << zoom << "#" << x << "#" << y << "#" << width << "#" << height << "#" string conversion = PSTRING() << "#map#" << zoom << "#" << x << "#" << y << "#" << width << "#" << height << "#"
<< scale << "#"; << scale << "#";
return register_generate( return register_generate_internal(
owner_dialog_id.get_type() == DialogType::SecretChat ? FileType::EncryptedThumbnail : FileType::Thumbnail, owner_dialog_id.get_type() == DialogType::SecretChat ? FileType::EncryptedThumbnail : FileType::Thumbnail,
FileLocationSource::FromUser, string(), std::move(conversion), owner_dialog_id, 0); FileLocationSource::FromUser, string(), std::move(conversion), owner_dialog_id, 0);
} }
vector<tl_object_ptr<telegram_api::InputDocument>> FileManager::get_input_documents(const vector<FileId> &file_ids) { vector<tl_object_ptr<telegram_api::InputDocument>> FileManager::get_input_documents(const vector<FileId> &file_ids) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
vector<tl_object_ptr<telegram_api::InputDocument>> result; vector<tl_object_ptr<telegram_api::InputDocument>> result;
result.reserve(file_ids.size()); result.reserve(file_ids.size());
for (auto file_id : file_ids) { for (auto file_id : file_ids) {
@ -3178,6 +3236,7 @@ bool FileManager::extract_was_thumbnail_uploaded(const tl_object_ptr<telegram_ap
} }
string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputMedia> &input_media) { string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputMedia> &input_media) {
// don't lock this method because now it only calls the other extract_file_reference
if (input_media == nullptr) { if (input_media == nullptr) {
return string(); return string();
} }
@ -3213,6 +3272,7 @@ bool FileManager::extract_was_uploaded(const tl_object_ptr<telegram_api::InputCh
} }
string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputChatPhoto> &input_chat_photo) { string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputChatPhoto> &input_chat_photo) {
// don't lock this method because now it only calls the other extract_file_reference
if (input_chat_photo == nullptr || input_chat_photo->get_id() != telegram_api::inputChatPhoto::ID) { if (input_chat_photo == nullptr || input_chat_photo->get_id() != telegram_api::inputChatPhoto::ID) {
return string(); return string();
} }
@ -3234,6 +3294,8 @@ FileManager::FileNodeId FileManager::next_file_node_id() {
} }
void FileManager::on_start_download(QueryId query_id) { void FileManager::on_start_download(QueryId query_id) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
if (is_closed_) { if (is_closed_) {
return; return;
} }
@ -3259,6 +3321,8 @@ void FileManager::on_start_download(QueryId query_id) {
void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLocation &partial_local, int64 ready_size, void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLocation &partial_local, int64 ready_size,
int64 size) { int64 size) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
if (is_closed_) { if (is_closed_) {
return; return;
} }
@ -3290,6 +3354,8 @@ void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLo
} }
void FileManager::on_hash(QueryId query_id, string hash) { void FileManager::on_hash(QueryId query_id, string hash) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
if (is_closed_) { if (is_closed_) {
return; return;
} }
@ -3315,6 +3381,8 @@ void FileManager::on_hash(QueryId query_id, string hash) {
void FileManager::on_partial_upload(QueryId query_id, const PartialRemoteFileLocation &partial_remote, void FileManager::on_partial_upload(QueryId query_id, const PartialRemoteFileLocation &partial_remote,
int64 ready_size) { int64 ready_size) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
if (is_closed_) { if (is_closed_) {
return; return;
} }
@ -3339,6 +3407,8 @@ void FileManager::on_partial_upload(QueryId query_id, const PartialRemoteFileLoc
} }
void FileManager::on_download_ok(QueryId query_id, const FullLocalFileLocation &local, int64 size, bool is_new) { void FileManager::on_download_ok(QueryId query_id, const FullLocalFileLocation &local, int64 size, bool is_new) {
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
if (is_closed_) { if (is_closed_) {
return; return;
} }
@ -3348,7 +3418,7 @@ void FileManager::on_download_ok(QueryId query_id, const FullLocalFileLocation &
std::tie(query, was_active) = finish_query(query_id); std::tie(query, was_active) = finish_query(query_id);
auto file_id = query.file_id_; auto file_id = query.file_id_;
LOG(INFO) << "ON DOWNLOAD OK of " << (is_new ? "new" : "checked") << " file " << file_id << " of size " << size; LOG(INFO) << "ON DOWNLOAD OK of " << (is_new ? "new" : "checked") << " file " << file_id << " of size " << size;
auto r_new_file_id = register_local(local, DialogId(), size, false, false, true); auto r_new_file_id = register_local_internal(local, DialogId(), size, false, false, true);
Status status = Status::OK(); Status status = Status::OK();
if (r_new_file_id.is_error()) { if (r_new_file_id.is_error()) {
status = Status::Error(PSLICE() << "Can't register local file after download: " << r_new_file_id.error().message()); status = Status::Error(PSLICE() << "Can't register local file after download: " << r_new_file_id.error().message());
@ -3356,7 +3426,7 @@ void FileManager::on_download_ok(QueryId query_id, const FullLocalFileLocation &
if (is_new) { if (is_new) {
context_->on_new_file(size, get_file_view(r_new_file_id.ok()).get_allocated_local_size(), 1); context_->on_new_file(size, get_file_view(r_new_file_id.ok()).get_allocated_local_size(), 1);
} }
auto r_file_id = merge(r_new_file_id.ok(), file_id); auto r_file_id = merge_internal(r_new_file_id.ok(), file_id);
if (r_file_id.is_error()) { if (r_file_id.is_error()) {
status = r_file_id.move_as_error(); status = r_file_id.move_as_error();
} }
@ -3451,8 +3521,8 @@ void FileManager::on_upload_full_ok(QueryId query_id, const FullRemoteFileLocati
auto file_id = finish_query(query_id).first.file_id_; auto file_id = finish_query(query_id).first.file_id_;
LOG(INFO) << "ON UPLOAD FULL OK for file " << file_id; LOG(INFO) << "ON UPLOAD FULL OK for file " << file_id;
auto new_file_id = register_remote(remote, FileLocationSource::FromServer, DialogId(), 0, 0, ""); auto new_file_id = register_remote_internal(remote, FileLocationSource::FromServer, DialogId(), 0, 0, "");
LOG_STATUS(merge(new_file_id, file_id)); LOG_STATUS(merge_internal(new_file_id, file_id));
} }
void FileManager::on_partial_generate(QueryId query_id, const PartialLocalFileLocation &partial_local, void FileManager::on_partial_generate(QueryId query_id, const PartialLocalFileLocation &partial_local,
@ -3512,12 +3582,12 @@ void FileManager::on_generate_ok(QueryId query_id, const FullLocalFileLocation &
auto old_upload_id = file_node->upload_id_; auto old_upload_id = file_node->upload_id_;
auto r_new_file_id = register_local(local, DialogId(), 0); auto r_new_file_id = register_local_internal(local, DialogId(), 0);
Status status; Status status;
if (r_new_file_id.is_error()) { if (r_new_file_id.is_error()) {
status = Status::Error(PSLICE() << "Can't register local file after generate: " << r_new_file_id.error()); status = Status::Error(PSLICE() << "Can't register local file after generate: " << r_new_file_id.error());
} else { } else {
auto result = merge(r_new_file_id.ok(), generate_file_id); auto result = merge_internal(r_new_file_id.ok(), generate_file_id);
if (result.is_error()) { if (result.is_error()) {
status = result.move_as_error(); status = result.move_as_error();
} }
@ -3636,7 +3706,7 @@ void FileManager::on_error_impl(FileNodePtr node, Query::Type type, bool was_act
LOG(ERROR) << "Unexpected error, file_reference will be deleted just in case " << status; LOG(ERROR) << "Unexpected error, file_reference will be deleted just in case " << status;
} }
CHECK(!node->file_ids_.empty()); CHECK(!node->file_ids_.empty());
delete_file_reference(node->file_ids_.back(), file_reference); delete_file_reference_internal(node->file_ids_.back(), file_reference);
run_download(node); run_download(node);
return; return;
} }
@ -3762,6 +3832,7 @@ void FileManager::destroy_query(int32 file_id) {
void FileManager::memory_cleanup() { void FileManager::memory_cleanup() {
/* DESTROY OLD file_id_info_ */ /* DESTROY OLD file_id_info_ */
{ {
std::lock_guard<std::shared_timed_mutex> writerLock(memory_cleanup_mutex);
auto it = file_id_info_.begin(); auto it = file_id_info_.begin();
auto time = std::time(nullptr); auto time = std::time(nullptr);
std::vector<int32> file_to_be_deleted = {}; std::vector<int32> file_to_be_deleted = {};
@ -3770,15 +3841,15 @@ void FileManager::memory_cleanup() {
if (it->second.node_id_ != 0) { if (it->second.node_id_ != 0) {
auto &node = file_nodes_[it->second.node_id_]; auto &node = file_nodes_[it->second.node_id_];
if (time - node.main_file_id_.get_time() > FILE_TTL) { if (time - node->main_file_id_.get_time() > FILE_TTL) {
auto can_reset = node.download_priority_ == 0; auto can_reset = node->download_priority_ == 0;
can_reset &= node.generate_download_priority_ == 0; can_reset &= node->generate_download_priority_ == 0;
can_reset &= node.download_id_ == 0; can_reset &= node->download_id_ == 0;
if (can_reset) { if (can_reset) {
auto file_ids_it = node.file_ids_.begin(); auto file_ids_it = node->file_ids_.begin();
while (file_ids_it != node.file_ids_.end() && can_reset) { while (file_ids_it != node->file_ids_.end() && can_reset) {
auto &file = file_id_info_[file_ids_it->fast_get()]; auto &file = file_id_info_[file_ids_it->fast_get()];
can_reset &= file.download_priority_ == 0; can_reset &= file.download_priority_ == 0;
can_reset &= time - file_ids_it->get_time() > FILE_TTL; can_reset &= time - file_ids_it->get_time() > FILE_TTL;
@ -3787,9 +3858,9 @@ void FileManager::memory_cleanup() {
} }
if (can_reset) { if (can_reset) {
node.main_file_id_.reset_time(); node->main_file_id_.reset_time();
for (auto &file_id : node.file_ids_) { for (auto &file_id : node->file_ids_) {
file_id.reset_time(); file_id.reset_time();
/* DESTROY ASSOCIATED QUERIES */ /* DESTROY ASSOCIATED QUERIES */
@ -3827,7 +3898,8 @@ void FileManager::memory_cleanup() {
auto it = file_id_info_.begin(); auto it = file_id_info_.begin();
while (it != file_id_info_.end()) { while (it != file_id_info_.end()) {
if (it->second.node_id_ != 0) { if (it->second.node_id_ != 0) {
if (file_nodes_[it->second.node_id_].empty) { auto find_file_node = file_nodes_.find(it->second.node_id_);
if (find_file_node == file_nodes_.end() || find_file_node->second->empty) {
destroy_query(it->first); destroy_query(it->first);
context_->destroy_file_source({it->first, 0}); context_->destroy_file_source({it->first, 0});
file_nodes_.erase(it->second.node_id_); file_nodes_.erase(it->second.node_id_);
@ -3845,18 +3917,18 @@ void FileManager::memory_cleanup() {
{ {
auto it = file_nodes_.begin(); auto it = file_nodes_.begin();
while (it != file_nodes_.end()) { while (it != file_nodes_.end()) {
if (it->second.empty) { if (it->second->empty) {
file_nodes_.erase(it++); file_nodes_.erase(it++);
} else { } else {
if (it->second.main_file_id_.empty()) { if (it->second->main_file_id_.empty()) {
file_nodes_.erase(it++); file_nodes_.erase(it++);
} else { } else {
if (file_id_info_[it->second.main_file_id_.get()].node_id_ == 0) { if (file_id_info_[it->second->main_file_id_.get()].node_id_ == 0) {
for (auto &file_id : it->second.file_ids_) { for (auto &file_id : it->second->file_ids_) {
context_->destroy_file_source(file_id); context_->destroy_file_source(file_id);
file_id_info_.erase(file_id.get()); file_id_info_.erase(file_id.get());
} }
file_id_info_.erase(it->second.main_file_id_.get()); file_id_info_.erase(it->second->main_file_id_.get());
file_nodes_.erase(it++); file_nodes_.erase(it++);
} else { } else {
++it; ++it;
@ -3871,7 +3943,8 @@ void FileManager::memory_cleanup() {
auto it = file_hash_to_file_id_.begin(); auto it = file_hash_to_file_id_.begin();
while (it != file_hash_to_file_id_.end()) { while (it != file_hash_to_file_id_.end()) {
auto &file = file_id_info_[it->second.fast_get()]; auto &file = file_id_info_[it->second.fast_get()];
if (file_nodes_[file.node_id_].empty) { auto find_file_node = file_nodes_.find(file.node_id_);
if (find_file_node == file_nodes_.end() || find_file_node->second->empty) {
file_hash_to_file_id_.erase(it++); file_hash_to_file_id_.erase(it++);
file_nodes_.erase(file.node_id_); file_nodes_.erase(file.node_id_);
} else { } else {
@ -3885,7 +3958,8 @@ void FileManager::memory_cleanup() {
auto it = local_location_to_file_id_.begin(); auto it = local_location_to_file_id_.begin();
while (it != local_location_to_file_id_.end()) { while (it != local_location_to_file_id_.end()) {
auto &file = file_id_info_[it->second.fast_get()]; auto &file = file_id_info_[it->second.fast_get()];
if (file_nodes_[file.node_id_].empty) { auto find_file_node = file_nodes_.find(file.node_id_);
if (find_file_node == file_nodes_.end() || find_file_node->second->empty) {
it = local_location_to_file_id_.erase(it++); it = local_location_to_file_id_.erase(it++);
file_nodes_.erase(file.node_id_); file_nodes_.erase(file.node_id_);
} else { } else {
@ -3899,7 +3973,8 @@ void FileManager::memory_cleanup() {
auto it = generate_location_to_file_id_.begin(); auto it = generate_location_to_file_id_.begin();
while (it != generate_location_to_file_id_.end()) { while (it != generate_location_to_file_id_.end()) {
auto &file = file_id_info_[it->second.fast_get()]; auto &file = file_id_info_[it->second.fast_get()];
if (file_nodes_[file.node_id_].empty) { auto find_file_node = file_nodes_.find(file.node_id_);
if (find_file_node == file_nodes_.end() || find_file_node->second->empty) {
it = generate_location_to_file_id_.erase(it++); it = generate_location_to_file_id_.erase(it++);
file_nodes_.erase(file.node_id_); file_nodes_.erase(file.node_id_);
} else { } else {
@ -3914,7 +3989,8 @@ void FileManager::memory_cleanup() {
auto it = map.begin(); auto it = map.begin();
while (it != map.end()) { while (it != map.end()) {
auto &file = file_id_info_[it->first.file_id_.fast_get()]; auto &file = file_id_info_[it->first.file_id_.fast_get()];
if (file_nodes_[file.node_id_].empty) { auto find_file_node = file_nodes_.find(file.node_id_);
if (find_file_node == file_nodes_.end() || find_file_node->second->empty) {
remote_location_info_.erase(it->second); remote_location_info_.erase(it->second);
map.erase(it++); map.erase(it++);
file_nodes_.erase(file.node_id_); file_nodes_.erase(file.node_id_);
@ -3928,7 +4004,8 @@ void FileManager::memory_cleanup() {
{ {
auto it = file_id_info_.begin(); auto it = file_id_info_.begin();
while (it != file_id_info_.end()) { while (it != file_id_info_.end()) {
if (file_nodes_[it->second.node_id_].empty) { auto find_file_node = file_nodes_.find(it->second.node_id_);
if (find_file_node == file_nodes_.end() || find_file_node->second->empty) {
context_->destroy_file_source({it->first, 0}); context_->destroy_file_source({it->first, 0});
file_id_info_.erase(it++); file_id_info_.erase(it++);
} else { } else {
@ -3937,9 +4014,9 @@ void FileManager::memory_cleanup() {
} }
} }
file_nodes_.rehash(0); file_nodes_.rehash(file_nodes_.size() + 1);
file_hash_to_file_id_.rehash(0); file_hash_to_file_id_.rehash(file_hash_to_file_id_.size() + 1);
file_id_info_.rehash(0); file_id_info_.rehash(file_id_info_.size() + 1);
LOG(ERROR) << "registered ids: " << file_id_info_.size() << " registered nodes: " << file_nodes_.size(); LOG(ERROR) << "registered ids: " << file_id_info_.size() << " registered nodes: " << file_nodes_.size();
} }

25
td/telegram/files/FileManager.h
View File

@ -41,6 +41,7 @@
#include <unordered_map> #include <unordered_map>
#include <unordered_set> #include <unordered_set>
#include <utility> #include <utility>
#include <shared_mutex>
namespace td { namespace td {
@ -72,7 +73,8 @@ class FileNode {
FileNode(LocalFileLocation local, NewRemoteFileLocation remote, unique_ptr<FullGenerateFileLocation> generate, FileNode(LocalFileLocation local, NewRemoteFileLocation remote, unique_ptr<FullGenerateFileLocation> generate,
int64 size, int64 expected_size, string remote_name, string url, DialogId owner_dialog_id, int64 size, int64 expected_size, string remote_name, string url, DialogId owner_dialog_id,
FileEncryptionKey key, FileId main_file_id, int8 main_file_id_priority) FileEncryptionKey key, FileId main_file_id, int8 main_file_id_priority)
: local_(std::move(local)) : empty(false)
, local_(std::move(local))
, remote_(std::move(remote)) , remote_(std::move(remote))
, generate_(std::move(generate)) , generate_(std::move(generate))
, size_(size) , size_(size)
@ -93,6 +95,7 @@ class FileNode {
void set_partial_remote_location(const PartialRemoteFileLocation &remote, int64 ready_size); void set_partial_remote_location(const PartialRemoteFileLocation &remote, int64 ready_size);
bool delete_file_reference(Slice file_reference); bool delete_file_reference(Slice file_reference);
bool delete_file_reference_internal(Slice file_reference);
void set_generate_location(unique_ptr<FullGenerateFileLocation> &&generate); void set_generate_location(unique_ptr<FullGenerateFileLocation> &&generate);
void set_size(int64 size); void set_size(int64 size);
void set_expected_size(int64 expected_size); void set_expected_size(int64 expected_size);
@ -419,23 +422,38 @@ class FileManager : public FileLoadManager::Callback {
FileId dup_file_id(FileId file_id); FileId dup_file_id(FileId file_id);
FileId dup_file_id_internal(FileId file_id);
void on_file_unlink(const FullLocalFileLocation &location); void on_file_unlink(const FullLocalFileLocation &location);
FileId register_empty(FileType type); FileId register_empty(FileType type);
Result<FileId> register_local(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size, Result<FileId> register_local(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size,
bool get_by_hash = false, bool force = false, bool get_by_hash = false, bool force = false,
bool skip_file_size_checks = false) TD_WARN_UNUSED_RESULT; bool skip_file_size_checks = false) TD_WARN_UNUSED_RESULT;
Result<FileId> register_local_internal(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size,
bool get_by_hash = false, bool force = false,
bool skip_file_size_checks = false) TD_WARN_UNUSED_RESULT;
FileId register_remote(const FullRemoteFileLocation &location, FileLocationSource file_location_source, FileId register_remote(const FullRemoteFileLocation &location, FileLocationSource file_location_source,
DialogId owner_dialog_id, int64 size, int64 expected_size, DialogId owner_dialog_id, int64 size, int64 expected_size,
string remote_name) TD_WARN_UNUSED_RESULT; string remote_name) TD_WARN_UNUSED_RESULT;
FileId register_remote_internal(const FullRemoteFileLocation &location, FileLocationSource file_location_source,
DialogId owner_dialog_id, int64 size, int64 expected_size,
string remote_name) TD_WARN_UNUSED_RESULT;
Result<FileId> register_generate(FileType file_type, FileLocationSource file_location_source, string original_path, Result<FileId> register_generate(FileType file_type, FileLocationSource file_location_source, string original_path,
string conversion, DialogId owner_dialog_id, string conversion, DialogId owner_dialog_id,
int64 expected_size) TD_WARN_UNUSED_RESULT; int64 expected_size) TD_WARN_UNUSED_RESULT;
Result<FileId> register_generate_internal(FileType file_type, FileLocationSource file_location_source, string original_path,
string conversion, DialogId owner_dialog_id,
int64 expected_size) TD_WARN_UNUSED_RESULT;
Result<FileId> merge(FileId x_file_id, FileId y_file_id, bool no_sync = false) TD_WARN_UNUSED_RESULT; Result<FileId> merge(FileId x_file_id, FileId y_file_id, bool no_sync = false) TD_WARN_UNUSED_RESULT;
Result<FileId> merge_internal(FileId x_file_id, FileId y_file_id, bool no_sync = false) TD_WARN_UNUSED_RESULT;
void add_file_source(FileId file_id, FileSourceId file_source_id); void add_file_source(FileId file_id, FileSourceId file_source_id);
void add_file_source_internal(FileId file_id, FileSourceId file_source_id);
void remove_file_source(FileId file_id, FileSourceId file_source_id); void remove_file_source(FileId file_id, FileSourceId file_source_id);
void change_files_source(FileSourceId file_source_id, const vector<FileId> &old_file_ids, void change_files_source(FileSourceId file_source_id, const vector<FileId> &old_file_ids,
@ -455,6 +473,7 @@ class FileManager : public FileLoadManager::Callback {
void cancel_upload(FileId file_id); void cancel_upload(FileId file_id);
bool delete_partial_remote_location(FileId file_id); bool delete_partial_remote_location(FileId file_id);
void delete_file_reference(FileId file_id, std::string file_reference); void delete_file_reference(FileId file_id, std::string file_reference);
void delete_file_reference_internal(FileId file_id, std::string file_reference);
void get_content(FileId file_id, Promise<BufferSlice> promise); void get_content(FileId file_id, Promise<BufferSlice> promise);
void read_file_part(FileId file_id, int32 offset, int32 count, int left_tries, void read_file_part(FileId file_id, int32 offset, int32 count, int left_tries,
@ -580,7 +599,7 @@ class FileManager : public FileLoadManager::Callback {
std::map<FileDbId, int32> pmc_id_to_file_node_id_; std::map<FileDbId, int32> pmc_id_to_file_node_id_;
std::unordered_map<int32, FileIdInfo> file_id_info_; std::unordered_map<int32, FileIdInfo> file_id_info_;
std::unordered_map<FileNodeId, FileNode> file_nodes_; std::unordered_map<FileNodeId, unique_ptr<FileNode>> file_nodes_;
ActorOwn<FileLoadManager> file_load_manager_; ActorOwn<FileLoadManager> file_load_manager_;
ActorOwn<FileGenerateManager> file_generate_manager_; ActorOwn<FileGenerateManager> file_generate_manager_;
@ -668,6 +687,8 @@ class FileManager : public FileLoadManager::Callback {
void hangup() override; void hangup() override;
void tear_down() override; void tear_down() override;
mutable std::shared_timed_mutex memory_cleanup_mutex;
friend class FileNodePtr; friend class FileNodePtr;
}; };