Increased files TTL to 120 seconds, fixed rehash buckets count, lock other methods when cleaning, unique_ptr for nodes
This commit is contained in:
parent
22192cda3c
commit
000375e21b
@ -49,7 +49,7 @@
|
||||
#include <unordered_set>
|
||||
#include <utility>
|
||||
|
||||
#define FILE_TTL 20
|
||||
#define FILE_TTL 120
|
||||
|
||||
namespace td {
|
||||
namespace {
|
||||
@ -300,7 +300,7 @@ void FileNode::set_partial_remote_location(const PartialRemoteFileLocation &remo
|
||||
on_changed();
|
||||
}
|
||||
|
||||
bool FileNode::delete_file_reference(Slice file_reference) {
|
||||
bool FileNode::delete_file_reference_internal(Slice file_reference) {
|
||||
if (!remote_.full) {
|
||||
VLOG(file_references) << "Can't delete file reference, because there is no remote location";
|
||||
return false;
|
||||
@ -1038,6 +1038,11 @@ FileManager::FileIdInfo *FileManager::get_file_id_info(FileId file_id) {
|
||||
}
|
||||
|
||||
FileId FileManager::dup_file_id(FileId file_id) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
return dup_file_id_internal(file_id);
|
||||
}
|
||||
|
||||
FileId FileManager::dup_file_id_internal(FileId file_id) {
|
||||
int32 file_node_id;
|
||||
auto *file_node = get_file_node_raw(file_id, &file_node_id);
|
||||
if (!file_node) {
|
||||
@ -1073,7 +1078,7 @@ void FileManager::try_forget_file_id(FileId file_id) {
|
||||
}
|
||||
|
||||
FileId FileManager::register_empty(FileType type) {
|
||||
return register_local(FullLocalFileLocation(type, "", 0), DialogId(), 0, false, true).ok();
|
||||
return register_local_internal(FullLocalFileLocation(type, "", 0), DialogId(), 0, false, true).ok();
|
||||
}
|
||||
|
||||
void FileManager::on_file_unlink(const FullLocalFileLocation &location) {
|
||||
@ -1093,6 +1098,12 @@ void FileManager::on_file_unlink(const FullLocalFileLocation &location) {
|
||||
|
||||
Result<FileId> FileManager::register_local(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size,
|
||||
bool get_by_hash, bool force, bool skip_file_size_checks) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
return register_local_internal(location, owner_dialog_id, size, get_by_hash, force, skip_file_size_checks);
|
||||
}
|
||||
|
||||
Result<FileId> FileManager::register_local_internal(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size,
|
||||
bool get_by_hash, bool force, bool skip_file_size_checks) {
|
||||
// TODO: use get_by_hash
|
||||
FileData data;
|
||||
data.local_ = LocalFileLocation(std::move(location));
|
||||
@ -1103,6 +1114,12 @@ Result<FileId> FileManager::register_local(FullLocalFileLocation location, Dialo
|
||||
}
|
||||
|
||||
FileId FileManager::register_remote(const FullRemoteFileLocation &location, FileLocationSource file_location_source,
|
||||
DialogId owner_dialog_id, int64 size, int64 expected_size, string remote_name) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
return register_remote_internal(location, file_location_source, owner_dialog_id, size, expected_size, std::move(remote_name));
|
||||
}
|
||||
|
||||
FileId FileManager::register_remote_internal(const FullRemoteFileLocation &location, FileLocationSource file_location_source,
|
||||
DialogId owner_dialog_id, int64 size, int64 expected_size, string remote_name) {
|
||||
FileData data;
|
||||
data.remote_ = RemoteFileLocation(location);
|
||||
@ -1123,7 +1140,7 @@ FileId FileManager::register_remote(const FullRemoteFileLocation &location, File
|
||||
|
||||
FileId FileManager::register_url(string url, FileType file_type, FileLocationSource file_location_source,
|
||||
DialogId owner_dialog_id) {
|
||||
auto file_id = register_generate(file_type, file_location_source, url, "#url#", owner_dialog_id, 0).ok();
|
||||
auto file_id = register_generate_internal(file_type, file_location_source, url, "#url#", owner_dialog_id, 0).ok();
|
||||
auto file_node = get_file_node(file_id);
|
||||
CHECK(file_node);
|
||||
file_node->set_url(url);
|
||||
@ -1133,6 +1150,13 @@ FileId FileManager::register_url(string url, FileType file_type, FileLocationSou
|
||||
Result<FileId> FileManager::register_generate(FileType file_type, FileLocationSource file_location_source,
|
||||
string original_path, string conversion, DialogId owner_dialog_id,
|
||||
int64 expected_size) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
return register_generate_internal(file_type, file_location_source, std::move(original_path), std::move(conversion), owner_dialog_id, expected_size);
|
||||
}
|
||||
|
||||
Result<FileId> FileManager::register_generate_internal(FileType file_type, FileLocationSource file_location_source,
|
||||
string original_path, string conversion, DialogId owner_dialog_id,
|
||||
int64 expected_size) {
|
||||
// add #mtime# into conversion
|
||||
if (!original_path.empty() && conversion[0] != '#' && PathView(original_path).is_absolute()) {
|
||||
auto file_paths = log_interface->get_file_paths();
|
||||
@ -1190,13 +1214,13 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi
|
||||
// create FileNode
|
||||
auto file_node_id = next_file_node_id();
|
||||
auto &node = file_nodes_[file_node_id];
|
||||
node = FileNode(std::move(data.local_), NewRemoteFileLocation(data.remote_, file_location_source),
|
||||
node = td::make_unique<FileNode>(std::move(data.local_), NewRemoteFileLocation(data.remote_, file_location_source),
|
||||
std::move(data.generate_), data.size_, data.expected_size_,
|
||||
std::move(data.remote_name_), std::move(data.url_), data.owner_dialog_id_,
|
||||
std::move(data.encryption_key_), file_id, static_cast<int8>(has_remote));
|
||||
node.pmc_id_ = FileDbId(data.pmc_id_);
|
||||
node->pmc_id_ = FileDbId(data.pmc_id_);
|
||||
get_file_id_info(file_id)->node_id_ = file_node_id;
|
||||
node.file_ids_.push_back(file_id);
|
||||
node->file_ids_.push_back(file_id);
|
||||
|
||||
FileView file_view(get_file_node(file_id));
|
||||
|
||||
@ -1243,12 +1267,12 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi
|
||||
|
||||
int new_cnt = new_remote + new_local + new_generate;
|
||||
if (data.pmc_id_ == 0 && file_db_ && new_cnt > 0) {
|
||||
node.need_load_from_pmc_ = true;
|
||||
node->need_load_from_pmc_ = true;
|
||||
}
|
||||
bool no_sync_merge = to_merge.size() == 1 && new_cnt == 0;
|
||||
for (auto id : to_merge) {
|
||||
// may invalidate node
|
||||
merge(file_id, id, no_sync_merge).ignore();
|
||||
merge_internal(file_id, id, no_sync_merge).ignore();
|
||||
}
|
||||
|
||||
try_flush_node(get_file_node(file_id), "register_file");
|
||||
@ -1422,6 +1446,11 @@ void FileManager::do_cancel_generate(FileNodePtr node) {
|
||||
}
|
||||
|
||||
Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
return merge_internal(x_file_id, y_file_id, no_sync);
|
||||
}
|
||||
|
||||
Result<FileId> FileManager::merge_internal(FileId x_file_id, FileId y_file_id, bool no_sync) {
|
||||
LOG(DEBUG) << "Merge new file " << x_file_id << " and old file " << y_file_id;
|
||||
|
||||
if (!x_file_id.is_valid()) {
|
||||
@ -1636,7 +1665,7 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
|
||||
file_id_info->node_id_ = node_ids[node_i];
|
||||
send_updates_flag |= file_id_info->send_updates_flag_;
|
||||
}
|
||||
other_node = {this};
|
||||
other_node = {};
|
||||
|
||||
if (send_updates_flag) {
|
||||
// node might not changed, but other_node might changed, so we need to send update anyway
|
||||
@ -1679,6 +1708,11 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
|
||||
}
|
||||
|
||||
void FileManager::add_file_source(FileId file_id, FileSourceId file_source_id) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
return add_file_source_internal(file_id, file_source_id);
|
||||
}
|
||||
|
||||
void FileManager::add_file_source_internal(FileId file_id, FileSourceId file_source_id) {
|
||||
auto node = get_file_node(file_id);
|
||||
if (!node) {
|
||||
return;
|
||||
@ -1689,6 +1723,7 @@ void FileManager::add_file_source(FileId file_id, FileSourceId file_source_id) {
|
||||
node->on_pmc_changed();
|
||||
try_flush_node_pmc(node, "add_file_source");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void FileManager::remove_file_source(FileId file_id, FileSourceId file_source_id) {
|
||||
@ -1706,6 +1741,7 @@ void FileManager::remove_file_source(FileId file_id, FileSourceId file_source_id
|
||||
|
||||
void FileManager::change_files_source(FileSourceId file_source_id, const vector<FileId> &old_file_ids,
|
||||
const vector<FileId> &new_file_ids) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
if (old_file_ids == new_file_ids) {
|
||||
return;
|
||||
}
|
||||
@ -1722,7 +1758,7 @@ void FileManager::change_files_source(FileSourceId file_source_id, const vector<
|
||||
}
|
||||
}
|
||||
for (auto file_id : new_main_file_ids) {
|
||||
add_file_source(file_id, file_source_id);
|
||||
add_file_source_internal(file_id, file_source_id);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1881,7 +1917,7 @@ FileNode *FileManager::get_file_node_raw(FileId file_id, FileNodeId *file_node_i
|
||||
if (file_node_id != nullptr) {
|
||||
*file_node_id = node_id;
|
||||
}
|
||||
return &file_nodes_[node_id];
|
||||
return file_nodes_[node_id].get();
|
||||
}
|
||||
|
||||
FileNodePtr FileManager::get_sync_file_node(FileId file_id) {
|
||||
@ -1927,7 +1963,7 @@ void FileManager::load_from_pmc(FileNodePtr node, bool new_remote, bool new_loca
|
||||
TRY_RESULT(file_data, file_db_->get_file_data_sync(location));
|
||||
TRY_RESULT(new_file_id,
|
||||
register_file(std::move(file_data), FileLocationSource::FromDatabase, "load_from_pmc", false));
|
||||
TRY_RESULT(main_file_id, merge(file_id, new_file_id));
|
||||
TRY_RESULT(main_file_id, merge_internal(file_id, new_file_id));
|
||||
file_id = main_file_id;
|
||||
return Status::OK();
|
||||
};
|
||||
@ -1960,6 +1996,7 @@ bool FileManager::set_encryption_key(FileId file_id, FileEncryptionKey key) {
|
||||
}
|
||||
|
||||
bool FileManager::set_content(FileId file_id, BufferSlice bytes) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
if (G()->shared_config().get_option_boolean("ignore_inline_thumbnails")) {
|
||||
return false;
|
||||
}
|
||||
@ -1994,6 +2031,7 @@ bool FileManager::set_content(FileId file_id, BufferSlice bytes) {
|
||||
}
|
||||
|
||||
void FileManager::get_content(FileId file_id, Promise<BufferSlice> promise) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
auto node = get_sync_file_node(file_id);
|
||||
if (!node) {
|
||||
return promise.set_error(Status::Error("Unknown file_id"));
|
||||
@ -2011,6 +2049,7 @@ void FileManager::get_content(FileId file_id, Promise<BufferSlice> promise) {
|
||||
|
||||
void FileManager::read_file_part(FileId file_id, int32 offset, int32 count, int left_tries,
|
||||
Promise<td_api::object_ptr<td_api::filePart>> promise) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
if (G()->close_flag()) {
|
||||
return promise.set_error(Status::Error(500, "Request aborted"));
|
||||
}
|
||||
@ -2089,6 +2128,7 @@ void FileManager::read_file_part(FileId file_id, int32 offset, int32 count, int
|
||||
}
|
||||
|
||||
void FileManager::delete_file(FileId file_id, Promise<Unit> promise, const char *source) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
LOG(INFO) << "Trying to delete file " << file_id << " from " << source;
|
||||
auto node = get_sync_file_node(file_id);
|
||||
if (!node) {
|
||||
@ -2125,6 +2165,7 @@ void FileManager::delete_file(FileId file_id, Promise<Unit> promise, const char
|
||||
|
||||
void FileManager::download(FileId file_id, std::shared_ptr<DownloadCallback> callback, int32 new_priority, int64 offset,
|
||||
int64 limit) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
LOG(INFO) << "Download file " << file_id << " with priority " << new_priority;
|
||||
auto node = get_sync_file_node(file_id);
|
||||
if (!node) {
|
||||
@ -2516,6 +2557,7 @@ void FileManager::resume_upload(FileId file_id, std::vector<int> bad_parts, std:
|
||||
}
|
||||
|
||||
bool FileManager::delete_partial_remote_location(FileId file_id) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
auto node = get_sync_file_node(file_id);
|
||||
if (!node) {
|
||||
LOG(INFO) << "Wrong file identifier " << file_id;
|
||||
@ -2550,6 +2592,11 @@ bool FileManager::delete_partial_remote_location(FileId file_id) {
|
||||
}
|
||||
|
||||
void FileManager::delete_file_reference(FileId file_id, string file_reference) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
return delete_file_reference_internal(file_id, file_reference);
|
||||
}
|
||||
|
||||
void FileManager::delete_file_reference_internal(FileId file_id, string file_reference) {
|
||||
VLOG(file_references) << "Delete file reference of file " << file_id << " "
|
||||
<< tag("reference_base64", base64_encode(file_reference));
|
||||
auto node = get_sync_file_node(file_id);
|
||||
@ -2557,7 +2604,7 @@ void FileManager::delete_file_reference(FileId file_id, string file_reference) {
|
||||
LOG(ERROR) << "Wrong file identifier " << file_id;
|
||||
return;
|
||||
}
|
||||
node->delete_file_reference(file_reference);
|
||||
node->delete_file_reference_internal(file_reference);
|
||||
auto remote = get_remote(file_id.get_remote());
|
||||
if (remote != nullptr) {
|
||||
VLOG(file_references) << "Do delete file reference of remote file " << file_id;
|
||||
@ -2572,17 +2619,20 @@ void FileManager::delete_file_reference(FileId file_id, string file_reference) {
|
||||
}
|
||||
|
||||
void FileManager::external_file_generate_write_part(int64 id, int32 offset, string data, Promise<> promise) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_write_part, id, offset,
|
||||
std::move(data), std::move(promise));
|
||||
}
|
||||
|
||||
void FileManager::external_file_generate_progress(int64 id, int32 expected_size, int32 local_prefix_size,
|
||||
Promise<> promise) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_progress, id, expected_size,
|
||||
local_prefix_size, std::move(promise));
|
||||
}
|
||||
|
||||
void FileManager::external_file_generate_finish(int64 id, Status status, Promise<> promise) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_finish, id, std::move(status),
|
||||
std::move(promise));
|
||||
}
|
||||
@ -2784,10 +2834,12 @@ void FileManager::run_upload(FileNodePtr node, std::vector<int> bad_parts) {
|
||||
|
||||
void FileManager::upload(FileId file_id, std::shared_ptr<UploadCallback> callback, int32 new_priority,
|
||||
uint64 upload_order) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
return resume_upload(file_id, std::vector<int>(), std::move(callback), new_priority, upload_order);
|
||||
}
|
||||
|
||||
void FileManager::cancel_upload(FileId file_id) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
return resume_upload(file_id, std::vector<int>(), nullptr, 0, 0);
|
||||
}
|
||||
|
||||
@ -2912,6 +2964,7 @@ FileView FileManager::get_sync_file_view(FileId file_id) {
|
||||
}
|
||||
|
||||
td_api::object_ptr<td_api::file> FileManager::get_file_object(FileId file_id, bool with_main_file_id) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
auto file_view = get_sync_file_view(file_id);
|
||||
|
||||
if (file_view.empty()) {
|
||||
@ -2955,6 +3008,7 @@ td_api::object_ptr<td_api::file> FileManager::get_file_object(FileId file_id, bo
|
||||
|
||||
vector<int32> FileManager::get_file_ids_object(const vector<FileId> &file_ids, bool with_main_file_id) {
|
||||
return transform(file_ids, [this, with_main_file_id](FileId file_id) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
auto file_view = get_sync_file_view(file_id);
|
||||
auto result_file_id = file_id;
|
||||
auto *file_info = get_file_id_info(result_file_id);
|
||||
@ -2996,7 +3050,7 @@ Result<FileId> FileManager::check_input_file_id(FileType type, Result<FileId> re
|
||||
if (!file_view.has_remote_location()) {
|
||||
// TODO why not return file_id here? We will dup it anyway
|
||||
// But it will not be duped if has_input_media(), so for now we can't return main_file_id
|
||||
return dup_file_id(file_id);
|
||||
return dup_file_id_internal(file_id);
|
||||
}
|
||||
|
||||
int32 remote_id = file_id.get_remote();
|
||||
@ -3012,6 +3066,7 @@ Result<FileId> FileManager::check_input_file_id(FileType type, Result<FileId> re
|
||||
|
||||
Result<FileId> FileManager::get_input_thumbnail_file_id(const tl_object_ptr<td_api::InputFile> &thumbnail_input_file,
|
||||
DialogId owner_dialog_id, bool is_encrypted) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
if (thumbnail_input_file == nullptr) {
|
||||
return Status::Error(6, "inputThumbnail not specified");
|
||||
}
|
||||
@ -3019,7 +3074,7 @@ Result<FileId> FileManager::get_input_thumbnail_file_id(const tl_object_ptr<td_a
|
||||
switch (thumbnail_input_file->get_id()) {
|
||||
case td_api::inputFileLocal::ID: {
|
||||
const string &path = static_cast<const td_api::inputFileLocal *>(thumbnail_input_file.get())->path_;
|
||||
return register_local(
|
||||
return register_local_internal(
|
||||
FullLocalFileLocation(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail, path, 0),
|
||||
owner_dialog_id, 0, false);
|
||||
}
|
||||
@ -3029,7 +3084,7 @@ Result<FileId> FileManager::get_input_thumbnail_file_id(const tl_object_ptr<td_a
|
||||
return Status::Error(6, "InputFileRemote is not supported for thumbnails");
|
||||
case td_api::inputFileGenerated::ID: {
|
||||
auto *generated_thumbnail = static_cast<const td_api::inputFileGenerated *>(thumbnail_input_file.get());
|
||||
return register_generate(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail,
|
||||
return register_generate_internal(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail,
|
||||
FileLocationSource::FromUser, generated_thumbnail->original_path_,
|
||||
generated_thumbnail->conversion_, owner_dialog_id, generated_thumbnail->expected_size_);
|
||||
}
|
||||
@ -3042,6 +3097,7 @@ Result<FileId> FileManager::get_input_thumbnail_file_id(const tl_object_ptr<td_a
|
||||
Result<FileId> FileManager::get_input_file_id(FileType type, const tl_object_ptr<td_api::InputFile> &file,
|
||||
DialogId owner_dialog_id, bool allow_zero, bool is_encrypted,
|
||||
bool get_by_hash, bool is_secure) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
if (file == nullptr) {
|
||||
if (allow_zero) {
|
||||
return FileId();
|
||||
@ -3079,7 +3135,7 @@ Result<FileId> FileManager::get_input_file_id(FileType type, const tl_object_ptr
|
||||
}
|
||||
}
|
||||
}
|
||||
TRY_RESULT(file_id, register_local(FullLocalFileLocation(new_type, path, 0), owner_dialog_id, 0, get_by_hash));
|
||||
TRY_RESULT(file_id, register_local_internal(FullLocalFileLocation(new_type, path, 0), owner_dialog_id, 0, get_by_hash));
|
||||
if (!hash.empty()) {
|
||||
file_hash_to_file_id_[hash] = file_id;
|
||||
}
|
||||
@ -3101,7 +3157,7 @@ Result<FileId> FileManager::get_input_file_id(FileType type, const tl_object_ptr
|
||||
}
|
||||
case td_api::inputFileGenerated::ID: {
|
||||
auto *generated_file = static_cast<const td_api::inputFileGenerated *>(file.get());
|
||||
return register_generate(new_type, FileLocationSource::FromUser, generated_file->original_path_,
|
||||
return register_generate_internal(new_type, FileLocationSource::FromUser, generated_file->original_path_,
|
||||
generated_file->conversion_, owner_dialog_id, generated_file->expected_size_);
|
||||
}
|
||||
default:
|
||||
@ -3115,6 +3171,7 @@ Result<FileId> FileManager::get_input_file_id(FileType type, const tl_object_ptr
|
||||
|
||||
Result<FileId> FileManager::get_map_thumbnail_file_id(Location location, int32 zoom, int32 width, int32 height,
|
||||
int32 scale, DialogId owner_dialog_id) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
if (!location.is_valid_map_point()) {
|
||||
return Status::Error(6, "Invalid location specified");
|
||||
}
|
||||
@ -3141,12 +3198,13 @@ Result<FileId> FileManager::get_map_thumbnail_file_id(Location location, int32 z
|
||||
|
||||
string conversion = PSTRING() << "#map#" << zoom << "#" << x << "#" << y << "#" << width << "#" << height << "#"
|
||||
<< scale << "#";
|
||||
return register_generate(
|
||||
return register_generate_internal(
|
||||
owner_dialog_id.get_type() == DialogType::SecretChat ? FileType::EncryptedThumbnail : FileType::Thumbnail,
|
||||
FileLocationSource::FromUser, string(), std::move(conversion), owner_dialog_id, 0);
|
||||
}
|
||||
|
||||
vector<tl_object_ptr<telegram_api::InputDocument>> FileManager::get_input_documents(const vector<FileId> &file_ids) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
vector<tl_object_ptr<telegram_api::InputDocument>> result;
|
||||
result.reserve(file_ids.size());
|
||||
for (auto file_id : file_ids) {
|
||||
@ -3178,6 +3236,7 @@ bool FileManager::extract_was_thumbnail_uploaded(const tl_object_ptr<telegram_ap
|
||||
}
|
||||
|
||||
string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputMedia> &input_media) {
|
||||
// don't lock this method because now it only calls the other extract_file_reference
|
||||
if (input_media == nullptr) {
|
||||
return string();
|
||||
}
|
||||
@ -3213,6 +3272,7 @@ bool FileManager::extract_was_uploaded(const tl_object_ptr<telegram_api::InputCh
|
||||
}
|
||||
|
||||
string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::InputChatPhoto> &input_chat_photo) {
|
||||
// don't lock this method because now it only calls the other extract_file_reference
|
||||
if (input_chat_photo == nullptr || input_chat_photo->get_id() != telegram_api::inputChatPhoto::ID) {
|
||||
return string();
|
||||
}
|
||||
@ -3234,6 +3294,8 @@ FileManager::FileNodeId FileManager::next_file_node_id() {
|
||||
}
|
||||
|
||||
void FileManager::on_start_download(QueryId query_id) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
|
||||
if (is_closed_) {
|
||||
return;
|
||||
}
|
||||
@ -3259,6 +3321,8 @@ void FileManager::on_start_download(QueryId query_id) {
|
||||
|
||||
void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLocation &partial_local, int64 ready_size,
|
||||
int64 size) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
|
||||
if (is_closed_) {
|
||||
return;
|
||||
}
|
||||
@ -3290,6 +3354,8 @@ void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLo
|
||||
}
|
||||
|
||||
void FileManager::on_hash(QueryId query_id, string hash) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
|
||||
if (is_closed_) {
|
||||
return;
|
||||
}
|
||||
@ -3315,6 +3381,8 @@ void FileManager::on_hash(QueryId query_id, string hash) {
|
||||
|
||||
void FileManager::on_partial_upload(QueryId query_id, const PartialRemoteFileLocation &partial_remote,
|
||||
int64 ready_size) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
|
||||
if (is_closed_) {
|
||||
return;
|
||||
}
|
||||
@ -3339,6 +3407,8 @@ void FileManager::on_partial_upload(QueryId query_id, const PartialRemoteFileLoc
|
||||
}
|
||||
|
||||
void FileManager::on_download_ok(QueryId query_id, const FullLocalFileLocation &local, int64 size, bool is_new) {
|
||||
std::shared_lock<std::shared_timed_mutex> readerLock(memory_cleanup_mutex);
|
||||
|
||||
if (is_closed_) {
|
||||
return;
|
||||
}
|
||||
@ -3348,7 +3418,7 @@ void FileManager::on_download_ok(QueryId query_id, const FullLocalFileLocation &
|
||||
std::tie(query, was_active) = finish_query(query_id);
|
||||
auto file_id = query.file_id_;
|
||||
LOG(INFO) << "ON DOWNLOAD OK of " << (is_new ? "new" : "checked") << " file " << file_id << " of size " << size;
|
||||
auto r_new_file_id = register_local(local, DialogId(), size, false, false, true);
|
||||
auto r_new_file_id = register_local_internal(local, DialogId(), size, false, false, true);
|
||||
Status status = Status::OK();
|
||||
if (r_new_file_id.is_error()) {
|
||||
status = Status::Error(PSLICE() << "Can't register local file after download: " << r_new_file_id.error().message());
|
||||
@ -3356,7 +3426,7 @@ void FileManager::on_download_ok(QueryId query_id, const FullLocalFileLocation &
|
||||
if (is_new) {
|
||||
context_->on_new_file(size, get_file_view(r_new_file_id.ok()).get_allocated_local_size(), 1);
|
||||
}
|
||||
auto r_file_id = merge(r_new_file_id.ok(), file_id);
|
||||
auto r_file_id = merge_internal(r_new_file_id.ok(), file_id);
|
||||
if (r_file_id.is_error()) {
|
||||
status = r_file_id.move_as_error();
|
||||
}
|
||||
@ -3451,8 +3521,8 @@ void FileManager::on_upload_full_ok(QueryId query_id, const FullRemoteFileLocati
|
||||
|
||||
auto file_id = finish_query(query_id).first.file_id_;
|
||||
LOG(INFO) << "ON UPLOAD FULL OK for file " << file_id;
|
||||
auto new_file_id = register_remote(remote, FileLocationSource::FromServer, DialogId(), 0, 0, "");
|
||||
LOG_STATUS(merge(new_file_id, file_id));
|
||||
auto new_file_id = register_remote_internal(remote, FileLocationSource::FromServer, DialogId(), 0, 0, "");
|
||||
LOG_STATUS(merge_internal(new_file_id, file_id));
|
||||
}
|
||||
|
||||
void FileManager::on_partial_generate(QueryId query_id, const PartialLocalFileLocation &partial_local,
|
||||
@ -3512,12 +3582,12 @@ void FileManager::on_generate_ok(QueryId query_id, const FullLocalFileLocation &
|
||||
|
||||
auto old_upload_id = file_node->upload_id_;
|
||||
|
||||
auto r_new_file_id = register_local(local, DialogId(), 0);
|
||||
auto r_new_file_id = register_local_internal(local, DialogId(), 0);
|
||||
Status status;
|
||||
if (r_new_file_id.is_error()) {
|
||||
status = Status::Error(PSLICE() << "Can't register local file after generate: " << r_new_file_id.error());
|
||||
} else {
|
||||
auto result = merge(r_new_file_id.ok(), generate_file_id);
|
||||
auto result = merge_internal(r_new_file_id.ok(), generate_file_id);
|
||||
if (result.is_error()) {
|
||||
status = result.move_as_error();
|
||||
}
|
||||
@ -3636,7 +3706,7 @@ void FileManager::on_error_impl(FileNodePtr node, Query::Type type, bool was_act
|
||||
LOG(ERROR) << "Unexpected error, file_reference will be deleted just in case " << status;
|
||||
}
|
||||
CHECK(!node->file_ids_.empty());
|
||||
delete_file_reference(node->file_ids_.back(), file_reference);
|
||||
delete_file_reference_internal(node->file_ids_.back(), file_reference);
|
||||
run_download(node);
|
||||
return;
|
||||
}
|
||||
@ -3762,6 +3832,7 @@ void FileManager::destroy_query(int32 file_id) {
|
||||
void FileManager::memory_cleanup() {
|
||||
/* DESTROY OLD file_id_info_ */
|
||||
{
|
||||
std::lock_guard<std::shared_timed_mutex> writerLock(memory_cleanup_mutex);
|
||||
auto it = file_id_info_.begin();
|
||||
auto time = std::time(nullptr);
|
||||
std::vector<int32> file_to_be_deleted = {};
|
||||
@ -3770,15 +3841,15 @@ void FileManager::memory_cleanup() {
|
||||
if (it->second.node_id_ != 0) {
|
||||
auto &node = file_nodes_[it->second.node_id_];
|
||||
|
||||
if (time - node.main_file_id_.get_time() > FILE_TTL) {
|
||||
auto can_reset = node.download_priority_ == 0;
|
||||
can_reset &= node.generate_download_priority_ == 0;
|
||||
can_reset &= node.download_id_ == 0;
|
||||
if (time - node->main_file_id_.get_time() > FILE_TTL) {
|
||||
auto can_reset = node->download_priority_ == 0;
|
||||
can_reset &= node->generate_download_priority_ == 0;
|
||||
can_reset &= node->download_id_ == 0;
|
||||
|
||||
if (can_reset) {
|
||||
auto file_ids_it = node.file_ids_.begin();
|
||||
auto file_ids_it = node->file_ids_.begin();
|
||||
|
||||
while (file_ids_it != node.file_ids_.end() && can_reset) {
|
||||
while (file_ids_it != node->file_ids_.end() && can_reset) {
|
||||
auto &file = file_id_info_[file_ids_it->fast_get()];
|
||||
can_reset &= file.download_priority_ == 0;
|
||||
can_reset &= time - file_ids_it->get_time() > FILE_TTL;
|
||||
@ -3787,9 +3858,9 @@ void FileManager::memory_cleanup() {
|
||||
}
|
||||
|
||||
if (can_reset) {
|
||||
node.main_file_id_.reset_time();
|
||||
node->main_file_id_.reset_time();
|
||||
|
||||
for (auto &file_id : node.file_ids_) {
|
||||
for (auto &file_id : node->file_ids_) {
|
||||
file_id.reset_time();
|
||||
|
||||
/* DESTROY ASSOCIATED QUERIES */
|
||||
@ -3827,7 +3898,8 @@ void FileManager::memory_cleanup() {
|
||||
auto it = file_id_info_.begin();
|
||||
while (it != file_id_info_.end()) {
|
||||
if (it->second.node_id_ != 0) {
|
||||
if (file_nodes_[it->second.node_id_].empty) {
|
||||
auto find_file_node = file_nodes_.find(it->second.node_id_);
|
||||
if (find_file_node == file_nodes_.end() || find_file_node->second->empty) {
|
||||
destroy_query(it->first);
|
||||
context_->destroy_file_source({it->first, 0});
|
||||
file_nodes_.erase(it->second.node_id_);
|
||||
@ -3845,18 +3917,18 @@ void FileManager::memory_cleanup() {
|
||||
{
|
||||
auto it = file_nodes_.begin();
|
||||
while (it != file_nodes_.end()) {
|
||||
if (it->second.empty) {
|
||||
if (it->second->empty) {
|
||||
file_nodes_.erase(it++);
|
||||
} else {
|
||||
if (it->second.main_file_id_.empty()) {
|
||||
if (it->second->main_file_id_.empty()) {
|
||||
file_nodes_.erase(it++);
|
||||
} else {
|
||||
if (file_id_info_[it->second.main_file_id_.get()].node_id_ == 0) {
|
||||
for (auto &file_id : it->second.file_ids_) {
|
||||
if (file_id_info_[it->second->main_file_id_.get()].node_id_ == 0) {
|
||||
for (auto &file_id : it->second->file_ids_) {
|
||||
context_->destroy_file_source(file_id);
|
||||
file_id_info_.erase(file_id.get());
|
||||
}
|
||||
file_id_info_.erase(it->second.main_file_id_.get());
|
||||
file_id_info_.erase(it->second->main_file_id_.get());
|
||||
file_nodes_.erase(it++);
|
||||
} else {
|
||||
++it;
|
||||
@ -3871,7 +3943,8 @@ void FileManager::memory_cleanup() {
|
||||
auto it = file_hash_to_file_id_.begin();
|
||||
while (it != file_hash_to_file_id_.end()) {
|
||||
auto &file = file_id_info_[it->second.fast_get()];
|
||||
if (file_nodes_[file.node_id_].empty) {
|
||||
auto find_file_node = file_nodes_.find(file.node_id_);
|
||||
if (find_file_node == file_nodes_.end() || find_file_node->second->empty) {
|
||||
file_hash_to_file_id_.erase(it++);
|
||||
file_nodes_.erase(file.node_id_);
|
||||
} else {
|
||||
@ -3885,7 +3958,8 @@ void FileManager::memory_cleanup() {
|
||||
auto it = local_location_to_file_id_.begin();
|
||||
while (it != local_location_to_file_id_.end()) {
|
||||
auto &file = file_id_info_[it->second.fast_get()];
|
||||
if (file_nodes_[file.node_id_].empty) {
|
||||
auto find_file_node = file_nodes_.find(file.node_id_);
|
||||
if (find_file_node == file_nodes_.end() || find_file_node->second->empty) {
|
||||
it = local_location_to_file_id_.erase(it++);
|
||||
file_nodes_.erase(file.node_id_);
|
||||
} else {
|
||||
@ -3899,7 +3973,8 @@ void FileManager::memory_cleanup() {
|
||||
auto it = generate_location_to_file_id_.begin();
|
||||
while (it != generate_location_to_file_id_.end()) {
|
||||
auto &file = file_id_info_[it->second.fast_get()];
|
||||
if (file_nodes_[file.node_id_].empty) {
|
||||
auto find_file_node = file_nodes_.find(file.node_id_);
|
||||
if (find_file_node == file_nodes_.end() || find_file_node->second->empty) {
|
||||
it = generate_location_to_file_id_.erase(it++);
|
||||
file_nodes_.erase(file.node_id_);
|
||||
} else {
|
||||
@ -3914,7 +3989,8 @@ void FileManager::memory_cleanup() {
|
||||
auto it = map.begin();
|
||||
while (it != map.end()) {
|
||||
auto &file = file_id_info_[it->first.file_id_.fast_get()];
|
||||
if (file_nodes_[file.node_id_].empty) {
|
||||
auto find_file_node = file_nodes_.find(file.node_id_);
|
||||
if (find_file_node == file_nodes_.end() || find_file_node->second->empty) {
|
||||
remote_location_info_.erase(it->second);
|
||||
map.erase(it++);
|
||||
file_nodes_.erase(file.node_id_);
|
||||
@ -3928,7 +4004,8 @@ void FileManager::memory_cleanup() {
|
||||
{
|
||||
auto it = file_id_info_.begin();
|
||||
while (it != file_id_info_.end()) {
|
||||
if (file_nodes_[it->second.node_id_].empty) {
|
||||
auto find_file_node = file_nodes_.find(it->second.node_id_);
|
||||
if (find_file_node == file_nodes_.end() || find_file_node->second->empty) {
|
||||
context_->destroy_file_source({it->first, 0});
|
||||
file_id_info_.erase(it++);
|
||||
} else {
|
||||
@ -3937,9 +4014,9 @@ void FileManager::memory_cleanup() {
|
||||
}
|
||||
}
|
||||
|
||||
file_nodes_.rehash(0);
|
||||
file_hash_to_file_id_.rehash(0);
|
||||
file_id_info_.rehash(0);
|
||||
file_nodes_.rehash(file_nodes_.size() + 1);
|
||||
file_hash_to_file_id_.rehash(file_hash_to_file_id_.size() + 1);
|
||||
file_id_info_.rehash(file_id_info_.size() + 1);
|
||||
|
||||
LOG(ERROR) << "registered ids: " << file_id_info_.size() << " registered nodes: " << file_nodes_.size();
|
||||
}
|
||||
|
||||
@ -41,6 +41,7 @@
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <utility>
|
||||
#include <shared_mutex>
|
||||
|
||||
namespace td {
|
||||
|
||||
@ -72,7 +73,8 @@ class FileNode {
|
||||
FileNode(LocalFileLocation local, NewRemoteFileLocation remote, unique_ptr<FullGenerateFileLocation> generate,
|
||||
int64 size, int64 expected_size, string remote_name, string url, DialogId owner_dialog_id,
|
||||
FileEncryptionKey key, FileId main_file_id, int8 main_file_id_priority)
|
||||
: local_(std::move(local))
|
||||
: empty(false)
|
||||
, local_(std::move(local))
|
||||
, remote_(std::move(remote))
|
||||
, generate_(std::move(generate))
|
||||
, size_(size)
|
||||
@ -93,6 +95,7 @@ class FileNode {
|
||||
void set_partial_remote_location(const PartialRemoteFileLocation &remote, int64 ready_size);
|
||||
|
||||
bool delete_file_reference(Slice file_reference);
|
||||
bool delete_file_reference_internal(Slice file_reference);
|
||||
void set_generate_location(unique_ptr<FullGenerateFileLocation> &&generate);
|
||||
void set_size(int64 size);
|
||||
void set_expected_size(int64 expected_size);
|
||||
@ -419,23 +422,38 @@ class FileManager : public FileLoadManager::Callback {
|
||||
|
||||
FileId dup_file_id(FileId file_id);
|
||||
|
||||
FileId dup_file_id_internal(FileId file_id);
|
||||
|
||||
void on_file_unlink(const FullLocalFileLocation &location);
|
||||
|
||||
FileId register_empty(FileType type);
|
||||
Result<FileId> register_local(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size,
|
||||
bool get_by_hash = false, bool force = false,
|
||||
bool skip_file_size_checks = false) TD_WARN_UNUSED_RESULT;
|
||||
Result<FileId> register_local_internal(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size,
|
||||
bool get_by_hash = false, bool force = false,
|
||||
bool skip_file_size_checks = false) TD_WARN_UNUSED_RESULT;
|
||||
FileId register_remote(const FullRemoteFileLocation &location, FileLocationSource file_location_source,
|
||||
DialogId owner_dialog_id, int64 size, int64 expected_size,
|
||||
string remote_name) TD_WARN_UNUSED_RESULT;
|
||||
FileId register_remote_internal(const FullRemoteFileLocation &location, FileLocationSource file_location_source,
|
||||
DialogId owner_dialog_id, int64 size, int64 expected_size,
|
||||
string remote_name) TD_WARN_UNUSED_RESULT;
|
||||
Result<FileId> register_generate(FileType file_type, FileLocationSource file_location_source, string original_path,
|
||||
string conversion, DialogId owner_dialog_id,
|
||||
int64 expected_size) TD_WARN_UNUSED_RESULT;
|
||||
Result<FileId> register_generate_internal(FileType file_type, FileLocationSource file_location_source, string original_path,
|
||||
string conversion, DialogId owner_dialog_id,
|
||||
int64 expected_size) TD_WARN_UNUSED_RESULT;
|
||||
|
||||
Result<FileId> merge(FileId x_file_id, FileId y_file_id, bool no_sync = false) TD_WARN_UNUSED_RESULT;
|
||||
|
||||
Result<FileId> merge_internal(FileId x_file_id, FileId y_file_id, bool no_sync = false) TD_WARN_UNUSED_RESULT;
|
||||
|
||||
void add_file_source(FileId file_id, FileSourceId file_source_id);
|
||||
|
||||
void add_file_source_internal(FileId file_id, FileSourceId file_source_id);
|
||||
|
||||
void remove_file_source(FileId file_id, FileSourceId file_source_id);
|
||||
|
||||
void change_files_source(FileSourceId file_source_id, const vector<FileId> &old_file_ids,
|
||||
@ -455,6 +473,7 @@ class FileManager : public FileLoadManager::Callback {
|
||||
void cancel_upload(FileId file_id);
|
||||
bool delete_partial_remote_location(FileId file_id);
|
||||
void delete_file_reference(FileId file_id, std::string file_reference);
|
||||
void delete_file_reference_internal(FileId file_id, std::string file_reference);
|
||||
void get_content(FileId file_id, Promise<BufferSlice> promise);
|
||||
|
||||
void read_file_part(FileId file_id, int32 offset, int32 count, int left_tries,
|
||||
@ -580,7 +599,7 @@ class FileManager : public FileLoadManager::Callback {
|
||||
std::map<FileDbId, int32> pmc_id_to_file_node_id_;
|
||||
|
||||
std::unordered_map<int32, FileIdInfo> file_id_info_;
|
||||
std::unordered_map<FileNodeId, FileNode> file_nodes_;
|
||||
std::unordered_map<FileNodeId, unique_ptr<FileNode>> file_nodes_;
|
||||
ActorOwn<FileLoadManager> file_load_manager_;
|
||||
ActorOwn<FileGenerateManager> file_generate_manager_;
|
||||
|
||||
@ -668,6 +687,8 @@ class FileManager : public FileLoadManager::Callback {
|
||||
void hangup() override;
|
||||
void tear_down() override;
|
||||
|
||||
mutable std::shared_timed_mutex memory_cleanup_mutex;
|
||||
|
||||
friend class FileNodePtr;
|
||||
};
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user