diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index 8339734a0..83176323f 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -49,7 +49,7 @@ #include #include -#define FILE_TTL 20 +#define FILE_TTL 120 namespace td { namespace { @@ -300,7 +300,7 @@ void FileNode::set_partial_remote_location(const PartialRemoteFileLocation &remo on_changed(); } -bool FileNode::delete_file_reference(Slice file_reference) { +bool FileNode::delete_file_reference_internal(Slice file_reference) { if (!remote_.full) { VLOG(file_references) << "Can't delete file reference, because there is no remote location"; return false; @@ -1038,6 +1038,11 @@ FileManager::FileIdInfo *FileManager::get_file_id_info(FileId file_id) { } FileId FileManager::dup_file_id(FileId file_id) { + std::shared_lock readerLock(memory_cleanup_mutex); + return dup_file_id_internal(file_id); +} + +FileId FileManager::dup_file_id_internal(FileId file_id) { int32 file_node_id; auto *file_node = get_file_node_raw(file_id, &file_node_id); if (!file_node) { @@ -1073,7 +1078,7 @@ void FileManager::try_forget_file_id(FileId file_id) { } FileId FileManager::register_empty(FileType type) { - return register_local(FullLocalFileLocation(type, "", 0), DialogId(), 0, false, true).ok(); + return register_local_internal(FullLocalFileLocation(type, "", 0), DialogId(), 0, false, true).ok(); } void FileManager::on_file_unlink(const FullLocalFileLocation &location) { @@ -1093,6 +1098,12 @@ void FileManager::on_file_unlink(const FullLocalFileLocation &location) { Result FileManager::register_local(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size, bool get_by_hash, bool force, bool skip_file_size_checks) { + std::shared_lock readerLock(memory_cleanup_mutex); + return register_local_internal(location, owner_dialog_id, size, get_by_hash, force, skip_file_size_checks); +} + +Result FileManager::register_local_internal(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size, + bool get_by_hash, bool force, bool skip_file_size_checks) { // TODO: use get_by_hash FileData data; data.local_ = LocalFileLocation(std::move(location)); @@ -1103,6 +1114,12 @@ Result FileManager::register_local(FullLocalFileLocation location, Dialo } FileId FileManager::register_remote(const FullRemoteFileLocation &location, FileLocationSource file_location_source, + DialogId owner_dialog_id, int64 size, int64 expected_size, string remote_name) { + std::shared_lock readerLock(memory_cleanup_mutex); + return register_remote_internal(location, file_location_source, owner_dialog_id, size, expected_size, std::move(remote_name)); +} + +FileId FileManager::register_remote_internal(const FullRemoteFileLocation &location, FileLocationSource file_location_source, DialogId owner_dialog_id, int64 size, int64 expected_size, string remote_name) { FileData data; data.remote_ = RemoteFileLocation(location); @@ -1123,7 +1140,7 @@ FileId FileManager::register_remote(const FullRemoteFileLocation &location, File FileId FileManager::register_url(string url, FileType file_type, FileLocationSource file_location_source, DialogId owner_dialog_id) { - auto file_id = register_generate(file_type, file_location_source, url, "#url#", owner_dialog_id, 0).ok(); + auto file_id = register_generate_internal(file_type, file_location_source, url, "#url#", owner_dialog_id, 0).ok(); auto file_node = get_file_node(file_id); CHECK(file_node); file_node->set_url(url); @@ -1133,6 +1150,13 @@ FileId FileManager::register_url(string url, FileType file_type, FileLocationSou Result FileManager::register_generate(FileType file_type, FileLocationSource file_location_source, string original_path, string conversion, DialogId owner_dialog_id, int64 expected_size) { + std::shared_lock readerLock(memory_cleanup_mutex); + return register_generate_internal(file_type, file_location_source, std::move(original_path), std::move(conversion), owner_dialog_id, expected_size); +} + +Result FileManager::register_generate_internal(FileType file_type, FileLocationSource file_location_source, + string original_path, string conversion, DialogId owner_dialog_id, + int64 expected_size) { // add #mtime# into conversion if (!original_path.empty() && conversion[0] != '#' && PathView(original_path).is_absolute()) { auto file_paths = log_interface->get_file_paths(); @@ -1190,13 +1214,13 @@ Result FileManager::register_file(FileData &&data, FileLocationSource fi // create FileNode auto file_node_id = next_file_node_id(); auto &node = file_nodes_[file_node_id]; - node = FileNode(std::move(data.local_), NewRemoteFileLocation(data.remote_, file_location_source), + node = td::make_unique(std::move(data.local_), NewRemoteFileLocation(data.remote_, file_location_source), std::move(data.generate_), data.size_, data.expected_size_, std::move(data.remote_name_), std::move(data.url_), data.owner_dialog_id_, std::move(data.encryption_key_), file_id, static_cast(has_remote)); - node.pmc_id_ = FileDbId(data.pmc_id_); + node->pmc_id_ = FileDbId(data.pmc_id_); get_file_id_info(file_id)->node_id_ = file_node_id; - node.file_ids_.push_back(file_id); + node->file_ids_.push_back(file_id); FileView file_view(get_file_node(file_id)); @@ -1243,12 +1267,12 @@ Result FileManager::register_file(FileData &&data, FileLocationSource fi int new_cnt = new_remote + new_local + new_generate; if (data.pmc_id_ == 0 && file_db_ && new_cnt > 0) { - node.need_load_from_pmc_ = true; + node->need_load_from_pmc_ = true; } bool no_sync_merge = to_merge.size() == 1 && new_cnt == 0; for (auto id : to_merge) { // may invalidate node - merge(file_id, id, no_sync_merge).ignore(); + merge_internal(file_id, id, no_sync_merge).ignore(); } try_flush_node(get_file_node(file_id), "register_file"); @@ -1422,6 +1446,11 @@ void FileManager::do_cancel_generate(FileNodePtr node) { } Result FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) { + std::shared_lock readerLock(memory_cleanup_mutex); + return merge_internal(x_file_id, y_file_id, no_sync); +} + +Result FileManager::merge_internal(FileId x_file_id, FileId y_file_id, bool no_sync) { LOG(DEBUG) << "Merge new file " << x_file_id << " and old file " << y_file_id; if (!x_file_id.is_valid()) { @@ -1636,7 +1665,7 @@ Result FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy file_id_info->node_id_ = node_ids[node_i]; send_updates_flag |= file_id_info->send_updates_flag_; } - other_node = {this}; + other_node = {}; if (send_updates_flag) { // node might not changed, but other_node might changed, so we need to send update anyway @@ -1679,6 +1708,11 @@ Result FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy } void FileManager::add_file_source(FileId file_id, FileSourceId file_source_id) { + std::shared_lock readerLock(memory_cleanup_mutex); + return add_file_source_internal(file_id, file_source_id); +} + +void FileManager::add_file_source_internal(FileId file_id, FileSourceId file_source_id) { auto node = get_file_node(file_id); if (!node) { return; @@ -1689,6 +1723,7 @@ void FileManager::add_file_source(FileId file_id, FileSourceId file_source_id) { node->on_pmc_changed(); try_flush_node_pmc(node, "add_file_source"); } + } void FileManager::remove_file_source(FileId file_id, FileSourceId file_source_id) { @@ -1706,6 +1741,7 @@ void FileManager::remove_file_source(FileId file_id, FileSourceId file_source_id void FileManager::change_files_source(FileSourceId file_source_id, const vector &old_file_ids, const vector &new_file_ids) { + std::shared_lock readerLock(memory_cleanup_mutex); if (old_file_ids == new_file_ids) { return; } @@ -1722,7 +1758,7 @@ void FileManager::change_files_source(FileSourceId file_source_id, const vector< } } for (auto file_id : new_main_file_ids) { - add_file_source(file_id, file_source_id); + add_file_source_internal(file_id, file_source_id); } } @@ -1881,7 +1917,7 @@ FileNode *FileManager::get_file_node_raw(FileId file_id, FileNodeId *file_node_i if (file_node_id != nullptr) { *file_node_id = node_id; } - return &file_nodes_[node_id]; + return file_nodes_[node_id].get(); } FileNodePtr FileManager::get_sync_file_node(FileId file_id) { @@ -1927,7 +1963,7 @@ void FileManager::load_from_pmc(FileNodePtr node, bool new_remote, bool new_loca TRY_RESULT(file_data, file_db_->get_file_data_sync(location)); TRY_RESULT(new_file_id, register_file(std::move(file_data), FileLocationSource::FromDatabase, "load_from_pmc", false)); - TRY_RESULT(main_file_id, merge(file_id, new_file_id)); + TRY_RESULT(main_file_id, merge_internal(file_id, new_file_id)); file_id = main_file_id; return Status::OK(); }; @@ -1960,6 +1996,7 @@ bool FileManager::set_encryption_key(FileId file_id, FileEncryptionKey key) { } bool FileManager::set_content(FileId file_id, BufferSlice bytes) { + std::shared_lock readerLock(memory_cleanup_mutex); if (G()->shared_config().get_option_boolean("ignore_inline_thumbnails")) { return false; } @@ -1994,6 +2031,7 @@ bool FileManager::set_content(FileId file_id, BufferSlice bytes) { } void FileManager::get_content(FileId file_id, Promise promise) { + std::shared_lock readerLock(memory_cleanup_mutex); auto node = get_sync_file_node(file_id); if (!node) { return promise.set_error(Status::Error("Unknown file_id")); @@ -2011,6 +2049,7 @@ void FileManager::get_content(FileId file_id, Promise promise) { void FileManager::read_file_part(FileId file_id, int32 offset, int32 count, int left_tries, Promise> promise) { + std::shared_lock readerLock(memory_cleanup_mutex); if (G()->close_flag()) { return promise.set_error(Status::Error(500, "Request aborted")); } @@ -2089,6 +2128,7 @@ void FileManager::read_file_part(FileId file_id, int32 offset, int32 count, int } void FileManager::delete_file(FileId file_id, Promise promise, const char *source) { + std::shared_lock readerLock(memory_cleanup_mutex); LOG(INFO) << "Trying to delete file " << file_id << " from " << source; auto node = get_sync_file_node(file_id); if (!node) { @@ -2125,6 +2165,7 @@ void FileManager::delete_file(FileId file_id, Promise promise, const char void FileManager::download(FileId file_id, std::shared_ptr callback, int32 new_priority, int64 offset, int64 limit) { + std::shared_lock readerLock(memory_cleanup_mutex); LOG(INFO) << "Download file " << file_id << " with priority " << new_priority; auto node = get_sync_file_node(file_id); if (!node) { @@ -2516,6 +2557,7 @@ void FileManager::resume_upload(FileId file_id, std::vector bad_parts, std: } bool FileManager::delete_partial_remote_location(FileId file_id) { + std::shared_lock readerLock(memory_cleanup_mutex); auto node = get_sync_file_node(file_id); if (!node) { LOG(INFO) << "Wrong file identifier " << file_id; @@ -2550,6 +2592,11 @@ bool FileManager::delete_partial_remote_location(FileId file_id) { } void FileManager::delete_file_reference(FileId file_id, string file_reference) { + std::shared_lock readerLock(memory_cleanup_mutex); + return delete_file_reference_internal(file_id, file_reference); +} + +void FileManager::delete_file_reference_internal(FileId file_id, string file_reference) { VLOG(file_references) << "Delete file reference of file " << file_id << " " << tag("reference_base64", base64_encode(file_reference)); auto node = get_sync_file_node(file_id); @@ -2557,7 +2604,7 @@ void FileManager::delete_file_reference(FileId file_id, string file_reference) { LOG(ERROR) << "Wrong file identifier " << file_id; return; } - node->delete_file_reference(file_reference); + node->delete_file_reference_internal(file_reference); auto remote = get_remote(file_id.get_remote()); if (remote != nullptr) { VLOG(file_references) << "Do delete file reference of remote file " << file_id; @@ -2572,17 +2619,20 @@ void FileManager::delete_file_reference(FileId file_id, string file_reference) { } void FileManager::external_file_generate_write_part(int64 id, int32 offset, string data, Promise<> promise) { + std::shared_lock readerLock(memory_cleanup_mutex); send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_write_part, id, offset, std::move(data), std::move(promise)); } void FileManager::external_file_generate_progress(int64 id, int32 expected_size, int32 local_prefix_size, Promise<> promise) { + std::shared_lock readerLock(memory_cleanup_mutex); send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_progress, id, expected_size, local_prefix_size, std::move(promise)); } void FileManager::external_file_generate_finish(int64 id, Status status, Promise<> promise) { + std::shared_lock readerLock(memory_cleanup_mutex); send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_finish, id, std::move(status), std::move(promise)); } @@ -2784,10 +2834,12 @@ void FileManager::run_upload(FileNodePtr node, std::vector bad_parts) { void FileManager::upload(FileId file_id, std::shared_ptr callback, int32 new_priority, uint64 upload_order) { + std::shared_lock readerLock(memory_cleanup_mutex); return resume_upload(file_id, std::vector(), std::move(callback), new_priority, upload_order); } void FileManager::cancel_upload(FileId file_id) { + std::shared_lock readerLock(memory_cleanup_mutex); return resume_upload(file_id, std::vector(), nullptr, 0, 0); } @@ -2912,6 +2964,7 @@ FileView FileManager::get_sync_file_view(FileId file_id) { } td_api::object_ptr FileManager::get_file_object(FileId file_id, bool with_main_file_id) { + std::shared_lock readerLock(memory_cleanup_mutex); auto file_view = get_sync_file_view(file_id); if (file_view.empty()) { @@ -2955,6 +3008,7 @@ td_api::object_ptr FileManager::get_file_object(FileId file_id, bo vector FileManager::get_file_ids_object(const vector &file_ids, bool with_main_file_id) { return transform(file_ids, [this, with_main_file_id](FileId file_id) { + std::shared_lock readerLock(memory_cleanup_mutex); auto file_view = get_sync_file_view(file_id); auto result_file_id = file_id; auto *file_info = get_file_id_info(result_file_id); @@ -2996,7 +3050,7 @@ Result FileManager::check_input_file_id(FileType type, Result re if (!file_view.has_remote_location()) { // TODO why not return file_id here? We will dup it anyway // But it will not be duped if has_input_media(), so for now we can't return main_file_id - return dup_file_id(file_id); + return dup_file_id_internal(file_id); } int32 remote_id = file_id.get_remote(); @@ -3012,6 +3066,7 @@ Result FileManager::check_input_file_id(FileType type, Result re Result FileManager::get_input_thumbnail_file_id(const tl_object_ptr &thumbnail_input_file, DialogId owner_dialog_id, bool is_encrypted) { + std::shared_lock readerLock(memory_cleanup_mutex); if (thumbnail_input_file == nullptr) { return Status::Error(6, "inputThumbnail not specified"); } @@ -3019,7 +3074,7 @@ Result FileManager::get_input_thumbnail_file_id(const tl_object_ptrget_id()) { case td_api::inputFileLocal::ID: { const string &path = static_cast(thumbnail_input_file.get())->path_; - return register_local( + return register_local_internal( FullLocalFileLocation(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail, path, 0), owner_dialog_id, 0, false); } @@ -3029,7 +3084,7 @@ Result FileManager::get_input_thumbnail_file_id(const tl_object_ptr(thumbnail_input_file.get()); - return register_generate(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail, + return register_generate_internal(is_encrypted ? FileType::EncryptedThumbnail : FileType::Thumbnail, FileLocationSource::FromUser, generated_thumbnail->original_path_, generated_thumbnail->conversion_, owner_dialog_id, generated_thumbnail->expected_size_); } @@ -3042,6 +3097,7 @@ Result FileManager::get_input_thumbnail_file_id(const tl_object_ptr FileManager::get_input_file_id(FileType type, const tl_object_ptr &file, DialogId owner_dialog_id, bool allow_zero, bool is_encrypted, bool get_by_hash, bool is_secure) { + std::shared_lock readerLock(memory_cleanup_mutex); if (file == nullptr) { if (allow_zero) { return FileId(); @@ -3079,7 +3135,7 @@ Result FileManager::get_input_file_id(FileType type, const tl_object_ptr } } } - TRY_RESULT(file_id, register_local(FullLocalFileLocation(new_type, path, 0), owner_dialog_id, 0, get_by_hash)); + TRY_RESULT(file_id, register_local_internal(FullLocalFileLocation(new_type, path, 0), owner_dialog_id, 0, get_by_hash)); if (!hash.empty()) { file_hash_to_file_id_[hash] = file_id; } @@ -3101,7 +3157,7 @@ Result FileManager::get_input_file_id(FileType type, const tl_object_ptr } case td_api::inputFileGenerated::ID: { auto *generated_file = static_cast(file.get()); - return register_generate(new_type, FileLocationSource::FromUser, generated_file->original_path_, + return register_generate_internal(new_type, FileLocationSource::FromUser, generated_file->original_path_, generated_file->conversion_, owner_dialog_id, generated_file->expected_size_); } default: @@ -3115,6 +3171,7 @@ Result FileManager::get_input_file_id(FileType type, const tl_object_ptr Result FileManager::get_map_thumbnail_file_id(Location location, int32 zoom, int32 width, int32 height, int32 scale, DialogId owner_dialog_id) { + std::shared_lock readerLock(memory_cleanup_mutex); if (!location.is_valid_map_point()) { return Status::Error(6, "Invalid location specified"); } @@ -3141,12 +3198,13 @@ Result FileManager::get_map_thumbnail_file_id(Location location, int32 z string conversion = PSTRING() << "#map#" << zoom << "#" << x << "#" << y << "#" << width << "#" << height << "#" << scale << "#"; - return register_generate( + return register_generate_internal( owner_dialog_id.get_type() == DialogType::SecretChat ? FileType::EncryptedThumbnail : FileType::Thumbnail, FileLocationSource::FromUser, string(), std::move(conversion), owner_dialog_id, 0); } vector> FileManager::get_input_documents(const vector &file_ids) { + std::shared_lock readerLock(memory_cleanup_mutex); vector> result; result.reserve(file_ids.size()); for (auto file_id : file_ids) { @@ -3178,6 +3236,7 @@ bool FileManager::extract_was_thumbnail_uploaded(const tl_object_ptr &input_media) { + // don't lock this method because now it only calls the other extract_file_reference if (input_media == nullptr) { return string(); } @@ -3213,6 +3272,7 @@ bool FileManager::extract_was_uploaded(const tl_object_ptr &input_chat_photo) { + // don't lock this method because now it only calls the other extract_file_reference if (input_chat_photo == nullptr || input_chat_photo->get_id() != telegram_api::inputChatPhoto::ID) { return string(); } @@ -3234,6 +3294,8 @@ FileManager::FileNodeId FileManager::next_file_node_id() { } void FileManager::on_start_download(QueryId query_id) { + std::shared_lock readerLock(memory_cleanup_mutex); + if (is_closed_) { return; } @@ -3259,6 +3321,8 @@ void FileManager::on_start_download(QueryId query_id) { void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLocation &partial_local, int64 ready_size, int64 size) { + std::shared_lock readerLock(memory_cleanup_mutex); + if (is_closed_) { return; } @@ -3290,6 +3354,8 @@ void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLo } void FileManager::on_hash(QueryId query_id, string hash) { + std::shared_lock readerLock(memory_cleanup_mutex); + if (is_closed_) { return; } @@ -3315,6 +3381,8 @@ void FileManager::on_hash(QueryId query_id, string hash) { void FileManager::on_partial_upload(QueryId query_id, const PartialRemoteFileLocation &partial_remote, int64 ready_size) { + std::shared_lock readerLock(memory_cleanup_mutex); + if (is_closed_) { return; } @@ -3339,6 +3407,8 @@ void FileManager::on_partial_upload(QueryId query_id, const PartialRemoteFileLoc } void FileManager::on_download_ok(QueryId query_id, const FullLocalFileLocation &local, int64 size, bool is_new) { + std::shared_lock readerLock(memory_cleanup_mutex); + if (is_closed_) { return; } @@ -3348,7 +3418,7 @@ void FileManager::on_download_ok(QueryId query_id, const FullLocalFileLocation & std::tie(query, was_active) = finish_query(query_id); auto file_id = query.file_id_; LOG(INFO) << "ON DOWNLOAD OK of " << (is_new ? "new" : "checked") << " file " << file_id << " of size " << size; - auto r_new_file_id = register_local(local, DialogId(), size, false, false, true); + auto r_new_file_id = register_local_internal(local, DialogId(), size, false, false, true); Status status = Status::OK(); if (r_new_file_id.is_error()) { status = Status::Error(PSLICE() << "Can't register local file after download: " << r_new_file_id.error().message()); @@ -3356,7 +3426,7 @@ void FileManager::on_download_ok(QueryId query_id, const FullLocalFileLocation & if (is_new) { context_->on_new_file(size, get_file_view(r_new_file_id.ok()).get_allocated_local_size(), 1); } - auto r_file_id = merge(r_new_file_id.ok(), file_id); + auto r_file_id = merge_internal(r_new_file_id.ok(), file_id); if (r_file_id.is_error()) { status = r_file_id.move_as_error(); } @@ -3451,8 +3521,8 @@ void FileManager::on_upload_full_ok(QueryId query_id, const FullRemoteFileLocati auto file_id = finish_query(query_id).first.file_id_; LOG(INFO) << "ON UPLOAD FULL OK for file " << file_id; - auto new_file_id = register_remote(remote, FileLocationSource::FromServer, DialogId(), 0, 0, ""); - LOG_STATUS(merge(new_file_id, file_id)); + auto new_file_id = register_remote_internal(remote, FileLocationSource::FromServer, DialogId(), 0, 0, ""); + LOG_STATUS(merge_internal(new_file_id, file_id)); } void FileManager::on_partial_generate(QueryId query_id, const PartialLocalFileLocation &partial_local, @@ -3512,12 +3582,12 @@ void FileManager::on_generate_ok(QueryId query_id, const FullLocalFileLocation & auto old_upload_id = file_node->upload_id_; - auto r_new_file_id = register_local(local, DialogId(), 0); + auto r_new_file_id = register_local_internal(local, DialogId(), 0); Status status; if (r_new_file_id.is_error()) { status = Status::Error(PSLICE() << "Can't register local file after generate: " << r_new_file_id.error()); } else { - auto result = merge(r_new_file_id.ok(), generate_file_id); + auto result = merge_internal(r_new_file_id.ok(), generate_file_id); if (result.is_error()) { status = result.move_as_error(); } @@ -3636,7 +3706,7 @@ void FileManager::on_error_impl(FileNodePtr node, Query::Type type, bool was_act LOG(ERROR) << "Unexpected error, file_reference will be deleted just in case " << status; } CHECK(!node->file_ids_.empty()); - delete_file_reference(node->file_ids_.back(), file_reference); + delete_file_reference_internal(node->file_ids_.back(), file_reference); run_download(node); return; } @@ -3762,6 +3832,7 @@ void FileManager::destroy_query(int32 file_id) { void FileManager::memory_cleanup() { /* DESTROY OLD file_id_info_ */ { + std::lock_guard writerLock(memory_cleanup_mutex); auto it = file_id_info_.begin(); auto time = std::time(nullptr); std::vector file_to_be_deleted = {}; @@ -3770,15 +3841,15 @@ void FileManager::memory_cleanup() { if (it->second.node_id_ != 0) { auto &node = file_nodes_[it->second.node_id_]; - if (time - node.main_file_id_.get_time() > FILE_TTL) { - auto can_reset = node.download_priority_ == 0; - can_reset &= node.generate_download_priority_ == 0; - can_reset &= node.download_id_ == 0; + if (time - node->main_file_id_.get_time() > FILE_TTL) { + auto can_reset = node->download_priority_ == 0; + can_reset &= node->generate_download_priority_ == 0; + can_reset &= node->download_id_ == 0; if (can_reset) { - auto file_ids_it = node.file_ids_.begin(); + auto file_ids_it = node->file_ids_.begin(); - while (file_ids_it != node.file_ids_.end() && can_reset) { + while (file_ids_it != node->file_ids_.end() && can_reset) { auto &file = file_id_info_[file_ids_it->fast_get()]; can_reset &= file.download_priority_ == 0; can_reset &= time - file_ids_it->get_time() > FILE_TTL; @@ -3787,9 +3858,9 @@ void FileManager::memory_cleanup() { } if (can_reset) { - node.main_file_id_.reset_time(); + node->main_file_id_.reset_time(); - for (auto &file_id : node.file_ids_) { + for (auto &file_id : node->file_ids_) { file_id.reset_time(); /* DESTROY ASSOCIATED QUERIES */ @@ -3827,7 +3898,8 @@ void FileManager::memory_cleanup() { auto it = file_id_info_.begin(); while (it != file_id_info_.end()) { if (it->second.node_id_ != 0) { - if (file_nodes_[it->second.node_id_].empty) { + auto find_file_node = file_nodes_.find(it->second.node_id_); + if (find_file_node == file_nodes_.end() || find_file_node->second->empty) { destroy_query(it->first); context_->destroy_file_source({it->first, 0}); file_nodes_.erase(it->second.node_id_); @@ -3845,18 +3917,18 @@ void FileManager::memory_cleanup() { { auto it = file_nodes_.begin(); while (it != file_nodes_.end()) { - if (it->second.empty) { + if (it->second->empty) { file_nodes_.erase(it++); } else { - if (it->second.main_file_id_.empty()) { + if (it->second->main_file_id_.empty()) { file_nodes_.erase(it++); } else { - if (file_id_info_[it->second.main_file_id_.get()].node_id_ == 0) { - for (auto &file_id : it->second.file_ids_) { + if (file_id_info_[it->second->main_file_id_.get()].node_id_ == 0) { + for (auto &file_id : it->second->file_ids_) { context_->destroy_file_source(file_id); file_id_info_.erase(file_id.get()); } - file_id_info_.erase(it->second.main_file_id_.get()); + file_id_info_.erase(it->second->main_file_id_.get()); file_nodes_.erase(it++); } else { ++it; @@ -3871,7 +3943,8 @@ void FileManager::memory_cleanup() { auto it = file_hash_to_file_id_.begin(); while (it != file_hash_to_file_id_.end()) { auto &file = file_id_info_[it->second.fast_get()]; - if (file_nodes_[file.node_id_].empty) { + auto find_file_node = file_nodes_.find(file.node_id_); + if (find_file_node == file_nodes_.end() || find_file_node->second->empty) { file_hash_to_file_id_.erase(it++); file_nodes_.erase(file.node_id_); } else { @@ -3885,7 +3958,8 @@ void FileManager::memory_cleanup() { auto it = local_location_to_file_id_.begin(); while (it != local_location_to_file_id_.end()) { auto &file = file_id_info_[it->second.fast_get()]; - if (file_nodes_[file.node_id_].empty) { + auto find_file_node = file_nodes_.find(file.node_id_); + if (find_file_node == file_nodes_.end() || find_file_node->second->empty) { it = local_location_to_file_id_.erase(it++); file_nodes_.erase(file.node_id_); } else { @@ -3899,7 +3973,8 @@ void FileManager::memory_cleanup() { auto it = generate_location_to_file_id_.begin(); while (it != generate_location_to_file_id_.end()) { auto &file = file_id_info_[it->second.fast_get()]; - if (file_nodes_[file.node_id_].empty) { + auto find_file_node = file_nodes_.find(file.node_id_); + if (find_file_node == file_nodes_.end() || find_file_node->second->empty) { it = generate_location_to_file_id_.erase(it++); file_nodes_.erase(file.node_id_); } else { @@ -3914,7 +3989,8 @@ void FileManager::memory_cleanup() { auto it = map.begin(); while (it != map.end()) { auto &file = file_id_info_[it->first.file_id_.fast_get()]; - if (file_nodes_[file.node_id_].empty) { + auto find_file_node = file_nodes_.find(file.node_id_); + if (find_file_node == file_nodes_.end() || find_file_node->second->empty) { remote_location_info_.erase(it->second); map.erase(it++); file_nodes_.erase(file.node_id_); @@ -3928,7 +4004,8 @@ void FileManager::memory_cleanup() { { auto it = file_id_info_.begin(); while (it != file_id_info_.end()) { - if (file_nodes_[it->second.node_id_].empty) { + auto find_file_node = file_nodes_.find(it->second.node_id_); + if (find_file_node == file_nodes_.end() || find_file_node->second->empty) { context_->destroy_file_source({it->first, 0}); file_id_info_.erase(it++); } else { @@ -3937,9 +4014,9 @@ void FileManager::memory_cleanup() { } } - file_nodes_.rehash(0); - file_hash_to_file_id_.rehash(0); - file_id_info_.rehash(0); + file_nodes_.rehash(file_nodes_.size() + 1); + file_hash_to_file_id_.rehash(file_hash_to_file_id_.size() + 1); + file_id_info_.rehash(file_id_info_.size() + 1); LOG(ERROR) << "registered ids: " << file_id_info_.size() << " registered nodes: " << file_nodes_.size(); } diff --git a/td/telegram/files/FileManager.h b/td/telegram/files/FileManager.h index 62950f9de..1da8da163 100644 --- a/td/telegram/files/FileManager.h +++ b/td/telegram/files/FileManager.h @@ -41,6 +41,7 @@ #include #include #include +#include namespace td { @@ -72,7 +73,8 @@ class FileNode { FileNode(LocalFileLocation local, NewRemoteFileLocation remote, unique_ptr generate, int64 size, int64 expected_size, string remote_name, string url, DialogId owner_dialog_id, FileEncryptionKey key, FileId main_file_id, int8 main_file_id_priority) - : local_(std::move(local)) + : empty(false) + , local_(std::move(local)) , remote_(std::move(remote)) , generate_(std::move(generate)) , size_(size) @@ -93,6 +95,7 @@ class FileNode { void set_partial_remote_location(const PartialRemoteFileLocation &remote, int64 ready_size); bool delete_file_reference(Slice file_reference); + bool delete_file_reference_internal(Slice file_reference); void set_generate_location(unique_ptr &&generate); void set_size(int64 size); void set_expected_size(int64 expected_size); @@ -419,23 +422,38 @@ class FileManager : public FileLoadManager::Callback { FileId dup_file_id(FileId file_id); + FileId dup_file_id_internal(FileId file_id); + void on_file_unlink(const FullLocalFileLocation &location); FileId register_empty(FileType type); Result register_local(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size, bool get_by_hash = false, bool force = false, bool skip_file_size_checks = false) TD_WARN_UNUSED_RESULT; + Result register_local_internal(FullLocalFileLocation location, DialogId owner_dialog_id, int64 size, + bool get_by_hash = false, bool force = false, + bool skip_file_size_checks = false) TD_WARN_UNUSED_RESULT; FileId register_remote(const FullRemoteFileLocation &location, FileLocationSource file_location_source, DialogId owner_dialog_id, int64 size, int64 expected_size, string remote_name) TD_WARN_UNUSED_RESULT; + FileId register_remote_internal(const FullRemoteFileLocation &location, FileLocationSource file_location_source, + DialogId owner_dialog_id, int64 size, int64 expected_size, + string remote_name) TD_WARN_UNUSED_RESULT; Result register_generate(FileType file_type, FileLocationSource file_location_source, string original_path, string conversion, DialogId owner_dialog_id, int64 expected_size) TD_WARN_UNUSED_RESULT; + Result register_generate_internal(FileType file_type, FileLocationSource file_location_source, string original_path, + string conversion, DialogId owner_dialog_id, + int64 expected_size) TD_WARN_UNUSED_RESULT; Result merge(FileId x_file_id, FileId y_file_id, bool no_sync = false) TD_WARN_UNUSED_RESULT; + Result merge_internal(FileId x_file_id, FileId y_file_id, bool no_sync = false) TD_WARN_UNUSED_RESULT; + void add_file_source(FileId file_id, FileSourceId file_source_id); + void add_file_source_internal(FileId file_id, FileSourceId file_source_id); + void remove_file_source(FileId file_id, FileSourceId file_source_id); void change_files_source(FileSourceId file_source_id, const vector &old_file_ids, @@ -455,6 +473,7 @@ class FileManager : public FileLoadManager::Callback { void cancel_upload(FileId file_id); bool delete_partial_remote_location(FileId file_id); void delete_file_reference(FileId file_id, std::string file_reference); + void delete_file_reference_internal(FileId file_id, std::string file_reference); void get_content(FileId file_id, Promise promise); void read_file_part(FileId file_id, int32 offset, int32 count, int left_tries, @@ -580,7 +599,7 @@ class FileManager : public FileLoadManager::Callback { std::map pmc_id_to_file_node_id_; std::unordered_map file_id_info_; - std::unordered_map file_nodes_; + std::unordered_map> file_nodes_; ActorOwn file_load_manager_; ActorOwn file_generate_manager_; @@ -668,6 +687,8 @@ class FileManager : public FileLoadManager::Callback { void hangup() override; void tear_down() override; + mutable std::shared_timed_mutex memory_cleanup_mutex; + friend class FileNodePtr; };