diff --git a/td/telegram/files/FileDownloader.cpp b/td/telegram/files/FileDownloader.cpp index a1164154..d9343dad 100644 --- a/td/telegram/files/FileDownloader.cpp +++ b/td/telegram/files/FileDownloader.cpp @@ -26,7 +26,7 @@ namespace td { FileDownloader::FileDownloader(const FullRemoteFileLocation &remote, const LocalFileLocation &local, int64 size, - string name, const FileEncryptionKey &encryption_key, bool is_small, + string name, const FileEncryptionKey &encryption_key, bool is_small, bool search_file, std::unique_ptr callback) : remote_(remote) , local_(local) @@ -34,7 +34,8 @@ FileDownloader::FileDownloader(const FullRemoteFileLocation &remote, const Local , name_(std::move(name)) , encryption_key_(encryption_key) , callback_(std::move(callback)) - , is_small_(is_small) { + , is_small_(is_small) + , search_file_(search_file) { if (!encryption_key.empty()) { set_ordered_flag(true); } @@ -47,14 +48,14 @@ Result FileDownloader::init() { if (local_.type() == LocalFileLocation::Type::Full) { return Status::Error("File is already downloaded"); } - int offset = 0; + int ready_part_count = 0; int32 part_size = 0; if (local_.type() == LocalFileLocation::Type::Partial) { const auto &partial = local_.partial(); path_ = partial.path_; auto result_fd = FileFd::open(path_, FileFd::Write | FileFd::Read); // TODO: check timestamps.. - if (!result_fd.is_error()) { + if (result_fd.is_ok()) { if (!encryption_key_.empty()) { CHECK(partial.iv_.size() == 32) << partial.iv_.size(); encryption_key_.mutable_iv() = as(partial.iv_.data()); @@ -62,11 +63,27 @@ Result FileDownloader::init() { } fd_ = result_fd.move_as_ok(); part_size = partial.part_size_; - offset = partial.ready_part_count_; + ready_part_count = partial.ready_part_count_; } } - std::vector parts(offset); - for (int i = 0; i < offset; i++) { + if (search_file_ && fd_.empty() && size_ > 0 && size_ < 1000 * (1 << 20) && encryption_key_.empty() && + !remote_.is_web()) { + [&] { + TRY_RESULT(path, search_file(get_files_dir(remote_.file_type_), name_, size_)); + TRY_RESULT(fd, FileFd::open(path, FileFd::Read)); + LOG(INFO) << "Check hash of local file " << path; + path_ = std::move(path); + fd_ = std::move(fd); + need_check_ = true; + only_check_ = true; + part_size = 32 * (1 << 10); + ready_part_count = narrow_cast((size_ + part_size - 1) / part_size); + return Status::OK(); + }(); + } + + std::vector parts(ready_part_count); + for (int i = 0; i < ready_part_count; i++) { parts[i] = i; } @@ -76,14 +93,21 @@ Result FileDownloader::init() { res.part_size = part_size; res.ready_parts = std::move(parts); res.use_part_count_limit = false; + res.only_check_ = only_check_; return res; } Status FileDownloader::on_ok(int64 size) { auto dir = get_files_dir(remote_.file_type_); - TRY_RESULT(perm_path, create_from_temp(path_, dir, name_)); + std::string path; + if (only_check_) { + path = path_; + } else { + TRY_RESULT(perm_path, create_from_temp(path_, dir, name_)); + path = std::move(perm_path); + } fd_.close(); - callback_->on_ok(FullLocalFileLocation(remote_.file_type_, std::move(perm_path), 0), size); + callback_->on_ok(FullLocalFileLocation(remote_.file_type_, std::move(path), 0), size); return Status::OK(); } void FileDownloader::on_error(Status status) { @@ -368,16 +392,20 @@ Result FileDownloader::check_loop(int64 checked_prefix_si sha256(slice.as_slice(), hash); if (hash != it->hash) { + if (only_check_) { + return Status::Error("FILE_DOWNLOAD_RESTART"); + } return Status::Error("Hash mismatch"); } checked_prefix_size = end_offset; + info.changed = true; continue; } - if (!has_hash_query_ && use_cdn_) { + if (!has_hash_query_) { has_hash_query_ = true; auto query = - telegram_api::upload_getCdnFileHashes(BufferSlice(cdn_file_token_), narrow_cast(checked_prefix_size)); + telegram_api::upload_getFileHashes(remote_.as_input_file_location(), narrow_cast(checked_prefix_size)); auto net_query = G()->net_query_creator().create( create_storer(query), remote_.get_dc_id(), is_small_ ? NetQuery::Type::DownloadSmall : NetQuery::Type::Download, NetQuery::AuthFlag::On); @@ -419,7 +447,7 @@ Status FileDownloader::acquire_fd() { TRY_RESULT(file_path, open_temp_file(remote_.file_type_)); std::tie(fd_, path_) = std::move(file_path); } else { - TRY_RESULT(fd, FileFd::open(path_, FileFd::Write | FileFd::Read)); + TRY_RESULT(fd, FileFd::open(path_, (only_check_ ? 0 : FileFd::Write) | FileFd::Read)); fd_ = std::move(fd); } } diff --git a/td/telegram/files/FileDownloader.h b/td/telegram/files/FileDownloader.h index 088d0771..6a45ca56 100644 --- a/td/telegram/files/FileDownloader.h +++ b/td/telegram/files/FileDownloader.h @@ -35,7 +35,8 @@ class FileDownloader : public FileLoader { }; FileDownloader(const FullRemoteFileLocation &remote, const LocalFileLocation &local, int64 size, string name, - const FileEncryptionKey &encryption_key, bool is_small, std::unique_ptr callback); + const FileEncryptionKey &encryption_key, bool is_small, bool search_file, + std::unique_ptr callback); // Should just implement all parent pure virtual methods. // Must not call any of them... @@ -48,6 +49,7 @@ class FileDownloader : public FileLoader { string name_; FileEncryptionKey encryption_key_; std::unique_ptr callback_; + bool only_check_{false}; string path_; FileFd fd_; @@ -55,6 +57,7 @@ class FileDownloader : public FileLoader { int32 next_part_ = 0; bool next_part_stop_ = false; bool is_small_; + bool search_file_{false}; bool use_cdn_ = false; DcId cdn_dc_id_; diff --git a/td/telegram/files/FileLoadManager.cpp b/td/telegram/files/FileLoadManager.cpp index fd9fddab..10246d7f 100644 --- a/td/telegram/files/FileLoadManager.cpp +++ b/td/telegram/files/FileLoadManager.cpp @@ -30,7 +30,7 @@ void FileLoadManager::start_up() { void FileLoadManager::download(QueryId id, const FullRemoteFileLocation &remote_location, const LocalFileLocation &local, int64 size, string name, - const FileEncryptionKey &encryption_key, int8 priority) { + const FileEncryptionKey &encryption_key, bool search_file, int8 priority) { if (stop_flag_) { return; } @@ -42,7 +42,7 @@ void FileLoadManager::download(QueryId id, const FullRemoteFileLocation &remote_ auto callback = make_unique(actor_shared(this, node_id)); bool is_small = size < 20 * 1024; node->loader_ = create_actor("Downloader", remote_location, local, size, std::move(name), - encryption_key, is_small, std::move(callback)); + encryption_key, is_small, search_file, std::move(callback)); auto &resource_manager = is_small ? download_small_resource_manager_ : download_resource_manager_; send_closure(resource_manager, &ResourceManager::register_worker, ActorShared(node->loader_.get(), static_cast(-1)), priority); diff --git a/td/telegram/files/FileLoadManager.h b/td/telegram/files/FileLoadManager.h index efa381d3..e6f892ba 100644 --- a/td/telegram/files/FileLoadManager.h +++ b/td/telegram/files/FileLoadManager.h @@ -44,7 +44,7 @@ class FileLoadManager final : public Actor { explicit FileLoadManager(ActorShared callback, ActorShared<> parent); void download(QueryId id, const FullRemoteFileLocation &remote_location, const LocalFileLocation &local, int64 size, - string name, const FileEncryptionKey &encryption_key, int8 priority); + string name, const FileEncryptionKey &encryption_key, bool search_file, int8 priority); void upload(QueryId id, const LocalFileLocation &local_location, const RemoteFileLocation &remote_location, int64 size, const FileEncryptionKey &encryption_key, int8 priority, vector bad_parts); void upload_by_hash(QueryId id, const FullLocalFileLocation &local_location, int64 size, int8 priority); diff --git a/td/telegram/files/FileLoader.cpp b/td/telegram/files/FileLoader.cpp index 24299093..f0c789b8 100644 --- a/td/telegram/files/FileLoader.cpp +++ b/td/telegram/files/FileLoader.cpp @@ -78,6 +78,9 @@ void FileLoader::start_up() { auto &ready_parts = file_info.ready_parts; auto use_part_count_limit = file_info.use_part_count_limit; auto status = parts_manager_.init(size, expected_size, is_size_final, part_size, ready_parts, use_part_count_limit); + if (file_info.only_check_) { + parts_manager_.set_checked_prefix_size(0); + } if (status.is_error()) { on_error(std::move(status)); stop_flag_ = true; @@ -110,6 +113,9 @@ Status FileLoader::do_loop() { TRY_RESULT(check_info, check_loop(parts_manager_.get_checked_prefix_size(), parts_manager_.get_unchecked_ready_prefix_size(), parts_manager_.unchecked_ready())); + if (check_info.changed) { + on_progress_impl(narrow_cast(parts_manager_.get_ready_size())); + } for (auto &query : check_info.queries) { G()->net_query_dispatcher().dispatch_with_callback( std::move(query), actor_shared(this, UniqueId::next(UniqueId::Type::Default, CommonQueryKey))); diff --git a/td/telegram/files/FileLoader.h b/td/telegram/files/FileLoader.h index 10d97fab..952da5a0 100644 --- a/td/telegram/files/FileLoader.h +++ b/td/telegram/files/FileLoader.h @@ -53,6 +53,7 @@ class FileLoader : public FileLoaderActor { int32 part_size; std::vector ready_parts; bool use_part_count_limit = true; + bool only_check_ = false; }; virtual Result init() TD_WARN_UNUSED_RESULT = 0; virtual Status on_ok(int64 size) TD_WARN_UNUSED_RESULT = 0; @@ -79,6 +80,7 @@ class FileLoader : public FileLoaderActor { } struct CheckInfo { bool need_check{false}; + bool changed{false}; int64 checked_prefix_size{0}; std::vector queries; }; diff --git a/td/telegram/files/FileLoaderUtils.cpp b/td/telegram/files/FileLoaderUtils.cpp index f6c603d6..bc932959 100644 --- a/td/telegram/files/FileLoaderUtils.cpp +++ b/td/telegram/files/FileLoaderUtils.cpp @@ -28,6 +28,11 @@ Result> try_create_new_file(Result result_name TRY_RESULT(fd, FileFd::open(name, FileFd::Read | FileFd::Write | FileFd::CreateNew, 0640)); return std::make_pair(std::move(fd), name.str()); } +Result> try_open_file(Result result_name) { + TRY_RESULT(name, std::move(result_name)); + TRY_RESULT(fd, FileFd::open(name, FileFd::Read, 0640)); + return std::make_pair(std::move(fd), name.str()); +} struct RandSuff { int len; @@ -63,32 +68,47 @@ Result> open_temp_file(const FileType &file_type) { return res; } -Result create_from_temp(CSlice temp_path, CSlice dir, CSlice name) { - LOG(INFO) << "Create file in directory " << dir << " with suggested name " << name << " from temporary file " - << temp_path; +template +bool for_suggested_file_name(CSlice name, bool use_pmc, bool use_random, F &&callback) { + auto try_callback = [&](Result r_path) { + if (r_path.is_error()) { + return true; + } + return callback(r_path.move_as_ok()); + }; auto cleaned_name = clean_filename(name); PathView path_view(cleaned_name); auto stem = path_view.file_stem(); auto ext = path_view.extension(); - Result> res; + bool active = true; if (!stem.empty() && !G()->parameters().ignore_file_names) { - res = try_create_new_file(PSLICE_SAFE() << dir << stem << Ext{ext}); - for (int i = 0; res.is_error() && i < 10; i++) { - res = try_create_new_file(PSLICE_SAFE() << dir << stem << "_(" << i << ")" << Ext{ext}); + active = try_callback(PSLICE_SAFE() << stem << Ext{ext}); + for (int i = 0; active && i < 10; i++) { + active = try_callback(PSLICE_SAFE() << stem << "_(" << i << ")" << Ext{ext}); } - for (int i = 2; res.is_error() && i < 12; i++) { - res = try_create_new_file(PSLICE_SAFE() << dir << stem << "_(" << RandSuff{i} << ")" << Ext{ext}); + for (int i = 2; active && i < 12 && use_random; i++) { + active = try_callback(PSLICE_SAFE() << stem << "_(" << RandSuff{i} << ")" << Ext{ext}); } - } else { + } else if (use_pmc) { auto pmc = G()->td_db()->get_binlog_pmc(); int32 file_id = to_integer(pmc->get("perm_file_id")); pmc->set("perm_file_id", to_string(file_id + 1)); - res = try_create_new_file(PSLICE_SAFE() << dir << "file_" << file_id << Ext{ext}); - if (res.is_error()) { - res = try_create_new_file(PSLICE_SAFE() << dir << "file_" << file_id << "_" << RandSuff{6} << Ext{ext}); + active = try_callback(PSLICE_SAFE() << "file_" << file_id << Ext{ext}); + if (active) { + active = try_callback(PSLICE_SAFE() << "file_" << file_id << "_" << RandSuff{6} << Ext{ext}); } } + return active; +} +Result create_from_temp(CSlice temp_path, CSlice dir, CSlice name) { + LOG(INFO) << "Create file in directory " << dir << " with suggested name " << name << " from temporary file " + << temp_path; + Result> res = Status::Error(); + for_suggested_file_name(name, true, true, [&](CSlice suggested_name) { + res = try_create_new_file(PSLICE_SAFE() << dir << suggested_name); + return res.is_error(); + }); TRY_RESULT(tmp, std::move(res)); tmp.first.close(); auto perm_path = std::move(tmp.second); @@ -96,6 +116,26 @@ Result create_from_temp(CSlice temp_path, CSlice dir, CSlice name) { return perm_path; } +Result search_file(CSlice dir, CSlice name, int64 expected_size) { + Result res = Status::Error(); + for_suggested_file_name(name, false, false, [&](CSlice suggested_name) { + auto r_pair = try_open_file(PSLICE_SAFE() << dir << suggested_name); + if (r_pair.is_error()) { + return false; + } + FileFd fd; + std::string path; + std::tie(fd, path) = r_pair.move_as_ok(); + if (fd.stat().size_ != expected_size) { + return true; + } + fd.close(); + res = std::move(path); + return false; + }); + return res; +} + const char *file_type_name[file_type_size] = {"thumbnails", "profile_photos", "photos", "voice", "videos", "documents", "secret", "temp", "stickers", "music", "animations", "secret_thumbnails", diff --git a/td/telegram/files/FileLoaderUtils.h b/td/telegram/files/FileLoaderUtils.h index 15621b29..7eaf1e12 100644 --- a/td/telegram/files/FileLoaderUtils.h +++ b/td/telegram/files/FileLoaderUtils.h @@ -18,6 +18,7 @@ enum class FileType : int8; Result> open_temp_file(const FileType &file_type) TD_WARN_UNUSED_RESULT; Result create_from_temp(CSlice temp_path, CSlice dir, CSlice name) TD_WARN_UNUSED_RESULT; +Result search_file(CSlice dir, CSlice name, int64 expected_size) TD_WARN_UNUSED_RESULT; string get_files_base_dir(const FileType &file_type); string get_files_temp_dir(const FileType &file_type); string get_files_dir(const FileType &file_type); diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index ba0adb3a..8b50e280 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -1037,6 +1037,7 @@ Result FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy nodes[node_i]->set_encryption_key(nodes[encryption_key_i]->encryption_key_); } node->need_load_from_pmc_ |= other_node->need_load_from_pmc_; + node->can_search_locally_ &= other_node->can_search_locally_; if (main_file_id_i == other_node_i) { node->main_file_id_ = other_node->main_file_id_; @@ -1450,7 +1451,7 @@ void FileManager::run_download(FileNodePtr node) { node->download_id_ = id; node->is_download_started_ = false; send_closure(file_load_manager_, &FileLoadManager::download, id, node->remote_.full(), node->local_, node->size_, - node->suggested_name(), node->encryption_key_, priority); + node->suggested_name(), node->encryption_key_, node->can_search_locally_, priority); } void FileManager::resume_upload(FileId file_id, std::vector bad_parts, std::shared_ptr callback, @@ -2236,6 +2237,11 @@ void FileManager::on_error_impl(FileNodePtr node, FileManager::Query::Type type, run_upload(node, {}); return; } + if (status.message() == "FILE_DOWNLOAD_RESTART") { + node->can_search_locally_ = false; + run_download(node); + return; + } if (!was_active) { return; diff --git a/td/telegram/files/FileManager.h b/td/telegram/files/FileManager.h index 4caf995d..a1a9d1a6 100644 --- a/td/telegram/files/FileManager.h +++ b/td/telegram/files/FileManager.h @@ -114,6 +114,7 @@ class FileNode { FileLocationSource remote_source_ = FileLocationSource::FromUser; bool get_by_hash_ = false; + bool can_search_locally_{true}; bool is_download_started_ = false; bool generate_was_update_ = false;