From da6a7ec512f9c14d12f02d7fa1bf0e3516a6adfc Mon Sep 17 00:00:00 2001 From: Arseny Smirnov Date: Thu, 27 Dec 2018 00:42:26 +0300 Subject: [PATCH] Files: pass file size from FileDownloader to FileManager GitOrigin-RevId: eb02bc323973b26b0306a84c5efadbb581bbef82 --- td/telegram/files/FileDownloader.cpp | 21 +++++++++++---------- td/telegram/files/FileDownloader.h | 5 ++--- td/telegram/files/FileLoadManager.cpp | 4 ++-- td/telegram/files/FileLoadManager.h | 9 +++++---- td/telegram/files/FileLoader.cpp | 20 +++++++++++++------- td/telegram/files/FileLoader.h | 14 +++++++++++--- td/telegram/files/FileManager.cpp | 7 +++++-- td/telegram/files/FileManager.h | 3 ++- td/telegram/files/FileUploader.cpp | 14 +++++++------- td/telegram/files/FileUploader.h | 3 +-- 10 files changed, 59 insertions(+), 41 deletions(-) diff --git a/td/telegram/files/FileDownloader.cpp b/td/telegram/files/FileDownloader.cpp index c6b8aaf4..6f748815 100644 --- a/td/telegram/files/FileDownloader.cpp +++ b/td/telegram/files/FileDownloader.cpp @@ -346,27 +346,28 @@ Result FileDownloader::process_part(Part part, NetQueryPtr net_query) { } return written; } -void FileDownloader::on_progress(int32 part_count, int32 part_size, int32 ready_part_count, const string &ready_bitmask, - bool is_ready, int64 ready_size) { - if (is_ready) { +void FileDownloader::on_progress(Progress progress) { + if (progress.is_ready) { // do not send partial location. will lead to wrong local_size return; } - if (ready_size == 0 || path_.empty()) { + if (progress.ready_size == 0 || path_.empty()) { return; } if (encryption_key_.empty() || encryption_key_.is_secure()) { - callback_->on_partial_download(PartialLocalFileLocation{remote_.file_type_, part_size, path_, "", ready_bitmask}, - ready_size); + callback_->on_partial_download( + PartialLocalFileLocation{remote_.file_type_, progress.part_size, path_, "", std::move(progress.ready_bitmask)}, + progress.ready_size, progress.size); } else if (encryption_key_.is_secret()) { UInt256 iv; - if (ready_part_count == next_part_) { + if (progress.ready_part_count == next_part_) { iv = encryption_key_.mutable_iv(); } else { - LOG(FATAL) << tag("ready_part_count", ready_part_count) << tag("next_part", next_part_); + LOG(FATAL) << tag("ready_part_count", progress.ready_part_count) << tag("next_part", next_part_); } - callback_->on_partial_download( - PartialLocalFileLocation{remote_.file_type_, part_size, path_, as_slice(iv).str(), ready_bitmask}, ready_size); + callback_->on_partial_download(PartialLocalFileLocation{remote_.file_type_, progress.part_size, path_, + as_slice(iv).str(), std::move(progress.ready_bitmask)}, + progress.ready_size, progress.size); } else { UNREACHABLE(); } diff --git a/td/telegram/files/FileDownloader.h b/td/telegram/files/FileDownloader.h index 59a0fe5f..52c24ece 100644 --- a/td/telegram/files/FileDownloader.h +++ b/td/telegram/files/FileDownloader.h @@ -27,7 +27,7 @@ class FileDownloader : public FileLoader { class Callback : public FileLoader::Callback { public: virtual void on_start_download() = 0; - virtual void on_partial_download(const PartialLocalFileLocation &partial_local, int64 ready_size) = 0; + virtual void on_partial_download(const PartialLocalFileLocation &partial_local, int64 ready_size, int64 size) = 0; virtual void on_ok(const FullLocalFileLocation &full_local, int64 size) = 0; virtual void on_error(Status status) = 0; }; @@ -85,8 +85,7 @@ class FileDownloader : public FileLoader { Result should_restart_part(Part part, NetQueryPtr &net_query) override TD_WARN_UNUSED_RESULT; Result> start_part(Part part, int32 part_count) override TD_WARN_UNUSED_RESULT; Result process_part(Part part, NetQueryPtr net_query) override TD_WARN_UNUSED_RESULT; - void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, const string &ready_bitmask, - bool is_ready, int64 ready_size) override; + void on_progress(Progress progress) override; FileLoader::Callback *get_callback() override; Status process_check_query(NetQueryPtr net_query) override; Result check_loop(int64 checked_prefix_size, int64 ready_prefix_size, bool is_ready) override; diff --git a/td/telegram/files/FileLoadManager.cpp b/td/telegram/files/FileLoadManager.cpp index e1c0f724..c393ea1f 100644 --- a/td/telegram/files/FileLoadManager.cpp +++ b/td/telegram/files/FileLoadManager.cpp @@ -187,14 +187,14 @@ void FileLoadManager::on_start_download() { } } -void FileLoadManager::on_partial_download(const PartialLocalFileLocation &partial_local, int64 ready_size) { +void FileLoadManager::on_partial_download(const PartialLocalFileLocation &partial_local, int64 ready_size, int64 size) { auto node_id = get_link_token(); auto node = nodes_container_.get(node_id); if (node == nullptr) { return; } if (!stop_flag_) { - send_closure(callback_, &Callback::on_partial_download, node->query_id_, partial_local, ready_size); + send_closure(callback_, &Callback::on_partial_download, node->query_id_, partial_local, ready_size, size); } } diff --git a/td/telegram/files/FileLoadManager.h b/td/telegram/files/FileLoadManager.h index a7c416b6..8cdb1b83 100644 --- a/td/telegram/files/FileLoadManager.h +++ b/td/telegram/files/FileLoadManager.h @@ -35,7 +35,8 @@ class FileLoadManager final : public Actor { Callback &operator=(const Callback &) = delete; ~Callback() override = default; virtual void on_start_download(QueryId id) = 0; - virtual void on_partial_download(QueryId id, const PartialLocalFileLocation &partial_local, int64 ready_size) = 0; + virtual void on_partial_download(QueryId id, const PartialLocalFileLocation &partial_local, int64 ready_size, + int64 size) = 0; virtual void on_partial_upload(QueryId id, const PartialRemoteFileLocation &partial_remote, int64 ready_size) = 0; virtual void on_hash(QueryId id, string hash) = 0; virtual void on_upload_ok(QueryId id, FileType file_type, const PartialRemoteFileLocation &remtoe, int64 size) = 0; @@ -84,7 +85,7 @@ class FileLoadManager final : public Actor { ActorOwn &get_download_resource_manager(bool is_small, DcId dc_id); void on_start_download(); - void on_partial_download(const PartialLocalFileLocation &partial_local, int64 ready_size); + void on_partial_download(const PartialLocalFileLocation &partial_local, int64 ready_size, int64 size); void on_partial_upload(const PartialRemoteFileLocation &partial_remote, int64 ready_size); void on_hash(string hash); void on_ok_download(const FullLocalFileLocation &local, int64 size); @@ -104,8 +105,8 @@ class FileLoadManager final : public Actor { void on_start_download() override { send_closure(actor_id_, &FileLoadManager::on_start_download); } - void on_partial_download(const PartialLocalFileLocation &partial_local, int64 ready_size) override { - send_closure(actor_id_, &FileLoadManager::on_partial_download, partial_local, ready_size); + void on_partial_download(const PartialLocalFileLocation &partial_local, int64 ready_size, int64 size) override { + send_closure(actor_id_, &FileLoadManager::on_partial_download, partial_local, ready_size, size); } void on_ok(const FullLocalFileLocation &full_local, int64 size) override { send_closure(std::move(actor_id_), &FileLoadManager::on_ok_download, full_local, size); diff --git a/td/telegram/files/FileLoader.cpp b/td/telegram/files/FileLoader.cpp index 77410670..137453e9 100644 --- a/td/telegram/files/FileLoader.cpp +++ b/td/telegram/files/FileLoader.cpp @@ -106,7 +106,7 @@ void FileLoader::start_up() { } resource_state_.set_unit_size(parts_manager_.get_part_size()); update_estimated_limit(); - on_progress_impl(narrow_cast(parts_manager_.get_ready_size())); + on_progress_impl(); yield(); } @@ -129,7 +129,7 @@ Status FileLoader::do_loop() { check_loop(parts_manager_.get_checked_prefix_size(), parts_manager_.get_unchecked_ready_prefix_size(), parts_manager_.unchecked_ready())); if (check_info.changed) { - on_progress_impl(narrow_cast(parts_manager_.get_ready_size())); + on_progress_impl(); } for (auto &query : check_info.queries) { G()->net_query_dispatcher().dispatch_with_callback( @@ -294,14 +294,20 @@ Status FileLoader::try_on_part_query(Part part, NetQueryPtr query) { debug_bad_parts_.push_back(part.id); debug_bad_part_order_++; } - on_progress_impl(size); + on_progress_impl(); return Status::OK(); } -void FileLoader::on_progress_impl(size_t size) { - on_progress(parts_manager_.get_part_count(), static_cast(parts_manager_.get_part_size()), - parts_manager_.get_ready_prefix_count(), parts_manager_.get_bitmask(), parts_manager_.ready(), - parts_manager_.get_ready_size()); +void FileLoader::on_progress_impl() { + Progress progress; + progress.part_count = parts_manager_.get_part_count(); + progress.part_size = static_cast(parts_manager_.get_part_size()); + progress.ready_part_count = parts_manager_.get_ready_prefix_count(); + progress.ready_bitmask = parts_manager_.get_bitmask(); + progress.is_ready = parts_manager_.ready(); + progress.ready_size = parts_manager_.get_ready_size(); + progress.size = parts_manager_.get_size_or_zero(); + on_progress(std::move(progress)); } } // namespace td diff --git a/td/telegram/files/FileLoader.h b/td/telegram/files/FileLoader.h index a487c959..0d810578 100644 --- a/td/telegram/files/FileLoader.h +++ b/td/telegram/files/FileLoader.h @@ -70,8 +70,16 @@ class FileLoader : public FileLoaderActor { virtual void after_start_parts() { } virtual Result process_part(Part part, NetQueryPtr net_query) TD_WARN_UNUSED_RESULT = 0; - virtual void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, const string &ready_bitmask, - bool is_ready, int64 ready_size) = 0; + struct Progress { + int32 part_count{0}; + int32 part_size{0}; + int32 ready_part_count{0}; + string ready_bitmask; + bool is_ready{false}; + int64 ready_size{0}; + int64 size{0}; + }; + virtual void on_progress(Progress progress) = 0; virtual Callback *get_callback() = 0; virtual Result on_update_local_location(const LocalFileLocation &location, int64 file_size) TD_WARN_UNUSED_RESULT { @@ -121,7 +129,7 @@ class FileLoader : public FileLoaderActor { void tear_down() override; void update_estimated_limit(); - void on_progress_impl(size_t size); + void on_progress_impl(); void on_result(NetQueryPtr query) override; void on_part_query(Part part, NetQueryPtr query); diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index f8fc0f16..ffb5bbf5 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -2271,8 +2271,8 @@ void FileManager::on_start_download(QueryId query_id) { file_node->is_download_started_ = true; } -void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLocation &partial_local, - int64 ready_size) { +void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLocation &partial_local, int64 ready_size, + int64 size) { if (is_closed_) { return; } @@ -2290,6 +2290,9 @@ void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLo return; } + if (size != 0) { + file_node->set_size(size); + } file_node->set_local_location(LocalFileLocation(partial_local), ready_size, -1, -1 /* TODO */); try_flush_node(file_node); } diff --git a/td/telegram/files/FileManager.h b/td/telegram/files/FileManager.h index 950e8fde..59bac652 100644 --- a/td/telegram/files/FileManager.h +++ b/td/telegram/files/FileManager.h @@ -495,7 +495,8 @@ class FileManager : public FileLoadManager::Callback { void run_generate(FileNodePtr node); void on_start_download(QueryId query_id) override; - void on_partial_download(QueryId query_id, const PartialLocalFileLocation &partial_local, int64 ready_size) override; + void on_partial_download(QueryId query_id, const PartialLocalFileLocation &partial_local, int64 ready_size, + int64 size) override; void on_hash(QueryId query_id, string hash) override; void on_partial_upload(QueryId query_id, const PartialRemoteFileLocation &partial_remote, int64 ready_size) override; void on_download_ok(QueryId query_id, const FullLocalFileLocation &local, int64 size) override; diff --git a/td/telegram/files/FileUploader.cpp b/td/telegram/files/FileUploader.cpp index 0f279e96..abc7d484 100644 --- a/td/telegram/files/FileUploader.cpp +++ b/td/telegram/files/FileUploader.cpp @@ -255,7 +255,6 @@ Result> FileUploader::start_part(Part part, int32 p } if (size != part.size) { - LOG(ERROR) << "Need to read " << part.size << " bytes, but read " << size << " bytes instead"; return Status::Error("Failed to read file part"); } @@ -295,13 +294,14 @@ Result FileUploader::process_part(Part part, NetQueryPtr net_query) { return part.size; } -void FileUploader::on_progress(int32 part_count, int32 part_size, int32 ready_part_count, const string &ready_bitmask, - bool is_ready, int64 ready_size) { - callback_->on_partial_upload(PartialRemoteFileLocation{file_id_, part_count, part_size, ready_part_count, big_flag_}, - ready_size); - if (is_ready) { +void FileUploader::on_progress(Progress progress) { + callback_->on_partial_upload(PartialRemoteFileLocation{file_id_, progress.part_count, progress.part_size, + progress.ready_part_count, big_flag_}, + progress.ready_size); + if (progress.is_ready) { callback_->on_ok(file_type_, - PartialRemoteFileLocation{file_id_, part_count, part_size, ready_part_count, big_flag_}, + PartialRemoteFileLocation{file_id_, progress.part_count, progress.part_size, + progress.ready_part_count, big_flag_}, local_size_); } } diff --git a/td/telegram/files/FileUploader.h b/td/telegram/files/FileUploader.h index 1ec12053..18f5feae 100644 --- a/td/telegram/files/FileUploader.h +++ b/td/telegram/files/FileUploader.h @@ -63,8 +63,7 @@ class FileUploader : public FileLoader { void after_start_parts() override; Result> start_part(Part part, int32 part_count) override TD_WARN_UNUSED_RESULT; Result process_part(Part part, NetQueryPtr net_query) override TD_WARN_UNUSED_RESULT; - void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, const string &ready_bitmask, - bool is_ready, int64 ready_size) override; + void on_progress(Progress progress) override; FileLoader::Callback *get_callback() override; Result on_update_local_location(const LocalFileLocation &location, int64 file_size) override TD_WARN_UNUSED_RESULT;