From ac3fa705a295d0a4009cfdb09d4be3f9664c80d6 Mon Sep 17 00:00:00 2001 From: levlam Date: Wed, 26 Dec 2018 19:11:15 +0300 Subject: [PATCH] File streaming fixes. GitOrigin-RevId: 86c6d1394a31d6bb108a12e14f4d981133ece235 --- memprof/memprof.cpp | 2 +- td/generate/scheme/td_api.tl | 16 ++--- td/generate/scheme/td_api.tlo | Bin 139804 -> 139808 bytes td/telegram/MessagesManager.cpp | 3 + td/telegram/SecretChatActor.h | 2 +- td/telegram/StickersManager.cpp | 2 +- td/telegram/Td.cpp | 4 +- td/telegram/Td.h | 2 +- td/telegram/cli.cpp | 40 +++++------ td/telegram/files/FileBitmask.cpp | 67 +++++++++++------- td/telegram/files/FileBitmask.h | 22 +++--- td/telegram/files/FileDownloader.cpp | 17 ++--- td/telegram/files/FileDownloader.h | 4 +- td/telegram/files/FileGenerateManager.cpp | 4 +- td/telegram/files/FileLoader.cpp | 4 +- td/telegram/files/FileLoader.h | 8 ++- td/telegram/files/FileLoaderActor.h | 2 +- td/telegram/files/FileLocation.h | 6 +- td/telegram/files/FileManager.cpp | 82 ++++++++++++---------- td/telegram/files/FileManager.h | 16 +++-- td/telegram/files/FileUploader.cpp | 11 +-- td/telegram/files/FileUploader.h | 9 ++- td/telegram/files/PartsManager.cpp | 4 +- td/telegram/files/PartsManager.h | 4 +- td/telegram/logevent/SecretChatEvent.h | 2 +- td/telegram/net/Session.h | 2 +- tdutils/td/utils/misc.cpp | 24 ++++--- tdutils/td/utils/misc.h | 12 ++-- 28 files changed, 206 insertions(+), 165 deletions(-) diff --git a/memprof/memprof.cpp b/memprof/memprof.cpp index 11822ee00..9be278af5 100644 --- a/memprof/memprof.cpp +++ b/memprof/memprof.cpp @@ -280,7 +280,7 @@ void *memalign(std::size_t aligment, std::size_t size) { } } -// c++14 guarantees than it is enough to override this two operators. +// c++14 guarantees that it is enough to override these two operators. void *operator new(std::size_t count) { return malloc_with_frame(count, get_backtrace()); } diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index 9671e1785..f0a29fa04 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -126,8 +126,8 @@ temporaryPasswordState has_password:Bool valid_for:int32 = TemporaryPasswordStat //@can_be_deleted True, if the file can be deleted //@is_downloading_active True, if the file is currently being downloaded (or a local copy is being generated by some other means) //@is_downloading_completed True, if the local copy is fully available -//@download_offset Download will be started from this offset. downloaded_prefix_size is calculated from this offset. -//@downloaded_prefix_size If is_downloading_completed is false, then only some prefix of the file is ready to be read. downloaded_prefix_size is the size of that prefix +//@download_offset Download will be started from this offset. downloaded_prefix_size is calculated from this offset +//@downloaded_prefix_size If is_downloading_completed is false, then only some prefix of the file starting from download_offset is ready to be read. downloaded_prefix_size is the size of that prefix //@downloaded_size Total downloaded file bytes. Should be used only for calculating download progress. The actual file size may be bigger, and some parts of it may contain garbage localFile path:string can_be_downloaded:Bool can_be_deleted:Bool is_downloading_active:Bool is_downloading_completed:Bool download_offset:int32 downloaded_prefix_size:int32 downloaded_size:int32 = LocalFile; @@ -2949,18 +2949,14 @@ setPinnedChats chat_ids:vector = Ok; //@description Asynchronously downloads a file from the cloud. updateFile will be used to notify about the download progress and successful completion of the download. Returns file state just after the download has been started //@file_id Identifier of the file to download //@priority Priority of the download (1-32). The higher the priority, the earlier the file will be downloaded. If the priorities of two files are equal, then the last one for which downloadFile was called will be downloaded first -//@offset File will be downloaded starting from offset first. Supposed to be used for streaming. +//@offset File will be downloaded starting from that offset in bytes first. Supposed to be used for streaming downloadFile file_id:int32 priority:int32 offset:int32 = File; -//@description Set offset for file downloading -//@file_id Identifier of file -//@offset File download offset +//@description Sets offset for file downloading @file_id Identifier of the file to change download offset @offset New file download offset setFileDownloadOffset file_id:int32 offset:int32 = File; -//@description Get downloaded prefix from a given offset -//@file_id Identifier of file -//@offset Offset from which downloaded prefix is calculated -getFileDownloadedPrefix file_id:int32 offset:int32 = Count; +//@description Returns file downloaded prefix size from a given offset @file_id Identifier of the file @offset Offset from which downloaded prefix size should be calculated +getFileDownloadedPrefixSize file_id:int32 offset:int32 = Count; //@description Stops the downloading of a file. If a file has already been downloaded, does nothing @file_id Identifier of a file to stop downloading @only_if_pending Pass true to stop downloading only if it hasn't been started, i.e. request hasn't been sent to server cancelDownloadFile file_id:int32 only_if_pending:Bool = Ok; diff --git a/td/generate/scheme/td_api.tlo b/td/generate/scheme/td_api.tlo index 59d436559683959ba142d86d26b8210710db2c85..4c98d067cb9b928ee35e699a7f204b4e735397e9 100644 GIT binary patch delta 50 zcmbPpgk!-Gj)pCa4f_-psIpinned_message_notification_message_id; + if (!old_message_id.is_valid() && !message_id.is_valid()) { + return; + } CHECK(old_message_id != message_id); VLOG(notifications) << "Change pinned message notification in " << d->dialog_id << " from " << old_message_id << " to " << message_id; diff --git a/td/telegram/SecretChatActor.h b/td/telegram/SecretChatActor.h index f5226b056..f64a61b31 100644 --- a/td/telegram/SecretChatActor.h +++ b/td/telegram/SecretChatActor.h @@ -103,7 +103,7 @@ class SecretChatActor : public NetQueryCallback { SecretChatActor(int32 id, unique_ptr context, bool can_be_empty); - // First query to new chat must be on of this two + // First query to new chat must be on of these two void update_chat(telegram_api::object_ptr chat); void create_chat(int32 user_id, int64 user_access_hash, int32 random_id, Promise promise); void cancel_chat(Promise<> promise); diff --git a/td/telegram/StickersManager.cpp b/td/telegram/StickersManager.cpp index 3bb230e7e..6a3faf311 100644 --- a/td/telegram/StickersManager.cpp +++ b/td/telegram/StickersManager.cpp @@ -3040,7 +3040,7 @@ Result> StickersManager::prepare_input_file( if (file_view.has_url()) { is_url = true; } else { - if (file_view.has_local_location() && file_view.local_size() > MAX_STICKER_FILE_SIZE) { + if (file_view.has_local_location() && file_view.expected_size() > MAX_STICKER_FILE_SIZE) { return Status::Error(400, "File is too big"); } is_local = true; diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index d69c870e5..9510db850 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -4858,13 +4858,13 @@ void Td::on_request(uint64 id, const td_api::getFile &request) { send_closure(actor_id(this), &Td::send_result, id, file_manager_->get_file_object(FileId(request.file_id_, 0))); } -void Td::on_request(uint64 id, const td_api::getFileDownloadedPrefix &request) { +void Td::on_request(uint64 id, const td_api::getFileDownloadedPrefixSize &request) { auto file_view = file_manager_->get_file_view(FileId(request.file_id_, 0)); if (file_view.empty()) { return send_closure(actor_id(this), &Td::send_error, id, Status::Error(10, "Unknown file id")); } send_closure(actor_id(this), &Td::send_result, id, - td_api::make_object(static_cast(file_view.downloaded_prefix(request.offset_)))); + td_api::make_object(narrow_cast(file_view.downloaded_prefix(request.offset_)))); } void Td::on_request(uint64 id, td_api::getRemoteFile &request) { diff --git a/td/telegram/Td.h b/td/telegram/Td.h index c7b4f82cd..73556a757 100644 --- a/td/telegram/Td.h +++ b/td/telegram/Td.h @@ -442,7 +442,7 @@ class Td final : public NetQueryCallback { void on_request(uint64 id, const td_api::getFile &request); - void on_request(uint64 id, const td_api::getFileDownloadedPrefix &request); + void on_request(uint64 id, const td_api::getFileDownloadedPrefixSize &request); void on_request(uint64 id, td_api::getRemoteFile &request); diff --git a/td/telegram/cli.cpp b/td/telegram/cli.cpp index 7a73a2346..677095a00 100644 --- a/td/telegram/cli.cpp +++ b/td/telegram/cli.cpp @@ -2251,11 +2251,11 @@ class CliClient final : public Actor { send_request(make_tl_object(as_chat_id(chat_id), to_integer(date))); } else if (op == "gf" || op == "GetFile") { send_request(make_tl_object(as_file_id(args))); - } else if (op == "gfp" || op == "GetFileDownloadedPrefix") { + } else if (op == "gfdps") { string file_id; string offset; std::tie(file_id, offset) = split(args); - send_request(make_tl_object(as_file_id(file_id), to_integer(offset))); + send_request(make_tl_object(as_file_id(file_id), to_integer(offset))); } else if (op == "grf") { send_request(make_tl_object(args, nullptr)); } else if (op == "gmtf") { @@ -2277,36 +2277,34 @@ class CliClient final : public Actor { as_location(latitude, longitude), to_integer(zoom), to_integer(width), to_integer(height), to_integer(scale), as_chat_id(chat_id))); } else if (op == "sfdo" || op == "SetDownloadFileOffset") { - string file_id_str; + string file_id; string offset; - std::tie(file_id_str, offset) = split(args); + std::tie(file_id, offset) = split(args); - auto file_id = as_file_id(file_id_str); - send_request(make_tl_object(file_id, to_integer(offset))); + send_request(make_tl_object(as_file_id(file_id), to_integer(offset))); } else if (op == "df" || op == "DownloadFile") { - string file_id_str; - string priority; - string offset; - std::tie(file_id_str, args) = split(args); - std::tie(priority, offset) = split(args); - if (priority.empty()) { - priority = "1"; - } - - auto file_id = as_file_id(file_id_str); - send_request( - make_tl_object(file_id, to_integer(priority), to_integer(offset))); - } else if (op == "dff") { string file_id; string priority; string offset; std::tie(file_id, args) = split(args); - std::tie(priority, offset) = split(args); + std::tie(offset, priority) = split(args); if (priority.empty()) { priority = "1"; } - for (int i = 1; i <= as_file_id(file_id); i++) { + send_request(make_tl_object(as_file_id(file_id), to_integer(priority), + to_integer(offset))); + } else if (op == "dff") { + string max_file_id; + string priority; + string offset; + std::tie(max_file_id, args) = split(args); + std::tie(offset, priority) = split(args); + if (priority.empty()) { + priority = "1"; + } + + for (int i = 1; i <= as_file_id(max_file_id); i++) { send_request(make_tl_object(i, to_integer(priority), to_integer(offset))); } } else if (op == "cdf") { diff --git a/td/telegram/files/FileBitmask.cpp b/td/telegram/files/FileBitmask.cpp index 004d5f401..8e4c6f43f 100644 --- a/td/telegram/files/FileBitmask.cpp +++ b/td/telegram/files/FileBitmask.cpp @@ -5,80 +5,97 @@ // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // #include "td/telegram/files/FileBitmask.h" + +#include "td/utils/common.h" #include "td/utils/misc.h" + namespace td { + Bitmask::Bitmask(Decode, Slice data) : data_(zero_one_decode(data)) { } -Bitmask::Bitmask(Ones, int64 count) : data_((count + 7) / 8, '\0') { + +Bitmask::Bitmask(Ones, int64 count) : data_(narrow_cast((count + 7) / 8), '\0') { for (int64 i = 0; i < count; i++) { set(i); } } + std::string Bitmask::encode() const { - // remove zeroes in the end to make encoding deteministic + // remove zeroes in the end to make encoding deterministic td::Slice data(data_); - while (!data.empty() && data.back() == 0) { + while (!data.empty() && data.back() == '\0') { data.remove_suffix(1); } - return zero_one_encode(data_); + return zero_one_encode(data); } -Bitmask::ReadySize Bitmask::get_ready_size(int64 offset, int64 part_size) const { - ReadySize res; - res.offset = offset; + +int64 Bitmask::get_ready_prefix_size(int64 offset, int64 part_size, int64 file_size) const { auto offset_part = offset / part_size; auto ones = get_ready_parts(offset_part); if (ones == 0) { - res.ready_size = 0; - } else { - res.ready_size = (offset_part + ones) * part_size - offset; + return 0; } - CHECK(res.ready_size >= 0); + auto ready_parts_end = (offset_part + ones) * part_size; + if (file_size != 0 && ready_parts_end > file_size) { + ready_parts_end = file_size; + if (offset > file_size) { + offset = file_size; + } + } + auto res = ready_parts_end - offset; + CHECK(res >= 0); return res; } + int64 Bitmask::get_total_size(int64 part_size) const { int64 res = 0; for (int64 i = 0; i < size(); i++) { - res += get(i); + res += static_cast(get(i)); } return res * part_size; } -bool Bitmask::get(int64 offset) const { - if (offset < 0) { + +bool Bitmask::get(int64 offset_part) const { + if (offset_part < 0) { return 0; } - if (offset / 8 >= narrow_cast(data_.size())) { + auto index = narrow_cast(offset_part / 8); + if (index >= data_.size()) { return 0; } - return (data_[offset / 8] & (1 << (offset % 8))) != 0; + return (static_cast(data_[index]) & (1 << static_cast(offset_part % 8))) != 0; } -int64 Bitmask::get_ready_parts(int64 offset) const { +int64 Bitmask::get_ready_parts(int64 offset_part) const { int64 res = 0; - while (get(offset + res)) { + while (get(offset_part + res)) { res++; } return res; -}; +} std::vector Bitmask::as_vector() const { std::vector res; - for (int32 i = 0; i < narrow_cast(data_.size() * 8); i++) { + auto size = narrow_cast(data_.size() * 8); + for (int32 i = 0; i < size; i++) { if (get(i)) { res.push_back(i); } } return res; } -void Bitmask::set(int64 offset) { - auto need_size = narrow_cast(offset / 8 + 1); + +void Bitmask::set(int64 offset_part) { + CHECK(offset_part >= 0); + auto need_size = narrow_cast(offset_part / 8 + 1); if (need_size > data_.size()) { - data_.resize(need_size, 0); + data_.resize(need_size, '\0'); } - data_[need_size - 1] |= (1 << (offset % 8)); + data_[need_size - 1] |= (1 << (offset_part % 8)); } int64 Bitmask::size() const { - return data_.size() * 8; + return static_cast(data_.size() * 8); } } // namespace td diff --git a/td/telegram/files/FileBitmask.h b/td/telegram/files/FileBitmask.h index 72ba398d9..65d4bf45f 100644 --- a/td/telegram/files/FileBitmask.h +++ b/td/telegram/files/FileBitmask.h @@ -5,34 +5,29 @@ // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // #pragma once + #include "td/utils/common.h" #include "td/utils/Slice.h" #include "td/utils/StringBuilder.h" namespace td { + class Bitmask { public: - struct ReadySize { - int64 offset{-1}; - int64 ready_size{-1}; - bool empty() const { - return offset == -1; - } - }; struct Decode {}; struct Ones {}; Bitmask() = default; Bitmask(Decode, Slice data); Bitmask(Ones, int64 count); std::string encode() const; - ReadySize get_ready_size(int64 offset, int64 part_size) const; + int64 get_ready_prefix_size(int64 offset, int64 part_size, int64 file_size) const; int64 get_total_size(int64 part_size) const; - bool get(int64 offset) const; + bool get(int64 offset_part) const; - int64 get_ready_parts(int64 offset) const; + int64 get_ready_parts(int64 offset_part) const; std::vector as_vector() const; - void set(int64 offset); + void set(int64 offset_part); int64 size() const; private: @@ -40,11 +35,10 @@ class Bitmask { }; inline StringBuilder &operator<<(StringBuilder &sb, const Bitmask &mask) { - std::string res; for (int64 i = 0; i < mask.size(); i++) { - res += mask.get(i) ? "1" : "0"; + sb << (mask.get(i) ? '1' : '0'); } - return sb << res; + return sb; } } // namespace td diff --git a/td/telegram/files/FileDownloader.cpp b/td/telegram/files/FileDownloader.cpp index 25cb88a19..c6b8aaf43 100644 --- a/td/telegram/files/FileDownloader.cpp +++ b/td/telegram/files/FileDownloader.cpp @@ -104,10 +104,11 @@ Result FileDownloader::init() { res.ready_parts = bitmask.as_vector(); res.use_part_count_limit = false; res.only_check = only_check_; - res.need_delay = !is_small_ && (remote_.file_type_ == FileType::VideoNote || - remote_.file_type_ == FileType::VoiceNote || remote_.file_type_ == FileType::Audio || - remote_.file_type_ == FileType::Video || remote_.file_type_ == FileType::Animation || - (remote_.file_type_ == FileType::Encrypted && size_ > (1 << 20))); + res.need_delay = + !is_small_ && (remote_.file_type_ == FileType::VideoNote || remote_.file_type_ == FileType::Document || + remote_.file_type_ == FileType::VoiceNote || remote_.file_type_ == FileType::Audio || + remote_.file_type_ == FileType::Video || remote_.file_type_ == FileType::Animation || + (remote_.file_type_ == FileType::Encrypted && size_ > (1 << 20))); res.offset = offset_; return res; } @@ -304,7 +305,6 @@ Result FileDownloader::process_part(Part part, NetQueryPtr net_query) { if (encryption_key_.is_secret()) { padded_size = (part.size + 15) & ~15; } - LOG(INFO) << "Got " << bytes.size() << " bytes, padded_size = " << padded_size << " for " << path_; if (bytes.size() > padded_size) { return Status::Error("Part size is more than requested"); } @@ -338,6 +338,7 @@ Result FileDownloader::process_part(Part part, NetQueryPtr net_query) { auto slice = bytes.as_slice().truncate(part.size); TRY_STATUS(acquire_fd()); + LOG(INFO) << "Got " << slice.size() << " bytes at " << part.offset << " for \"" << path_ << '"'; TRY_RESULT(written, fd_.pwrite(slice, part.offset)); // may write less than part.size, when size of downloadable file is unknown if (written != slice.size()) { @@ -345,7 +346,7 @@ Result FileDownloader::process_part(Part part, NetQueryPtr net_query) { } return written; } -void FileDownloader::on_progress(int32 part_count, int32 part_size, int32 ready_part_count, string ready_bitmask, +void FileDownloader::on_progress(int32 part_count, int32 part_size, int32 ready_part_count, const string &ready_bitmask, bool is_ready, int64 ready_size) { if (is_ready) { // do not send partial location. will lead to wrong local_size @@ -355,7 +356,7 @@ void FileDownloader::on_progress(int32 part_count, int32 part_size, int32 ready_ return; } if (encryption_key_.empty() || encryption_key_.is_secure()) { - callback_->on_partial_download(PartialLocalFileLocation{remote_.file_type_, path_, part_size, "", ready_bitmask}, + callback_->on_partial_download(PartialLocalFileLocation{remote_.file_type_, part_size, path_, "", ready_bitmask}, ready_size); } else if (encryption_key_.is_secret()) { UInt256 iv; @@ -365,7 +366,7 @@ void FileDownloader::on_progress(int32 part_count, int32 part_size, int32 ready_ LOG(FATAL) << tag("ready_part_count", ready_part_count) << tag("next_part", next_part_); } callback_->on_partial_download( - PartialLocalFileLocation{remote_.file_type_, path_, part_size, as_slice(iv).str(), ready_bitmask}, ready_size); + PartialLocalFileLocation{remote_.file_type_, part_size, path_, as_slice(iv).str(), ready_bitmask}, ready_size); } else { UNREACHABLE(); } diff --git a/td/telegram/files/FileDownloader.h b/td/telegram/files/FileDownloader.h index 4c5d23e28..59a0fe5f5 100644 --- a/td/telegram/files/FileDownloader.h +++ b/td/telegram/files/FileDownloader.h @@ -85,8 +85,8 @@ class FileDownloader : public FileLoader { Result should_restart_part(Part part, NetQueryPtr &net_query) override TD_WARN_UNUSED_RESULT; Result> start_part(Part part, int32 part_count) override TD_WARN_UNUSED_RESULT; Result process_part(Part part, NetQueryPtr net_query) override TD_WARN_UNUSED_RESULT; - void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, string ready_bitmask, bool is_ready, - int64 ready_size) override; + void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, const string &ready_bitmask, + bool is_ready, int64 ready_size) override; FileLoader::Callback *get_callback() override; Status process_check_query(NetQueryPtr net_query) override; Result check_loop(int64 checked_prefix_size, int64 ready_prefix_size, bool is_ready) override; diff --git a/td/telegram/files/FileGenerateManager.cpp b/td/telegram/files/FileGenerateManager.cpp index 5ec070537..d2f13fd4c 100644 --- a/td/telegram/files/FileGenerateManager.cpp +++ b/td/telegram/files/FileGenerateManager.cpp @@ -303,8 +303,8 @@ class FileExternalGenerateActor : public FileGenerateActor { if (local_prefix_size < 0) { return Status::Error(1, "Invalid local prefix size"); } - callback_->on_partial_generate(PartialLocalFileLocation{generate_location_.file_type_, path_, local_prefix_size, - Bitmask(Bitmask::Ones{}, 1).encode(), ""}, + callback_->on_partial_generate(PartialLocalFileLocation{generate_location_.file_type_, local_prefix_size, path_, "", + Bitmask(Bitmask::Ones{}, 1).encode()}, expected_size); return Status::OK(); } diff --git a/td/telegram/files/FileLoader.cpp b/td/telegram/files/FileLoader.cpp index 1f1a9dcd2..774106704 100644 --- a/td/telegram/files/FileLoader.cpp +++ b/td/telegram/files/FileLoader.cpp @@ -20,6 +20,7 @@ #include namespace td { + void FileLoader::set_resource_manager(ActorShared resource_manager) { resource_manager_ = std::move(resource_manager); send_closure(resource_manager_, &ResourceManager::update_resources, resource_state_); @@ -47,7 +48,7 @@ void FileLoader::hangup() { } void FileLoader::update_local_file_location(const LocalFileLocation &local) { - auto r_prefix_info = on_update_local_location(local); + auto r_prefix_info = on_update_local_location(local, parts_manager_.get_size_or_zero()); if (r_prefix_info.is_error()) { on_error(r_prefix_info.move_as_error()); stop_flag_ = true; @@ -302,4 +303,5 @@ void FileLoader::on_progress_impl(size_t size) { parts_manager_.get_ready_prefix_count(), parts_manager_.get_bitmask(), parts_manager_.ready(), parts_manager_.get_ready_size()); } + } // namespace td diff --git a/td/telegram/files/FileLoader.h b/td/telegram/files/FileLoader.h index a8f934ac7..a487c959c 100644 --- a/td/telegram/files/FileLoader.h +++ b/td/telegram/files/FileLoader.h @@ -24,6 +24,7 @@ #include namespace td { + class FileLoader : public FileLoaderActor { public: class Callback { @@ -69,10 +70,11 @@ class FileLoader : public FileLoaderActor { virtual void after_start_parts() { } virtual Result process_part(Part part, NetQueryPtr net_query) TD_WARN_UNUSED_RESULT = 0; - virtual void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, std::string ready_bitmask, + virtual void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, const string &ready_bitmask, bool is_ready, int64 ready_size) = 0; virtual Callback *get_callback() = 0; - virtual Result on_update_local_location(const LocalFileLocation &location) TD_WARN_UNUSED_RESULT { + virtual Result on_update_local_location(const LocalFileLocation &location, + int64 file_size) TD_WARN_UNUSED_RESULT { return Status::Error("unsupported"); } virtual Result should_restart_part(Part part, NetQueryPtr &net_query) TD_WARN_UNUSED_RESULT { @@ -103,7 +105,6 @@ class FileLoader : public FileLoaderActor { PartsManager parts_manager_; uint64 blocking_id_{0}; std::map>> part_map_; - // std::map> part_map_; bool ordered_flag_ = false; OrderedEventsProcessor> ordered_parts_; ActorOwn delay_dispatcher_; @@ -127,4 +128,5 @@ class FileLoader : public FileLoaderActor { void on_common_query(NetQueryPtr query); Status try_on_part_query(Part part, NetQueryPtr query); }; + } // namespace td diff --git a/td/telegram/files/FileLoaderActor.h b/td/telegram/files/FileLoaderActor.h index 27844564c..e42a9c3cf 100644 --- a/td/telegram/files/FileLoaderActor.h +++ b/td/telegram/files/FileLoaderActor.h @@ -22,7 +22,7 @@ class FileLoaderActor : public NetQueryCallback { virtual void update_priority(int8 priority) = 0; virtual void update_resources(const ResourceState &other) = 0; - // TODO: existence of this two functions is a dirty hack. Refactoring is highly appreciated + // TODO: existence of these two functions is a dirty hack. Refactoring is highly appreciated virtual void update_local_file_location(const LocalFileLocation &local) { } virtual void update_download_offset(int64 offset) { diff --git a/td/telegram/files/FileLocation.h b/td/telegram/files/FileLocation.h index a6221ae74..7a9456b75 100644 --- a/td/telegram/files/FileLocation.h +++ b/td/telegram/files/FileLocation.h @@ -6,11 +6,11 @@ // #pragma once -#include "td/telegram/files/FileBitmask.h" #include "td/telegram/td_api.h" #include "td/telegram/telegram_api.h" #include "td/telegram/DialogId.h" +#include "td/telegram/files/FileBitmask.h" #include "td/telegram/net/DcId.h" #include "td/telegram/SecureStorage.h" @@ -908,8 +908,8 @@ inline bool operator!=(const EmptyLocalFileLocation &lhs, const EmptyLocalFileLo struct PartialLocalFileLocation { FileType file_type_; - string path_; int32 part_size_; + string path_; string iv_; string ready_bitmask_; @@ -939,6 +939,8 @@ struct PartialLocalFileLocation { if (deprecated_ready_part_count == -1) { parse(ready_bitmask_, parser); } else { + CHECK(0 <= deprecated_ready_part_count); + CHECK(deprecated_ready_part_count <= (1 << 22)); ready_bitmask_ = Bitmask(Bitmask::Ones{}, deprecated_ready_part_count).encode(); } } diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index a3f9ad30b..f8fc0f16c 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -66,17 +66,16 @@ FileNodePtr::operator bool() const { return file_manager_ != nullptr && get_unsafe() != nullptr; } -void FileNode::recalc_ready_prefix_size(Bitmask::ReadySize ready_prefix_size) { +void FileNode::recalc_ready_prefix_size(int64 prefix_offset, int64 ready_prefix_size) { if (local_.type() != LocalFileLocation::Type::Partial) { return; } int64 new_local_ready_prefix_size; - if (download_offset_ == ready_prefix_size.offset) { - new_local_ready_prefix_size = ready_prefix_size.ready_size; + if (download_offset_ == prefix_offset) { + new_local_ready_prefix_size = ready_prefix_size; } else { new_local_ready_prefix_size = Bitmask(Bitmask::Decode{}, local_.partial().ready_bitmask_) - .get_ready_size(download_offset_, local_.partial().part_size_) - .ready_size; + .get_ready_prefix_size(download_offset_, local_.partial().part_size_, size_); } if (new_local_ready_prefix_size != local_ready_prefix_size_) { local_ready_prefix_size_ = new_local_ready_prefix_size; @@ -89,7 +88,7 @@ void FileNode::init_ready_size() { return; } auto bitmask = Bitmask(Bitmask::Decode{}, local_.partial().ready_bitmask_); - local_ready_prefix_size_ = bitmask.get_ready_size(0, local_.partial().part_size_).ready_size; + local_ready_prefix_size_ = bitmask.get_ready_prefix_size(0, local_.partial().part_size_, size_); local_ready_size_ = bitmask.get_total_size(local_.partial().part_size_); } @@ -102,22 +101,27 @@ void FileNode::set_download_offset(int64 download_offset) { } download_offset_ = download_offset; is_download_offset_dirty_ = true; - recalc_ready_prefix_size({}); + recalc_ready_prefix_size(-1, -1); on_info_changed(); } -void FileNode::set_local_location(const LocalFileLocation &local, int64 ready_size, - Bitmask::ReadySize ready_prefix_size) { +void FileNode::drop_local_location() { + set_local_location(LocalFileLocation(), 0, -1, -1); +} + +void FileNode::set_local_location(const LocalFileLocation &local, int64 ready_size, int64 prefix_offset, + int64 ready_prefix_size) { if (local_ready_size_ != ready_size) { + VLOG(update_file) << "File " << main_file_id_ << " has changed local ready size from " << local_ready_size_ + << " to " << ready_size; local_ready_size_ = ready_size; - VLOG(update_file) << "File " << main_file_id_ << " has changed local ready size"; on_info_changed(); } if (local_ != local) { VLOG(update_file) << "File " << main_file_id_ << " has changed local location"; local_ = local; - recalc_ready_prefix_size(ready_prefix_size); + recalc_ready_prefix_size(prefix_offset, ready_prefix_size); on_changed(); } @@ -357,22 +361,25 @@ int64 FileView::downloaded_prefix(int64 offset) const { } return 0; case LocalFileLocation::Type::Partial: + if (is_encrypted_secure()) { + // File is not decrypted and verified yet + return 0; + } return Bitmask(Bitmask::Decode{}, node_->local_.partial().ready_bitmask_) - .get_ready_size(offset, node_->local_.partial().part_size_) - .ready_size; + .get_ready_prefix_size(offset, node_->local_.partial().part_size_, node_->size_); default: UNREACHABLE(); return 0; } } -int64 FileView::local_size() const { +int64 FileView::local_prefix_size() const { switch (node_->local_.type()) { case LocalFileLocation::Type::Full: - return node_->size_; + return node_->download_offset_ <= node_->size_ ? node_->size_ - node_->download_offset_ : 0; case LocalFileLocation::Type::Partial: { if (is_encrypted_secure()) { - // File is not decrypted yet + // File is not decrypted and verified yet return 0; } return node_->local_ready_prefix_size_; @@ -660,7 +667,7 @@ Status FileManager::check_local_location(FileNodePtr node) { status = check_partial_local_location(node->local_.partial()); } if (status.is_error()) { - node->set_local_location(LocalFileLocation(), 0); + node->drop_local_location(); try_flush_node(node); } return status; @@ -719,7 +726,7 @@ void FileManager::on_file_unlink(const FullLocalFileLocation &location) { auto file_id = it->second; auto file_node = get_sync_file_node(file_id); CHECK(file_node); - file_node->set_local_location(LocalFileLocation(), 0); + file_node->drop_local_location(); try_flush_node_info(file_node); } @@ -1114,15 +1121,17 @@ Result FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy << ", main_file_id_i = " << main_file_id_i; if (local_i == other_node_i) { cancel_download(node); - node->set_local_location(other_node->local_, other_node->local_ready_size_); + node->set_download_offset(other_node->download_offset_); + node->set_local_location(other_node->local_, other_node->local_ready_size_, other_node->download_offset_, + other_node->local_ready_prefix_size_); node->download_id_ = other_node->download_id_; node->is_download_started_ |= other_node->is_download_started_; node->set_download_priority(other_node->download_priority_); - node->set_download_offset(other_node->download_offset_); other_node->download_id_ = 0; other_node->is_download_started_ = false; other_node->download_priority_ = 0; other_node->download_offset_ = 0; + other_node->local_ready_prefix_size_ = 0; //cancel_generate(node); //node->set_generate_location(std::move(other_node->generate_)); @@ -1500,7 +1509,7 @@ void FileManager::delete_file(FileId file_id, Promise promise, const char unlink(file_view.local_location().path_).ignore(); context_->on_new_file(-file_view.size()); - node->set_local_location(LocalFileLocation(), 0); + node->drop_local_location(); try_flush_node(node); } } else { @@ -1510,7 +1519,7 @@ void FileManager::delete_file(FileId file_id, Promise promise, const char if (node->local_.type() == LocalFileLocation::Type::Partial) { LOG(INFO) << "Unlink partial file " << file_id << " at " << node->local_.partial().path_; unlink(node->local_.partial().path_).ignore(); - node->set_local_location(LocalFileLocation(), 0); + node->drop_local_location(); try_flush_node(node); } } @@ -1566,6 +1575,10 @@ void FileManager::download(FileId file_id, std::shared_ptr cal } LOG(INFO) << "Change download priority of file " << file_id << " to " << new_priority; + if (file_view.is_encrypted_any()) { + // TODO: we need to set it always and just download from the beginning + offset = 0; + } node->set_download_offset(offset); auto *file_info = get_file_id_info(file_id); CHECK(new_priority == 0 || callback); @@ -1585,7 +1598,7 @@ void FileManager::download_set_offset(FileId file_id, int64 offset) { LOG(INFO) << "File " << file_id << " not found"; return; } - if (FileView(file_node).is_encrypted()) { + if (FileView(file_node).is_encrypted_any()) { offset = 0; } file_node->set_download_offset(offset); @@ -1622,10 +1635,6 @@ void FileManager::run_download(FileNodePtr node) { } return; } - if (file_view.is_encrypted()) { - node->set_download_offset(0); - } - auto offset = node->download_offset_; bool need_update_offset = node->is_download_offset_dirty_; node->is_download_offset_dirty_ = false; @@ -1633,7 +1642,8 @@ void FileManager::run_download(FileNodePtr node) { CHECK(node->download_id_ != 0); send_closure(file_load_manager_, &FileLoadManager::update_priority, node->download_id_, priority); if (need_update_offset) { - send_closure(file_load_manager_, &FileLoadManager::update_download_offset, node->download_id_, offset); + send_closure(file_load_manager_, &FileLoadManager::update_download_offset, node->download_id_, + node->download_offset_); } return; } @@ -2030,7 +2040,7 @@ tl_object_ptr FileManager::get_file_object(FileId file_id, bool wi int32 size = narrow_cast(file_view.size()); int32 expected_size = narrow_cast(file_view.expected_size()); int32 download_offset = narrow_cast(file_view.download_offset()); - int32 local_size = narrow_cast(file_view.local_size()); + int32 local_prefix_size = narrow_cast(file_view.local_prefix_size()); int32 local_total_size = narrow_cast(file_view.local_total_size()); int32 remote_size = narrow_cast(file_view.remote_size()); string path = file_view.path(); @@ -2053,7 +2063,7 @@ tl_object_ptr FileManager::get_file_object(FileId file_id, bool wi result_file_id.get(), size, expected_size, td_api::make_object(std::move(path), can_be_downloaded, can_be_deleted, file_view.is_downloading(), file_view.has_local_location(), - download_offset, local_size, local_total_size), + download_offset, local_prefix_size, local_total_size), td_api::make_object(std::move(persistent_file_id), file_view.is_uploading(), is_uploading_completed, remote_size)); } @@ -2272,9 +2282,7 @@ void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLo auto file_id = query->file_id_; auto file_node = get_file_node(file_id); - LOG(DEBUG) << "Receive on_parial_download for file " << file_id; - LOG(ERROR) << "Receive on_parital_generate for file " << file_id << ": " << partial_local.path_ << " " - << Bitmask(Bitmask::Decode{}, partial_local.ready_bitmask_); + LOG(DEBUG) << "Receive on_partial_download for file " << file_id; if (!file_node) { return; } @@ -2282,7 +2290,7 @@ void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLo return; } - file_node->set_local_location(LocalFileLocation(partial_local), ready_size); + file_node->set_local_location(LocalFileLocation(partial_local), ready_size, -1, -1 /* TODO */); try_flush_node(file_node); } @@ -2445,7 +2453,7 @@ void FileManager::on_partial_generate(QueryId query_id, const PartialLocalFileLo auto file_id = query->file_id_; auto file_node = get_file_node(file_id); - LOG(DEBUG) << "Receive on_parital_generate for file " << file_id << ": " << partial_local.path_ << " " + LOG(DEBUG) << "Receive on_partial_generate for file " << file_id << ": " << partial_local.path_ << " " << Bitmask(Bitmask::Decode{}, partial_local.ready_bitmask_); if (!file_node) { return; @@ -2453,7 +2461,7 @@ void FileManager::on_partial_generate(QueryId query_id, const PartialLocalFileLo if (file_node->generate_id_ != query_id) { return; } - file_node->set_local_location(LocalFileLocation(partial_local), 0); + file_node->set_local_location(LocalFileLocation(partial_local), 0, -1, -1 /* TODO */); // TODO check for size and local_size, abort generation if needed if (expected_size > 0) { file_node->set_expected_size(expected_size); @@ -2551,7 +2559,7 @@ void FileManager::on_error_impl(FileNodePtr node, FileManager::Query::Type type, if (node->local_.type() == LocalFileLocation::Type::Partial && status.message() != "FILE_UPLOAD_RESTART") { LOG(INFO) << "Unlink file " << node->local_.partial().path_; unlink(node->local_.partial().path_).ignore(); - node->set_local_location(LocalFileLocation(), 0); + node->drop_local_location(); } if (node->remote_.type() == RemoteFileLocation::Type::Partial) { node->set_remote_location(RemoteFileLocation(), FileLocationSource::None, 0); diff --git a/td/telegram/files/FileManager.h b/td/telegram/files/FileManager.h index f508d6c71..950e8fde0 100644 --- a/td/telegram/files/FileManager.h +++ b/td/telegram/files/FileManager.h @@ -58,7 +58,9 @@ class FileNode { , main_file_id_priority_(main_file_id_priority) { init_ready_size(); } - void set_local_location(const LocalFileLocation &local, int64 ready_size, Bitmask::ReadySize ready_prefix_size = {}); + void drop_local_location(); + void set_local_location(const LocalFileLocation &local, int64 ready_size, int64 prefix_offset, + int64 ready_prefix_size); void set_remote_location(const RemoteFileLocation &remote, FileLocationSource source, int64 ready_size); void set_generate_location(unique_ptr &&generate); void set_size(int64 size); @@ -73,7 +75,6 @@ class FileNode { void set_generate_priority(int8 download_priority, int8 upload_priority); void set_download_offset(int64 download_offset); - void recalc_ready_prefix_size(Bitmask::ReadySize ready_prefix_size); void on_changed(); void on_info_changed(); @@ -95,7 +96,7 @@ class FileNode { FileLoadManager::QueryId upload_id_ = 0; int64 local_ready_size_ = 0; int64 download_offset_ = 0; - int64 local_ready_prefix_size_ = 0; + int64 local_ready_prefix_size_ = 0; // PartialLocal only RemoteFileLocation remote_; FileLoadManager::QueryId download_id_ = 0; @@ -124,10 +125,11 @@ class FileNode { int8 generate_upload_priority_ = 0; int8 main_file_id_priority_ = 0; - bool is_download_offset_dirty_ = false; FileLocationSource remote_source_ = FileLocationSource::FromUser; + bool is_download_offset_dirty_ = false; + bool get_by_hash_ = false; bool can_search_locally_{true}; @@ -138,9 +140,10 @@ class FileNode { bool pmc_changed_flag_{false}; bool info_changed_flag_{false}; - bool offset_changed_flags_{false}; void init_ready_size(); + + void recalc_ready_prefix_size(int64 prefix_offset, int64 ready_prefix_size); }; class FileManager; @@ -221,7 +224,7 @@ class FileView { bool is_downloading() const; int64 download_offset() const; int64 downloaded_prefix(int64 offset) const; - int64 local_size() const; + int64 local_prefix_size() const; int64 local_total_size() const; bool is_uploading() const; int64 remote_size() const; @@ -515,4 +518,5 @@ class FileManager : public FileLoadManager::Callback { friend class FileNodePtr; }; + } // namespace td diff --git a/td/telegram/files/FileUploader.cpp b/td/telegram/files/FileUploader.cpp index 9acb05e94..0f279e963 100644 --- a/td/telegram/files/FileUploader.cpp +++ b/td/telegram/files/FileUploader.cpp @@ -24,6 +24,7 @@ #include "td/utils/ScopeGuard.h" namespace td { + FileUploader::FileUploader(const LocalFileLocation &local, const RemoteFileLocation &remote, int64 expected_size, const FileEncryptionKey &encryption_key, std::vector bad_parts, unique_ptr callback) @@ -47,7 +48,7 @@ Result FileUploader::init() { return Status::Error("File is already uploaded"); } - TRY_RESULT(prefix_info, on_update_local_location(local_)); + TRY_RESULT(prefix_info, on_update_local_location(local_, 0 /* TODO(now) */)); (void)prefix_info; int offset = 0; @@ -86,7 +87,8 @@ Result FileUploader::init() { return res; } -Result FileUploader::on_update_local_location(const LocalFileLocation &location) { +Result FileUploader::on_update_local_location(const LocalFileLocation &location, + int64 file_size) { SCOPE_EXIT { try_release_fd(); }; @@ -108,8 +110,7 @@ Result FileUploader::on_update_local_location(const Loca } else if (location.type() == LocalFileLocation::Type::Partial) { path = location.partial().path_; local_size = Bitmask(Bitmask::Decode{}, location.partial().ready_bitmask_) - .get_ready_size(0, location.partial().part_size_) - .ready_size; + .get_ready_prefix_size(0, location.partial().part_size_, file_size); local_is_ready = false; file_type = location.partial().file_type_; } else { @@ -294,7 +295,7 @@ Result FileUploader::process_part(Part part, NetQueryPtr net_query) { return part.size; } -void FileUploader::on_progress(int32 part_count, int32 part_size, int32 ready_part_count, std::string bitmask, +void FileUploader::on_progress(int32 part_count, int32 part_size, int32 ready_part_count, const string &ready_bitmask, bool is_ready, int64 ready_size) { callback_->on_partial_upload(PartialRemoteFileLocation{file_id_, part_count, part_size, ready_part_count, big_flag_}, ready_size); diff --git a/td/telegram/files/FileUploader.h b/td/telegram/files/FileUploader.h index 9d846c4a3..1ec12053a 100644 --- a/td/telegram/files/FileUploader.h +++ b/td/telegram/files/FileUploader.h @@ -16,6 +16,7 @@ #include namespace td { + class FileUploader : public FileLoader { public: class Callback : public FileLoader::Callback { @@ -62,10 +63,11 @@ class FileUploader : public FileLoader { void after_start_parts() override; Result> start_part(Part part, int32 part_count) override TD_WARN_UNUSED_RESULT; Result process_part(Part part, NetQueryPtr net_query) override TD_WARN_UNUSED_RESULT; - void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, string ready_bitmask, bool is_ready, - int64 ready_size) override; + void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, const string &ready_bitmask, + bool is_ready, int64 ready_size) override; FileLoader::Callback *get_callback() override; - Result on_update_local_location(const LocalFileLocation &location) override TD_WARN_UNUSED_RESULT; + Result on_update_local_location(const LocalFileLocation &location, + int64 file_size) override TD_WARN_UNUSED_RESULT; Status generate_iv_map(); @@ -74,4 +76,5 @@ class FileUploader : public FileLoader { void try_release_fd(); Status acquire_fd() TD_WARN_UNUSED_RESULT; }; + } // namespace td diff --git a/td/telegram/files/PartsManager.cpp b/td/telegram/files/PartsManager.cpp index 74500a740..f51f78527 100644 --- a/td/telegram/files/PartsManager.cpp +++ b/td/telegram/files/PartsManager.cpp @@ -30,13 +30,13 @@ Status PartsManager::init_known_prefix(int64 known_prefix, size_t part_size, con } void PartsManager::set_streaming_offset(int64 offset) { - if (need_check_ || (!unknown_size_flag_ && get_size() < offset)) { + if (offset < 0 || need_check_ || (!unknown_size_flag_ && get_size() < offset)) { streaming_offset_ = 0; return; } auto part_i = offset / part_size_; - if (part_i > MAX_PART_COUNT || part_i < 0) { + if (part_i >= MAX_PART_COUNT) { streaming_offset_ = 0; // error? return; diff --git a/td/telegram/files/PartsManager.h b/td/telegram/files/PartsManager.h index 22a1b2a93..e0e56fabd 100644 --- a/td/telegram/files/PartsManager.h +++ b/td/telegram/files/PartsManager.h @@ -6,11 +6,11 @@ // #pragma once +#include "td/telegram/files/FileBitmask.h" + #include "td/utils/common.h" #include "td/utils/Status.h" -#include "td/telegram/files/FileBitmask.h" - namespace td { /*** PartsManager***/ diff --git a/td/telegram/logevent/SecretChatEvent.h b/td/telegram/logevent/SecretChatEvent.h index 4ad444eed..7a43d3acd 100644 --- a/td/telegram/logevent/SecretChatEvent.h +++ b/td/telegram/logevent/SecretChatEvent.h @@ -323,7 +323,7 @@ class OutboundSecretMessage : public LogEventHelper void store(T &storer) const { using td::store; diff --git a/td/telegram/net/Session.h b/td/telegram/net/Session.h index c5882f929..2ddb82654 100644 --- a/td/telegram/net/Session.h +++ b/td/telegram/net/Session.h @@ -115,7 +115,7 @@ class Session final std::unordered_set unknown_queries_; std::vector to_cancel_; - // Do not invalidate iterators of this two containers! + // Do not invalidate iterators of these two containers! // TODO: better data structures std::deque pending_queries_; std::map sent_queries_; diff --git a/tdutils/td/utils/misc.cpp b/tdutils/td/utils/misc.cpp index 95970429b..8e27f03ee 100644 --- a/tdutils/td/utils/misc.cpp +++ b/tdutils/td/utils/misc.cpp @@ -125,7 +125,9 @@ string url_encode(Slice str) { CHECK(result.size() == length); return result; } -namespace detail { + +namespace { + template string x_decode(Slice s, F &&f) { string res; @@ -139,6 +141,7 @@ string x_decode(Slice s, F &&f) { } return res; } + template string x_encode(Slice s, F &&f) { string res; @@ -149,31 +152,34 @@ string x_encode(Slice s, F &&f) { while (cnt < 250 && i + cnt < n && s[i + cnt] == s[i]) { cnt++; } - res.push_back(cnt); + res.push_back(static_cast(cnt)); i += cnt - 1; } } return res; } -bool is_zero(uint8 c) { + +bool is_zero(unsigned char c) { return c == 0; } -bool is_zero_or_one(uint8 c) { + +bool is_zero_or_one(unsigned char c) { return c == 0 || c == 0xff; } -} // namespace detail + +} // namespace std::string zero_encode(Slice data) { - return detail::x_encode(data, detail::is_zero); + return x_encode(data, is_zero); } std::string zero_decode(Slice data) { - return detail::x_decode(data, detail::is_zero); + return x_decode(data, is_zero); } std::string zero_one_encode(Slice data) { - return detail::x_encode(data, detail::is_zero_or_one); + return x_encode(data, is_zero_or_one); } std::string zero_one_decode(Slice data) { - return detail::x_decode(data, detail::is_zero_or_one); + return x_decode(data, is_zero_or_one); } } // namespace td diff --git a/tdutils/td/utils/misc.h b/tdutils/td/utils/misc.h index dbc1fc8b8..6cb8a1ae7 100644 --- a/tdutils/td/utils/misc.h +++ b/tdutils/td/utils/misc.h @@ -380,8 +380,12 @@ detail::reversion_wrapper reversed(T &iterable) { return {iterable}; } -std::string zero_encode(Slice data); -std::string zero_decode(Slice data); -std::string zero_one_encode(Slice data); -std::string zero_one_decode(Slice data); +string zero_encode(Slice data); + +string zero_decode(Slice data); + +string zero_one_encode(Slice data); + +string zero_one_decode(Slice data); + } // namespace td