From be06d10b398e45e6f91dcbd3dd779de584270f57 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 26 Apr 2019 01:03:31 +0300 Subject: [PATCH] Add td_api::readFilePart. GitOrigin-RevId: 8d44ecfa62dc39d288232248e6db5aad82870729 --- td/generate/scheme/td_api.tl | 8 ++++ td/generate/scheme/td_api.tlo | Bin 151228 -> 151504 bytes td/telegram/Td.cpp | 30 +++++++----- td/telegram/Td.h | 6 ++- td/telegram/cli.cpp | 9 ++++ td/telegram/files/FileManager.cpp | 76 ++++++++++++++++++++++++++++++ td/telegram/files/FileManager.h | 3 ++ 7 files changed, 118 insertions(+), 14 deletions(-) diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index 68e30432..f849de67 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -2200,6 +2200,10 @@ chatReportReasonCustom text:string = ChatReportReason; publicMessageLink link:string html:string = PublicMessageLink; +//@description Contains a part of a file @data File bytes +filePart data:bytes = FilePart; + + //@class FileType @description Represents the type of a file //@description The data is not a file @@ -3251,6 +3255,10 @@ setFileGenerationProgress generation_id:int64 expected_size:int32 local_prefix_s //@error If set, means that file generation has failed and should be terminated finishFileGeneration generation_id:int64 error:error = Ok; +//@description Reads a part of a file from the TDLib file cache and returns read bytes. This method is intended to be used only if the client has no direct access to the TDLib's file system, because it is usually slower than a direct read of a data from the file +//@file_id Identifier of the file. The file must be located in the TDLib file cache @offset The offset from which to read the file @count Number of bytes to read. An error will be returned if there are not enough bytes available in the file from the specified position +readFilePart file_id:int32 offset:int32 count:int32 = FilePart; + //@description Deletes a file from the TDLib file cache @file_id Identifier of the file to delete deleteFile file_id:int32 = Ok; diff --git a/td/generate/scheme/td_api.tlo b/td/generate/scheme/td_api.tlo index e1458e7b33b38fe3d7228fa88ad5af286a20f897..26d93b75bc64a701a48d0f9e9690cb712a8c3e3c 100644 GIT binary patch delta 165 zcmdlpm-E7WPTohe^{p77;ORzQZ!TUlpGY$fx6GW>fW)Gb$%)+3nW!Lbp|XTM)Qi|?JJ5ISC}*UZVz0|I6pzw&UVvto}$#m6qtcfEjNz| n`7lpse8H$Zd4mV*^xhYY($ftNF|tmMOj8CMwfzY<(+wv8mgO~p delta 59 zcmcaGpL5S#PTohe^{p77;K@c_Z?4S^Tr9GT`!=_#hx;@elx#OBVVq;m=(GLoYR35q L+gW&-&Nu-8H?|Xw diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index d31e3c6f..61037b32 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -4946,18 +4946,6 @@ void Td::on_request(uint64 id, const td_api::getFile &request) { send_closure(actor_id(this), &Td::send_result, id, file_manager_->get_file_object(FileId(request.file_id_, 0))); } -void Td::on_request(uint64 id, const td_api::getFileDownloadedPrefixSize &request) { - if (request.offset_ < 0) { - return send_error_raw(id, 5, "Prefix offset must be non-negative"); - } - auto file_view = file_manager_->get_file_view(FileId(request.file_id_, 0)); - if (file_view.empty()) { - return send_closure(actor_id(this), &Td::send_error, id, Status::Error(10, "Unknown file id")); - } - send_closure(actor_id(this), &Td::send_result, id, - td_api::make_object(narrow_cast(file_view.downloaded_prefix(request.offset_)))); -} - void Td::on_request(uint64 id, td_api::getRemoteFile &request) { CLEAN_INPUT_STRING(request.remote_file_id_); auto file_type = request.file_type_ == nullptr ? FileType::Temp : from_td_api(*request.file_type_); @@ -5847,6 +5835,18 @@ void Td::on_file_download_finished(FileId file_id) { pending_file_downloads_.erase(it); } +void Td::on_request(uint64 id, const td_api::getFileDownloadedPrefixSize &request) { + if (request.offset_ < 0) { + return send_error_raw(id, 5, "Parameter offset must be non-negative"); + } + auto file_view = file_manager_->get_file_view(FileId(request.file_id_, 0)); + if (file_view.empty()) { + return send_closure(actor_id(this), &Td::send_error, id, Status::Error(10, "Unknown file ID")); + } + send_closure(actor_id(this), &Td::send_result, id, + td_api::make_object(narrow_cast(file_view.downloaded_prefix(request.offset_)))); +} + void Td::on_request(uint64 id, const td_api::cancelDownloadFile &request) { file_manager_->download(FileId(request.file_id_, 0), nullptr, request.only_if_pending_ ? -1 : 0, -1, -1); @@ -5898,6 +5898,12 @@ void Td::on_request(uint64 id, td_api::finishFileGeneration &request) { std::move(status), std::move(promise)); } +void Td::on_request(uint64 id, const td_api::readFilePart &request) { + CREATE_REQUEST_PROMISE(); + send_closure(file_manager_actor_, &FileManager::read_file_part, FileId(request.file_id_, 0), request.offset_, + request.count_, 2, std::move(promise)); +} + void Td::on_request(uint64 id, const td_api::deleteFile &request) { CREATE_OK_REQUEST_PROMISE(); send_closure(file_manager_actor_, &FileManager::delete_file, FileId(request.file_id_, 0), std::move(promise), diff --git a/td/telegram/Td.h b/td/telegram/Td.h index 9059958a..91ffa251 100644 --- a/td/telegram/Td.h +++ b/td/telegram/Td.h @@ -473,8 +473,6 @@ class Td final : public NetQueryCallback { void on_request(uint64 id, const td_api::getFile &request); - void on_request(uint64 id, const td_api::getFileDownloadedPrefixSize &request); - void on_request(uint64 id, td_api::getRemoteFile &request); void on_request(uint64 id, td_api::getStorageStatistics &request); @@ -683,6 +681,8 @@ class Td final : public NetQueryCallback { void on_request(uint64 id, const td_api::downloadFile &request); + void on_request(uint64 id, const td_api::getFileDownloadedPrefixSize &request); + void on_request(uint64 id, const td_api::cancelDownloadFile &request); void on_request(uint64 id, td_api::uploadFile &request); @@ -693,6 +693,8 @@ class Td final : public NetQueryCallback { void on_request(uint64 id, td_api::finishFileGeneration &request); + void on_request(uint64 id, const td_api::readFilePart &request); + void on_request(uint64 id, const td_api::deleteFile &request); void on_request(uint64 id, const td_api::blockUser &request); diff --git a/td/telegram/cli.cpp b/td/telegram/cli.cpp index 76f7a53a..85cbed54 100644 --- a/td/telegram/cli.cpp +++ b/td/telegram/cli.cpp @@ -2358,6 +2358,15 @@ class CliClient final : public Actor { std::tie(file_id, offset) = split(args); send_request( td_api::make_object(as_file_id(file_id), to_integer(offset))); + } else if (op == "rfp") { + string file_id; + string offset; + string count; + std::tie(file_id, args) = split(args); + std::tie(offset, count) = split(args); + + send_request(td_api::make_object(as_file_id(file_id), to_integer(offset), + to_integer(count))); } else if (op == "grf") { send_request(td_api::make_object(args, nullptr)); } else if (op == "gmtf") { diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index b30d46b2..ed434306 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -20,6 +20,8 @@ #include "td/telegram/SecureStorage.h" #include "td/telegram/TdDb.h" +#include "td/actor/SleepActor.h" + #include "td/utils/base64.h" #include "td/utils/format.h" #include "td/utils/HttpUrl.h" @@ -1813,6 +1815,80 @@ void FileManager::get_content(FileId file_id, Promise promise) { send_closure(file_load_manager_, &FileLoadManager::get_content, node->local_.full(), std::move(promise)); } +void FileManager::read_file_part(FileId file_id, int32 offset, int32 count, int left_tries, + Promise> promise) { + if (G()->close_flag()) { + return promise.set_error(Status::Error(500, "Request aborted")); + } + + if (!file_id.is_valid()) { + return promise.set_error(Status::Error(400, "File ID is invalid")); + } + auto node = get_sync_file_node(file_id); + if (!node) { + return promise.set_error(Status::Error(400, "File not found")); + } + if (offset < 0) { + return promise.set_error(Status::Error(400, "Parameter offset must be non-negative")); + } + if (count <= 0) { + return promise.set_error(Status::Error(400, "Parameter count must be positive")); + } + + auto file_view = FileView(node); + + // TODO this check is safer to do in another thread + if (file_view.downloaded_prefix(offset) < static_cast(count)) { + return promise.set_error(Status::Error(400, "There is not enough downloaded bytes in the file to read")); + } + + const string *path = nullptr; + bool is_partial = false; + if (file_view.has_local_location()) { + path = &file_view.local_location().path_; + if (!begins_with(*path, get_files_dir(file_view.get_type()))) { + return promise.set_error(Status::Error(400, "File is not inside the cache")); + } + } else { + CHECK(node->local_.type() == LocalFileLocation::Type::Partial); + path = &node->local_.partial().path_; + is_partial = true; + } + + // TODO move file reading to another thread + auto r_bytes = [&]() -> Result { + TRY_RESULT(fd, FileFd::open(*path, FileFd::Read)); + string data; + data.resize(count); + TRY_RESULT(read_bytes, fd.pread(data, offset)); + if (read_bytes != static_cast(count)) { + return Status::Error("Read less bytes than expected"); + } + return std::move(data); + }(); + if (r_bytes.is_error()) { + LOG(INFO) << "Failed to read file bytes: " << r_bytes.error(); + if (--left_tries == 0 || !is_partial) { + return promise.set_error(Status::Error(400, "Failed to read the file")); + } + + // the temporary file could be moved from temp to persistent folder + // we need to wait for the corresponding update and repeat the reading + create_actor("RepeatReadFilePartActor", 0.01, + PromiseCreator::lambda([actor_id = actor_id(this), file_id, offset, count, left_tries, + promise = std::move(promise)](Result result) mutable { + send_closure(actor_id, &FileManager::read_file_part, file_id, offset, count, left_tries, + std::move(promise)); + })) + .release(); + return; + } + + auto result = td_api::make_object(); + result->data_ = r_bytes.move_as_ok(); + promise.set_value(std::move(result)); +} + void FileManager::delete_file(FileId file_id, Promise promise, const char *source) { LOG(INFO) << "Trying to delete file " << file_id << " from " << source; auto node = get_sync_file_node(file_id); diff --git a/td/telegram/files/FileManager.h b/td/telegram/files/FileManager.h index 13b82aaa..c4783720 100644 --- a/td/telegram/files/FileManager.h +++ b/td/telegram/files/FileManager.h @@ -410,6 +410,9 @@ class FileManager : public FileLoadManager::Callback { void delete_file_reference(FileId file_id, std::string file_reference); void get_content(FileId file_id, Promise promise); + void read_file_part(FileId file_id, int32 offset, int32 count, int left_tries, + Promise> promise); + void delete_file(FileId file_id, Promise promise, const char *source); void external_file_generate_progress(int64 id, int32 expected_size, int32 local_prefix_size, Promise<> promise);