diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index 4ab66072..4955f7a1 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -3245,6 +3245,10 @@ uploadFile file:InputFile file_type:FileType priority:int32 = File; //@description Stops the uploading of a file. Supported only for files uploaded by using uploadFile. For other files the behavior is undefined @file_id Identifier of the file to stop uploading cancelUploadFile file_id:int32 = Ok; +//@description Writes a part of a generated file. This method is intended to be used only if the client has no direct access to the TDLib's file system, because it is usually slower than a direct write to the destination file +//@generation_id The identifier of the generation process @offset The offset from which to write the data to the file @data The data to write +writeGeneratedFilePart generation_id:int64 offset:int32 data:bytes = Ok; + //@description Informs TDLib on a file generation prograss //@generation_id The identifier of the generation process //@expected_size Expected size of the generated file, in bytes; 0 if unknown @@ -3256,7 +3260,7 @@ setFileGenerationProgress generation_id:int64 expected_size:int32 local_prefix_s //@error If set, means that file generation has failed and should be terminated finishFileGeneration generation_id:int64 error:error = Ok; -//@description Reads a part of a file from the TDLib file cache and returns read bytes. This method is intended to be used only if the client has no direct access to the TDLib's file system, because it is usually slower than a direct read of a data from the file +//@description Reads a part of a file from the TDLib file cache and returns read bytes. This method is intended to be used only if the client has no direct access to the TDLib's file system, because it is usually slower than a direct read from the file //@file_id Identifier of the file. The file must be located in the TDLib file cache @offset The offset from which to read the file @count Number of bytes to read. An error will be returned if there are not enough bytes available in the file from the specified position readFilePart file_id:int32 offset:int32 count:int32 = FilePart; diff --git a/td/generate/scheme/td_api.tlo b/td/generate/scheme/td_api.tlo index b9db2cde..a5d7b069 100644 Binary files a/td/generate/scheme/td_api.tlo and b/td/generate/scheme/td_api.tlo differ diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index 61037b32..262e0ecb 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -5881,6 +5881,12 @@ void Td::on_request(uint64 id, const td_api::cancelUploadFile &request) { send_closure(actor_id(this), &Td::send_result, id, make_tl_object()); } +void Td::on_request(uint64 id, td_api::writeGeneratedFilePart &request) { + CREATE_OK_REQUEST_PROMISE(); + send_closure(file_manager_actor_, &FileManager::external_file_generate_write_part, request.generation_id_, + request.offset_, std::move(request.data_), std::move(promise)); +} + void Td::on_request(uint64 id, const td_api::setFileGenerationProgress &request) { CREATE_OK_REQUEST_PROMISE(); send_closure(file_manager_actor_, &FileManager::external_file_generate_progress, request.generation_id_, diff --git a/td/telegram/Td.h b/td/telegram/Td.h index 91ffa251..aaee2207 100644 --- a/td/telegram/Td.h +++ b/td/telegram/Td.h @@ -689,6 +689,8 @@ class Td final : public NetQueryCallback { void on_request(uint64 id, const td_api::cancelUploadFile &request); + void on_request(uint64 id, td_api::writeGeneratedFilePart &request); + void on_request(uint64 id, const td_api::setFileGenerationProgress &request); void on_request(uint64 id, td_api::finishFileGeneration &request); diff --git a/td/telegram/cli.cpp b/td/telegram/cli.cpp index 85cbed54..c7b81e4a 100644 --- a/td/telegram/cli.cpp +++ b/td/telegram/cli.cpp @@ -32,6 +32,7 @@ #include "td/utils/port/Stat.h" #include "td/utils/port/StdStreams.h" #include "td/utils/port/thread_local.h" +#include "td/utils/Random.h" #include "td/utils/ScopeGuard.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" @@ -3742,8 +3743,13 @@ class CliClient final : public Actor { } BufferSlice block(it->part_size); FileFd::open(it->source, FileFd::Flags::Read).move_as_ok().pread(block.as_slice(), it->local_size).ensure(); - auto open_flags = FileFd::Flags::Write | (it->local_size ? 1 : FileFd::Flags::Truncate | FileFd::Flags::Create); - FileFd::open(it->destination, open_flags).move_as_ok().pwrite(block.as_slice(), it->local_size).ensure(); + if (Random::fast(0, 1) == 0) { + auto open_flags = FileFd::Flags::Write | (it->local_size ? 0 : FileFd::Flags::Truncate | FileFd::Flags::Create); + FileFd::open(it->destination, open_flags).move_as_ok().pwrite(block.as_slice(), it->local_size).ensure(); + } else { + send_request( + td_api::make_object(it->id, it->local_size, block.as_slice().str())); + } it->local_size += it->part_size; if (it->local_size == it->size) { send_request(td_api::make_object(it->id, it->size, it->size)); diff --git a/td/telegram/files/FileGenerateManager.cpp b/td/telegram/files/FileGenerateManager.cpp index 127c816d..3fba5203 100644 --- a/td/telegram/files/FileGenerateManager.cpp +++ b/td/telegram/files/FileGenerateManager.cpp @@ -41,6 +41,9 @@ class FileGenerateActor : public Actor { FileGenerateActor(FileGenerateActor &&) = delete; FileGenerateActor &operator=(FileGenerateActor &&) = delete; ~FileGenerateActor() override = default; + virtual void file_generate_write_part(int32 offset, string data, Promise<> promise) { + LOG(ERROR) << "Receive unexpected file_generate_write_part"; + } virtual void file_generate_progress(int32 expected_size, int32 local_prefix_size, Promise<> promise) = 0; virtual void file_generate_finish(Status status, Promise<> promise) = 0; }; @@ -249,9 +252,14 @@ class FileExternalGenerateActor : public FileGenerateActor { , parent_(std::move(parent)) { } + void file_generate_write_part(int32 offset, string data, Promise<> promise) override { + check_status(do_file_generate_write_part(offset, data), std::move(promise)); + } + void file_generate_progress(int32 expected_size, int32 local_prefix_size, Promise<> promise) override { check_status(do_file_generate_progress(expected_size, local_prefix_size), std::move(promise)); } + void file_generate_finish(Status status, Promise<> promise) override { if (status.is_error()) { check_status(std::move(status)); @@ -300,6 +308,20 @@ class FileExternalGenerateActor : public FileGenerateActor { check_status(Status::Error(1, "Cancelled")); } + Status do_file_generate_write_part(int32 offset, const string &data) { + if (offset < 0) { + return Status::Error("Wrong offset specified"); + } + + auto size = data.size(); + TRY_RESULT(fd, FileFd::open(path_, FileFd::Create | FileFd::Write)); + TRY_RESULT(written, fd.pwrite(data, offset)); + if (written != size) { + return Status::Error(PSLICE() << "Failed to write file: written " << written << " bytes instead of " << size); + } + return Status::OK(); + } + Status do_file_generate_progress(int32 expected_size, int32 local_prefix_size) { if (local_prefix_size < 0) { return Status::Error(1, "Invalid local prefix size"); @@ -418,6 +440,16 @@ void FileGenerateManager::cancel(uint64 query_id) { it->second.worker_.reset(); } +void FileGenerateManager::external_file_generate_write_part(uint64 query_id, int32 offset, string data, + Promise<> promise) { + auto it = query_id_to_query_.find(query_id); + if (it == query_id_to_query_.end()) { + return promise.set_error(Status::Error(400, "Unknown generation_id")); + } + send_closure(it->second.worker_, &FileGenerateActor::file_generate_write_part, offset, std::move(data), + std::move(promise)); +} + void FileGenerateManager::external_file_generate_progress(uint64 query_id, int32 expected_size, int32 local_prefix_size, Promise<> promise) { auto it = query_id_to_query_.find(query_id); diff --git a/td/telegram/files/FileGenerateManager.h b/td/telegram/files/FileGenerateManager.h index 48e82c62..62ace9e8 100644 --- a/td/telegram/files/FileGenerateManager.h +++ b/td/telegram/files/FileGenerateManager.h @@ -41,6 +41,7 @@ class FileGenerateManager : public Actor { void cancel(uint64 query_id); // external updates about file generation state + void external_file_generate_write_part(uint64 query_id, int32 offset, string data, Promise<> promise); void external_file_generate_progress(uint64 query_id, int32 expected_size, int32 local_prefix_size, Promise<> promise); void external_file_generate_finish(uint64 query_id, Status status, Promise<> promise); diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index ed434306..de693bf4 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -2318,6 +2318,11 @@ void FileManager::delete_file_reference(FileId file_id, string file_reference) { try_flush_node_pmc(node, "delete_file_reference"); } +void FileManager::external_file_generate_write_part(int64 id, int32 offset, string data, Promise<> promise) { + send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_write_part, id, offset, + std::move(data), std::move(promise)); +} + void FileManager::external_file_generate_progress(int64 id, int32 expected_size, int32 local_prefix_size, Promise<> promise) { send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_progress, id, expected_size, diff --git a/td/telegram/files/FileManager.h b/td/telegram/files/FileManager.h index c4783720..3222d2d4 100644 --- a/td/telegram/files/FileManager.h +++ b/td/telegram/files/FileManager.h @@ -415,6 +415,7 @@ class FileManager : public FileLoadManager::Callback { void delete_file(FileId file_id, Promise promise, const char *source); + void external_file_generate_write_part(int64 id, int32 offset, string data, Promise<> promise); void external_file_generate_progress(int64 id, int32 expected_size, int32 local_prefix_size, Promise<> promise); void external_file_generate_finish(int64 id, Status status, Promise<> promise); diff --git a/tdutils/td/utils/filesystem.cpp b/tdutils/td/utils/filesystem.cpp index 2782d436..57502ef9 100644 --- a/tdutils/td/utils/filesystem.cpp +++ b/tdutils/td/utils/filesystem.cpp @@ -75,7 +75,7 @@ Status write_file(CSlice to, Slice data) { auto size = data.size(); TRY_RESULT(to_file, FileFd::open(to, FileFd::Truncate | FileFd::Create | FileFd::Write)); TRY_RESULT(written, to_file.write(data)); - if (written != static_cast(size)) { + if (written != size) { return Status::Error(PSLICE() << "Failed to write file: written " << written << " bytes instead of " << size); } to_file.close();