tdlight-team/tdlight
tdlight-team/tdlight
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Use int53 for file size in TDLib API.

Browse Source
This commit is contained in:
levlam 2022-05-11 17:17:20 +03:00
parent 880a582959
commit 0909f88bc3
21 changed files with 204 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
td/generate/scheme/td_api.tl
View File

@ -142,7 +142,7 @@ temporaryPasswordState has_password:Bool valid_for:int32 = TemporaryPasswordStat
//@download_offset Download will be started from this offset. downloaded_prefix_size is calculated from this offset
//@downloaded_prefix_size If is_downloading_completed is false, then only some prefix of the file starting from download_offset is ready to be read. downloaded_prefix_size is the size of that prefix in bytes
//@downloaded_size Total downloaded file size, in bytes. Can be used only for calculating download progress. The actual file size may be bigger, and some parts of it may contain garbage
localFile path:string can_be_downloaded:Bool can_be_deleted:Bool is_downloading_active:Bool is_downloading_completed:Bool download_offset:int32 downloaded_prefix_size:int32 downloaded_size:int32 = LocalFile;
localFile path:string can_be_downloaded:Bool can_be_deleted:Bool is_downloading_active:Bool is_downloading_completed:Bool download_offset:int53 downloaded_prefix_size:int53 downloaded_size:int53 = LocalFile;
//@description Represents a remote file
//@id Remote file identifier; may be empty. Can be used by the current user across application restarts or even from other devices. Uniquely identifies a file, but a file can have a lot of different valid identifiers.
@ -152,7 +152,7 @@ localFile path:string can_be_downloaded:Bool can_be_deleted:Bool is_downloading_
//@is_uploading_active True, if the file is currently being uploaded (or a remote copy is being generated by some other means)
//@is_uploading_completed True, if a remote copy is fully available
//@uploaded_size Size of the remote available part of the file, in bytes; 0 if unknown
remoteFile id:string unique_id:string is_uploading_active:Bool is_uploading_completed:Bool uploaded_size:int32 = RemoteFile;
remoteFile id:string unique_id:string is_uploading_active:Bool is_uploading_completed:Bool uploaded_size:int53 = RemoteFile;
//@description Represents a file
//@id Unique file identifier
@ -160,7 +160,7 @@ remoteFile id:string unique_id:string is_uploading_active:Bool is_uploading_comp
//@expected_size Approximate file size in bytes in case the exact file size is unknown. Can be used to show download/upload progress
//@local Information about the local copy of the file
//@remote Information about the remote copy of the file
file id:int32 size:int32 expected_size:int32 local:localFile remote:remoteFile = File;
file id:int32 size:int53 expected_size:int53 local:localFile remote:remoteFile = File;
//@class InputFile @description Points to a file
@ -179,7 +179,7 @@ inputFileLocal path:string = InputFile;
//@description A file generated by the application @original_path Local path to a file from which the file is generated; may be empty if there is no such file
//@conversion String specifying the conversion applied to the original file; must be persistent across application restarts. Conversions beginning with '#' are reserved for internal TDLib usage
//@expected_size Expected size of the generated file, in bytes; 0 if unknown
inputFileGenerated original_path:string conversion:string expected_size:int32 = InputFile;
inputFileGenerated original_path:string conversion:string expected_size:int53 = InputFile;
//@description Describes an image in JPEG format @type Image type (see https://core.telegram.org/constructor/photoSize)
@ -3782,6 +3782,9 @@ text text:string = Text;
//@description Contains a value representing a number of seconds @seconds Number of seconds
seconds seconds:double = Seconds;
//@description Contains size of downloaded prefix of a file @size The prefix size
fileDownloadedPrefixSize size:int53 = FileDownloadedPrefixSize;
//@description Contains information about a tg: deep link @text Text to be shown to the user @need_update_application True, if the user must be asked to update the application
deepLinkInfo text:formattedText need_update_application:Bool = DeepLinkInfo;
@ -5319,10 +5322,10 @@ toggleBotIsAddedToAttachmentMenu bot_user_id:int53 is_added:Bool = Ok;
//@offset The starting position from which the file needs to be downloaded
//@limit Number of bytes which need to be downloaded starting from the "offset" position before the download will automatically be canceled; use 0 to download without a limit
//@synchronous Pass true to return response only after the file download has succeeded, has failed, has been canceled, or a new downloadFile request with different offset/limit parameters was sent; pass false to return file state immediately, just after the download has been started
downloadFile file_id:int32 priority:int32 offset:int32 limit:int32 synchronous:Bool = File;
downloadFile file_id:int32 priority:int32 offset:int53 limit:int53 synchronous:Bool = File;
//@description Returns file downloaded prefix size from a given offset, in bytes @file_id Identifier of the file @offset Offset from which downloaded prefix size needs to be calculated
getFileDownloadedPrefixSize file_id:int32 offset:int32 = Count;
getFileDownloadedPrefixSize file_id:int32 offset:int53 = FileDownloadedPrefixSize;
//@description Stops the downloading of a file. If a file has already been downloaded, does nothing @file_id Identifier of a file to stop downloading @only_if_pending Pass true to stop downloading only if it hasn't been started, i.e. request hasn't been sent to server
cancelDownloadFile file_id:int32 only_if_pending:Bool = Ok;
@ -5341,13 +5344,13 @@ cancelUploadFile file_id:int32 = Ok;
//@description Writes a part of a generated file. This method is intended to be used only if the application has no direct access to TDLib's file system, because it is usually slower than a direct write to the destination file
//@generation_id The identifier of the generation process @offset The offset from which to write the data to the file @data The data to write
writeGeneratedFilePart generation_id:int64 offset:int32 data:bytes = Ok;
writeGeneratedFilePart generation_id:int64 offset:int53 data:bytes = Ok;
//@description Informs TDLib on a file generation progress
//@generation_id The identifier of the generation process
//@expected_size Expected size of the generated file, in bytes; 0 if unknown
//@local_prefix_size The number of bytes already generated
setFileGenerationProgress generation_id:int64 expected_size:int32 local_prefix_size:int32 = Ok;
setFileGenerationProgress generation_id:int64 expected_size:int53 local_prefix_size:int53 = Ok;
//@description Finishes the file generation
//@generation_id The identifier of the generation process
@ -5358,7 +5361,7 @@ finishFileGeneration generation_id:int64 error:error = Ok;
//@file_id Identifier of the file. The file must be located in the TDLib file cache
//@offset The offset from which to read the file
//@count Number of bytes to read. An error will be returned if there are not enough bytes available in the file from the specified position. Pass 0 to read all available data from the specified position
readFilePart file_id:int32 offset:int32 count:int32 = FilePart;
readFilePart file_id:int32 offset:int53 count:int53 = FilePart;
//@description Deletes a file from the TDLib file cache @file_id Identifier of the file to delete
deleteFile file_id:int32 = Ok;

6
td/telegram/MessageContent.cpp
View File

@ -1801,7 +1801,11 @@ static Result<InputMessageContent> create_input_message_content(
PhotoSize s;
s.type = type;
s.dimensions = get_dimensions(input_photo->width_, input_photo->height_, nullptr);
s.size = static_cast<int32>(file_view.size());
auto size = file_view.size();
if (size < 0 || size >= 1000000000) {
return Status::Error(400, "Wrong photo size");
}
s.size = static_cast<int32>(size);
s.file_id = file_id;
if (thumbnail.file_id.is_valid()) {

13
td/telegram/Photo.cpp
View File

@ -483,12 +483,15 @@ SecretInputMedia photo_get_secret_input_media(FileManager *file_manager, const P
if (thumbnail_file_id.is_valid() && thumbnail.empty()) {
return {};
}
auto size = file_view.size();
if (size < 0 || size >= 1000000000) {
size = 0;
}
return SecretInputMedia{
std::move(input_file),
make_tl_object<secret_api::decryptedMessageMediaPhoto>(
std::move(thumbnail), thumbnail_width, thumbnail_height, width, height, narrow_cast<int32>(file_view.size()),
BufferSlice(encryption_key.key_slice()), BufferSlice(encryption_key.iv_slice()), caption)};
return SecretInputMedia{std::move(input_file), make_tl_object<secret_api::decryptedMessageMediaPhoto>(
std::move(thumbnail), thumbnail_width, thumbnail_height, width,
height, static_cast<int32>(size), BufferSlice(encryption_key.key_slice()),
BufferSlice(encryption_key.iv_slice()), caption)};
}
vector<FileId> photo_get_file_ids(const Photo &photo) {

4
td/telegram/StickersManager.cpp
View File

@ -2789,6 +2789,10 @@ SecretInputMedia StickersManager::get_secret_input_media(FileId sticker_file_id,
LOG(ERROR) << "Have a web sticker in " << sticker->set_id;
return {};
}
if (file_view.size() > 1000000000) {
LOG(ERROR) << "Have a sticker of size " << file_view.size() << " in " << sticker->set_id;
return {};
}
return SecretInputMedia{
nullptr, make_tl_object<secret_api::decryptedMessageMediaExternalDocument>(
remote_location.get_id(), remote_location.get_access_hash(), 0 /*date*/,

4
td/telegram/Td.cpp
View File

@ -6491,7 +6491,7 @@ void Td::on_file_download_finished(FileId file_id) {
auto file_size = file_object->size_;
auto limit = it->second.limit;
if (limit == 0) {
limit = std::numeric_limits<int32>::max();
limit = std::numeric_limits<int64>::max();
}
if (file_object->local_->is_downloading_completed_ ||
(download_offset <= it->second.offset && download_offset + downloaded_size >= it->second.offset &&
@ -6514,7 +6514,7 @@ void Td::on_request(uint64 id, const td_api::getFileDownloadedPrefixSize &reques
return send_closure(actor_id(this), &Td::send_error, id, Status::Error(400, "Unknown file ID"));
}
send_closure(actor_id(this), &Td::send_result, id,
td_api::make_object<td_api::count>(narrow_cast<int32>(file_view.downloaded_prefix(request.offset_))));
td_api::make_object<td_api::fileDownloadedPrefixSize>(file_view.downloaded_prefix(request.offset_)));
}
void Td::on_request(uint64 id, const td_api::cancelDownloadFile &request) {

4
td/telegram/Td.h
View File

@ -312,8 +312,8 @@ class Td final : public Actor {
TermsOfService pending_terms_of_service_;
struct DownloadInfo {
int32 offset = -1;
int32 limit = -1;
int64 offset = -1;
int64 limit = -1;
vector<uint64> request_ids;
};
FlatHashMap<FileId, DownloadInfo, FileIdHash> pending_file_downloads_;

24
td/telegram/cli.cpp
View File

@ -352,9 +352,9 @@ class CliClient final : public Actor {
int64 id = 0;
string destination;
string source;
int32 part_size = 0;
int32 local_size = 0;
int32 size = 0;
int64 part_size = 0;
int64 local_size = 0;
int64 size = 0;
bool test_local_size_decrease = false;
};
@ -372,14 +372,14 @@ class CliClient final : public Actor {
return;
} else {
file_generation.source = update.original_path_;
file_generation.part_size = to_integer<int32>(update.conversion_);
file_generation.part_size = to_integer<int64>(update.conversion_);
file_generation.test_local_size_decrease = !update.conversion_.empty() && update.conversion_.back() == 't';
}
auto r_stat = stat(file_generation.source);
if (r_stat.is_ok()) {
auto size = r_stat.ok().size_;
if (size <= 0 || size > (2000 << 20)) {
if (size <= 0 || size > (static_cast<int64>(4000) << 20)) {
r_stat = Status::Error(400, size == 0 ? Slice("File is empty") : Slice("File is too big"));
}
}
@ -635,7 +635,7 @@ class CliClient final : public Actor {
}
static td_api::object_ptr<td_api::InputFile> as_generated_file(string original_path, string conversion,
int32 expected_size = 0) {
int64 expected_size = 0) {
return td_api::make_object<td_api::inputFileGenerated>(trim(std::move(original_path)), trim(std::move(conversion)),
expected_size);
}
@ -2881,13 +2881,13 @@ class CliClient final : public Actor {
send_request(td_api::make_object<td_api::getFile>(file_id));
} else if (op == "gfdps") {
FileId file_id;
int32 offset;
int64 offset;
get_args(args, file_id, offset);
send_request(td_api::make_object<td_api::getFileDownloadedPrefixSize>(file_id, offset));
} else if (op == "rfp") {
FileId file_id;
int32 offset;
int32 count;
int64 offset;
int64 count;
get_args(args, file_id, offset, count);
send_request(td_api::make_object<td_api::readFilePart>(file_id, offset, count));
} else if (op == "grf") {
@ -2905,8 +2905,8 @@ class CliClient final : public Actor {
width, height, scale, chat_id));
} else if (op == "df" || op == "DownloadFile" || op == "dff" || op == "dfs") {
FileId file_id;
int32 offset;
int32 limit;
int64 offset;
int64 limit;
int32 priority;
get_args(args, file_id, offset, limit, priority);
if (priority <= 0) {
@ -3940,7 +3940,7 @@ class CliClient final : public Actor {
ChatId chat_id;
string photo_path;
string conversion;
int32 expected_size;
int64 expected_size;
get_args(args, chat_id, photo_path, conversion, expected_size);
send_message(chat_id, td_api::make_object<td_api::inputMessagePhoto>(
as_generated_file(photo_path, conversion, expected_size), nullptr, vector<int32>(), 0,

8
td/telegram/files/FileData.hpp
View File

@ -115,10 +115,10 @@ void FileData::parse(ParserT &parser, bool register_file_sources) {
parser);
if (has_sources && register_file_sources) {
Td *td = G()->td().get_actor_unsafe();
int32 size;
parse(size, parser);
if (0 < size && size < 5) {
for (int i = 0; i < size; i++) {
int32 file_source_count;
parse(file_source_count, parser);
if (0 < file_source_count && file_source_count < 5) {
for (int i = 0; i < file_source_count; i++) {
if (parser.get_error()) {
return;
}

11
td/telegram/files/FileDownloader.cpp
View File

@ -81,7 +81,10 @@ Result<FileLoader::FileInfo> FileDownloader::init() {
next_part_ = narrow_cast<int32>(bitmask.get_ready_parts(0));
}
fd_ = result_fd.move_as_ok();
part_size = partial.part_size_;
CHECK(partial.part_size_ <= (1 << 20));
CHECK(0 <= partial.part_size_);
part_size = static_cast<int32>(partial.part_size_);
CHECK((part_size & (part_size - 1)) == 0);
}
}
if (search_file_ && fd_.empty() && size_ > 0 && size_ < 1000 * (1 << 20) && encryption_key_.empty() &&
@ -251,13 +254,13 @@ Result<std::pair<NetQueryPtr, bool>> FileDownloader::start_part(Part part, int32
remote_.is_web()
? G()->net_query_creator().create(
id,
telegram_api::upload_getWebFile(remote_.as_input_web_file_location(), static_cast<int32>(part.offset),
static_cast<int32>(size)),
telegram_api::upload_getWebFile(remote_.as_input_web_file_location(), narrow_cast<int32>(part.offset),
narrow_cast<int32>(size)),
{}, dc_id, net_query_type, NetQuery::AuthFlag::On)
: G()->net_query_creator().create(
id,
telegram_api::upload_getFile(flags, false /*ignored*/, false /*ignored*/,
remote_.as_input_file_location(), part.offset, static_cast<int32>(size)),
remote_.as_input_file_location(), part.offset, narrow_cast<int32>(size)),
{}, dc_id, net_query_type, NetQuery::AuthFlag::On);
} else {
if (remote_.is_web()) {

22
td/telegram/files/FileGenerateManager.cpp
View File

@ -36,10 +36,10 @@ namespace td {
class FileGenerateActor : public Actor {
public:
virtual void file_generate_write_part(int32 offset, string data, Promise<> promise) {
virtual void file_generate_write_part(int64 offset, string data, Promise<> promise) {
LOG(ERROR) << "Receive unexpected file_generate_write_part";
}
virtual void file_generate_progress(int32 expected_size, int32 local_prefix_size, Promise<> promise) = 0;
virtual void file_generate_progress(int64 expected_size, int64 local_prefix_size, Promise<> promise) = 0;
virtual void file_generate_finish(Status status, Promise<> promise) = 0;
};
@ -49,7 +49,7 @@ class FileDownloadGenerateActor final : public FileGenerateActor {
ActorShared<> parent)
: file_type_(file_type), file_id_(file_id), callback_(std::move(callback)), parent_(std::move(parent)) {
}
void file_generate_progress(int32 expected_size, int32 local_prefix_size, Promise<> promise) final {
void file_generate_progress(int64 expected_size, int64 local_prefix_size, Promise<> promise) final {
UNREACHABLE();
}
void file_generate_finish(Status status, Promise<> promise) final {
@ -118,7 +118,7 @@ class MapDownloadGenerateActor final : public FileGenerateActor {
MapDownloadGenerateActor(string conversion, unique_ptr<FileGenerateCallback> callback, ActorShared<> parent)
: conversion_(std::move(conversion)), callback_(std::move(callback)), parent_(std::move(parent)) {
}
void file_generate_progress(int32 expected_size, int32 local_prefix_size, Promise<> promise) final {
void file_generate_progress(int64 expected_size, int64 local_prefix_size, Promise<> promise) final {
UNREACHABLE();
}
void file_generate_finish(Status status, Promise<> promise) final {
@ -250,11 +250,11 @@ class FileExternalGenerateActor final : public FileGenerateActor {
, parent_(std::move(parent)) {
}
void file_generate_write_part(int32 offset, string data, Promise<> promise) final {
void file_generate_write_part(int64 offset, string data, Promise<> promise) final {
check_status(do_file_generate_write_part(offset, data), std::move(promise));
}
void file_generate_progress(int32 expected_size, int32 local_prefix_size, Promise<> promise) final {
void file_generate_progress(int64 expected_size, int64 local_prefix_size, Promise<> promise) final {
check_status(do_file_generate_progress(expected_size, local_prefix_size), std::move(promise));
}
@ -306,7 +306,7 @@ class FileExternalGenerateActor final : public FileGenerateActor {
check_status(Status::Error(1, "Canceled"));
}
Status do_file_generate_write_part(int32 offset, const string &data) {
Status do_file_generate_write_part(int64 offset, const string &data) {
if (offset < 0) {
return Status::Error("Wrong offset specified");
}
@ -320,9 +320,9 @@ class FileExternalGenerateActor final : public FileGenerateActor {
return Status::OK();
}
Status do_file_generate_progress(int32 expected_size, int32 local_prefix_size) {
Status do_file_generate_progress(int64 expected_size, int64 local_prefix_size) {
if (local_prefix_size < 0) {
return Status::Error(1, "Invalid local prefix size");
return Status::Error(400, "Invalid local prefix size");
}
callback_->on_partial_generate(PartialLocalFileLocation{generate_location_.file_type_, local_prefix_size, path_, "",
Bitmask(Bitmask::Ones{}, 1).encode()},
@ -438,7 +438,7 @@ void FileGenerateManager::cancel(uint64 query_id) {
it->second.worker_.reset();
}
void FileGenerateManager::external_file_generate_write_part(uint64 query_id, int32 offset, string data,
void FileGenerateManager::external_file_generate_write_part(uint64 query_id, int64 offset, string data,
Promise<> promise) {
auto it = query_id_to_query_.find(query_id);
if (it == query_id_to_query_.end()) {
@ -448,7 +448,7 @@ void FileGenerateManager::external_file_generate_write_part(uint64 query_id, int
std::move(promise));
}
void FileGenerateManager::external_file_generate_progress(uint64 query_id, int32 expected_size, int32 local_prefix_size,
void FileGenerateManager::external_file_generate_progress(uint64 query_id, int64 expected_size, int64 local_prefix_size,
Promise<> promise) {
auto it = query_id_to_query_.find(query_id);
if (it == query_id_to_query_.end()) {

6
td/telegram/files/FileGenerateManager.h
View File

@ -26,7 +26,7 @@ class FileGenerateCallback {
FileGenerateCallback &operator=(const FileGenerateCallback &) = delete;
virtual ~FileGenerateCallback() = default;
virtual void on_partial_generate(PartialLocalFileLocation partial_local, int32 expected_size) = 0;
virtual void on_partial_generate(PartialLocalFileLocation partial_local, int64 expected_size) = 0;
virtual void on_ok(FullLocalFileLocation local) = 0;
virtual void on_error(Status error) = 0;
};
@ -41,8 +41,8 @@ class FileGenerateManager final : public Actor {
void cancel(uint64 query_id);
// external updates about file generation state
void external_file_generate_write_part(uint64 query_id, int32 offset, string data, Promise<> promise);
void external_file_generate_progress(uint64 query_id, int32 expected_size, int32 local_prefix_size,
void external_file_generate_write_part(uint64 query_id, int64 offset, string data, Promise<> promise);
void external_file_generate_progress(uint64 query_id, int64 expected_size, int64 local_prefix_size,
Promise<> promise);
void external_file_generate_finish(uint64 query_id, Status status, Promise<> promise);

6
td/telegram/files/FileLoader.cpp
View File

@ -76,8 +76,8 @@ void FileLoader::update_downloaded_part(int64 offset, int64 limit) {
if (parts_manager_.get_streaming_offset() != offset) {
auto begin_part_id = parts_manager_.set_streaming_offset(offset, limit);
auto new_end_part_id = limit <= 0 ? parts_manager_.get_part_count()
: static_cast<int32>((offset + limit - 1) / parts_manager_.get_part_size()) + 1;
auto max_parts = static_cast<int32>(ResourceManager::MAX_RESOURCE_LIMIT / parts_manager_.get_part_size());
: narrow_cast<int32>((offset + limit - 1) / parts_manager_.get_part_size()) + 1;
auto max_parts = narrow_cast<int32>(ResourceManager::MAX_RESOURCE_LIMIT / parts_manager_.get_part_size());
auto end_part_id = begin_part_id + td::min(max_parts, new_end_part_id - begin_part_id);
VLOG(file_loader) << "Protect parts " << begin_part_id << " ... " << end_part_id - 1;
for (auto &it : part_map_) {
@ -196,7 +196,7 @@ Status FileLoader::do_loop() {
if (blocking_id_ != 0) {
break;
}
if (resource_state_.unused() < static_cast<int64>(parts_manager_.get_part_size())) {
if (resource_state_.unused() < narrow_cast<int64>(parts_manager_.get_part_size())) {
VLOG(file_loader) << "Got only " << resource_state_.unused() << " resource";
break;
}

2
td/telegram/files/FileLocation.h
View File

@ -684,7 +684,7 @@ inline bool operator!=(const EmptyLocalFileLocation &lhs, const EmptyLocalFileLo
struct PartialLocalFileLocation {
FileType file_type_;
int32 part_size_;
int64 part_size_;
string path_;
string iv_;
string ready_bitmask_;

19
td/telegram/files/FileLocation.hpp
View File

@ -313,11 +313,15 @@ void PartialLocalFileLocation::store(StorerT &storer) const {
using td::store;
store(file_type_, storer);
store(path_, storer);
store(part_size_, storer);
int32 deprecated_ready_part_count = -1;
store(static_cast<int32>(part_size_ & 0x7FFFFFFF), storer);
int32 deprecated_ready_part_count = part_size_ > 0x7FFFFFFF ? -2 : -1;
store(deprecated_ready_part_count, storer);
store(iv_, storer);
store(ready_bitmask_, storer);
if (deprecated_ready_part_count == -2) {
CHECK(part_size_ < (1ll << 62));
store(static_cast<int32>(part_size_ >> 31), storer);
}
}
template <class ParserT>
@ -328,12 +332,19 @@ void PartialLocalFileLocation::parse(ParserT &parser) {
return parser.set_error("Invalid type in PartialLocalFileLocation");
}
parse(path_, parser);
parse(part_size_, parser);
int32 part_size_low;
parse(part_size_low, parser);
part_size_ = part_size_low;
int32 deprecated_ready_part_count;
parse(deprecated_ready_part_count, parser);
parse(iv_, parser);
if (deprecated_ready_part_count == -1) {
if (deprecated_ready_part_count == -1 || deprecated_ready_part_count == -2) {
parse(ready_bitmask_, parser);
if (deprecated_ready_part_count == -2) {
int32 part_size_high;
parse(part_size_high, parser);
part_size_ += static_cast<int64>(part_size_high) << 31;
}
} else {
CHECK(0 <= deprecated_ready_part_count);
CHECK(deprecated_ready_part_count <= (1 << 22));

46
td/telegram/files/FileManager.cpp
View File

@ -53,7 +53,7 @@
namespace td {
namespace {
constexpr int64 MAX_FILE_SIZE = 2000 * (1 << 20) /* 2000MB */;
constexpr int64 MAX_FILE_SIZE = static_cast<int64>(4000) << 20; // 4000MB
} // namespace
int VERBOSITY_NAME(update_file) = VERBOSITY_NAME(INFO);
@ -222,6 +222,9 @@ void FileNode::set_download_limit(int64 download_limit) {
// KEEP_DOWNLOAD_LIMIT is handled here
return;
}
if (download_limit > MAX_FILE_SIZE) {
download_limit = MAX_FILE_SIZE;
}
auto old_download_limit = get_download_limit();
private_download_limit_ = download_limit;
update_effective_download_limit(old_download_limit);
@ -691,7 +694,7 @@ string FileView::path() const {
case LocalFileLocation::Type::Partial:
return node_->local_.partial().path_;
default:
return "";
return string();
}
}
@ -1056,11 +1059,11 @@ bool FileManager::try_fix_partial_local_location(FileNodePtr node) {
LOG(INFO) << " failed - partial location has nonempty iv";
return false;
}
if (partial.part_size_ >= 512 * (1 << 10)) {
if (partial.part_size_ >= 512 * (1 << 10) || (partial.part_size_ & (partial.part_size_ - 1)) != 0) {
LOG(INFO) << " failed - too big part_size already: " << partial.part_size_;
return false;
}
auto old_part_size = partial.part_size_;
auto old_part_size = narrow_cast<int32>(partial.part_size_);
int new_part_size = 512 * (1 << 10);
auto k = new_part_size / old_part_size;
Bitmask mask(Bitmask::Decode(), partial.ready_bitmask_);
@ -2077,7 +2080,7 @@ void FileManager::get_content(FileId file_id, Promise<BufferSlice> promise) {
send_closure(file_load_manager_, &FileLoadManager::get_content, node->local_.full(), std::move(promise));
}
void FileManager::read_file_part(FileId file_id, int32 offset, int32 count, int left_tries,
void FileManager::read_file_part(FileId file_id, int64 offset, int64 count, int left_tries,
Promise<td_api::object_ptr<td_api::filePart>> promise) {
TRY_STATUS_PROMISE(promise, G()->close_status());
@ -2098,14 +2101,17 @@ void FileManager::read_file_part(FileId file_id, int32 offset, int32 count, int
auto file_view = FileView(node);
if (count == 0) {
count = narrow_cast<int32>(file_view.downloaded_prefix(offset));
count = file_view.downloaded_prefix(offset);
if (count == 0) {
return promise.set_value(td_api::make_object<td_api::filePart>());
}
} else if (file_view.downloaded_prefix(offset) < static_cast<int64>(count)) {
} else if (file_view.downloaded_prefix(offset) < count) {
// TODO this check is safer to do in another thread
return promise.set_error(Status::Error(400, "There is not enough downloaded bytes in the file to read"));
}
if (count >= static_cast<int64>(std::numeric_limits<size_t>::max() / 2 - 1)) {
return promise.set_error(Status::Error(400, "Part length is too big"));
}
const string *path = nullptr;
bool is_partial = false;
@ -2124,7 +2130,7 @@ void FileManager::read_file_part(FileId file_id, int32 offset, int32 count, int
auto r_bytes = [&]() -> Result<string> {
TRY_RESULT(fd, FileFd::open(*path, FileFd::Read));
string data;
data.resize(count);
data.resize(narrow_cast<size_t>(count));
TRY_RESULT(read_bytes, fd.pread(data, offset));
if (read_bytes != static_cast<size_t>(count)) {
return Status::Error("Read less bytes than expected");
@ -2318,7 +2324,7 @@ void FileManager::run_download(FileNodePtr node, bool force_update_priority) {
auto download_limit = node->get_download_limit();
if (file_view.is_encrypted_any()) {
CHECK(download_offset <= MAX_FILE_SIZE);
CHECK(download_limit <= std::numeric_limits<int32>::max());
CHECK(download_limit <= MAX_FILE_SIZE);
download_limit += download_offset;
download_offset = 0;
}
@ -2387,7 +2393,7 @@ void FileManager::run_download(FileNodePtr node, bool force_update_priority) {
auto download_limit = node->get_download_limit();
if (file_view.is_encrypted_any()) {
CHECK(download_offset <= MAX_FILE_SIZE);
CHECK(download_limit <= std::numeric_limits<int32>::max());
CHECK(download_limit <= MAX_FILE_SIZE);
download_limit += download_offset;
download_offset = 0;
}
@ -2672,12 +2678,12 @@ void FileManager::delete_file_reference(FileId file_id, Slice file_reference) {
try_flush_node_pmc(node, "delete_file_reference");
}
void FileManager::external_file_generate_write_part(int64 id, int32 offset, string data, Promise<> promise) {
void FileManager::external_file_generate_write_part(int64 id, int64 offset, string data, Promise<> promise) {
send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_write_part, id, offset,
std::move(data), std::move(promise));
}
void FileManager::external_file_generate_progress(int64 id, int32 expected_size, int32 local_prefix_size,
void FileManager::external_file_generate_progress(int64 id, int64 expected_size, int64 local_prefix_size,
Promise<> promise) {
send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_progress, id, expected_size,
local_prefix_size, std::move(promise));
@ -2753,7 +2759,7 @@ void FileManager::run_generate(FileNodePtr node) {
public:
Callback(ActorId<FileManager> actor, QueryId id) : actor_(std::move(actor)), query_id_(id) {
}
void on_partial_generate(PartialLocalFileLocation partial_local, int32 expected_size) final {
void on_partial_generate(PartialLocalFileLocation partial_local, int64 expected_size) final {
send_closure(actor_, &FileManager::on_partial_generate, query_id_, std::move(partial_local),
expected_size);
}
@ -3023,12 +3029,12 @@ td_api::object_ptr<td_api::file> FileManager::get_file_object(FileId file_id, bo
string persistent_file_id = file_view.get_persistent_file_id();
string unique_file_id = file_view.get_unique_file_id();
bool is_uploading_completed = !persistent_file_id.empty();
auto size = narrow_cast<int32>(file_view.size());
auto expected_size = narrow_cast<int32>(file_view.expected_size());
auto download_offset = narrow_cast<int32>(file_view.download_offset());
auto local_prefix_size = narrow_cast<int32>(file_view.local_prefix_size());
auto local_total_size = narrow_cast<int32>(file_view.local_total_size());
auto remote_size = narrow_cast<int32>(file_view.remote_size());
auto size = file_view.size();
auto expected_size = file_view.expected_size();
auto download_offset = file_view.download_offset();
auto local_prefix_size = file_view.local_prefix_size();
auto local_total_size = file_view.local_total_size();
auto remote_size = file_view.remote_size();
string path = file_view.path();
bool can_be_downloaded = file_view.can_download_from_server() || file_view.can_generate();
bool can_be_deleted = file_view.can_delete();
@ -3624,7 +3630,7 @@ void FileManager::on_upload_full_ok(QueryId query_id, FullRemoteFileLocation rem
LOG_STATUS(merge(new_file_id, file_id));
}
void FileManager::on_partial_generate(QueryId query_id, PartialLocalFileLocation partial_local, int32 expected_size) {
void FileManager::on_partial_generate(QueryId query_id, PartialLocalFileLocation partial_local, int64 expected_size) {
if (is_closed_) {
return;
}

8
td/telegram/files/FileManager.h
View File

@ -459,13 +459,13 @@ class FileManager final : public FileLoadManager::Callback {
Result<string> get_suggested_file_name(FileId file_id, const string &directory);
void read_file_part(FileId file_id, int32 offset, int32 count, int left_tries,
void read_file_part(FileId file_id, int64 offset, int64 count, int left_tries,
Promise<td_api::object_ptr<td_api::filePart>> promise);
void delete_file(FileId file_id, Promise<Unit> promise, const char *source);
void external_file_generate_write_part(int64 id, int32 offset, string data, Promise<> promise);
void external_file_generate_progress(int64 id, int32 expected_size, int32 local_prefix_size, Promise<> promise);
void external_file_generate_write_part(int64 id, int64 offset, string data, Promise<> promise);
void external_file_generate_progress(int64 id, int64 expected_size, int64 local_prefix_size, Promise<> promise);
void external_file_generate_finish(int64 id, Status status, Promise<> promise);
Result<FileId> from_persistent_id(CSlice persistent_id, FileType file_type) TD_WARN_UNUSED_RESULT;
@ -656,7 +656,7 @@ class FileManager final : public FileLoadManager::Callback {
void on_error_impl(FileNodePtr node, Query::Type type, bool was_active, Status status);
void on_partial_generate(QueryId, PartialLocalFileLocation partial_local, int32 expected_size);
void on_partial_generate(QueryId, PartialLocalFileLocation partial_local, int64 expected_size);
void on_generate_ok(QueryId, FullLocalFileLocation local);
std::pair<Query, bool> finish_query(QueryId query_id);

65
td/telegram/files/FileManager.hpp
View File

@ -20,6 +20,8 @@
#include "td/utils/SliceBuilder.h"
#include "td/utils/tl_helpers.h"
#include <limits>
namespace td {
enum class FileStoreType : int32 { Empty, Url, Generate, Local, Remote };
@ -45,13 +47,21 @@ void FileManager::store_file(FileId file_id, StorerT &storer, int32 ttl) const {
bool has_expected_size =
file_store_type == FileStoreType::Remote && file_view.size() == 0 && file_view.expected_size() != 0;
bool has_secure_key = false;
int64 size = 0;
bool has_64bit_size = false;
if (file_store_type != FileStoreType::Empty) {
has_encryption_key = !file_view.empty() && file_view.is_encrypted_secret();
has_secure_key = !file_view.empty() && file_view.is_encrypted_secure();
if (file_store_type != FileStoreType::Url) {
size = has_expected_size || file_store_type == FileStoreType::Generate ? file_view.expected_size()
: file_view.size();
has_64bit_size = (size > std::numeric_limits<int32>::max());
}
BEGIN_STORE_FLAGS();
STORE_FLAG(has_encryption_key);
STORE_FLAG(has_expected_size);
STORE_FLAG(has_secure_key);
STORE_FLAG(has_64bit_size);
END_STORE_FLAGS();
}
@ -65,10 +75,10 @@ void FileManager::store_file(FileId file_id, StorerT &storer, int32 ttl) const {
break;
case FileStoreType::Remote: {
store(file_view.remote_location(), storer);
if (has_expected_size) {
store(narrow_cast<int32>(file_view.expected_size()), storer);
if (has_64bit_size) {
store(size, storer);
} else {
store(narrow_cast<int32>(file_view.size()), storer);
store(narrow_cast<int32>(size), storer);
}
store(file_view.remote_name(), storer);
store(file_view.owner_dialog_id(), storer);
@ -76,7 +86,11 @@ void FileManager::store_file(FileId file_id, StorerT &storer, int32 ttl) const {
}
case FileStoreType::Local: {
store(file_view.local_location(), storer);
store(narrow_cast<int32>(file_view.size()), storer);
if (has_64bit_size) {
store(size, storer);
} else {
store(narrow_cast<int32>(size), storer);
}
store(static_cast<int32>(file_view.get_by_hash()), storer);
store(file_view.owner_dialog_id(), storer);
break;
@ -95,8 +109,12 @@ void FileManager::store_file(FileId file_id, StorerT &storer, int32 ttl) const {
have_file_id = true;
}
store(generate_location, storer);
store(static_cast<int32>(file_view.expected_size()), storer);
store(static_cast<int32>(0), storer);
if (has_64bit_size) {
store(size, storer);
} else {
store(narrow_cast<int32>(size), storer);
store(static_cast<int32>(0), storer); // legacy
}
store(file_view.owner_dialog_id(), storer);
if (have_file_id) {
@ -124,12 +142,14 @@ FileId FileManager::parse_file(ParserT &parser) {
bool has_encryption_key = false;
bool has_expected_size = false;
bool has_secure_key = false;
bool has_64bit_size = false;
if (file_store_type != FileStoreType::Empty) {
if (parser.version() >= static_cast<int32>(Version::StoreFileEncryptionKey)) {
BEGIN_PARSE_FLAGS();
PARSE_FLAG(has_encryption_key);
PARSE_FLAG(has_expected_size);
PARSE_FLAG(has_secure_key);
PARSE_FLAG(has_64bit_size);
END_PARSE_FLAGS();
}
}
@ -141,13 +161,16 @@ FileId FileManager::parse_file(ParserT &parser) {
case FileStoreType::Remote: {
FullRemoteFileLocation full_remote_location;
parse(full_remote_location, parser);
int32 size = 0;
int32 expected_size = 0;
if (has_expected_size) {
parse(expected_size, parser);
int64 stored_size;
if (has_64bit_size) {
parse(stored_size, parser);
} else {
parse(size, parser);
int32 int_size;
parse(int_size, parser);
stored_size = int_size;
}
int64 size = has_expected_size ? 0 : stored_size;
int64 expected_size = has_expected_size ? stored_size : 0;
string name;
parse(name, parser);
DialogId owner_dialog_id;
@ -160,8 +183,14 @@ FileId FileManager::parse_file(ParserT &parser) {
case FileStoreType::Local: {
FullLocalFileLocation full_local_location;
parse(full_local_location, parser);
int32 size;
parse(size, parser);
int64 size;
if (has_64bit_size) {
parse(size, parser);
} else {
int32 int_size;
parse(int_size, parser);
size = int_size;
}
int32 get_by_hash;
parse(get_by_hash, parser);
DialogId owner_dialog_id;
@ -179,8 +208,14 @@ FileId FileManager::parse_file(ParserT &parser) {
case FileStoreType::Generate: {
FullGenerateFileLocation full_generated_location;
parse(full_generated_location, parser);
int32 expected_size;
parse(expected_size, parser);
int64 expected_size;
if (has_64bit_size) {
parse(expected_size, parser);
} else {
int32 int_size;
parse(int_size, parser);
expected_size = int_size;
}
int32 zero;
parse(zero, parser);
DialogId owner_dialog_id;

18
td/telegram/files/PartsManager.cpp
View File

@ -47,7 +47,7 @@ int32 PartsManager::set_streaming_offset(int64 offset, int64 limit) {
}
auto part_i = offset / part_size_;
if (use_part_count_limit_ && part_i >= MAX_PART_COUNT) {
if (use_part_count_limit_ && part_i >= MAX_PART_COUNT_PREMIUM) {
streaming_offset_ = 0;
LOG(ERROR) << "Ignore streaming_offset " << offset << " in part " << part_i;
@ -92,9 +92,8 @@ Status PartsManager::init_no_size(size_t part_size, const std::vector<int> &read
part_size_ = part_size;
} else {
part_size_ = 32 << 10;
while (calc_part_count(expected_size_, part_size_) > MAX_PART_COUNT) {
while (part_size_ < MAX_PART_SIZE && calc_part_count(expected_size_, part_size_) > MAX_PART_COUNT) {
part_size_ *= 2;
CHECK(part_size_ <= MAX_PART_SIZE);
}
// just in case if expected_size_ is wrong
if (part_size_ < MAX_PART_SIZE) {
@ -128,19 +127,19 @@ Status PartsManager::init(int64 size, int64 expected_size, bool is_size_final, s
if (part_size != 0) {
part_size_ = part_size;
if (use_part_count_limit_ && calc_part_count(expected_size_, part_size_) > MAX_PART_COUNT) {
if (use_part_count_limit_ && part_size_ < MAX_PART_SIZE &&
calc_part_count(expected_size_, part_size_) > MAX_PART_COUNT) {
CHECK(is_upload_);
return Status::Error("FILE_UPLOAD_RESTART");
}
} else {
part_size_ = 64 << 10;
while (calc_part_count(expected_size_, part_size_) > MAX_PART_COUNT) {
while (part_size_ < MAX_PART_SIZE && calc_part_count(expected_size_, part_size_) > MAX_PART_COUNT) {
part_size_ *= 2;
CHECK(part_size_ <= MAX_PART_SIZE);
}
}
LOG_CHECK(1 <= size_) << tag("size_", size_);
LOG_CHECK(!use_part_count_limit || calc_part_count(expected_size_, part_size_) <= MAX_PART_COUNT)
LOG_CHECK(!use_part_count_limit || calc_part_count(expected_size_, part_size_) <= MAX_PART_COUNT_PREMIUM)
<< tag("size_", size_) << tag("expected_size", size_) << tag("is_size_final", is_size_final)
<< tag("part_size_", part_size_) << tag("ready_parts", ready_parts.size());
part_count_ = static_cast<int>(calc_part_count(size_, part_size_));
@ -287,7 +286,7 @@ Result<Part> PartsManager::start_part() {
if (part_i == part_count_) {
if (unknown_size_flag_) {
part_count_++;
if (part_count_ > MAX_PART_COUNT + (use_part_count_limit_ ? 0 : 64)) {
if (part_count_ > MAX_PART_COUNT_PREMIUM + (use_part_count_limit_ ? 0 : 64)) {
if (!is_upload_) {
// Caller will try to increase part size if it is possible
return Status::Error("FILE_DOWNLOAD_RESTART_INCREASE_PART_SIZE");
@ -334,7 +333,8 @@ Status PartsManager::set_known_prefix(size_t size, bool is_ready) {
LOG_CHECK(static_cast<size_t>(part_count_) >= part_status_.size())
<< size << " " << is_ready << " " << part_count_ << " " << part_size_ << " " << part_status_.size();
part_status_.resize(part_count_);
if (use_part_count_limit_ && calc_part_count(expected_size_, part_size_) > MAX_PART_COUNT) {
if (use_part_count_limit_ && part_size_ < MAX_PART_SIZE &&
calc_part_count(expected_size_, part_size_) > MAX_PART_COUNT) {
CHECK(is_upload_);
return Status::Error("FILE_UPLOAD_RESTART");
}

5
td/telegram/files/PartsManager.h
View File

@ -55,8 +55,9 @@ class PartsManager {
private:
static constexpr int MAX_PART_COUNT = 4000;
static constexpr size_t MAX_PART_SIZE = 512 * (1 << 10);
static constexpr int64 MAX_FILE_SIZE = static_cast<int64>(MAX_PART_SIZE) * MAX_PART_COUNT;
static constexpr int MAX_PART_COUNT_PREMIUM = 8000;
static constexpr size_t MAX_PART_SIZE = 512 << 10;
static constexpr int64 MAX_FILE_SIZE = static_cast<int64>(MAX_PART_SIZE) * MAX_PART_COUNT_PREMIUM;
enum class PartStatus : int32 { Empty, Pending, Ready };

2
tdnet/td/net/HttpReader.h
View File

@ -105,7 +105,7 @@ class HttpReader {
static constexpr size_t MAX_TOTAL_PARAMETERS_LENGTH = 1 << 20; // Some reasonable limit
static constexpr size_t MAX_TOTAL_HEADERS_LENGTH = 1 << 18; // Some reasonable limit
static constexpr size_t MAX_BOUNDARY_LENGTH = 70; // As defined by RFC1341
static constexpr int64 MAX_FILE_SIZE = 2000 << 20; // Telegram server file size limit
static constexpr int64 MAX_FILE_SIZE = static_cast<int64>(4000) << 20; // Telegram server file size limit
static constexpr const char TEMP_DIRECTORY_PREFIX[] = "tdlib-server-tmp";
};

38
test/online.cpp
View File

@ -4,26 +4,31 @@
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include <iostream>
#include <map>
#include <tdutils/td/utils/FileLog.h>
#include <tdutils/td/utils/OptionParser.h>
#include <tdutils/td/utils/port/path.h>
#include <td/telegram/ClientActor.h>
#include <tdutils/td/utils/filesystem.h>
#include "td/telegram/TdCallback.h"
#include "td/utils/port/signals.h"
#include "td/telegram/ClientActor.h"
#include "td/telegram/Log.h"
#include "td/utils/crypto.h"
#include "td/utils/misc.h"
#include "td/utils/Random.h"
#include "td/telegram/td_api_json.h"
#include "td/telegram/TdCallback.h"
#include "td/actor/actor.h"
#include "td/actor/ConcurrentScheduler.h"
#include "td/actor/PromiseFuture.h"
#include "td/actor/MultiPromise.h"
#include "td/telegram/td_api_json.h"
#include "td/actor/PromiseFuture.h"
#include "td/utils/crypto.h"
#include "td/utils/FileLog.h"
#include "td/utils/filesystem.h"
#include "td/utils/misc.h"
#include "td/utils/OptionParser.h"
#include "td/utils/port/path.h"
#include "td/utils/port/signals.h"
#include "td/utils/Random.h"
#include <iostream>
#include <map>
#include <memory>
namespace td {
template <class T>
static void check_td_error(T &result) {
LOG_CHECK(result->get_id() != td::td_api::error::ID) << to_string(result);
@ -454,8 +459,9 @@ class TestDownloadFile : public Task {
}
void start_chunk() {
send_query(td::make_tl_object<td::td_api::downloadFile>(file_id_, 1, int(ranges_.back().begin),
int(ranges_.back().end - ranges_.back().begin), true),
send_query(td::make_tl_object<td::td_api::downloadFile>(
file_id_, 1, static_cast<int64>(ranges_.back().begin),
static_cast<int64>(ranges_.back().end - ranges_.back().begin), true),
[this](auto res) { got_chunk(*res.ok()); });
}
};