tdlight-team/tdlight
tdlight-team/tdlight
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Support 64-bit file size in server-side API.

Browse Source
This commit is contained in:
levlam 2022-05-10 17:13:16 +03:00
parent 4ba68b5672
commit 81a8f832d3
9 changed files with 85 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
td/generate/scheme/td_api.tl
View File

@ -3680,7 +3680,7 @@ networkStatistics since_date:int32 entries:vector<NetworkStatisticsEntry> = Netw
//@preload_large_videos True, if the beginning of video files needs to be preloaded for instant playback
//@preload_next_audio True, if the next audio track needs to be preloaded while the user is listening to an audio file
//@use_less_data_for_calls True, if "use less data for calls" option needs to be enabled
autoDownloadSettings is_auto_download_enabled:Bool max_photo_file_size:int32 max_video_file_size:int32 max_other_file_size:int32 video_upload_bitrate:int32 preload_large_videos:Bool preload_next_audio:Bool use_less_data_for_calls:Bool = AutoDownloadSettings;
autoDownloadSettings is_auto_download_enabled:Bool max_photo_file_size:int32 max_video_file_size:int53 max_other_file_size:int53 video_upload_bitrate:int32 preload_large_videos:Bool preload_next_audio:Bool use_less_data_for_calls:Bool = AutoDownloadSettings;
//@description Contains auto-download settings presets for the current user
//@low Preset with lowest settings; supposed to be used by default when roaming

8
td/telegram/AutoDownloadSettings.cpp
View File

@ -25,9 +25,13 @@ static td_api::object_ptr<td_api::autoDownloadSettings> convert_auto_download_se
auto video_preload_large = (flags & telegram_api::autoDownloadSettings::VIDEO_PRELOAD_LARGE_MASK) != 0;
auto audio_preload_next = (flags & telegram_api::autoDownloadSettings::AUDIO_PRELOAD_NEXT_MASK) != 0;
auto phonecalls_less_data = (flags & telegram_api::autoDownloadSettings::PHONECALLS_LESS_DATA_MASK) != 0;
constexpr int32 MAX_PHOTO_SIZE = 10 * (1 << 20) /* 10 MB */;
constexpr int64 MAX_DOCUMENT_SIZE = (1ll << 52);
return td_api::make_object<td_api::autoDownloadSettings>(
!disabled, settings->photo_size_max_, settings->video_size_max_, settings->file_size_max_,
settings->video_upload_maxbitrate_, video_preload_large, audio_preload_next, phonecalls_less_data);
!disabled, clamp(settings->photo_size_max_, static_cast<int32>(0), MAX_PHOTO_SIZE),
clamp(settings->video_size_max_, static_cast<int64>(0), MAX_DOCUMENT_SIZE),
clamp(settings->file_size_max_, static_cast<int64>(0), MAX_DOCUMENT_SIZE), settings->video_upload_maxbitrate_,
video_preload_large, audio_preload_next, phonecalls_less_data);
}
class GetAutoDownloadSettingsQuery final : public Td::ResultHandler {

4
td/telegram/AutoDownloadSettings.h
View File

@ -20,8 +20,8 @@ class Td;
class AutoDownloadSettings {
public:
int32 max_photo_file_size = 0;
int32 max_video_file_size = 0;
int32 max_other_file_size = 0;
int64 max_video_file_size = 0;
int64 max_other_file_size = 0;
int32 video_upload_bitrate = 0;
bool is_enabled = false;
bool preload_large_videos = false;

2
td/telegram/DocumentsManager.cpp
View File

@ -238,7 +238,7 @@ Document DocumentsManager::on_get_document(RemoteDocument remote_document, Dialo
int64 id;
int64 access_hash;
int32 dc_id;
int32 size;
int64 size;
int32 date = 0;
string mime_type;
string file_reference;

46
td/telegram/EncryptedFile.h
View File

@ -16,16 +16,16 @@
namespace td {
struct EncryptedFile {
static constexpr int32 MAGIC = 0x473d738a;
int64 id_ = 0;
int64 access_hash_ = 0;
int32 size_ = 0;
int64 size_ = 0;
int32 dc_id_ = 0;
int32 key_fingerprint_ = 0;
EncryptedFile() = default;
EncryptedFile(int64 id, int64 access_hash, int32 size, int32 dc_id, int32 key_fingerprint)
EncryptedFile(int64 id, int64 access_hash, int64 size, int32 dc_id, int32 key_fingerprint)
: id_(id), access_hash_(access_hash), size_(size), dc_id_(dc_id), key_fingerprint_(key_fingerprint) {
CHECK(size_ >= 0);
}
static unique_ptr<EncryptedFile> get_encrypted_file(tl_object_ptr<telegram_api::EncryptedFile> file_ptr) {
@ -33,17 +33,26 @@ struct EncryptedFile {
return nullptr;
}
auto file = move_tl_object_as<telegram_api::encryptedFile>(file_ptr);
return make_unique<EncryptedFile>(file->id_, file->access_hash_, static_cast<int32>(file->size_), file->dc_id_,
file->key_fingerprint_);
if (file->size_ < 0) {
return nullptr;
}
return make_unique<EncryptedFile>(file->id_, file->access_hash_, file->size_, file->dc_id_, file->key_fingerprint_);
}
template <class StorerT>
void store(StorerT &storer) const {
using td::store;
store(MAGIC, storer);
bool has_64bit_size = (size_ >= (1ll << 31));
BEGIN_STORE_FLAGS();
STORE_FLAG(has_64bit_size);
END_STORE_FLAGS();
store(id_, storer);
store(access_hash_, storer);
store(size_, storer);
if (has_64bit_size) {
store(size_, storer);
} else {
store(narrow_cast<int32>(size_), storer);
}
store(dc_id_, storer);
store(key_fingerprint_, storer);
}
@ -51,18 +60,25 @@ struct EncryptedFile {
template <class ParserT>
void parse(ParserT &parser) {
using td::parse;
int32 got_magic;
parse(got_magic, parser);
bool has_64bit_size;
BEGIN_PARSE_FLAGS();
constexpr int32 OLD_MAGIC = 0x473d738a;
if (flags_parse == OLD_MAGIC) {
flags_parse = 0;
}
PARSE_FLAG(has_64bit_size);
END_PARSE_FLAGS();
parse(id_, parser);
parse(access_hash_, parser);
parse(size_, parser);
if (has_64bit_size) {
parse(size_, parser);
} else {
int32 int_size;
parse(int_size, parser);
size_ = int_size;
}
parse(dc_id_, parser);
parse(key_fingerprint_, parser);
if (got_magic != MAGIC) {
parser.set_error("EncryptedFile magic mismatch");
}
}
};

19
td/telegram/PhotoSize.cpp
View File

@ -243,6 +243,10 @@ Variant<PhotoSize, string> get_photo_size(FileManager *file_manager, PhotoSizeSo
if (source.get_type("get_photo_size") == PhotoSizeSource::Type::Thumbnail) {
source.thumbnail().thumbnail_type = res.type;
}
if (res.size < 0 || res.size > 1000000000) {
LOG(ERROR) << "Receive photo of size " << res.size;
res.size = 0;
}
res.file_id = register_photo_size(file_manager, source, id, access_hash, std::move(file_reference), owner_dialog_id,
res.size, dc_id, format);
@ -276,6 +280,10 @@ AnimationSize get_animation_size(FileManager *file_manager, PhotoSizeSource sour
if (source.get_type("get_animation_size") == PhotoSizeSource::Type::Thumbnail) {
source.thumbnail().thumbnail_type = res.type;
}
if (res.size < 0 || res.size > 1000000000) {
LOG(ERROR) << "Receive animation of size " << res.size;
res.size = 0;
}
res.file_id = register_photo_size(file_manager, source, id, access_hash, std::move(file_reference), owner_dialog_id,
res.size, dc_id, PhotoFormat::Mpeg4);
@ -302,9 +310,9 @@ PhotoSize get_web_document_photo_size(FileManager *file_manager, FileType file_t
}
auto http_url = r_http_url.move_as_ok();
auto url = http_url.get_url();
file_id = file_manager->register_remote(FullRemoteFileLocation(file_type, url, web_document->access_hash_),
FileLocationSource::FromServer, owner_dialog_id, 0, web_document->size_,
get_url_query_file_name(http_url.query_));
file_id = file_manager->register_remote(
FullRemoteFileLocation(file_type, url, web_document->access_hash_), FileLocationSource::FromServer,
owner_dialog_id, 0, static_cast<uint32>(web_document->size_), get_url_query_file_name(http_url.query_));
size = web_document->size_;
mime_type = std::move(web_document->mime_type_);
attributes = std::move(web_document->attributes_);
@ -363,6 +371,11 @@ PhotoSize get_web_document_photo_size(FileManager *file_manager, FileType file_t
s.dimensions = dimensions;
s.size = size;
s.file_id = file_id;
if (s.size < 0 || s.size > 1000000000) {
LOG(ERROR) << "Receive web photo of size " << s.size;
s.size = 0;
}
return s;
}

36
td/telegram/files/FileDownloader.cpp
View File

@ -247,26 +247,25 @@ Result<std::pair<NetQueryPtr, bool>> FileDownloader::start_part(Part part, int32
#endif
DcId dc_id = remote_.is_web() ? G()->get_webfile_dc_id() : remote_.get_dc_id();
auto id = UniqueId::next(UniqueId::Type::Default, static_cast<uint8>(QueryType::Default));
net_query = remote_.is_web()
? G()->net_query_creator().create(
id,
telegram_api::upload_getWebFile(remote_.as_input_web_file_location(),
static_cast<int32>(part.offset), static_cast<int32>(size)),
{}, dc_id, net_query_type, NetQuery::AuthFlag::On)
: G()->net_query_creator().create(
id,
telegram_api::upload_getFile(flags, false /*ignored*/, false /*ignored*/,
remote_.as_input_file_location(),
static_cast<int32>(part.offset), static_cast<int32>(size)),
{}, dc_id, net_query_type, NetQuery::AuthFlag::On);
net_query =
remote_.is_web()
? G()->net_query_creator().create(
id,
telegram_api::upload_getWebFile(remote_.as_input_web_file_location(), static_cast<int32>(part.offset),
static_cast<int32>(size)),
{}, dc_id, net_query_type, NetQuery::AuthFlag::On)
: G()->net_query_creator().create(
id,
telegram_api::upload_getFile(flags, false /*ignored*/, false /*ignored*/,
remote_.as_input_file_location(), part.offset, static_cast<int32>(size)),
{}, dc_id, net_query_type, NetQuery::AuthFlag::On);
} else {
if (remote_.is_web()) {
return Status::Error("Can't download web file from CDN");
}
auto it = cdn_part_reupload_token_.find(part.id);
if (it == cdn_part_reupload_token_.end()) {
auto query = telegram_api::upload_getCdnFile(BufferSlice(cdn_file_token_), static_cast<int32>(part.offset),
static_cast<int32>(size));
auto query = telegram_api::upload_getCdnFile(BufferSlice(cdn_file_token_), part.offset, narrow_cast<int32>(size));
cdn_part_file_token_generation_[part.id] = cdn_file_token_generation_;
LOG(DEBUG) << part.id << " " << to_string(query);
net_query =
@ -313,7 +312,7 @@ Result<size_t> FileDownloader::process_part(Part part, NetQueryPtr net_query) {
TRY_RESULT(file_base, fetch_result<telegram_api::upload_getFile>(net_query->ok()));
CHECK(file_base->get_id() == telegram_api::upload_file::ID);
auto file = move_tl_object_as<telegram_api::upload_file>(file_base);
LOG(DEBUG) << part.id << " upload_getFile result " << to_string(file);
LOG(DEBUG) << part.id << " upload.getFile result " << to_string(file);
bytes = std::move(file->bytes_);
}
break;
@ -322,7 +321,7 @@ Result<size_t> FileDownloader::process_part(Part part, NetQueryPtr net_query) {
TRY_RESULT(file_base, fetch_result<telegram_api::upload_getCdnFile>(net_query->ok()));
CHECK(file_base->get_id() == telegram_api::upload_cdnFile::ID);
auto file = move_tl_object_as<telegram_api::upload_cdnFile>(file_base);
LOG(DEBUG) << part.id << " upload_getCdnFile result " << to_string(file);
LOG(DEBUG) << part.id << " upload.getCdnFile result " << to_string(file);
bytes = std::move(file->bytes_);
need_cdn_decrypt = true;
break;
@ -412,7 +411,7 @@ FileLoader::Callback *FileDownloader::get_callback() {
Status FileDownloader::process_check_query(NetQueryPtr net_query) {
has_hash_query_ = false;
TRY_STATUS(check_net_query(net_query));
TRY_RESULT(file_hashes, fetch_result<telegram_api::upload_getCdnFileHashes>(std::move(net_query)));
TRY_RESULT(file_hashes, fetch_result<telegram_api::upload_getFileHashes>(std::move(net_query)));
add_hash_info(file_hashes);
return Status::OK();
}
@ -467,8 +466,7 @@ Result<FileLoader::CheckInfo> FileDownloader::check_loop(int64 checked_prefix_si
}
if (!has_hash_query_) {
has_hash_query_ = true;
auto query =
telegram_api::upload_getFileHashes(remote_.as_input_file_location(), narrow_cast<int32>(checked_prefix_size));
auto query = telegram_api::upload_getFileHashes(remote_.as_input_file_location(), checked_prefix_size);
auto net_query_type = is_small_ ? NetQuery::Type::DownloadSmall : NetQuery::Type::Download;
auto net_query = G()->net_query_creator().create(query, {}, remote_.get_dc_id(), net_query_type);
info.queries.push_back(std::move(net_query));

5
td/telegram/files/FileHashUploader.cpp
View File

@ -66,12 +66,11 @@ Status FileHashUploader::loop_impl() {
TRY_STATUS(loop_sha());
}
if (state_ == State::NetRequest) {
// messages.getDocumentByHash#338e2464 sha256:bytes size:int mime_type:string = Document;
// messages.getDocumentByHash#338e2464 sha256:bytes size:long mime_type:string = Document;
auto hash = BufferSlice(32);
sha256_state_.extract(hash.as_slice(), true);
auto mime_type = MimeType::from_extension(PathView(local_.path_).extension(), "image/gif");
auto query =
telegram_api::messages_getDocumentByHash(std::move(hash), static_cast<int32>(size_), std::move(mime_type));
auto query = telegram_api::messages_getDocumentByHash(std::move(hash), size_, std::move(mime_type));
LOG(INFO) << "Send getDocumentByHash request: " << to_string(query);
auto ptr = G()->net_query_creator().create(query);
G()->net_query_dispatcher().dispatch_with_callback(std::move(ptr), actor_shared(this));

9
td/telegram/files/FileManager.cpp
View File

@ -1226,6 +1226,15 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi
return Status::Error(400, "No location");
}
if (data.size_ < 0) {
LOG(ERROR) << "Receive file of size " << data.size_;
data.size_ = 0;
}
if (data.expected_size_ < 0) {
LOG(ERROR) << "Receive file of expected size " << data.expected_size_;
data.expected_size_ = 0;
}
FileId file_id = next_file_id();
LOG(INFO) << "Register file data " << data << " as " << file_id << " from " << source;