diff --git a/td/telegram/DocumentsManager.cpp b/td/telegram/DocumentsManager.cpp index 4eee8e606..81955ef97 100644 --- a/td/telegram/DocumentsManager.cpp +++ b/td/telegram/DocumentsManager.cpp @@ -208,7 +208,7 @@ std::pair DocumentsManager::on_get_docum file_reference = document->file_reference_.as_slice().str(); if (document_type != DocumentType::VoiceNote) { - thumbnail = get_photo_size(td_->file_manager_.get(), FileType::Thumbnail, 0, 0, owner_dialog_id, + thumbnail = get_photo_size(td_->file_manager_.get(), FileType::Thumbnail, 0, 0, "", owner_dialog_id, std::move(document->thumb_), has_webp_thumbnail); } } else if (remote_document.secret_file != nullptr) { diff --git a/td/telegram/FileReferenceManager.cpp b/td/telegram/FileReferenceManager.cpp index 44e7a99e7..35136fdf5 100644 --- a/td/telegram/FileReferenceManager.cpp +++ b/td/telegram/FileReferenceManager.cpp @@ -190,7 +190,8 @@ void FileReferenceManager::send_query(Destination dest, FileSourceId file_source new_promise = std::move(new_promise)]() mutable { auto view = file_manager.get_actor_unsafe()->get_file_view(dest.node_id); CHECK(!view.empty()); - if (result.is_ok() && !view.has_active_remote_location()) { + if (result.is_ok() && + (!view.has_active_upload_remote_location() || !view.has_active_download_remote_location())) { result = Status::Error("No active remote location"); } if (result.is_error() && result.error().code() != 429 && result.error().code() < 500) { diff --git a/td/telegram/MessageContent.cpp b/td/telegram/MessageContent.cpp index a8c279248..dd903088b 100644 --- a/td/telegram/MessageContent.cpp +++ b/td/telegram/MessageContent.cpp @@ -2122,7 +2122,7 @@ tl_object_ptr get_input_media(const MessageContent *co } if (!was_uploaded) { auto file_reference = FileManager::extract_file_reference(input_media); - if (file_reference == FullRemoteFileLocation::invalid_file_reference()) { + if (file_reference == FileReferenceView::invalid_file_reference()) { return nullptr; } } @@ -2132,7 +2132,7 @@ tl_object_ptr get_input_media(const MessageContent *co tl_object_ptr get_input_media(const MessageContent *content, Td *td, int32 ttl) { auto input_media = get_input_media(content, td, nullptr, nullptr, ttl); auto file_reference = FileManager::extract_file_reference(input_media); - if (file_reference == FullRemoteFileLocation::invalid_file_reference()) { + if (file_reference == FileReferenceView::invalid_file_reference()) { return nullptr; } return input_media; @@ -2788,7 +2788,8 @@ void merge_message_contents(Td *td, MessageContent *old_content, MessageContent FileId file_id = td->file_manager_->register_remote( FullRemoteFileLocation(FileType::Photo, new_file_view.remote_location().get_id(), new_file_view.remote_location().get_access_hash(), 0, 0, 0, DcId::invalid(), - new_file_view.remote_location().get_file_reference()), + new_file_view.remote_location().get_upload_file_reference().str(), + new_file_view.remote_location().get_download_file_reference().str()), FileLocationSource::FromServer, dialog_id, old_photo->photos.back().size, 0, ""); LOG_STATUS(td->file_manager_->merge(file_id, old_file_id)); } diff --git a/td/telegram/Photo.cpp b/td/telegram/Photo.cpp index c8a385d01..c6acc385c 100644 --- a/td/telegram/Photo.cpp +++ b/td/telegram/Photo.cpp @@ -62,13 +62,14 @@ StringBuilder &operator<<(StringBuilder &string_builder, const Dimensions &dimen } static FileId register_photo(FileManager *file_manager, FileType file_type, int64 id, int64 access_hash, + std::string upload_file_reference, tl_object_ptr &&location_ptr, DialogId owner_dialog_id, int32 file_size, bool is_webp = false) { DcId dc_id; int32 local_id; int64 volume_id; int64 secret; - std::string file_reference; + std::string download_file_reference; switch (location_ptr->get_id()) { case telegram_api::fileLocationUnavailable::ID: { auto location = move_tl_object_as(location_ptr); @@ -88,7 +89,7 @@ static FileId register_photo(FileManager *file_manager, FileType file_type, int6 local_id = location->local_id_; volume_id = location->volume_id_; secret = location->secret_; - file_reference = location->file_reference_.as_slice().str(); + download_file_reference = location->file_reference_.as_slice().str(); break; } default: @@ -101,9 +102,10 @@ static FileId register_photo(FileManager *file_manager, FileType file_type, int6 << ")"; auto suggested_name = PSTRING() << static_cast(volume_id) << "_" << static_cast(local_id) << (is_webp ? ".webp" : ".jpg"); - return file_manager->register_remote( - FullRemoteFileLocation(file_type, id, access_hash, local_id, volume_id, secret, dc_id, file_reference), - FileLocationSource::FromServer, owner_dialog_id, file_size, 0, std::move(suggested_name)); + return file_manager->register_remote(FullRemoteFileLocation(file_type, id, access_hash, local_id, volume_id, secret, + dc_id, upload_file_reference, download_file_reference), + FileLocationSource::FromServer, owner_dialog_id, file_size, 0, + std::move(suggested_name)); } ProfilePhoto get_profile_photo(FileManager *file_manager, @@ -118,9 +120,9 @@ ProfilePhoto get_profile_photo(FileManager *file_manager, auto profile_photo = move_tl_object_as(profile_photo_ptr); result.id = profile_photo->photo_id_; - result.small_file_id = register_photo(file_manager, FileType::ProfilePhoto, result.id, 0, + result.small_file_id = register_photo(file_manager, FileType::ProfilePhoto, result.id, 0, "", std::move(profile_photo->photo_small_), DialogId(), 0); - result.big_file_id = register_photo(file_manager, FileType::ProfilePhoto, result.id, 0, + result.big_file_id = register_photo(file_manager, FileType::ProfilePhoto, result.id, 0, "", std::move(profile_photo->photo_big_), DialogId(), 0); break; } @@ -179,10 +181,10 @@ DialogPhoto get_dialog_photo(FileManager *file_manager, tl_object_ptr(chat_photo_ptr); - result.small_file_id = register_photo(file_manager, FileType::ProfilePhoto, 0, 0, + result.small_file_id = register_photo(file_manager, FileType::ProfilePhoto, 0, 0, "", std::move(chat_photo->photo_small_), DialogId(), 0); - result.big_file_id = - register_photo(file_manager, FileType::ProfilePhoto, 0, 0, std::move(chat_photo->photo_big_), DialogId(), 0); + result.big_file_id = register_photo(file_manager, FileType::ProfilePhoto, 0, 0, "", + std::move(chat_photo->photo_big_), DialogId(), 0); break; } @@ -242,7 +244,7 @@ PhotoSize get_thumbnail_photo_size(FileManager *file_manager, BufferSlice bytes, auto volume_id = Random::secure_int64(); auto secret = 0; res.file_id = file_manager->register_remote( - FullRemoteFileLocation(FileType::EncryptedThumbnail, 0, 0, local_id, volume_id, secret, dc_id, ""), + FullRemoteFileLocation(FileType::EncryptedThumbnail, 0, 0, local_id, volume_id, secret, dc_id, "", ""), FileLocationSource::FromServer, owner_dialog_id, res.size, 0, PSTRING() << static_cast(volume_id) << "_" << static_cast(local_id) << ".jpg"); file_manager->set_content(res.file_id, std::move(bytes)); @@ -251,7 +253,8 @@ PhotoSize get_thumbnail_photo_size(FileManager *file_manager, BufferSlice bytes, } PhotoSize get_photo_size(FileManager *file_manager, FileType file_type, int64 id, int64 access_hash, - DialogId owner_dialog_id, tl_object_ptr &&size_ptr, bool is_webp) { + std::string upload_file_reference, DialogId owner_dialog_id, + tl_object_ptr &&size_ptr, bool is_webp) { tl_object_ptr location_ptr; string type; @@ -289,8 +292,8 @@ PhotoSize get_photo_size(FileManager *file_manager, FileType file_type, int64 id break; } - res.file_id = register_photo(file_manager, file_type, id, access_hash, std::move(location_ptr), owner_dialog_id, - res.size, is_webp); + res.file_id = register_photo(file_manager, file_type, id, access_hash, upload_file_reference, std::move(location_ptr), + owner_dialog_id, res.size, is_webp); if (!content.empty()) { file_manager->set_content(res.file_id, std::move(content)); @@ -482,8 +485,9 @@ Photo get_photo(FileManager *file_manager, tl_object_ptr && // TODO use file_reference for (auto &size_ptr : photo->sizes_) { - res.photos.push_back(get_photo_size(file_manager, FileType::Photo, photo->id_, photo->access_hash_, owner_dialog_id, - std::move(size_ptr), false)); + res.photos.push_back(get_photo_size(file_manager, FileType::Photo, photo->id_, photo->access_hash_, + photo->file_reference_.as_slice().str(), owner_dialog_id, std::move(size_ptr), + false)); } return res; diff --git a/td/telegram/Photo.h b/td/telegram/Photo.h index 5cff517c1..296f53a79 100644 --- a/td/telegram/Photo.h +++ b/td/telegram/Photo.h @@ -83,7 +83,8 @@ StringBuilder &operator<<(StringBuilder &string_builder, const DialogPhoto &dial PhotoSize get_thumbnail_photo_size(FileManager *file_manager, BufferSlice bytes, DialogId owner_dialog_id, int32 width, int32 height); PhotoSize get_photo_size(FileManager *file_manager, FileType file_type, int64 id, int64 access_hash, - DialogId owner_dialog_id, tl_object_ptr &&size_ptr, bool is_webp); + std::string upload_file_reference, DialogId owner_dialog_id, + tl_object_ptr &&size_ptr, bool is_webp); PhotoSize get_web_document_photo_size(FileManager *file_manager, FileType file_type, DialogId owner_dialog_id, tl_object_ptr web_document_ptr); td_api::object_ptr get_photo_size_object(FileManager *file_manager, const PhotoSize *photo_size); diff --git a/td/telegram/StickersManager.cpp b/td/telegram/StickersManager.cpp index a2e811fc3..99ac1211b 100644 --- a/td/telegram/StickersManager.cpp +++ b/td/telegram/StickersManager.cpp @@ -1069,7 +1069,7 @@ std::pair StickersManager::on_get_sticker_document(tl_object_ptr< document->file_reference_.as_slice().str()), FileLocationSource::FromServer, DialogId(), document->size_, 0, PSTRING() << document_id << ".webp"); - PhotoSize thumbnail = get_photo_size(td_->file_manager_.get(), FileType::Thumbnail, 0, 0, DialogId(), + PhotoSize thumbnail = get_photo_size(td_->file_manager_.get(), FileType::Thumbnail, 0, 0, "", DialogId(), std::move(document->thumb_), has_webp_thumbnail(sticker)); create_sticker(sticker_id, std::move(thumbnail), dimensions, from_message, std::move(sticker), nullptr); diff --git a/td/telegram/WallpaperManager.cpp b/td/telegram/WallpaperManager.cpp index 32f13dd45..ce8ba3eff 100644 --- a/td/telegram/WallpaperManager.cpp +++ b/td/telegram/WallpaperManager.cpp @@ -97,10 +97,11 @@ void WallpaperManager::on_get_wallpapers(Resultget_id()) { case telegram_api::wallPaper::ID: { auto wallpaper = move_tl_object_as(wallpaper_ptr); - vector sizes = transform( - std::move(wallpaper->sizes_), [file_manager](tl_object_ptr &&photo_size) { - return get_photo_size(file_manager, FileType::Wallpaper, 0, 0, DialogId(), std::move(photo_size), false); - }); + vector sizes = transform(std::move(wallpaper->sizes_), + [file_manager](tl_object_ptr &&photo_size) { + return get_photo_size(file_manager, FileType::Wallpaper, 0, 0, "", + DialogId(), std::move(photo_size), false); + }); return Wallpaper{wallpaper->id_, std::move(sizes), wallpaper->color_}; } case telegram_api::wallPaperSolid::ID: { diff --git a/td/telegram/files/FileDownloader.cpp b/td/telegram/files/FileDownloader.cpp index e8a1c77e9..3fcbdd0bd 100644 --- a/td/telegram/files/FileDownloader.cpp +++ b/td/telegram/files/FileDownloader.cpp @@ -274,8 +274,8 @@ Status FileDownloader::check_net_query(NetQueryPtr &net_query) { if (net_query->is_error()) { auto error = net_query->move_as_error(); if (FileReferenceManager::is_file_reference_error(error)) { - error = Status::Error(error.code(), - PSLICE() << error.message() << "#BASE64" << base64_encode(remote_.get_file_reference())); + error = Status::Error(error.code(), PSLICE() << error.message() << "#BASE64" + << base64_encode(remote_.get_download_file_reference())); } return error; } diff --git a/td/telegram/files/FileLocation.h b/td/telegram/files/FileLocation.h index 6f7678396..6c2561b41 100644 --- a/td/telegram/files/FileLocation.h +++ b/td/telegram/files/FileLocation.h @@ -25,6 +25,93 @@ #include namespace td { +class FileReferenceView { + public: + static Slice invalid_file_reference() { + return "#"; + } + static std::string create_one(td::Slice first) { + char second_length = char(255); + return PSTRING() << second_length << first; + } + static std::string create_two(td::Slice first, td::Slice second = {}) { + if (second.size() >= 255) { + LOG(ERROR) << "File reference is too big " << base64_encode(second); + second = invalid_file_reference(); + } + char second_length = narrow_cast(second.size()); + return PSTRING() << second_length << first << second; + } + std::string create(td::Slice first, td::Slice second) const { + if (size() == 1) { + return create_one(first); + } + return create_two(first, second); + } + FileReferenceView(Slice data) { + if (data.empty()) { + size_ = 1; + return; + } + + unsigned char second_size = data.ubegin()[0]; + + if (second_size == 255) { + first_ = data.substr(1); + second_ = data.substr(1); + size_ = 1; + } else { + if (second_size > data.size() - 1) { + size_ = 1; + return; + } + auto first_size = data.size() - 1 - second_size; + first_ = data.substr(1, first_size); + second_ = data.substr(1 + first_size); + size_ = 2; + } + } + size_t size() const { + return size_; + } + Slice first() const { + return first_; + } + Slice second() const { + return second_; + } + bool has_first() const { + return first() != invalid_file_reference(); + } + bool has_second() const { + return second() != invalid_file_reference(); + } + std::pair delete_file_reference(Slice bad_file_reference) const { + if (bad_file_reference == FileReferenceView::invalid_file_reference()) { + return {"", false}; + } + auto first = this->first(); + auto second = this->second(); + bool changed = false; + if (first == bad_file_reference) { + first = invalid_file_reference(); + changed = true; + } + if (second == bad_file_reference) { + second = invalid_file_reference(); + changed = true; + } + if (!changed) { + return {"", false}; + } + return {create(first, second), true}; + } + + private: + Slice first_; + Slice second_; + int size_; +}; struct EmptyRemoteFileLocation { template @@ -399,25 +486,31 @@ class FullRemoteFileLocation { return 0; } } - static Slice invalid_file_reference() { - return "#"; - } bool delete_file_reference(Slice bad_file_reference) { - if (file_reference_ == invalid_file_reference()) { - return false; + auto res = FileReferenceView(file_reference_).delete_file_reference(bad_file_reference); + if (res.second) { + file_reference_ = res.first; } - if (file_reference_ != bad_file_reference) { - return false; - } - file_reference_ = invalid_file_reference().str(); - return true; + return res.second; } - bool has_file_reference() const { - return file_reference_ != invalid_file_reference(); + bool has_upload_file_reference() const { + return FileReferenceView(file_reference_).has_first(); } - const string &get_file_reference() const { + bool has_download_file_reference() const { + return FileReferenceView(file_reference_).has_second(); + } + bool has_any_file_reference() const { + return has_upload_file_reference() || has_download_file_reference(); + } + Slice get_raw_file_reference() const { return file_reference_; } + Slice get_upload_file_reference() const { + return FileReferenceView(file_reference_).first(); + } + Slice get_download_file_reference() const { + return FileReferenceView(file_reference_).second(); + } string get_url() const { if (is_web()) { return web().url_; @@ -460,8 +553,9 @@ class FullRemoteFileLocation { tl_object_ptr as_input_file_location() const { switch (location_type()) { case LocationType::Photo: - return make_tl_object(photo().volume_id_, photo().local_id_, photo().secret_, - BufferSlice(file_reference_)); + return make_tl_object( + photo().volume_id_, photo().local_id_, photo().secret_, + BufferSlice(FileReferenceView(file_reference_).second())); case LocationType::Common: if (is_encrypted_secret()) { return make_tl_object(common().id_, common().access_hash_); @@ -490,7 +584,8 @@ class FullRemoteFileLocation { #define as_input_photo() as_input_photo_impl(__FILE__, __LINE__) tl_object_ptr as_input_photo_impl(const char *file, int line) const { CHECK(is_photo()) << file << ' ' << line; - return make_tl_object(photo().id_, photo().access_hash_, BufferSlice(file_reference_)); + return make_tl_object(photo().id_, photo().access_hash_, + BufferSlice(FileReferenceView(file_reference_).first())); } tl_object_ptr as_input_encrypted_file() const { @@ -507,13 +602,14 @@ class FullRemoteFileLocation { // TODO: this constructor is just for immediate unserialize FullRemoteFileLocation() = default; FullRemoteFileLocation(FileType file_type, int64 id, int64 access_hash, int32 local_id, int64 volume_id, int64 secret, - DcId dc_id, std::string file_reference) + DcId dc_id, std::string upload_file_reference, std::string download_file_reference) : file_type_(file_type) , dc_id_(dc_id) - , file_reference_(std::move(file_reference)) + , file_reference_(FileReferenceView::create_two(upload_file_reference, download_file_reference)) , variant_(PhotoRemoteFileLocation{id, access_hash, volume_id, secret, local_id}) { CHECK(is_photo()); - if (file_reference_ == invalid_file_reference()) { + FileReferenceView view(file_reference_); + if (!view.has_first() || !view.has_second()) { LOG(ERROR) << "Tried to register file with invalid file reference"; file_reference_.clear(); } @@ -524,7 +620,8 @@ class FullRemoteFileLocation { , file_reference_(std::move(file_reference)) , variant_(CommonRemoteFileLocation{id, access_hash}) { CHECK(is_common()); - if (file_reference_ == invalid_file_reference()) { + FileReferenceView view(file_reference_); + if (!view.has_first() || !view.has_second()) { LOG(ERROR) << "Tried to register file with invalid file reference"; file_reference_.clear(); } @@ -580,7 +677,7 @@ class FullRemoteFileLocation { } static const int32 KEY_MAGIC = 0x64374632; -}; +}; // namespace td inline StringBuilder &operator<<(StringBuilder &string_builder, const FullRemoteFileLocation &full_remote_file_location) { @@ -659,9 +756,9 @@ class RemoteFileLocation { explicit RemoteFileLocation(const PartialRemoteFileLocation &partial) : variant_(partial) { } RemoteFileLocation(FileType file_type, int64 id, int64 access_hash, int32 local_id, int64 volume_id, int64 secret, - DcId dc_id, std::string file_reference) + DcId dc_id, std::string upload_file_reference, std::string download_file_reference) : variant_(FullRemoteFileLocation{file_type, id, access_hash, local_id, volume_id, secret, dc_id, - std::move(file_reference)}) { + std::move(upload_file_reference), std::move(download_file_reference)}) { } RemoteFileLocation(FileType file_type, int64 id, int64 access_hash, DcId dc_id, std::string file_reference) : variant_(FullRemoteFileLocation{file_type, id, access_hash, dc_id, std::move(file_reference)}) { diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index 73e2413cd..f5ee26952 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -148,7 +148,7 @@ void FileNode::set_remote_location(const RemoteFileLocation &remote, FileLocatio if (remote_ == remote) { if (remote_.type() == RemoteFileLocation::Type::Full) { if (remote_.full().get_access_hash() != remote.full().get_access_hash() || - remote_.full().get_file_reference() != remote.full().get_file_reference()) { + remote_.full().get_raw_file_reference() != remote.full().get_raw_file_reference()) { remote_ = remote; remote_source_ = source; on_pmc_changed(); @@ -352,14 +352,24 @@ bool FileView::has_remote_location() const { return node_->remote_.type() == RemoteFileLocation::Type::Full; } -bool FileView::has_active_remote_location() const { +bool FileView::has_active_upload_remote_location() const { if (!has_remote_location()) { return false; } if (remote_location().is_encrypted_any()) { return true; } - return remote_location().has_file_reference(); + return remote_location().has_upload_file_reference(); +} + +bool FileView::has_active_download_remote_location() const { + if (!has_remote_location()) { + return false; + } + if (remote_location().is_encrypted_any()) { + return true; + } + return remote_location().has_download_file_reference(); } const FullRemoteFileLocation &FileView::remote_location() const { @@ -986,13 +996,13 @@ static int merge_choose_remote_location(const RemoteFileLocation &x, int8 x_sour if (x.full().is_web() != y.full().is_web()) { return x.full().is_web(); // prefer non-web } - auto x_ref = x.full().has_file_reference(); - auto y_ref = y.full().has_file_reference(); + auto x_ref = x.full().has_any_file_reference(); + auto y_ref = y.full().has_any_file_reference(); if (x_ref || y_ref) { if (x_ref != y_ref) { return !x_ref; } - if (x.full().get_file_reference() != y.full().get_file_reference()) { + if (x.full().get_raw_file_reference() != y.full().get_raw_file_reference()) { return x_source < y_source; } } @@ -1799,7 +1809,7 @@ void FileManager::run_download(FileNodePtr node) { auto file_id = node->main_file_id_; // If file reference is needed - if (!file_view.has_active_remote_location()) { + if (!file_view.has_active_download_remote_location()) { VLOG(file_references) << "run_download: Do not have valid file_reference for file " << file_id; QueryId id = queries_container_.create(Query{file_id, Query::DownloadWaitFileReferece}); node->download_id_ = id; @@ -1850,8 +1860,8 @@ void FileManager::resume_upload(FileId file_id, std::vector bad_parts, std: node->set_upload_pause(FileId()); } FileView file_view(node); - if (file_view.has_active_remote_location() && file_view.get_type() != FileType::Thumbnail && - file_view.get_type() != FileType::EncryptedThumbnail && file_view.get_type() != FileType::Photo) { + if (file_view.has_active_upload_remote_location() && file_view.get_type() != FileType::Thumbnail && + file_view.get_type() != FileType::EncryptedThumbnail) { LOG(INFO) << "File " << file_id << " is already uploaded"; if (callback) { callback->on_upload_ok(file_id, nullptr); @@ -2085,9 +2095,9 @@ void FileManager::run_upload(FileNodePtr node, std::vector bad_parts) { } CHECK(node->upload_id_ == 0); - if (file_view.has_remote_location() && !file_view.has_active_remote_location() && - file_view.get_type() != FileType::Photo && file_view.get_type() != FileType::Thumbnail && - file_view.get_type() != FileType::EncryptedThumbnail && !node->upload_was_update_file_reference_) { + if (file_view.has_remote_location() && !file_view.has_active_upload_remote_location() && + file_view.get_type() != FileType::Thumbnail && file_view.get_type() != FileType::EncryptedThumbnail && + !node->upload_was_update_file_reference_) { QueryId id = queries_container_.create(Query{file_id, Query::UploadWaitFileReference}); node->upload_id_ = id; node->upload_was_update_file_reference_ = true; diff --git a/td/telegram/files/FileManager.h b/td/telegram/files/FileManager.h index c5631f379..8cb83033e 100644 --- a/td/telegram/files/FileManager.h +++ b/td/telegram/files/FileManager.h @@ -212,7 +212,8 @@ class FileView { bool has_local_location() const; const FullLocalFileLocation &local_location() const; bool has_remote_location() const; - bool has_active_remote_location() const; + bool has_active_upload_remote_location() const; + bool has_active_download_remote_location() const; const FullRemoteFileLocation &remote_location() const; bool has_generate_location() const; const FullGenerateFileLocation &generate_location() const;