From a75726d77ae48915448ad669930846d22b1f65a5 Mon Sep 17 00:00:00 2001 From: Arseny Smirnov Date: Wed, 5 Dec 2018 12:32:31 +0300 Subject: [PATCH] FileManager: file references draft GitOrigin-RevId: 8d37ee4b5520497df08f49e598ae91825e74e1ea --- CMakeLists.txt | 2 + td/generate/scheme/telegram_api.tl | 15 +-- td/generate/scheme/telegram_api.tlo | Bin 165576 -> 165908 bytes td/telegram/ContactsManager.cpp | 8 +- td/telegram/DocumentsManager.cpp | 6 +- td/telegram/FileReferenceManager.cpp | 72 ++++++++++++ td/telegram/FileReferenceManager.h | 31 ++++++ td/telegram/Global.h | 9 ++ td/telegram/MessageContent.cpp | 12 +- td/telegram/MessagesManager.cpp | 13 ++- td/telegram/Photo.cpp | 8 +- td/telegram/SecureValue.cpp | 14 ++- td/telegram/StickersManager.cpp | 3 +- td/telegram/Td.cpp | 9 ++ td/telegram/Td.h | 3 + td/telegram/files/FileDownloader.cpp | 15 ++- td/telegram/files/FileDownloader.h | 2 + td/telegram/files/FileHashUploader.cpp | 3 +- td/telegram/files/FileId.h | 2 + td/telegram/files/FileLocation.h | 64 ++++++++--- td/telegram/files/FileManager.cpp | 148 ++++++++++++++++++++++++- td/telegram/files/FileManager.h | 12 ++ td/telegram/net/MtprotoHeader.cpp | 2 +- 23 files changed, 402 insertions(+), 51 deletions(-) create mode 100644 td/telegram/FileReferenceManager.cpp create mode 100644 td/telegram/FileReferenceManager.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 0fadce5a3..8f6c42bb2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -358,6 +358,7 @@ set(TDLIB_SOURCE td/telegram/DialogParticipant.cpp td/telegram/DocumentsManager.cpp td/telegram/DraftMessage.cpp + td/telegram/FileReferenceManager.cpp td/telegram/files/FileBitmask.cpp td/telegram/files/FileDb.cpp td/telegram/files/FileDownloader.cpp @@ -481,6 +482,7 @@ set(TDLIB_SOURCE td/telegram/DialogParticipant.h td/telegram/DocumentsManager.h td/telegram/DraftMessage.h + td/telegram/FileReferenceManager.h td/telegram/files/FileBitmask.h td/telegram/files/FileDb.h td/telegram/files/FileDbId.h diff --git a/td/generate/scheme/telegram_api.tl b/td/generate/scheme/telegram_api.tl index 91537eabe..62aad71e9 100644 --- a/td/generate/scheme/telegram_api.tl +++ b/td/generate/scheme/telegram_api.tl @@ -64,11 +64,11 @@ inputGeoPointEmpty#e4c123d6 = InputGeoPoint; inputGeoPoint#f3b7acc9 lat:double long:double = InputGeoPoint; inputPhotoEmpty#1cd7bf0d = InputPhoto; -inputPhoto#fb95c6c4 id:long access_hash:long = InputPhoto; +inputPhoto#3bb3b94a id:long access_hash:long file_reference:bytes = InputPhoto; -inputFileLocation#14637196 volume_id:long local_id:int secret:long = InputFileLocation; +inputFileLocation#dfdaabe1 volume_id:long local_id:int secret:long file_reference:bytes = InputFileLocation; inputEncryptedFileLocation#f5235d55 id:long access_hash:long = InputFileLocation; -inputDocumentFileLocation#430f0724 id:long access_hash:long version:int = InputFileLocation; +inputDocumentFileLocation#196683d9 id:long access_hash:long file_reference:bytes = InputFileLocation; inputSecureFileLocation#cbc7ee28 id:long access_hash:long = InputFileLocation; inputTakeoutFileLocation#29be5899 = InputFileLocation; @@ -88,7 +88,7 @@ storage.fileMp4#b3cea0e4 = storage.FileType; storage.fileWebp#1081464c = storage.FileType; fileLocationUnavailable#7c596b46 volume_id:long local_id:int secret:long = FileLocation; -fileLocation#53d69076 dc_id:int volume_id:long local_id:int secret:long = FileLocation; +fileLocation#91d11eb dc_id:int volume_id:long local_id:int secret:long file_reference:bytes = FileLocation; userEmpty#200250ba id:int = User; user#2e13f4c3 flags:# self:flags.10?true contact:flags.11?true mutual_contact:flags.12?true deleted:flags.13?true bot:flags.14?true bot_chat_history:flags.15?true bot_nochats:flags.16?true verified:flags.17?true restricted:flags.18?true min:flags.20?true bot_inline_geo:flags.21?true id:int access_hash:flags.0?long first_name:flags.1?string last_name:flags.2?string username:flags.3?string phone:flags.4?string photo:flags.5?UserProfilePhoto status:flags.6?UserStatus bot_info_version:flags.14?int restriction_reason:flags.18?string bot_inline_placeholder:flags.19?string lang_code:flags.22?string = User; @@ -164,7 +164,7 @@ messageActionSecureValuesSent#d95c6154 types:Vector = MessageAc dialog#e4def5db flags:# pinned:flags.2?true unread_mark:flags.3?true peer:Peer top_message:int read_inbox_max_id:int read_outbox_max_id:int unread_count:int unread_mentions_count:int notify_settings:PeerNotifySettings pts:flags.0?int draft:flags.1?DraftMessage = Dialog; photoEmpty#2331b22d id:long = Photo; -photo#9288dd29 flags:# has_stickers:flags.0?true id:long access_hash:long date:int sizes:Vector = Photo; +photo#9c477dd8 flags:# has_stickers:flags.0?true id:long access_hash:long file_reference:bytes date:int sizes:Vector = Photo; photoSizeEmpty#e17e23c type:string = PhotoSize; photoSize#77bfb61b type:string location:FileLocation w:int h:int size:int = PhotoSize; @@ -382,10 +382,10 @@ messages.sentEncryptedMessage#560f8935 date:int = messages.SentEncryptedMessage; messages.sentEncryptedFile#9493ff32 date:int file:EncryptedFile = messages.SentEncryptedMessage; inputDocumentEmpty#72f0eaae = InputDocument; -inputDocument#18798952 id:long access_hash:long = InputDocument; +inputDocument#1abfb575 id:long access_hash:long file_reference:bytes = InputDocument; documentEmpty#36f8c871 id:long = Document; -document#87232bc7 id:long access_hash:long date:int mime_type:string size:int thumb:PhotoSize dc_id:int version:int attributes:Vector = Document; +document#59534e4c id:long access_hash:long file_reference:bytes date:int mime_type:string size:int thumb:PhotoSize dc_id:int attributes:Vector = Document; help.support#17c6b5f6 phone_number:string user:User = help.Support; @@ -1003,6 +1003,7 @@ users.getUsers#d91a548 id:Vector = Vector; users.getFullUser#ca30a5b1 id:InputUser = UserFull; users.setSecureValueErrors#90c894b5 id:InputUser errors:Vector = Bool; +contacts.getContactIDs#2caa4a42 hash:int = Vector; contacts.getStatuses#c4a353ee = Vector; contacts.getContacts#c023849f hash:int = contacts.Contacts; contacts.importContacts#2c800be5 contacts:Vector = contacts.ImportedContacts; diff --git a/td/generate/scheme/telegram_api.tlo b/td/generate/scheme/telegram_api.tlo index 3bc0ab2d029a450843a89a549338d34b259a52a3..8e28ba45da2af99bdcc5500d1e48cba53c8068fb 100644 GIT binary patch delta 1579 zcmZ`(eP~-%6wkSP>0V=MKV(~)rt7m57WE-1R;|ovw{^Cb-~x-TgF-XY)D&w{+86ES z3bndgF+%jt-hvgo(X#3?2;9wAb&wY4s36#)LOKg(B7&G<|4^tdpzq$B)UB-Vuk+65 z@0@#nXYn1&>9{4iZkF7m=OZ*{PJ?veM01p;x95=bW9Oj|PFs@F>X*MR)7V{?L24=L z+Cx1hl3e`Pw=bv6-cUrt@77jPZAV|NnW;=jlZwQwbgA99 zjZ1i`Jx=F3ri*wz><=h1D{L}8Yz;W*bl{g#4vNG}K&|7sn9=xt*lp?XJahP{+!YB# zUkZjbL&xlRb-+`4cXv;3u(Lnd9qbQ=13{8{80^@ct{Hy|>PTh`6dcoODTIvPq;6tHE)cl5Vs9qA85HFX68W`KPZ##y?*}MPvt!R zsoR=od%$W-`0tNi+_HD(h5M#dzBYLHL+@iuBhU$aE3bD~R^Ep1&ULoN%4vJ zXS!RPYWs}tweY4GDmZyXL>I6sv;wt<+HGwQPkT z55wGe!-n|cUDqZ|16aGEOjsb2r}(TcPw& z%cQxFH)3M8@g7u9N^B6>ygHm=WXJ|bM7FdI=FFT(c;S^NiT^Sb;jm`MQ>sjs>`+Fm zc4+_K0sCK5u4|gj=RF2vjKgGnWc71!=vTgg>0%Mo`xVIf_`$1? z5F+um8dX{sA(l6#zXol)Xx6(7lR{cshWQQ2r_0}gQVJFNeLus$4a)wuPt=`(NLV`- z&;}lSA*dY?^3%-&rX5!f10_G&+ms6%^U}ud*Noj5>L?}rv=(hqqtS(4&NydmG%`ez)fD-_eJrha=e8L zJsL9mPK6|ND=%A-R)zC8;zIJ6qqER%CiP5cdqKA1F6ikF0Rrlb{TErI;MCaS!<35oSi zU3+J;Qs1V+fWWcLbK5SZady!|r&w7ju%-9Oa zI{qsc@%_zxe4=~7O1YD54Vg$B5vk}dJJms+E&B9WKLf3QT)`k_9#3bP$x@rp_qVx9 z{M}pz8*KQ2pnRQoKss5ZpU~HNm*PvzSXrFcJS-$?0%~8VTk2IrP0OY(y50gsNHQk# z-Tbk&(;TK;P4r2Lb#5GUfH1P}6tH#!xNU}RB#h-OFiJNPi`k$A%WFV8J*xUq-a?7s zANzGDc_(U9DRA#%gr@u+b*=%}*qbNMklhm+LC``|)JQvU19ow|1Jtcyrq~Y(Gnsic z2F5MSESUg&GS~MM3^H?M(y&zdIgnCV-}QHNqQRHI|JQNq6|m!+S0MPur|Ld;8c0{c zXYk9TN)9-> input_photo_ids; - input_photo_ids.push_back(make_tl_object(profile_photo_id, 0)); + input_photo_ids.push_back(make_tl_object(profile_photo_id, 0, BufferSlice(""))); send_query( G()->net_query_creator().create(create_storer(telegram_api::photos_deletePhotos(std::move(input_photo_ids))))); } @@ -5436,8 +5436,10 @@ ContactsManager::User *ContactsManager::get_user_force(UserId user_id) { telegram_api::user::PHONE_MASK | telegram_api::user::PHOTO_MASK | telegram_api::user::VERIFIED_MASK; auto profile_photo = telegram_api::make_object( 3337190045231018, - telegram_api::make_object(1, 702229962, 26779, 5859320227133863146), - telegram_api::make_object(1, 702229962, 26781, -3695031185685824216)); + telegram_api::make_object(1, 702229962, 26779, 5859320227133863146, + BufferSlice("0")), + telegram_api::make_object(1, 702229962, 26781, -3695031185685824216, + BufferSlice("0"))); if (G()->is_test_dc()) { profile_photo = nullptr; flags -= telegram_api::user::PHOTO_MASK; diff --git a/td/telegram/DocumentsManager.cpp b/td/telegram/DocumentsManager.cpp index 7c815d4ab..5534df7f4 100644 --- a/td/telegram/DocumentsManager.cpp +++ b/td/telegram/DocumentsManager.cpp @@ -189,6 +189,7 @@ std::pair DocumentsManager::on_get_docum int32 dc_id; int32 size; string mime_type; + string file_reference; PhotoSize thumbnail; FileEncryptionKey encryption_key; bool is_web = false; @@ -202,6 +203,7 @@ std::pair DocumentsManager::on_get_docum dc_id = document->dc_id_; size = document->size_; mime_type = std::move(document->mime_type_); + file_reference = document->file_reference_.as_slice().str(); if (document_type != DocumentType::VoiceNote) { thumbnail = get_photo_size(td_->file_manager_.get(), FileType::Thumbnail, 0, 0, owner_dialog_id, @@ -291,8 +293,8 @@ std::pair DocumentsManager::on_get_docum FileId file_id; if (!is_web) { file_id = td_->file_manager_->register_remote( - FullRemoteFileLocation(file_type, id, access_hash, DcId::internal(dc_id)), FileLocationSource::FromServer, - owner_dialog_id, size, 0, suggested_file_name); + FullRemoteFileLocation(file_type, id, access_hash, DcId::internal(dc_id), std::move(file_reference)), + FileLocationSource::FromServer, owner_dialog_id, size, 0, suggested_file_name); if (!encryption_key.empty()) { td_->file_manager_->set_encryption_key(file_id, std::move(encryption_key)); } diff --git a/td/telegram/FileReferenceManager.cpp b/td/telegram/FileReferenceManager.cpp new file mode 100644 index 000000000..2434878ee --- /dev/null +++ b/td/telegram/FileReferenceManager.cpp @@ -0,0 +1,72 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/telegram/FileReferenceManager.h" + +#include "td/telegram/Global.h" +#include "td/telegram/MessagesManager.h" + +#include "td/actor/MultiPromise.h" + +#include "td/utils/format.h" + +namespace td { + +FileSourceId FileReferenceManager::create_file_source(FullMessageId full_message_id) { + auto it = from_full_message_id_.find(full_message_id); + if (it != from_full_message_id_.end()) { + return it->second; + } + ++last_file_source_id_; + to_full_message_id_[last_file_source_id_] = full_message_id; + from_full_message_id_[full_message_id] = FileSourceId{last_file_source_id_}; + return FileSourceId{last_file_source_id_}; +} + +void FileReferenceManager::update_file_reference(FileId file_id, std::vector sources, Promise<> promise) { + LOG(ERROR) << "update file reference: " << file_id << " " << format::as_array(sources); + //if (td::Random::fast(0, 3) == 0) { + //promise.set_error(td::Status::Error("Error")); + //return; + //} + + //if (td::Random::fast(0, 3) == 0) { + //promise.set_value(Unit()); + //return; + //} + + MultiPromiseActorSafe mpas{"UpdateFileReferenceMultiPromiseActor"}; + mpas.add_promise(std::move(promise)); + auto lock = mpas.get_promise(); + SCOPE_EXIT { + lock.set_value(Unit()); + }; + for (auto &source_id : sources) { + auto it = to_full_message_id_.find(source_id); + if (it != to_full_message_id_.end()) { + std::vector message_ids = {it->second}; + auto new_promise = PromiseCreator::lambda([promise = mpas.get_promise(), file_id, source_id, + file_manager = G()->file_manager()](Result res) mutable { + if (res.is_error()) { + send_closure(file_manager, &FileManager::remove_file_source, file_id, source_id); + send_lambda(file_manager, [promise = std::move(promise)]() mutable { promise.set_value({}); }); + } else { + promise.set_value(Unit()); + } + }); + + LOG(ERROR) << "Ask for " << it->second; + send_closure_later(G()->messages_manager(), &MessagesManager::get_messages_from_server, std::move(message_ids), + std::move(new_promise), nullptr); + } else { + LOG(ERROR) << "Invalid source id " << source_id << " " << file_id; + send_closure(G()->file_manager(), &FileManager::remove_file_source, file_id, source_id); + send_lambda(G()->file_manager(), [promise = mpas.get_promise()]() mutable { promise.set_value(Unit()); }); + } + } +} + +} // namespace td diff --git a/td/telegram/FileReferenceManager.h b/td/telegram/FileReferenceManager.h new file mode 100644 index 000000000..4a6107c37 --- /dev/null +++ b/td/telegram/FileReferenceManager.h @@ -0,0 +1,31 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/actor.h" +#include "td/actor/PromiseFuture.h" + +#include "td/telegram/files/FileManager.h" +#include "td/telegram/MessageId.h" + +#include +#include + +namespace td { + +class FileReferenceManager : public Actor { + public: + FileSourceId create_file_source(FullMessageId full_message_id); + void update_file_reference(FileId file_id, std::vector sources, Promise<> promise); + + private: + td::int32 last_file_source_id_{0}; + std::map to_full_message_id_; + std::unordered_map from_full_message_id_; +}; + +} // namespace td diff --git a/td/telegram/Global.h b/td/telegram/Global.h index f4d080624..ec24889d4 100644 --- a/td/telegram/Global.h +++ b/td/telegram/Global.h @@ -36,6 +36,7 @@ class ConfigManager; class ConfigShared; class ConnectionCreator; class ContactsManager; +class FileReferenceManager; class FileManager; class LanguagePackManager; class MessagesManager; @@ -172,6 +173,13 @@ class Global : public ActorContext { contacts_manager_ = contacts_manager; } + ActorId file_reference_manager() const { + return file_reference_manager_; + } + void set_file_reference_manager(ActorId file_reference_manager) { + file_reference_manager_ = std::move(file_reference_manager); + } + ActorId file_manager() const { return file_manager_; } @@ -348,6 +356,7 @@ class Global : public ActorContext { ActorId call_manager_; ActorId config_manager_; ActorId contacts_manager_; + ActorId file_reference_manager_; ActorId file_manager_; ActorId language_pack_manager_; ActorId messages_manager_; diff --git a/td/telegram/MessageContent.cpp b/td/telegram/MessageContent.cpp index 133c0d352..994fb8583 100644 --- a/td/telegram/MessageContent.cpp +++ b/td/telegram/MessageContent.cpp @@ -2745,7 +2745,8 @@ void merge_message_contents(Td *td, MessageContent *old_content, MessageContent CHECK(!new_file_view.remote_location().is_web()); FileId file_id = td->file_manager_->register_remote( FullRemoteFileLocation(FileType::Photo, new_file_view.remote_location().get_id(), - new_file_view.remote_location().get_access_hash(), 0, 0, 0, DcId::invalid()), + new_file_view.remote_location().get_access_hash(), 0, 0, 0, DcId::invalid(), + new_file_view.remote_location().get_file_reference()), FileLocationSource::FromServer, dialog_id, old_photo->photos.back().size, 0, ""); LOG_STATUS(td->file_manager_->merge(file_id, old_file_id)); } @@ -3157,7 +3158,8 @@ static auto secret_to_telegram(secret_api::fileLocationUnavailable &file_locatio // fileLocation#53d69076 dc_id:int volume_id:long local_id:int secret:long = FileLocation; static auto secret_to_telegram(secret_api::fileLocation &file_location) { return make_tl_object(file_location.dc_id_, file_location.volume_id_, - file_location.local_id_, file_location.secret_); + file_location.local_id_, file_location.secret_, + BufferSlice() /*FIXME*/); } // photoSizeEmpty#e17e23c type:string = PhotoSize; @@ -3305,9 +3307,9 @@ static auto secret_to_telegram_document(secret_api::decryptedMessageMediaExterna if (!clean_input_string(from.mime_type_)) { from.mime_type_.clear(); } - return make_tl_object(from.id_, from.access_hash_, from.date_, from.mime_type_, from.size_, - secret_to_telegram(*from.thumb_), from.dc_id_, - 0, secret_to_telegram(from.attributes_)); + return make_tl_object( + from.id_, from.access_hash_, BufferSlice() /*FIXME*/, from.date_, from.mime_type_, from.size_, + secret_to_telegram(*from.thumb_), from.dc_id_, secret_to_telegram(from.attributes_)); } template diff --git a/td/telegram/MessagesManager.cpp b/td/telegram/MessagesManager.cpp index c60a99e32..dac1d0974 100644 --- a/td/telegram/MessagesManager.cpp +++ b/td/telegram/MessagesManager.cpp @@ -13,6 +13,7 @@ #include "td/telegram/DialogDb.h" #include "td/telegram/DraftMessage.h" #include "td/telegram/DraftMessage.hpp" +#include "td/telegram/FileReferenceManager.h" #include "td/telegram/files/FileId.hpp" #include "td/telegram/files/FileManager.h" #include "td/telegram/Global.h" @@ -8956,7 +8957,7 @@ void MessagesManager::on_send_secret_message_success(int64 random_id, MessageId } new_file_id = td_->file_manager_->register_remote( - FullRemoteFileLocation(FileType::Encrypted, file->id_, file->access_hash_, DcId::internal(file->dc_id_)), + FullRemoteFileLocation(FileType::Encrypted, file->id_, file->access_hash_, DcId::internal(file->dc_id_), ""), FileLocationSource::FromServer, owner_dialog_id, 0, 0, ""); } } @@ -20970,6 +20971,16 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, uniq cancel_user_dialog_action(dialog_id, message.get()); } + if (message_id.is_server()) { + auto file_ids = get_message_content_file_ids(message->content.get(), td_); + if (!file_ids.empty()) { + auto file_source_id = td_->file_reference_manager_->create_file_source(FullMessageId(dialog_id, message_id)); + for (auto file_id : file_ids) { + td_->file_manager_->add_file_source(file_id, file_source_id); + } + } + } + { unique_ptr *v = find_message(&d->messages, message_id); if (*v == nullptr && !message->from_database) { diff --git a/td/telegram/Photo.cpp b/td/telegram/Photo.cpp index 06f3b6590..5f4be744b 100644 --- a/td/telegram/Photo.cpp +++ b/td/telegram/Photo.cpp @@ -68,6 +68,7 @@ static FileId register_photo(FileManager *file_manager, FileType file_type, int6 int32 local_id; int64 volume_id; int64 secret; + std::string file_reference; switch (location_id) { case telegram_api::fileLocationUnavailable::ID: { auto location = move_tl_object_as(location_ptr); @@ -87,6 +88,7 @@ static FileId register_photo(FileManager *file_manager, FileType file_type, int6 local_id = location->local_id_; volume_id = location->volume_id_; secret = location->secret_; + file_reference = location->file_reference_.as_slice().str(); break; } default: @@ -100,7 +102,7 @@ static FileId register_photo(FileManager *file_manager, FileType file_type, int6 auto suggested_name = PSTRING() << static_cast(volume_id) << "_" << static_cast(local_id) << (is_webp ? ".webp" : ".jpg"); return file_manager->register_remote( - FullRemoteFileLocation(file_type, id, access_hash, local_id, volume_id, secret, dc_id), + FullRemoteFileLocation(file_type, id, access_hash, local_id, volume_id, secret, dc_id, file_reference), FileLocationSource::FromServer, owner_dialog_id, file_size, 0, std::move(suggested_name)); } @@ -229,7 +231,7 @@ PhotoSize get_thumbnail_photo_size(FileManager *file_manager, BufferSlice bytes, auto volume_id = Random::secure_int64(); auto secret = 0; res.file_id = file_manager->register_remote( - FullRemoteFileLocation(FileType::EncryptedThumbnail, 0, 0, local_id, volume_id, secret, dc_id), + FullRemoteFileLocation(FileType::EncryptedThumbnail, 0, 0, local_id, volume_id, secret, dc_id, ""), FileLocationSource::FromServer, owner_dialog_id, res.size, 0, PSTRING() << static_cast(volume_id) << "_" << static_cast(local_id) << ".jpg"); file_manager->set_content(res.file_id, std::move(bytes)); @@ -437,7 +439,7 @@ Photo get_photo(FileManager *file_manager, tl_object_ptr &&photo, DialogId owner_dialog_id) { CHECK(DcId::is_valid(file->dc_id_)); FileId file_id = file_manager->register_remote( - FullRemoteFileLocation(FileType::Encrypted, file->id_, file->access_hash_, DcId::internal(file->dc_id_)), + FullRemoteFileLocation(FileType::Encrypted, file->id_, file->access_hash_, DcId::internal(file->dc_id_), ""), FileLocationSource::FromServer, owner_dialog_id, photo->size_, 0, PSTRING() << static_cast(file->id_) << ".jpg"); file_manager->set_encryption_key(file_id, FileEncryptionKey{photo->key_.as_slice(), photo->iv_.as_slice()}); diff --git a/td/telegram/SecureValue.cpp b/td/telegram/SecureValue.cpp index 57ba65933..aa467b4e9 100644 --- a/td/telegram/SecureValue.cpp +++ b/td/telegram/SecureValue.cpp @@ -354,7 +354,8 @@ EncryptedSecureFile get_encrypted_secure_file(FileManager *file_manager, break; } result.file.file_id = file_manager->register_remote( - FullRemoteFileLocation(FileType::Secure, secure_file->id_, secure_file->access_hash_, DcId::internal(dc_id)), + FullRemoteFileLocation(FileType::Secure, secure_file->id_, secure_file->access_hash_, DcId::internal(dc_id), + ""), FileLocationSource::FromServer, DialogId(), 0, secure_file->size_, PSTRING() << secure_file->id_ << ".jpg"); result.file.date = secure_file->date_; if (result.file.date < 0) { @@ -425,11 +426,12 @@ static td_api::object_ptr get_dated_file_object(FileManager * LOG(ERROR) << "Have wrong file in get_dated_file_object"; return nullptr; } - dated_file.file_id = file_manager->register_remote( - FullRemoteFileLocation(FileType::SecureRaw, file_view.remote_location().get_id(), - file_view.remote_location().get_access_hash(), file_view.remote_location().get_dc_id()), - FileLocationSource::FromServer, DialogId(), file_view.size(), file_view.expected_size(), - file_view.suggested_name()); + dated_file.file_id = + file_manager->register_remote(FullRemoteFileLocation(FileType::SecureRaw, file_view.remote_location().get_id(), + file_view.remote_location().get_access_hash(), + file_view.remote_location().get_dc_id(), ""), + FileLocationSource::FromServer, DialogId(), file_view.size(), + file_view.expected_size(), file_view.suggested_name()); return get_dated_file_object(file_manager, dated_file); } diff --git a/td/telegram/StickersManager.cpp b/td/telegram/StickersManager.cpp index 8b2bcf006..2c91052c1 100644 --- a/td/telegram/StickersManager.cpp +++ b/td/telegram/StickersManager.cpp @@ -1029,7 +1029,8 @@ std::pair StickersManager::on_get_sticker_document(tl_object_ptr< int64 document_id = document->id_; FileId sticker_id = td_->file_manager_->register_remote( - FullRemoteFileLocation(FileType::Sticker, document_id, document->access_hash_, DcId::internal(document->dc_id_)), + FullRemoteFileLocation(FileType::Sticker, document_id, document->access_hash_, DcId::internal(document->dc_id_), + document->file_reference_.as_slice().str()), FileLocationSource::FromServer, DialogId(), document->size_, 0, PSTRING() << document_id << ".webp"); PhotoSize thumbnail = get_photo_size(td_->file_manager_.get(), FileType::Thumbnail, 0, 0, DialogId(), diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index 4b046aa0e..140176c66 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -32,6 +32,7 @@ #include "td/telegram/DialogId.h" #include "td/telegram/DialogParticipant.h" #include "td/telegram/DocumentsManager.h" +#include "td/telegram/FileReferenceManager.h" #include "td/telegram/files/FileGcParameters.h" #include "td/telegram/files/FileId.h" #include "td/telegram/files/FileManager.h" @@ -3632,6 +3633,8 @@ void Td::dec_actor_refcnt() { LOG(DEBUG) << "ContactsManager was cleared " << timer; documents_manager_.reset(); LOG(DEBUG) << "DocumentsManager was cleared " << timer; + file_reference_manager_.reset(); + LOG(DEBUG) << "FileReferenceManager was cleared " << timer; file_manager_.reset(); LOG(DEBUG) << "FileManager was cleared " << timer; inline_queries_manager_.reset(); @@ -3805,6 +3808,8 @@ void Td::clear() { LOG(DEBUG) << "AuthManager actor was cleared " << timer; contacts_manager_actor_.reset(); LOG(DEBUG) << "ContactsManager actor was cleared " << timer; + file_reference_manager_actor_.reset(); + LOG(DEBUG) << "FileReferenceManager actor was cleared " << timer; file_manager_actor_.reset(); LOG(DEBUG) << "FileManager actor was cleared " << timer; inline_queries_manager_actor_.reset(); @@ -4082,6 +4087,10 @@ Status Td::init(DbKey key) { private: Td *td_; }; + file_reference_manager_ = make_unique(); + file_reference_manager_actor_ = register_actor("FileReferenceManager", file_reference_manager_.get()); + G()->set_file_reference_manager(file_reference_manager_actor_.get()); + file_manager_ = make_unique(make_unique(this)); file_manager_actor_ = register_actor("FileManager", file_manager_.get()); file_manager_->init_actor(); diff --git a/td/telegram/Td.h b/td/telegram/Td.h index 42eec1cec..4a1795676 100644 --- a/td/telegram/Td.h +++ b/td/telegram/Td.h @@ -44,6 +44,7 @@ class ConfigManager; class ContactsManager; class DeviceTokenManager; class DocumentsManager; +class FileReferenceManager; class FileManager; class InlineQueriesManager; class HashtagHints; @@ -138,6 +139,8 @@ class Td final : public NetQueryCallback { ActorOwn auth_manager_actor_; unique_ptr contacts_manager_; ActorOwn contacts_manager_actor_; + unique_ptr file_reference_manager_; + ActorOwn file_reference_manager_actor_; unique_ptr file_manager_; ActorOwn file_manager_actor_; unique_ptr inline_queries_manager_; diff --git a/td/telegram/files/FileDownloader.cpp b/td/telegram/files/FileDownloader.cpp index ee38e476c..7ec31d4cf 100644 --- a/td/telegram/files/FileDownloader.cpp +++ b/td/telegram/files/FileDownloader.cpp @@ -15,6 +15,7 @@ #include "td/telegram/UniqueId.h" #include "td/utils/as.h" +#include "td/utils/base64.h" #include "td/utils/buffer.h" #include "td/utils/common.h" #include "td/utils/crypto.h" @@ -267,10 +268,19 @@ Result> FileDownloader::start_part(Part part, int32 return std::make_pair(std::move(net_query), false); } -Result FileDownloader::process_part(Part part, NetQueryPtr net_query) { +Status FileDownloader::check_net_query(NetQueryPtr &net_query) { if (net_query->is_error()) { - return std::move(net_query->error()); + auto error = net_query->move_as_error(); + if (begins_with(error.message(), "FILE_REFERENCE_")) { + error = Status::Error(400, PSLICE() << "FILE_REFERENCE_BASE64" << base64_encode(remote_.get_file_reference())); + } + return error; } + return Status::OK(); +} + +Result FileDownloader::process_part(Part part, NetQueryPtr net_query) { + TRY_STATUS(check_net_query(net_query)); BufferSlice bytes; bool need_cdn_decrypt = false; @@ -380,6 +390,7 @@ FileLoader::Callback *FileDownloader::get_callback() { Status FileDownloader::process_check_query(NetQueryPtr net_query) { has_hash_query_ = false; + TRY_STATUS(check_net_query(net_query)); TRY_RESULT(file_hashes, fetch_result(std::move(net_query))); add_hash_info(file_hashes); return Status::OK(); diff --git a/td/telegram/files/FileDownloader.h b/td/telegram/files/FileDownloader.h index 969f959d6..fc7003c63 100644 --- a/td/telegram/files/FileDownloader.h +++ b/td/telegram/files/FileDownloader.h @@ -96,5 +96,7 @@ class FileDownloader : public FileLoader { void keep_fd_flag(bool keep_fd) override; void try_release_fd(); Status acquire_fd() TD_WARN_UNUSED_RESULT; + + Status check_net_query(NetQueryPtr &net_query); }; } // namespace td diff --git a/td/telegram/files/FileHashUploader.cpp b/td/telegram/files/FileHashUploader.cpp index 5679190a7..bf524c8d2 100644 --- a/td/telegram/files/FileHashUploader.cpp +++ b/td/telegram/files/FileHashUploader.cpp @@ -133,7 +133,8 @@ Status FileHashUploader::on_result_impl(NetQueryPtr net_query) { case telegram_api::document::ID: { auto document = move_tl_object_as(res); callback_->on_ok(FullRemoteFileLocation(FileType::Document, document->id_, document->access_hash_, - DcId::internal(document->dc_id_))); + DcId::internal(document->dc_id_), + document->file_reference_.as_slice().str())); stop_flag_ = true; return Status::OK(); diff --git a/td/telegram/files/FileId.h b/td/telegram/files/FileId.h index 32aa075a1..8814a33ee 100644 --- a/td/telegram/files/FileId.h +++ b/td/telegram/files/FileId.h @@ -14,6 +14,8 @@ namespace td { +using FileSourceId = int32; + class FileId { int32 id = 0; int32 remote_id = 0; diff --git a/td/telegram/files/FileLocation.h b/td/telegram/files/FileLocation.h index c520b4253..ea76dd52d 100644 --- a/td/telegram/files/FileLocation.h +++ b/td/telegram/files/FileLocation.h @@ -366,8 +366,10 @@ class FullRemoteFileLocation { private: static constexpr int32 WEB_LOCATION_FLAG = 1 << 24; + static constexpr int32 FILE_REFERENCE_FLAG = 1 << 25; bool web_location_flag_{false}; DcId dc_id_; + std::string file_reference_; enum class LocationType : int32 { Web, Photo, Common, None }; Variant variant_; @@ -429,6 +431,9 @@ class FullRemoteFileLocation { if (is_web()) { type |= WEB_LOCATION_FLAG; } + if (!file_reference_.empty()) { + type |= FILE_REFERENCE_FLAG; + } return type; } @@ -438,6 +443,9 @@ class FullRemoteFileLocation { using ::td::store; store(full_type(), storer); store(dc_id_.get_value(), storer); + if (!file_reference_.empty()) { + store(file_reference_, storer); + } variant_.visit([&](auto &&value) { using td::store; store(value, storer); @@ -450,6 +458,8 @@ class FullRemoteFileLocation { parse(raw_type, parser); web_location_flag_ = (raw_type & WEB_LOCATION_FLAG) != 0; raw_type &= ~WEB_LOCATION_FLAG; + bool has_file_reference = (raw_type & FILE_REFERENCE_FLAG) != 0; + raw_type &= ~FILE_REFERENCE_FLAG; if (raw_type < 0 || raw_type >= static_cast(FileType::Size)) { return parser.set_error("Invalid FileType in FullRemoteFileLocation"); } @@ -458,6 +468,10 @@ class FullRemoteFileLocation { parse(dc_id_value, parser); dc_id_ = DcId::from_value(dc_id_value); + if (has_file_reference) { + parse(file_reference_, parser); + } + switch (location_type()) { case LocationType::Web: { variant_ = WebRemoteFileLocation(); @@ -525,6 +539,19 @@ class FullRemoteFileLocation { return 0; } } + bool delete_file_reference(Slice bad_file_reference) { + if (file_reference_.empty()) { + return false; + } + if (!bad_file_reference.empty() && file_reference_ != bad_file_reference) { + return false; + } + file_reference_ = {}; + return true; + } + string get_file_reference() const { + return file_reference_; + } string get_url() const { if (is_web()) { return web().url_; @@ -567,14 +594,16 @@ class FullRemoteFileLocation { tl_object_ptr as_input_file_location() const { switch (location_type()) { case LocationType::Photo: - return make_tl_object(photo().volume_id_, photo().local_id_, photo().secret_); + return make_tl_object(photo().volume_id_, photo().local_id_, photo().secret_, + BufferSlice(file_reference_)); case LocationType::Common: if (is_encrypted_secret()) { return make_tl_object(common().id_, common().access_hash_); } else if (is_secure()) { return make_tl_object(common().id_, common().access_hash_); } else { - return make_tl_object(common().id_, common().access_hash_, 0); + return make_tl_object(common().id_, common().access_hash_, + BufferSlice(file_reference_)); } case LocationType::Web: case LocationType::None: @@ -588,13 +617,14 @@ class FullRemoteFileLocation { tl_object_ptr as_input_document_impl(const char *file, int line) const { CHECK(is_common()) << file << ' ' << line; CHECK(is_document()) << file << ' ' << line; - return make_tl_object(common().id_, common().access_hash_); + return make_tl_object(common().id_, common().access_hash_, + BufferSlice(file_reference_)); } #define as_input_photo() as_input_photo_impl(__FILE__, __LINE__) tl_object_ptr as_input_photo_impl(const char *file, int line) const { CHECK(is_photo()) << file << ' ' << line; - return make_tl_object(photo().id_, photo().access_hash_); + return make_tl_object(photo().id_, photo().access_hash_, BufferSlice(file_reference_)); } tl_object_ptr as_input_encrypted_file() const { @@ -611,14 +641,18 @@ class FullRemoteFileLocation { // TODO: this constructor is just for immediate unserialize FullRemoteFileLocation() = default; FullRemoteFileLocation(FileType file_type, int64 id, int64 access_hash, int32 local_id, int64 volume_id, int64 secret, - DcId dc_id) + DcId dc_id, std::string file_reference) : file_type_(file_type) , dc_id_(dc_id) + , file_reference_(std::move(file_reference)) , variant_(PhotoRemoteFileLocation{id, access_hash, volume_id, secret, local_id}) { CHECK(is_photo()); } - FullRemoteFileLocation(FileType file_type, int64 id, int64 access_hash, DcId dc_id) - : file_type_(file_type), dc_id_(dc_id), variant_(CommonRemoteFileLocation{id, access_hash}) { + FullRemoteFileLocation(FileType file_type, int64 id, int64 access_hash, DcId dc_id, std::string file_reference) + : file_type_(file_type) + , dc_id_(dc_id) + , file_reference_(std::move(file_reference)) + , variant_(CommonRemoteFileLocation{id, access_hash}) { CHECK(is_common()); } FullRemoteFileLocation(FileType file_type, string url, int64 access_hash) @@ -680,6 +714,10 @@ inline StringBuilder &operator<<(StringBuilder &string_builder, if (!full_remote_file_location.is_web()) { string_builder << ", " << full_remote_file_location.get_dc_id(); } + if (!full_remote_file_location.file_reference_.empty()) { + string_builder << ", " + << tag("file_reference", format::as_hex_dump<0>(Slice(full_remote_file_location.file_reference_))); + } string_builder << ", location = "; if (full_remote_file_location.is_web()) { @@ -748,11 +786,12 @@ class RemoteFileLocation { explicit RemoteFileLocation(const PartialRemoteFileLocation &partial) : variant_(partial) { } RemoteFileLocation(FileType file_type, int64 id, int64 access_hash, int32 local_id, int64 volume_id, int64 secret, - DcId dc_id) - : variant_(FullRemoteFileLocation{file_type, id, access_hash, local_id, volume_id, secret, dc_id}) { + DcId dc_id, std::string file_reference) + : variant_(FullRemoteFileLocation{file_type, id, access_hash, local_id, volume_id, secret, dc_id, + std::move(file_reference)}) { } - RemoteFileLocation(FileType file_type, int64 id, int64 access_hash, DcId dc_id) - : variant_(FullRemoteFileLocation{file_type, id, access_hash, dc_id}) { + RemoteFileLocation(FileType file_type, int64 id, int64 access_hash, DcId dc_id, std::string file_reference) + : variant_(FullRemoteFileLocation{file_type, id, access_hash, dc_id, std::move(file_reference)}) { } private: @@ -1006,8 +1045,7 @@ class LocalFileLocation { LocalFileLocation() : variant_{EmptyLocalFileLocation()} { } - explicit LocalFileLocation(const PartialLocalFileLocation &partial) - : variant_(PartialLocalFileLocationPtr(partial)) { + explicit LocalFileLocation(const PartialLocalFileLocation &partial) : variant_(PartialLocalFileLocationPtr(partial)) { } explicit LocalFileLocation(const FullLocalFileLocation &full) : variant_(full) { } diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index f4ff376f9..a1e2b73d3 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -9,6 +9,7 @@ #include "td/telegram/telegram_api.h" #include "td/telegram/ConfigShared.h" +#include "td/telegram/FileReferenceManager.h" #include "td/telegram/files/FileDb.h" #include "td/telegram/files/FileLoaderUtils.h" #include "td/telegram/files/FileLocation.h" @@ -158,6 +159,12 @@ void FileNode::set_remote_location(const RemoteFileLocation &remote, FileLocatio on_changed(); } +void FileNode::delete_file_reference(Slice file_reference) { + if (remote_.type() == RemoteFileLocation::Type::Full && remote_.full().delete_file_reference(file_reference)) { + on_pmc_changed(); + } +} + void FileNode::set_generate_location(unique_ptr &&generate) { bool is_changed = generate_ == nullptr ? generate != nullptr : generate == nullptr || *generate_ != *generate; if (is_changed) { @@ -246,6 +253,21 @@ void FileNode::set_generate_priority(int8 download_priority, int8 upload_priorit generate_upload_priority_ = upload_priority; } +void FileNode::add_file_source(FileSourceId file_source_id) { + if (std::find(file_source_ids_.begin(), file_source_ids_.end(), file_source_id) != file_source_ids_.end()) { + return; + } + file_source_ids_.push_back(file_source_id); +} + +void FileNode::remove_file_source(FileSourceId file_source_id) { + auto it = std::find(file_source_ids_.begin(), file_source_ids_.end(), file_source_id); + if (it == file_source_ids_.end()) { + return; + } + file_source_ids_.erase(it); +} + void FileNode::on_changed() { on_pmc_changed(); on_info_changed(); @@ -333,6 +355,15 @@ const FullLocalFileLocation &FileView::local_location() const { bool FileView::has_remote_location() const { return node_->remote_.type() == RemoteFileLocation::Type::Full; } +bool FileView::has_active_remote_location() const { + if (!has_remote_location()) { + return false; + } + if (remote_location().is_encrypted_any()) { + return true; + } + return !remote_location().get_file_reference().empty(); +} const FullRemoteFileLocation &FileView::remote_location() const { CHECK(has_remote_location()); auto *remote = node_.get_remote(); @@ -952,6 +983,14 @@ static int merge_choose_remote_location(const RemoteFileLocation &x, int8 x_sour if (x.full().is_web() != y.full().is_web()) { return x.full().is_web(); // prefer non-web } + auto x_ref = !x.full().get_file_reference().empty(); + auto y_ref = !y.full().get_file_reference().empty(); + if (x_ref || y_ref) { + if (x_ref != y_ref) { + return !x_ref; + } + return x_source < y_source; + } if (x.full().get_access_hash() != y.full().get_access_hash()) { return x_source < y_source; } @@ -1064,7 +1103,7 @@ void FileManager::cancel_generate(FileNodePtr node) { } Result FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) { - LOG(DEBUG) << x_file_id << " VS " << y_file_id; + LOG(ERROR) << x_file_id << " VS " << y_file_id; if (!x_file_id.is_valid()) { return Status::Error("First file_id is invalid"); @@ -1158,6 +1197,8 @@ Result FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy << ", generate_i = " << generate_i << ", size_i = " << size_i << ", remote_name_i = " << remote_name_i << ", url_i = " << url_i << ", owner_i = " << owner_i << ", encryption_key_i = " << encryption_key_i << ", main_file_id_i = " << main_file_id_i; + LOG(ERROR) << FileView(node).has_active_remote_location(); + LOG(ERROR) << FileView(other_node).has_active_remote_location(); if (local_i == other_node_i) { cancel_download(node); node->set_download_offset(other_node->download_offset_); @@ -1238,6 +1279,9 @@ Result FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy } node->need_load_from_pmc_ |= other_node->need_load_from_pmc_; node->can_search_locally_ &= other_node->can_search_locally_; + for (auto source_id : other_node->file_source_ids_) { + node->add_file_source(source_id); + } if (main_file_id_i == other_node_i) { node->main_file_id_ = other_node->main_file_id_; @@ -1297,6 +1341,24 @@ Result FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy return node->main_file_id_; } +void FileManager::add_file_source(FileId file_id, FileSourceId file_source_id) { + LOG(ERROR) << "Add file source " << file_id << " " << file_source_id; + auto node = get_file_node(file_id); + if (!node) { + return; + } + node->add_file_source(file_source_id); +} + +void FileManager::remove_file_source(FileId file_id, FileSourceId file_source_id) { + LOG(ERROR) << "Remove file source " << file_id, file_source_id; + auto node = get_file_node(file_id); + if (!node) { + return; + } + node->remove_file_source(file_source_id); +} + void FileManager::try_flush_node_full(FileNodePtr node, bool new_remote, bool new_local, bool new_generate, FileDbId other_pmc_id) { if (node->need_pmc_flush()) { @@ -1682,6 +1744,31 @@ void FileManager::run_download(FileNodePtr node) { CHECK(node->download_id_ == 0); CHECK(!node->file_ids_.empty()); auto file_id = node->file_ids_.back(); + + // If file reference is needed + if (!file_view.has_active_remote_location()) { + LOG(ERROR) << "run_download: no active location"; + QueryId id = queries_container_.create(Query{file_id, Query::Download}); + node->download_id_ = id; + if (node->file_source_ids_.empty()) { + on_error(id, Status::Error("Can't download file: no valid file refernce and no valid source id")); + return; + } + + send_closure(G()->file_reference_manager(), &FileReferenceManager::update_file_reference, file_id, + node->file_source_ids_, PromiseCreator::lambda([id, actor_id = actor_id(this)](Result res) { + Status error; + LOG(ERROR) << "run_download: update_file_reference finished"; + if (res.is_ok()) { + error = td::Status::Error("FILE_DOWNLOAD_RESTART_WITH_FILE_REFERENCE"); + } else { + error = res.move_as_error(); + } + send_closure(actor_id, &FileManager::on_error, id, std::move(error)); + })); + return; + } + QueryId id = queries_container_.create(Query{file_id, Query::Download}); node->download_id_ = id; node->is_download_started_ = false; @@ -1708,7 +1795,7 @@ void FileManager::resume_upload(FileId file_id, std::vector bad_parts, std: node->set_upload_pause(FileId()); } FileView file_view(node); - if (file_view.has_remote_location() && file_view.get_type() != FileType::Thumbnail && + if (file_view.has_active_remote_location() && file_view.get_type() != FileType::Thumbnail && file_view.get_type() != FileType::EncryptedThumbnail) { LOG(INFO) << "File " << file_id << " is already uploaded"; if (callback) { @@ -1779,6 +1866,17 @@ bool FileManager::delete_partial_remote_location(FileId file_id) { return true; } +void FileManager::delete_file_reference(FileId file_id, std::string file_reference) { + LOG(ERROR) << "Delete file reference " << file_id << tag("reference", format::as_hex_dump<0>(Slice(file_reference))); + auto node = get_sync_file_node(file_id); + if (!node) { + LOG(INFO) << "Wrong file id " << file_id; + return; + } + node->delete_file_reference(file_reference); + try_flush_node(node, "delete_file_reference"); +} + void FileManager::external_file_generate_progress(int64 id, int32 expected_size, int32 local_prefix_size, Promise<> promise) { send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_progress, id, expected_size, @@ -1930,6 +2028,18 @@ void FileManager::run_upload(FileNodePtr node, std::vector bad_parts) { } CHECK(node->upload_id_ == 0); + if (file_view.has_remote_location() && !file_view.has_active_remote_location() && + file_view.get_type() != FileType::Thumbnail && file_view.get_type() != FileType::EncryptedThumbnail && + !node->file_source_ids_.empty()) { + QueryId id = queries_container_.create(Query{file_id, Query::Upload}); + node->upload_id_ = id; + send_closure(G()->file_reference_manager(), &FileReferenceManager::update_file_reference, file_id, + node->file_source_ids_, PromiseCreator::lambda([id, actor_id = actor_id(this)](Result res) { + send_closure(actor_id, &FileManager::on_error, id, Status::Error("FILE_UPLOAD_RESTART")); + })); + return; + } + if (node->remote_.type() != RemoteFileLocation::Type::Partial && node->get_by_hash_) { LOG(INFO) << "Get file " << node->main_file_id_ << " by hash"; QueryId id = queries_container_.create(Query{file_id, Query::UploadByHash}); @@ -2604,6 +2714,7 @@ void FileManager::on_error_impl(FileNodePtr node, FileManager::Query::Type type, SCOPE_EXIT { try_flush_node(node, "on_error"); }; + //FIXME: should we apply error if !was_active? if (status.code() != 1 && !G()->close_flag()) { LOG(WARNING) << "Failed to upload/download/generate file: " << status << ". Query type = " << type << ". File type is " << FileView(node).get_type(); @@ -2645,15 +2756,40 @@ void FileManager::on_error_impl(FileNodePtr node, FileManager::Query::Type type, if (begins_with(status.message(), "FILE_GENERATE_LOCATION_INVALID")) { node->set_generate_location(nullptr); } + if (begins_with(status.message(), "FILE_REFERENCE_")) { + string file_reference; + Slice prefix = "FILE_REFERENCE_BASE64"; + if (begins_with(status.message(), prefix)) { + auto tmp = base64_decode(status.message().substr(prefix.size())); + LOG_IF(WARNING, tmp.is_error()) << "Can't decode file reference from error " << status << " " << tmp.error(); + if (tmp.is_ok()) { + file_reference = tmp.move_as_ok(); + } + } + CHECK(!node->file_ids_.empty()); + delete_file_reference(node->file_ids_.back(), file_reference); + run_download(node); + return; + } if (status.message() == "FILE_UPLOAD_RESTART") { run_upload(node, {}); return; } - if (status.message() == "FILE_DOWNLOAD_RESTART") { - node->can_search_locally_ = false; - run_download(node); - return; + if (begins_with(status.message(), "FILE_DOWNLOAD_RESTART")) { + if (ends_with(status.message(), "WITH_FILE_REFERENCE")) { + if (FileView(node).has_active_remote_location()) { + LOG(ERROR) << "??????"; + run_download(node); + return; + } + LOG_IF(WARNING, !node->file_source_ids_.empty()) + << "Got no active remote locations " << FileView(node).remote_location(); + } else { + node->can_search_locally_ = false; + run_download(node); + return; + } } if (!was_active) { diff --git a/td/telegram/files/FileManager.h b/td/telegram/files/FileManager.h index 4449b4536..87ce747e0 100644 --- a/td/telegram/files/FileManager.h +++ b/td/telegram/files/FileManager.h @@ -65,6 +65,7 @@ class FileNode { void set_local_location(const LocalFileLocation &local, int64 ready_size, int64 prefix_offset, int64 ready_prefix_size); void set_remote_location(const RemoteFileLocation &remote, FileLocationSource source, int64 ready_size); + void delete_file_reference(Slice file_reference); void set_generate_location(unique_ptr &&generate); void set_size(int64 size); void set_expected_size(int64 expected_size); @@ -80,6 +81,9 @@ class FileNode { void set_download_offset(int64 download_offset); + void add_file_source(FileSourceId file_source_id); + void remove_file_source(FileSourceId file_source_id); + void on_changed(); void on_info_changed(); void on_pmc_changed(); @@ -106,6 +110,8 @@ class FileNode { FileLoadManager::QueryId download_id_ = 0; int64 remote_ready_size_ = 0; + std::vector file_source_ids_; + unique_ptr generate_; FileLoadManager::QueryId generate_id_ = 0; @@ -204,6 +210,7 @@ class FileView { bool has_local_location() const; const FullLocalFileLocation &local_location() const; bool has_remote_location() const; + bool has_active_remote_location() const; const FullRemoteFileLocation &remote_location() const; bool has_generate_location() const; const FullGenerateFileLocation &generate_location() const; @@ -343,6 +350,10 @@ class FileManager : public FileLoadManager::Callback { Result merge(FileId x_file_id, FileId y_file_id, bool no_sync = false) TD_WARN_UNUSED_RESULT; + void add_file_source(FileId file_id, FileSourceId file_source_id); + + void remove_file_source(FileId file_id, FileSourceId file_source_id); + bool set_encryption_key(FileId file_id, FileEncryptionKey key); bool set_content(FileId file_id, BufferSlice bytes); @@ -351,6 +362,7 @@ class FileManager : public FileLoadManager::Callback { void resume_upload(FileId file_id, std::vector bad_parts, std::shared_ptr callback, int32 new_priority, uint64 upload_order); bool delete_partial_remote_location(FileId file_id); + void delete_file_reference(FileId file_id, std::string file_reference); void get_content(FileId file_id, Promise promise); void delete_file(FileId file_id, Promise promise, const char *source); diff --git a/td/telegram/net/MtprotoHeader.cpp b/td/telegram/net/MtprotoHeader.cpp index 056903a1b..e6d86ab7c 100644 --- a/td/telegram/net/MtprotoHeader.cpp +++ b/td/telegram/net/MtprotoHeader.cpp @@ -21,7 +21,7 @@ class HeaderStorer { } template void store(StorerT &storer) const { - constexpr int32 LAYER = 85; + constexpr int32 LAYER = 86; using td::store; // invokeWithLayer#da9b0d0d {X:Type} layer:int query:!X = X;