FileManager: file references draft

GitOrigin-RevId: 8d37ee4b5520497df08f49e598ae91825e74e1ea
This commit is contained in:
Arseny Smirnov 2018-12-05 12:32:31 +03:00
parent 3367067fa9
commit a75726d77a
23 changed files with 402 additions and 51 deletions

View File

@ -358,6 +358,7 @@ set(TDLIB_SOURCE
td/telegram/DialogParticipant.cpp
td/telegram/DocumentsManager.cpp
td/telegram/DraftMessage.cpp
td/telegram/FileReferenceManager.cpp
td/telegram/files/FileBitmask.cpp
td/telegram/files/FileDb.cpp
td/telegram/files/FileDownloader.cpp
@ -481,6 +482,7 @@ set(TDLIB_SOURCE
td/telegram/DialogParticipant.h
td/telegram/DocumentsManager.h
td/telegram/DraftMessage.h
td/telegram/FileReferenceManager.h
td/telegram/files/FileBitmask.h
td/telegram/files/FileDb.h
td/telegram/files/FileDbId.h

View File

@ -64,11 +64,11 @@ inputGeoPointEmpty#e4c123d6 = InputGeoPoint;
inputGeoPoint#f3b7acc9 lat:double long:double = InputGeoPoint;
inputPhotoEmpty#1cd7bf0d = InputPhoto;
inputPhoto#fb95c6c4 id:long access_hash:long = InputPhoto;
inputPhoto#3bb3b94a id:long access_hash:long file_reference:bytes = InputPhoto;
inputFileLocation#14637196 volume_id:long local_id:int secret:long = InputFileLocation;
inputFileLocation#dfdaabe1 volume_id:long local_id:int secret:long file_reference:bytes = InputFileLocation;
inputEncryptedFileLocation#f5235d55 id:long access_hash:long = InputFileLocation;
inputDocumentFileLocation#430f0724 id:long access_hash:long version:int = InputFileLocation;
inputDocumentFileLocation#196683d9 id:long access_hash:long file_reference:bytes = InputFileLocation;
inputSecureFileLocation#cbc7ee28 id:long access_hash:long = InputFileLocation;
inputTakeoutFileLocation#29be5899 = InputFileLocation;
@ -88,7 +88,7 @@ storage.fileMp4#b3cea0e4 = storage.FileType;
storage.fileWebp#1081464c = storage.FileType;
fileLocationUnavailable#7c596b46 volume_id:long local_id:int secret:long = FileLocation;
fileLocation#53d69076 dc_id:int volume_id:long local_id:int secret:long = FileLocation;
fileLocation#91d11eb dc_id:int volume_id:long local_id:int secret:long file_reference:bytes = FileLocation;
userEmpty#200250ba id:int = User;
user#2e13f4c3 flags:# self:flags.10?true contact:flags.11?true mutual_contact:flags.12?true deleted:flags.13?true bot:flags.14?true bot_chat_history:flags.15?true bot_nochats:flags.16?true verified:flags.17?true restricted:flags.18?true min:flags.20?true bot_inline_geo:flags.21?true id:int access_hash:flags.0?long first_name:flags.1?string last_name:flags.2?string username:flags.3?string phone:flags.4?string photo:flags.5?UserProfilePhoto status:flags.6?UserStatus bot_info_version:flags.14?int restriction_reason:flags.18?string bot_inline_placeholder:flags.19?string lang_code:flags.22?string = User;
@ -164,7 +164,7 @@ messageActionSecureValuesSent#d95c6154 types:Vector<SecureValueType> = MessageAc
dialog#e4def5db flags:# pinned:flags.2?true unread_mark:flags.3?true peer:Peer top_message:int read_inbox_max_id:int read_outbox_max_id:int unread_count:int unread_mentions_count:int notify_settings:PeerNotifySettings pts:flags.0?int draft:flags.1?DraftMessage = Dialog;
photoEmpty#2331b22d id:long = Photo;
photo#9288dd29 flags:# has_stickers:flags.0?true id:long access_hash:long date:int sizes:Vector<PhotoSize> = Photo;
photo#9c477dd8 flags:# has_stickers:flags.0?true id:long access_hash:long file_reference:bytes date:int sizes:Vector<PhotoSize> = Photo;
photoSizeEmpty#e17e23c type:string = PhotoSize;
photoSize#77bfb61b type:string location:FileLocation w:int h:int size:int = PhotoSize;
@ -382,10 +382,10 @@ messages.sentEncryptedMessage#560f8935 date:int = messages.SentEncryptedMessage;
messages.sentEncryptedFile#9493ff32 date:int file:EncryptedFile = messages.SentEncryptedMessage;
inputDocumentEmpty#72f0eaae = InputDocument;
inputDocument#18798952 id:long access_hash:long = InputDocument;
inputDocument#1abfb575 id:long access_hash:long file_reference:bytes = InputDocument;
documentEmpty#36f8c871 id:long = Document;
document#87232bc7 id:long access_hash:long date:int mime_type:string size:int thumb:PhotoSize dc_id:int version:int attributes:Vector<DocumentAttribute> = Document;
document#59534e4c id:long access_hash:long file_reference:bytes date:int mime_type:string size:int thumb:PhotoSize dc_id:int attributes:Vector<DocumentAttribute> = Document;
help.support#17c6b5f6 phone_number:string user:User = help.Support;
@ -1003,6 +1003,7 @@ users.getUsers#d91a548 id:Vector<InputUser> = Vector<User>;
users.getFullUser#ca30a5b1 id:InputUser = UserFull;
users.setSecureValueErrors#90c894b5 id:InputUser errors:Vector<SecureValueError> = Bool;
contacts.getContactIDs#2caa4a42 hash:int = Vector<int>;
contacts.getStatuses#c4a353ee = Vector<ContactStatus>;
contacts.getContacts#c023849f hash:int = contacts.Contacts;
contacts.importContacts#2c800be5 contacts:Vector<InputContact> = contacts.ImportedContacts;

Binary file not shown.

View File

@ -744,7 +744,7 @@ class DeleteProfilePhotoQuery : public Td::ResultHandler {
void send(int64 profile_photo_id) {
profile_photo_id_ = profile_photo_id;
vector<tl_object_ptr<telegram_api::InputPhoto>> input_photo_ids;
input_photo_ids.push_back(make_tl_object<telegram_api::inputPhoto>(profile_photo_id, 0));
input_photo_ids.push_back(make_tl_object<telegram_api::inputPhoto>(profile_photo_id, 0, BufferSlice("")));
send_query(
G()->net_query_creator().create(create_storer(telegram_api::photos_deletePhotos(std::move(input_photo_ids)))));
}
@ -5436,8 +5436,10 @@ ContactsManager::User *ContactsManager::get_user_force(UserId user_id) {
telegram_api::user::PHONE_MASK | telegram_api::user::PHOTO_MASK | telegram_api::user::VERIFIED_MASK;
auto profile_photo = telegram_api::make_object<telegram_api::userProfilePhoto>(
3337190045231018,
telegram_api::make_object<telegram_api::fileLocation>(1, 702229962, 26779, 5859320227133863146),
telegram_api::make_object<telegram_api::fileLocation>(1, 702229962, 26781, -3695031185685824216));
telegram_api::make_object<telegram_api::fileLocation>(1, 702229962, 26779, 5859320227133863146,
BufferSlice("0")),
telegram_api::make_object<telegram_api::fileLocation>(1, 702229962, 26781, -3695031185685824216,
BufferSlice("0")));
if (G()->is_test_dc()) {
profile_photo = nullptr;
flags -= telegram_api::user::PHOTO_MASK;

View File

@ -189,6 +189,7 @@ std::pair<DocumentsManager::DocumentType, FileId> DocumentsManager::on_get_docum
int32 dc_id;
int32 size;
string mime_type;
string file_reference;
PhotoSize thumbnail;
FileEncryptionKey encryption_key;
bool is_web = false;
@ -202,6 +203,7 @@ std::pair<DocumentsManager::DocumentType, FileId> DocumentsManager::on_get_docum
dc_id = document->dc_id_;
size = document->size_;
mime_type = std::move(document->mime_type_);
file_reference = document->file_reference_.as_slice().str();
if (document_type != DocumentType::VoiceNote) {
thumbnail = get_photo_size(td_->file_manager_.get(), FileType::Thumbnail, 0, 0, owner_dialog_id,
@ -291,8 +293,8 @@ std::pair<DocumentsManager::DocumentType, FileId> DocumentsManager::on_get_docum
FileId file_id;
if (!is_web) {
file_id = td_->file_manager_->register_remote(
FullRemoteFileLocation(file_type, id, access_hash, DcId::internal(dc_id)), FileLocationSource::FromServer,
owner_dialog_id, size, 0, suggested_file_name);
FullRemoteFileLocation(file_type, id, access_hash, DcId::internal(dc_id), std::move(file_reference)),
FileLocationSource::FromServer, owner_dialog_id, size, 0, suggested_file_name);
if (!encryption_key.empty()) {
td_->file_manager_->set_encryption_key(file_id, std::move(encryption_key));
}

View File

@ -0,0 +1,72 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/telegram/FileReferenceManager.h"
#include "td/telegram/Global.h"
#include "td/telegram/MessagesManager.h"
#include "td/actor/MultiPromise.h"
#include "td/utils/format.h"
namespace td {
FileSourceId FileReferenceManager::create_file_source(FullMessageId full_message_id) {
auto it = from_full_message_id_.find(full_message_id);
if (it != from_full_message_id_.end()) {
return it->second;
}
++last_file_source_id_;
to_full_message_id_[last_file_source_id_] = full_message_id;
from_full_message_id_[full_message_id] = FileSourceId{last_file_source_id_};
return FileSourceId{last_file_source_id_};
}
void FileReferenceManager::update_file_reference(FileId file_id, std::vector<FileSourceId> sources, Promise<> promise) {
LOG(ERROR) << "update file reference: " << file_id << " " << format::as_array(sources);
//if (td::Random::fast(0, 3) == 0) {
//promise.set_error(td::Status::Error("Error"));
//return;
//}
//if (td::Random::fast(0, 3) == 0) {
//promise.set_value(Unit());
//return;
//}
MultiPromiseActorSafe mpas{"UpdateFileReferenceMultiPromiseActor"};
mpas.add_promise(std::move(promise));
auto lock = mpas.get_promise();
SCOPE_EXIT {
lock.set_value(Unit());
};
for (auto &source_id : sources) {
auto it = to_full_message_id_.find(source_id);
if (it != to_full_message_id_.end()) {
std::vector<FullMessageId> message_ids = {it->second};
auto new_promise = PromiseCreator::lambda([promise = mpas.get_promise(), file_id, source_id,
file_manager = G()->file_manager()](Result<Unit> res) mutable {
if (res.is_error()) {
send_closure(file_manager, &FileManager::remove_file_source, file_id, source_id);
send_lambda(file_manager, [promise = std::move(promise)]() mutable { promise.set_value({}); });
} else {
promise.set_value(Unit());
}
});
LOG(ERROR) << "Ask for " << it->second;
send_closure_later(G()->messages_manager(), &MessagesManager::get_messages_from_server, std::move(message_ids),
std::move(new_promise), nullptr);
} else {
LOG(ERROR) << "Invalid source id " << source_id << " " << file_id;
send_closure(G()->file_manager(), &FileManager::remove_file_source, file_id, source_id);
send_lambda(G()->file_manager(), [promise = mpas.get_promise()]() mutable { promise.set_value(Unit()); });
}
}
}
} // namespace td

View File

@ -0,0 +1,31 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/actor/actor.h"
#include "td/actor/PromiseFuture.h"
#include "td/telegram/files/FileManager.h"
#include "td/telegram/MessageId.h"
#include <map>
#include <unordered_map>
namespace td {
class FileReferenceManager : public Actor {
public:
FileSourceId create_file_source(FullMessageId full_message_id);
void update_file_reference(FileId file_id, std::vector<FileSourceId> sources, Promise<> promise);
private:
td::int32 last_file_source_id_{0};
std::map<FileSourceId, FullMessageId> to_full_message_id_;
std::unordered_map<FullMessageId, FileSourceId, FullMessageIdHash> from_full_message_id_;
};
} // namespace td

View File

@ -36,6 +36,7 @@ class ConfigManager;
class ConfigShared;
class ConnectionCreator;
class ContactsManager;
class FileReferenceManager;
class FileManager;
class LanguagePackManager;
class MessagesManager;
@ -172,6 +173,13 @@ class Global : public ActorContext {
contacts_manager_ = contacts_manager;
}
ActorId<FileReferenceManager> file_reference_manager() const {
return file_reference_manager_;
}
void set_file_reference_manager(ActorId<FileReferenceManager> file_reference_manager) {
file_reference_manager_ = std::move(file_reference_manager);
}
ActorId<FileManager> file_manager() const {
return file_manager_;
}
@ -348,6 +356,7 @@ class Global : public ActorContext {
ActorId<CallManager> call_manager_;
ActorId<ConfigManager> config_manager_;
ActorId<ContactsManager> contacts_manager_;
ActorId<FileReferenceManager> file_reference_manager_;
ActorId<FileManager> file_manager_;
ActorId<LanguagePackManager> language_pack_manager_;
ActorId<MessagesManager> messages_manager_;

View File

@ -2745,7 +2745,8 @@ void merge_message_contents(Td *td, MessageContent *old_content, MessageContent
CHECK(!new_file_view.remote_location().is_web());
FileId file_id = td->file_manager_->register_remote(
FullRemoteFileLocation(FileType::Photo, new_file_view.remote_location().get_id(),
new_file_view.remote_location().get_access_hash(), 0, 0, 0, DcId::invalid()),
new_file_view.remote_location().get_access_hash(), 0, 0, 0, DcId::invalid(),
new_file_view.remote_location().get_file_reference()),
FileLocationSource::FromServer, dialog_id, old_photo->photos.back().size, 0, "");
LOG_STATUS(td->file_manager_->merge(file_id, old_file_id));
}
@ -3157,7 +3158,8 @@ static auto secret_to_telegram(secret_api::fileLocationUnavailable &file_locatio
// fileLocation#53d69076 dc_id:int volume_id:long local_id:int secret:long = FileLocation;
static auto secret_to_telegram(secret_api::fileLocation &file_location) {
return make_tl_object<telegram_api::fileLocation>(file_location.dc_id_, file_location.volume_id_,
file_location.local_id_, file_location.secret_);
file_location.local_id_, file_location.secret_,
BufferSlice() /*FIXME*/);
}
// photoSizeEmpty#e17e23c type:string = PhotoSize;
@ -3305,9 +3307,9 @@ static auto secret_to_telegram_document(secret_api::decryptedMessageMediaExterna
if (!clean_input_string(from.mime_type_)) {
from.mime_type_.clear();
}
return make_tl_object<telegram_api::document>(from.id_, from.access_hash_, from.date_, from.mime_type_, from.size_,
secret_to_telegram<telegram_api::PhotoSize>(*from.thumb_), from.dc_id_,
0, secret_to_telegram(from.attributes_));
return make_tl_object<telegram_api::document>(
from.id_, from.access_hash_, BufferSlice() /*FIXME*/, from.date_, from.mime_type_, from.size_,
secret_to_telegram<telegram_api::PhotoSize>(*from.thumb_), from.dc_id_, secret_to_telegram(from.attributes_));
}
template <class ToT, class FromT>

View File

@ -13,6 +13,7 @@
#include "td/telegram/DialogDb.h"
#include "td/telegram/DraftMessage.h"
#include "td/telegram/DraftMessage.hpp"
#include "td/telegram/FileReferenceManager.h"
#include "td/telegram/files/FileId.hpp"
#include "td/telegram/files/FileManager.h"
#include "td/telegram/Global.h"
@ -8956,7 +8957,7 @@ void MessagesManager::on_send_secret_message_success(int64 random_id, MessageId
}
new_file_id = td_->file_manager_->register_remote(
FullRemoteFileLocation(FileType::Encrypted, file->id_, file->access_hash_, DcId::internal(file->dc_id_)),
FullRemoteFileLocation(FileType::Encrypted, file->id_, file->access_hash_, DcId::internal(file->dc_id_), ""),
FileLocationSource::FromServer, owner_dialog_id, 0, 0, "");
}
}
@ -20970,6 +20971,16 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, uniq
cancel_user_dialog_action(dialog_id, message.get());
}
if (message_id.is_server()) {
auto file_ids = get_message_content_file_ids(message->content.get(), td_);
if (!file_ids.empty()) {
auto file_source_id = td_->file_reference_manager_->create_file_source(FullMessageId(dialog_id, message_id));
for (auto file_id : file_ids) {
td_->file_manager_->add_file_source(file_id, file_source_id);
}
}
}
{
unique_ptr<Message> *v = find_message(&d->messages, message_id);
if (*v == nullptr && !message->from_database) {

View File

@ -68,6 +68,7 @@ static FileId register_photo(FileManager *file_manager, FileType file_type, int6
int32 local_id;
int64 volume_id;
int64 secret;
std::string file_reference;
switch (location_id) {
case telegram_api::fileLocationUnavailable::ID: {
auto location = move_tl_object_as<telegram_api::fileLocationUnavailable>(location_ptr);
@ -87,6 +88,7 @@ static FileId register_photo(FileManager *file_manager, FileType file_type, int6
local_id = location->local_id_;
volume_id = location->volume_id_;
secret = location->secret_;
file_reference = location->file_reference_.as_slice().str();
break;
}
default:
@ -100,7 +102,7 @@ static FileId register_photo(FileManager *file_manager, FileType file_type, int6
auto suggested_name = PSTRING() << static_cast<uint64>(volume_id) << "_" << static_cast<uint64>(local_id)
<< (is_webp ? ".webp" : ".jpg");
return file_manager->register_remote(
FullRemoteFileLocation(file_type, id, access_hash, local_id, volume_id, secret, dc_id),
FullRemoteFileLocation(file_type, id, access_hash, local_id, volume_id, secret, dc_id, file_reference),
FileLocationSource::FromServer, owner_dialog_id, file_size, 0, std::move(suggested_name));
}
@ -229,7 +231,7 @@ PhotoSize get_thumbnail_photo_size(FileManager *file_manager, BufferSlice bytes,
auto volume_id = Random::secure_int64();
auto secret = 0;
res.file_id = file_manager->register_remote(
FullRemoteFileLocation(FileType::EncryptedThumbnail, 0, 0, local_id, volume_id, secret, dc_id),
FullRemoteFileLocation(FileType::EncryptedThumbnail, 0, 0, local_id, volume_id, secret, dc_id, ""),
FileLocationSource::FromServer, owner_dialog_id, res.size, 0,
PSTRING() << static_cast<uint64>(volume_id) << "_" << static_cast<uint64>(local_id) << ".jpg");
file_manager->set_content(res.file_id, std::move(bytes));
@ -437,7 +439,7 @@ Photo get_photo(FileManager *file_manager, tl_object_ptr<telegram_api::encrypted
tl_object_ptr<secret_api::decryptedMessageMediaPhoto> &&photo, DialogId owner_dialog_id) {
CHECK(DcId::is_valid(file->dc_id_));
FileId file_id = file_manager->register_remote(
FullRemoteFileLocation(FileType::Encrypted, file->id_, file->access_hash_, DcId::internal(file->dc_id_)),
FullRemoteFileLocation(FileType::Encrypted, file->id_, file->access_hash_, DcId::internal(file->dc_id_), ""),
FileLocationSource::FromServer, owner_dialog_id, photo->size_, 0,
PSTRING() << static_cast<uint64>(file->id_) << ".jpg");
file_manager->set_encryption_key(file_id, FileEncryptionKey{photo->key_.as_slice(), photo->iv_.as_slice()});

View File

@ -354,7 +354,8 @@ EncryptedSecureFile get_encrypted_secure_file(FileManager *file_manager,
break;
}
result.file.file_id = file_manager->register_remote(
FullRemoteFileLocation(FileType::Secure, secure_file->id_, secure_file->access_hash_, DcId::internal(dc_id)),
FullRemoteFileLocation(FileType::Secure, secure_file->id_, secure_file->access_hash_, DcId::internal(dc_id),
""),
FileLocationSource::FromServer, DialogId(), 0, secure_file->size_, PSTRING() << secure_file->id_ << ".jpg");
result.file.date = secure_file->date_;
if (result.file.date < 0) {
@ -425,11 +426,12 @@ static td_api::object_ptr<td_api::datedFile> get_dated_file_object(FileManager *
LOG(ERROR) << "Have wrong file in get_dated_file_object";
return nullptr;
}
dated_file.file_id = file_manager->register_remote(
FullRemoteFileLocation(FileType::SecureRaw, file_view.remote_location().get_id(),
file_view.remote_location().get_access_hash(), file_view.remote_location().get_dc_id()),
FileLocationSource::FromServer, DialogId(), file_view.size(), file_view.expected_size(),
file_view.suggested_name());
dated_file.file_id =
file_manager->register_remote(FullRemoteFileLocation(FileType::SecureRaw, file_view.remote_location().get_id(),
file_view.remote_location().get_access_hash(),
file_view.remote_location().get_dc_id(), ""),
FileLocationSource::FromServer, DialogId(), file_view.size(),
file_view.expected_size(), file_view.suggested_name());
return get_dated_file_object(file_manager, dated_file);
}

View File

@ -1029,7 +1029,8 @@ std::pair<int64, FileId> StickersManager::on_get_sticker_document(tl_object_ptr<
int64 document_id = document->id_;
FileId sticker_id = td_->file_manager_->register_remote(
FullRemoteFileLocation(FileType::Sticker, document_id, document->access_hash_, DcId::internal(document->dc_id_)),
FullRemoteFileLocation(FileType::Sticker, document_id, document->access_hash_, DcId::internal(document->dc_id_),
document->file_reference_.as_slice().str()),
FileLocationSource::FromServer, DialogId(), document->size_, 0, PSTRING() << document_id << ".webp");
PhotoSize thumbnail = get_photo_size(td_->file_manager_.get(), FileType::Thumbnail, 0, 0, DialogId(),

View File

@ -32,6 +32,7 @@
#include "td/telegram/DialogId.h"
#include "td/telegram/DialogParticipant.h"
#include "td/telegram/DocumentsManager.h"
#include "td/telegram/FileReferenceManager.h"
#include "td/telegram/files/FileGcParameters.h"
#include "td/telegram/files/FileId.h"
#include "td/telegram/files/FileManager.h"
@ -3632,6 +3633,8 @@ void Td::dec_actor_refcnt() {
LOG(DEBUG) << "ContactsManager was cleared " << timer;
documents_manager_.reset();
LOG(DEBUG) << "DocumentsManager was cleared " << timer;
file_reference_manager_.reset();
LOG(DEBUG) << "FileReferenceManager was cleared " << timer;
file_manager_.reset();
LOG(DEBUG) << "FileManager was cleared " << timer;
inline_queries_manager_.reset();
@ -3805,6 +3808,8 @@ void Td::clear() {
LOG(DEBUG) << "AuthManager actor was cleared " << timer;
contacts_manager_actor_.reset();
LOG(DEBUG) << "ContactsManager actor was cleared " << timer;
file_reference_manager_actor_.reset();
LOG(DEBUG) << "FileReferenceManager actor was cleared " << timer;
file_manager_actor_.reset();
LOG(DEBUG) << "FileManager actor was cleared " << timer;
inline_queries_manager_actor_.reset();
@ -4082,6 +4087,10 @@ Status Td::init(DbKey key) {
private:
Td *td_;
};
file_reference_manager_ = make_unique<FileReferenceManager>();
file_reference_manager_actor_ = register_actor("FileReferenceManager", file_reference_manager_.get());
G()->set_file_reference_manager(file_reference_manager_actor_.get());
file_manager_ = make_unique<FileManager>(make_unique<FileManagerContext>(this));
file_manager_actor_ = register_actor("FileManager", file_manager_.get());
file_manager_->init_actor();

View File

@ -44,6 +44,7 @@ class ConfigManager;
class ContactsManager;
class DeviceTokenManager;
class DocumentsManager;
class FileReferenceManager;
class FileManager;
class InlineQueriesManager;
class HashtagHints;
@ -138,6 +139,8 @@ class Td final : public NetQueryCallback {
ActorOwn<AuthManager> auth_manager_actor_;
unique_ptr<ContactsManager> contacts_manager_;
ActorOwn<ContactsManager> contacts_manager_actor_;
unique_ptr<FileReferenceManager> file_reference_manager_;
ActorOwn<FileReferenceManager> file_reference_manager_actor_;
unique_ptr<FileManager> file_manager_;
ActorOwn<FileManager> file_manager_actor_;
unique_ptr<InlineQueriesManager> inline_queries_manager_;

View File

@ -15,6 +15,7 @@
#include "td/telegram/UniqueId.h"
#include "td/utils/as.h"
#include "td/utils/base64.h"
#include "td/utils/buffer.h"
#include "td/utils/common.h"
#include "td/utils/crypto.h"
@ -267,10 +268,19 @@ Result<std::pair<NetQueryPtr, bool>> FileDownloader::start_part(Part part, int32
return std::make_pair(std::move(net_query), false);
}
Result<size_t> FileDownloader::process_part(Part part, NetQueryPtr net_query) {
Status FileDownloader::check_net_query(NetQueryPtr &net_query) {
if (net_query->is_error()) {
return std::move(net_query->error());
auto error = net_query->move_as_error();
if (begins_with(error.message(), "FILE_REFERENCE_")) {
error = Status::Error(400, PSLICE() << "FILE_REFERENCE_BASE64" << base64_encode(remote_.get_file_reference()));
}
return error;
}
return Status::OK();
}
Result<size_t> FileDownloader::process_part(Part part, NetQueryPtr net_query) {
TRY_STATUS(check_net_query(net_query));
BufferSlice bytes;
bool need_cdn_decrypt = false;
@ -380,6 +390,7 @@ FileLoader::Callback *FileDownloader::get_callback() {
Status FileDownloader::process_check_query(NetQueryPtr net_query) {
has_hash_query_ = false;
TRY_STATUS(check_net_query(net_query));
TRY_RESULT(file_hashes, fetch_result<telegram_api::upload_getCdnFileHashes>(std::move(net_query)));
add_hash_info(file_hashes);
return Status::OK();

View File

@ -96,5 +96,7 @@ class FileDownloader : public FileLoader {
void keep_fd_flag(bool keep_fd) override;
void try_release_fd();
Status acquire_fd() TD_WARN_UNUSED_RESULT;
Status check_net_query(NetQueryPtr &net_query);
};
} // namespace td

View File

@ -133,7 +133,8 @@ Status FileHashUploader::on_result_impl(NetQueryPtr net_query) {
case telegram_api::document::ID: {
auto document = move_tl_object_as<telegram_api::document>(res);
callback_->on_ok(FullRemoteFileLocation(FileType::Document, document->id_, document->access_hash_,
DcId::internal(document->dc_id_)));
DcId::internal(document->dc_id_),
document->file_reference_.as_slice().str()));
stop_flag_ = true;
return Status::OK();

View File

@ -14,6 +14,8 @@
namespace td {
using FileSourceId = int32;
class FileId {
int32 id = 0;
int32 remote_id = 0;

View File

@ -366,8 +366,10 @@ class FullRemoteFileLocation {
private:
static constexpr int32 WEB_LOCATION_FLAG = 1 << 24;
static constexpr int32 FILE_REFERENCE_FLAG = 1 << 25;
bool web_location_flag_{false};
DcId dc_id_;
std::string file_reference_;
enum class LocationType : int32 { Web, Photo, Common, None };
Variant<WebRemoteFileLocation, PhotoRemoteFileLocation, CommonRemoteFileLocation> variant_;
@ -429,6 +431,9 @@ class FullRemoteFileLocation {
if (is_web()) {
type |= WEB_LOCATION_FLAG;
}
if (!file_reference_.empty()) {
type |= FILE_REFERENCE_FLAG;
}
return type;
}
@ -438,6 +443,9 @@ class FullRemoteFileLocation {
using ::td::store;
store(full_type(), storer);
store(dc_id_.get_value(), storer);
if (!file_reference_.empty()) {
store(file_reference_, storer);
}
variant_.visit([&](auto &&value) {
using td::store;
store(value, storer);
@ -450,6 +458,8 @@ class FullRemoteFileLocation {
parse(raw_type, parser);
web_location_flag_ = (raw_type & WEB_LOCATION_FLAG) != 0;
raw_type &= ~WEB_LOCATION_FLAG;
bool has_file_reference = (raw_type & FILE_REFERENCE_FLAG) != 0;
raw_type &= ~FILE_REFERENCE_FLAG;
if (raw_type < 0 || raw_type >= static_cast<int32>(FileType::Size)) {
return parser.set_error("Invalid FileType in FullRemoteFileLocation");
}
@ -458,6 +468,10 @@ class FullRemoteFileLocation {
parse(dc_id_value, parser);
dc_id_ = DcId::from_value(dc_id_value);
if (has_file_reference) {
parse(file_reference_, parser);
}
switch (location_type()) {
case LocationType::Web: {
variant_ = WebRemoteFileLocation();
@ -525,6 +539,19 @@ class FullRemoteFileLocation {
return 0;
}
}
bool delete_file_reference(Slice bad_file_reference) {
if (file_reference_.empty()) {
return false;
}
if (!bad_file_reference.empty() && file_reference_ != bad_file_reference) {
return false;
}
file_reference_ = {};
return true;
}
string get_file_reference() const {
return file_reference_;
}
string get_url() const {
if (is_web()) {
return web().url_;
@ -567,14 +594,16 @@ class FullRemoteFileLocation {
tl_object_ptr<telegram_api::InputFileLocation> as_input_file_location() const {
switch (location_type()) {
case LocationType::Photo:
return make_tl_object<telegram_api::inputFileLocation>(photo().volume_id_, photo().local_id_, photo().secret_);
return make_tl_object<telegram_api::inputFileLocation>(photo().volume_id_, photo().local_id_, photo().secret_,
BufferSlice(file_reference_));
case LocationType::Common:
if (is_encrypted_secret()) {
return make_tl_object<telegram_api::inputEncryptedFileLocation>(common().id_, common().access_hash_);
} else if (is_secure()) {
return make_tl_object<telegram_api::inputSecureFileLocation>(common().id_, common().access_hash_);
} else {
return make_tl_object<telegram_api::inputDocumentFileLocation>(common().id_, common().access_hash_, 0);
return make_tl_object<telegram_api::inputDocumentFileLocation>(common().id_, common().access_hash_,
BufferSlice(file_reference_));
}
case LocationType::Web:
case LocationType::None:
@ -588,13 +617,14 @@ class FullRemoteFileLocation {
tl_object_ptr<telegram_api::InputDocument> as_input_document_impl(const char *file, int line) const {
CHECK(is_common()) << file << ' ' << line;
CHECK(is_document()) << file << ' ' << line;
return make_tl_object<telegram_api::inputDocument>(common().id_, common().access_hash_);
return make_tl_object<telegram_api::inputDocument>(common().id_, common().access_hash_,
BufferSlice(file_reference_));
}
#define as_input_photo() as_input_photo_impl(__FILE__, __LINE__)
tl_object_ptr<telegram_api::InputPhoto> as_input_photo_impl(const char *file, int line) const {
CHECK(is_photo()) << file << ' ' << line;
return make_tl_object<telegram_api::inputPhoto>(photo().id_, photo().access_hash_);
return make_tl_object<telegram_api::inputPhoto>(photo().id_, photo().access_hash_, BufferSlice(file_reference_));
}
tl_object_ptr<telegram_api::InputEncryptedFile> as_input_encrypted_file() const {
@ -611,14 +641,18 @@ class FullRemoteFileLocation {
// TODO: this constructor is just for immediate unserialize
FullRemoteFileLocation() = default;
FullRemoteFileLocation(FileType file_type, int64 id, int64 access_hash, int32 local_id, int64 volume_id, int64 secret,
DcId dc_id)
DcId dc_id, std::string file_reference)
: file_type_(file_type)
, dc_id_(dc_id)
, file_reference_(std::move(file_reference))
, variant_(PhotoRemoteFileLocation{id, access_hash, volume_id, secret, local_id}) {
CHECK(is_photo());
}
FullRemoteFileLocation(FileType file_type, int64 id, int64 access_hash, DcId dc_id)
: file_type_(file_type), dc_id_(dc_id), variant_(CommonRemoteFileLocation{id, access_hash}) {
FullRemoteFileLocation(FileType file_type, int64 id, int64 access_hash, DcId dc_id, std::string file_reference)
: file_type_(file_type)
, dc_id_(dc_id)
, file_reference_(std::move(file_reference))
, variant_(CommonRemoteFileLocation{id, access_hash}) {
CHECK(is_common());
}
FullRemoteFileLocation(FileType file_type, string url, int64 access_hash)
@ -680,6 +714,10 @@ inline StringBuilder &operator<<(StringBuilder &string_builder,
if (!full_remote_file_location.is_web()) {
string_builder << ", " << full_remote_file_location.get_dc_id();
}
if (!full_remote_file_location.file_reference_.empty()) {
string_builder << ", "
<< tag("file_reference", format::as_hex_dump<0>(Slice(full_remote_file_location.file_reference_)));
}
string_builder << ", location = ";
if (full_remote_file_location.is_web()) {
@ -748,11 +786,12 @@ class RemoteFileLocation {
explicit RemoteFileLocation(const PartialRemoteFileLocation &partial) : variant_(partial) {
}
RemoteFileLocation(FileType file_type, int64 id, int64 access_hash, int32 local_id, int64 volume_id, int64 secret,
DcId dc_id)
: variant_(FullRemoteFileLocation{file_type, id, access_hash, local_id, volume_id, secret, dc_id}) {
DcId dc_id, std::string file_reference)
: variant_(FullRemoteFileLocation{file_type, id, access_hash, local_id, volume_id, secret, dc_id,
std::move(file_reference)}) {
}
RemoteFileLocation(FileType file_type, int64 id, int64 access_hash, DcId dc_id)
: variant_(FullRemoteFileLocation{file_type, id, access_hash, dc_id}) {
RemoteFileLocation(FileType file_type, int64 id, int64 access_hash, DcId dc_id, std::string file_reference)
: variant_(FullRemoteFileLocation{file_type, id, access_hash, dc_id, std::move(file_reference)}) {
}
private:
@ -1006,8 +1045,7 @@ class LocalFileLocation {
LocalFileLocation() : variant_{EmptyLocalFileLocation()} {
}
explicit LocalFileLocation(const PartialLocalFileLocation &partial)
: variant_(PartialLocalFileLocationPtr(partial)) {
explicit LocalFileLocation(const PartialLocalFileLocation &partial) : variant_(PartialLocalFileLocationPtr(partial)) {
}
explicit LocalFileLocation(const FullLocalFileLocation &full) : variant_(full) {
}

View File

@ -9,6 +9,7 @@
#include "td/telegram/telegram_api.h"
#include "td/telegram/ConfigShared.h"
#include "td/telegram/FileReferenceManager.h"
#include "td/telegram/files/FileDb.h"
#include "td/telegram/files/FileLoaderUtils.h"
#include "td/telegram/files/FileLocation.h"
@ -158,6 +159,12 @@ void FileNode::set_remote_location(const RemoteFileLocation &remote, FileLocatio
on_changed();
}
void FileNode::delete_file_reference(Slice file_reference) {
if (remote_.type() == RemoteFileLocation::Type::Full && remote_.full().delete_file_reference(file_reference)) {
on_pmc_changed();
}
}
void FileNode::set_generate_location(unique_ptr<FullGenerateFileLocation> &&generate) {
bool is_changed = generate_ == nullptr ? generate != nullptr : generate == nullptr || *generate_ != *generate;
if (is_changed) {
@ -246,6 +253,21 @@ void FileNode::set_generate_priority(int8 download_priority, int8 upload_priorit
generate_upload_priority_ = upload_priority;
}
void FileNode::add_file_source(FileSourceId file_source_id) {
if (std::find(file_source_ids_.begin(), file_source_ids_.end(), file_source_id) != file_source_ids_.end()) {
return;
}
file_source_ids_.push_back(file_source_id);
}
void FileNode::remove_file_source(FileSourceId file_source_id) {
auto it = std::find(file_source_ids_.begin(), file_source_ids_.end(), file_source_id);
if (it == file_source_ids_.end()) {
return;
}
file_source_ids_.erase(it);
}
void FileNode::on_changed() {
on_pmc_changed();
on_info_changed();
@ -333,6 +355,15 @@ const FullLocalFileLocation &FileView::local_location() const {
bool FileView::has_remote_location() const {
return node_->remote_.type() == RemoteFileLocation::Type::Full;
}
bool FileView::has_active_remote_location() const {
if (!has_remote_location()) {
return false;
}
if (remote_location().is_encrypted_any()) {
return true;
}
return !remote_location().get_file_reference().empty();
}
const FullRemoteFileLocation &FileView::remote_location() const {
CHECK(has_remote_location());
auto *remote = node_.get_remote();
@ -952,6 +983,14 @@ static int merge_choose_remote_location(const RemoteFileLocation &x, int8 x_sour
if (x.full().is_web() != y.full().is_web()) {
return x.full().is_web(); // prefer non-web
}
auto x_ref = !x.full().get_file_reference().empty();
auto y_ref = !y.full().get_file_reference().empty();
if (x_ref || y_ref) {
if (x_ref != y_ref) {
return !x_ref;
}
return x_source < y_source;
}
if (x.full().get_access_hash() != y.full().get_access_hash()) {
return x_source < y_source;
}
@ -1064,7 +1103,7 @@ void FileManager::cancel_generate(FileNodePtr node) {
}
Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) {
LOG(DEBUG) << x_file_id << " VS " << y_file_id;
LOG(ERROR) << x_file_id << " VS " << y_file_id;
if (!x_file_id.is_valid()) {
return Status::Error("First file_id is invalid");
@ -1158,6 +1197,8 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
<< ", generate_i = " << generate_i << ", size_i = " << size_i << ", remote_name_i = " << remote_name_i
<< ", url_i = " << url_i << ", owner_i = " << owner_i << ", encryption_key_i = " << encryption_key_i
<< ", main_file_id_i = " << main_file_id_i;
LOG(ERROR) << FileView(node).has_active_remote_location();
LOG(ERROR) << FileView(other_node).has_active_remote_location();
if (local_i == other_node_i) {
cancel_download(node);
node->set_download_offset(other_node->download_offset_);
@ -1238,6 +1279,9 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
}
node->need_load_from_pmc_ |= other_node->need_load_from_pmc_;
node->can_search_locally_ &= other_node->can_search_locally_;
for (auto source_id : other_node->file_source_ids_) {
node->add_file_source(source_id);
}
if (main_file_id_i == other_node_i) {
node->main_file_id_ = other_node->main_file_id_;
@ -1297,6 +1341,24 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
return node->main_file_id_;
}
void FileManager::add_file_source(FileId file_id, FileSourceId file_source_id) {
LOG(ERROR) << "Add file source " << file_id << " " << file_source_id;
auto node = get_file_node(file_id);
if (!node) {
return;
}
node->add_file_source(file_source_id);
}
void FileManager::remove_file_source(FileId file_id, FileSourceId file_source_id) {
LOG(ERROR) << "Remove file source " << file_id, file_source_id;
auto node = get_file_node(file_id);
if (!node) {
return;
}
node->remove_file_source(file_source_id);
}
void FileManager::try_flush_node_full(FileNodePtr node, bool new_remote, bool new_local, bool new_generate,
FileDbId other_pmc_id) {
if (node->need_pmc_flush()) {
@ -1682,6 +1744,31 @@ void FileManager::run_download(FileNodePtr node) {
CHECK(node->download_id_ == 0);
CHECK(!node->file_ids_.empty());
auto file_id = node->file_ids_.back();
// If file reference is needed
if (!file_view.has_active_remote_location()) {
LOG(ERROR) << "run_download: no active location";
QueryId id = queries_container_.create(Query{file_id, Query::Download});
node->download_id_ = id;
if (node->file_source_ids_.empty()) {
on_error(id, Status::Error("Can't download file: no valid file refernce and no valid source id"));
return;
}
send_closure(G()->file_reference_manager(), &FileReferenceManager::update_file_reference, file_id,
node->file_source_ids_, PromiseCreator::lambda([id, actor_id = actor_id(this)](Result<Unit> res) {
Status error;
LOG(ERROR) << "run_download: update_file_reference finished";
if (res.is_ok()) {
error = td::Status::Error("FILE_DOWNLOAD_RESTART_WITH_FILE_REFERENCE");
} else {
error = res.move_as_error();
}
send_closure(actor_id, &FileManager::on_error, id, std::move(error));
}));
return;
}
QueryId id = queries_container_.create(Query{file_id, Query::Download});
node->download_id_ = id;
node->is_download_started_ = false;
@ -1708,7 +1795,7 @@ void FileManager::resume_upload(FileId file_id, std::vector<int> bad_parts, std:
node->set_upload_pause(FileId());
}
FileView file_view(node);
if (file_view.has_remote_location() && file_view.get_type() != FileType::Thumbnail &&
if (file_view.has_active_remote_location() && file_view.get_type() != FileType::Thumbnail &&
file_view.get_type() != FileType::EncryptedThumbnail) {
LOG(INFO) << "File " << file_id << " is already uploaded";
if (callback) {
@ -1779,6 +1866,17 @@ bool FileManager::delete_partial_remote_location(FileId file_id) {
return true;
}
void FileManager::delete_file_reference(FileId file_id, std::string file_reference) {
LOG(ERROR) << "Delete file reference " << file_id << tag("reference", format::as_hex_dump<0>(Slice(file_reference)));
auto node = get_sync_file_node(file_id);
if (!node) {
LOG(INFO) << "Wrong file id " << file_id;
return;
}
node->delete_file_reference(file_reference);
try_flush_node(node, "delete_file_reference");
}
void FileManager::external_file_generate_progress(int64 id, int32 expected_size, int32 local_prefix_size,
Promise<> promise) {
send_closure(file_generate_manager_, &FileGenerateManager::external_file_generate_progress, id, expected_size,
@ -1930,6 +2028,18 @@ void FileManager::run_upload(FileNodePtr node, std::vector<int> bad_parts) {
}
CHECK(node->upload_id_ == 0);
if (file_view.has_remote_location() && !file_view.has_active_remote_location() &&
file_view.get_type() != FileType::Thumbnail && file_view.get_type() != FileType::EncryptedThumbnail &&
!node->file_source_ids_.empty()) {
QueryId id = queries_container_.create(Query{file_id, Query::Upload});
node->upload_id_ = id;
send_closure(G()->file_reference_manager(), &FileReferenceManager::update_file_reference, file_id,
node->file_source_ids_, PromiseCreator::lambda([id, actor_id = actor_id(this)](Result<Unit> res) {
send_closure(actor_id, &FileManager::on_error, id, Status::Error("FILE_UPLOAD_RESTART"));
}));
return;
}
if (node->remote_.type() != RemoteFileLocation::Type::Partial && node->get_by_hash_) {
LOG(INFO) << "Get file " << node->main_file_id_ << " by hash";
QueryId id = queries_container_.create(Query{file_id, Query::UploadByHash});
@ -2604,6 +2714,7 @@ void FileManager::on_error_impl(FileNodePtr node, FileManager::Query::Type type,
SCOPE_EXIT {
try_flush_node(node, "on_error");
};
//FIXME: should we apply error if !was_active?
if (status.code() != 1 && !G()->close_flag()) {
LOG(WARNING) << "Failed to upload/download/generate file: " << status << ". Query type = " << type
<< ". File type is " << FileView(node).get_type();
@ -2645,15 +2756,40 @@ void FileManager::on_error_impl(FileNodePtr node, FileManager::Query::Type type,
if (begins_with(status.message(), "FILE_GENERATE_LOCATION_INVALID")) {
node->set_generate_location(nullptr);
}
if (begins_with(status.message(), "FILE_REFERENCE_")) {
string file_reference;
Slice prefix = "FILE_REFERENCE_BASE64";
if (begins_with(status.message(), prefix)) {
auto tmp = base64_decode(status.message().substr(prefix.size()));
LOG_IF(WARNING, tmp.is_error()) << "Can't decode file reference from error " << status << " " << tmp.error();
if (tmp.is_ok()) {
file_reference = tmp.move_as_ok();
}
}
CHECK(!node->file_ids_.empty());
delete_file_reference(node->file_ids_.back(), file_reference);
run_download(node);
return;
}
if (status.message() == "FILE_UPLOAD_RESTART") {
run_upload(node, {});
return;
}
if (status.message() == "FILE_DOWNLOAD_RESTART") {
node->can_search_locally_ = false;
run_download(node);
return;
if (begins_with(status.message(), "FILE_DOWNLOAD_RESTART")) {
if (ends_with(status.message(), "WITH_FILE_REFERENCE")) {
if (FileView(node).has_active_remote_location()) {
LOG(ERROR) << "??????";
run_download(node);
return;
}
LOG_IF(WARNING, !node->file_source_ids_.empty())
<< "Got no active remote locations " << FileView(node).remote_location();
} else {
node->can_search_locally_ = false;
run_download(node);
return;
}
}
if (!was_active) {

View File

@ -65,6 +65,7 @@ class FileNode {
void set_local_location(const LocalFileLocation &local, int64 ready_size, int64 prefix_offset,
int64 ready_prefix_size);
void set_remote_location(const RemoteFileLocation &remote, FileLocationSource source, int64 ready_size);
void delete_file_reference(Slice file_reference);
void set_generate_location(unique_ptr<FullGenerateFileLocation> &&generate);
void set_size(int64 size);
void set_expected_size(int64 expected_size);
@ -80,6 +81,9 @@ class FileNode {
void set_download_offset(int64 download_offset);
void add_file_source(FileSourceId file_source_id);
void remove_file_source(FileSourceId file_source_id);
void on_changed();
void on_info_changed();
void on_pmc_changed();
@ -106,6 +110,8 @@ class FileNode {
FileLoadManager::QueryId download_id_ = 0;
int64 remote_ready_size_ = 0;
std::vector<FileSourceId> file_source_ids_;
unique_ptr<FullGenerateFileLocation> generate_;
FileLoadManager::QueryId generate_id_ = 0;
@ -204,6 +210,7 @@ class FileView {
bool has_local_location() const;
const FullLocalFileLocation &local_location() const;
bool has_remote_location() const;
bool has_active_remote_location() const;
const FullRemoteFileLocation &remote_location() const;
bool has_generate_location() const;
const FullGenerateFileLocation &generate_location() const;
@ -343,6 +350,10 @@ class FileManager : public FileLoadManager::Callback {
Result<FileId> merge(FileId x_file_id, FileId y_file_id, bool no_sync = false) TD_WARN_UNUSED_RESULT;
void add_file_source(FileId file_id, FileSourceId file_source_id);
void remove_file_source(FileId file_id, FileSourceId file_source_id);
bool set_encryption_key(FileId file_id, FileEncryptionKey key);
bool set_content(FileId file_id, BufferSlice bytes);
@ -351,6 +362,7 @@ class FileManager : public FileLoadManager::Callback {
void resume_upload(FileId file_id, std::vector<int> bad_parts, std::shared_ptr<UploadCallback> callback,
int32 new_priority, uint64 upload_order);
bool delete_partial_remote_location(FileId file_id);
void delete_file_reference(FileId file_id, std::string file_reference);
void get_content(FileId file_id, Promise<BufferSlice> promise);
void delete_file(FileId file_id, Promise<Unit> promise, const char *source);

View File

@ -21,7 +21,7 @@ class HeaderStorer {
}
template <class StorerT>
void store(StorerT &storer) const {
constexpr int32 LAYER = 85;
constexpr int32 LAYER = 86;
using td::store;
// invokeWithLayer#da9b0d0d {X:Type} layer:int query:!X = X;