tdlight-team/tdlight
tdlight-team/tdlight
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

FileManager: support two file references for photos

Browse Source 
GitOrigin-RevId: f27a9867315e12e79a456c12949d8d2580531bbe
This commit is contained in:
Arseny Smirnov 2019-01-30 20:08:50 +04:00
parent b481ce8465
commit 2032f89723
11 changed files with 181 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
td/telegram/DocumentsManager.cpp
View File

@ -208,7 +208,7 @@ std::pair<DocumentsManager::DocumentType, FileId> DocumentsManager::on_get_docum
file_reference = document->file_reference_.as_slice().str();
if (document_type != DocumentType::VoiceNote) {
thumbnail = get_photo_size(td_->file_manager_.get(), FileType::Thumbnail, 0, 0, owner_dialog_id,
thumbnail = get_photo_size(td_->file_manager_.get(), FileType::Thumbnail, 0, 0, "", owner_dialog_id,
std::move(document->thumb_), has_webp_thumbnail);
}
} else if (remote_document.secret_file != nullptr) {

3
td/telegram/FileReferenceManager.cpp
View File

@ -190,7 +190,8 @@ void FileReferenceManager::send_query(Destination dest, FileSourceId file_source
new_promise = std::move(new_promise)]() mutable {
auto view = file_manager.get_actor_unsafe()->get_file_view(dest.node_id);
CHECK(!view.empty());
if (result.is_ok() && !view.has_active_remote_location()) {
if (result.is_ok() &&
(!view.has_active_upload_remote_location() || !view.has_active_download_remote_location())) {
result = Status::Error("No active remote location");
}
if (result.is_error() && result.error().code() != 429 && result.error().code() < 500) {

7
td/telegram/MessageContent.cpp
View File

@ -2122,7 +2122,7 @@ tl_object_ptr<telegram_api::InputMedia> get_input_media(const MessageContent *co
}
if (!was_uploaded) {
auto file_reference = FileManager::extract_file_reference(input_media);
if (file_reference == FullRemoteFileLocation::invalid_file_reference()) {
if (file_reference == FileReferenceView::invalid_file_reference()) {
return nullptr;
}
}
@ -2132,7 +2132,7 @@ tl_object_ptr<telegram_api::InputMedia> get_input_media(const MessageContent *co
tl_object_ptr<telegram_api::InputMedia> get_input_media(const MessageContent *content, Td *td, int32 ttl) {
auto input_media = get_input_media(content, td, nullptr, nullptr, ttl);
auto file_reference = FileManager::extract_file_reference(input_media);
if (file_reference == FullRemoteFileLocation::invalid_file_reference()) {
if (file_reference == FileReferenceView::invalid_file_reference()) {
return nullptr;
}
return input_media;
@ -2788,7 +2788,8 @@ void merge_message_contents(Td *td, MessageContent *old_content, MessageContent
FileId file_id = td->file_manager_->register_remote(
FullRemoteFileLocation(FileType::Photo, new_file_view.remote_location().get_id(),
new_file_view.remote_location().get_access_hash(), 0, 0, 0, DcId::invalid(),
new_file_view.remote_location().get_file_reference()),
new_file_view.remote_location().get_upload_file_reference().str(),
new_file_view.remote_location().get_download_file_reference().str()),
FileLocationSource::FromServer, dialog_id, old_photo->photos.back().size, 0, "");
LOG_STATUS(td->file_manager_->merge(file_id, old_file_id));
}

36
td/telegram/Photo.cpp
View File

@ -62,13 +62,14 @@ StringBuilder &operator<<(StringBuilder &string_builder, const Dimensions &dimen
}
static FileId register_photo(FileManager *file_manager, FileType file_type, int64 id, int64 access_hash,
std::string upload_file_reference,
tl_object_ptr<telegram_api::FileLocation> &&location_ptr, DialogId owner_dialog_id,
int32 file_size, bool is_webp = false) {
DcId dc_id;
int32 local_id;
int64 volume_id;
int64 secret;
std::string file_reference;
std::string download_file_reference;
switch (location_ptr->get_id()) {
case telegram_api::fileLocationUnavailable::ID: {
auto location = move_tl_object_as<telegram_api::fileLocationUnavailable>(location_ptr);
@ -88,7 +89,7 @@ static FileId register_photo(FileManager *file_manager, FileType file_type, int6
local_id = location->local_id_;
volume_id = location->volume_id_;
secret = location->secret_;
file_reference = location->file_reference_.as_slice().str();
download_file_reference = location->file_reference_.as_slice().str();
break;
}
default:
@ -101,9 +102,10 @@ static FileId register_photo(FileManager *file_manager, FileType file_type, int6
<< ")";
auto suggested_name = PSTRING() << static_cast<uint64>(volume_id) << "_" << static_cast<uint64>(local_id)
<< (is_webp ? ".webp" : ".jpg");
return file_manager->register_remote(
FullRemoteFileLocation(file_type, id, access_hash, local_id, volume_id, secret, dc_id, file_reference),
FileLocationSource::FromServer, owner_dialog_id, file_size, 0, std::move(suggested_name));
return file_manager->register_remote(FullRemoteFileLocation(file_type, id, access_hash, local_id, volume_id, secret,
dc_id, upload_file_reference, download_file_reference),
FileLocationSource::FromServer, owner_dialog_id, file_size, 0,
std::move(suggested_name));
}
ProfilePhoto get_profile_photo(FileManager *file_manager,
@ -118,9 +120,9 @@ ProfilePhoto get_profile_photo(FileManager *file_manager,
auto profile_photo = move_tl_object_as<telegram_api::userProfilePhoto>(profile_photo_ptr);
result.id = profile_photo->photo_id_;
result.small_file_id = register_photo(file_manager, FileType::ProfilePhoto, result.id, 0,
result.small_file_id = register_photo(file_manager, FileType::ProfilePhoto, result.id, 0, "",
std::move(profile_photo->photo_small_), DialogId(), 0);
result.big_file_id = register_photo(file_manager, FileType::ProfilePhoto, result.id, 0,
result.big_file_id = register_photo(file_manager, FileType::ProfilePhoto, result.id, 0, "",
std::move(profile_photo->photo_big_), DialogId(), 0);
break;
}
@ -179,10 +181,10 @@ DialogPhoto get_dialog_photo(FileManager *file_manager, tl_object_ptr<telegram_a
case telegram_api::chatPhoto::ID: {
auto chat_photo = move_tl_object_as<telegram_api::chatPhoto>(chat_photo_ptr);
result.small_file_id = register_photo(file_manager, FileType::ProfilePhoto, 0, 0,
result.small_file_id = register_photo(file_manager, FileType::ProfilePhoto, 0, 0, "",
std::move(chat_photo->photo_small_), DialogId(), 0);
result.big_file_id =
register_photo(file_manager, FileType::ProfilePhoto, 0, 0, std::move(chat_photo->photo_big_), DialogId(), 0);
result.big_file_id = register_photo(file_manager, FileType::ProfilePhoto, 0, 0, "",
std::move(chat_photo->photo_big_), DialogId(), 0);
break;
}
@ -242,7 +244,7 @@ PhotoSize get_thumbnail_photo_size(FileManager *file_manager, BufferSlice bytes,
auto volume_id = Random::secure_int64();
auto secret = 0;
res.file_id = file_manager->register_remote(
FullRemoteFileLocation(FileType::EncryptedThumbnail, 0, 0, local_id, volume_id, secret, dc_id, ""),
FullRemoteFileLocation(FileType::EncryptedThumbnail, 0, 0, local_id, volume_id, secret, dc_id, "", ""),
FileLocationSource::FromServer, owner_dialog_id, res.size, 0,
PSTRING() << static_cast<uint64>(volume_id) << "_" << static_cast<uint64>(local_id) << ".jpg");
file_manager->set_content(res.file_id, std::move(bytes));
@ -251,7 +253,8 @@ PhotoSize get_thumbnail_photo_size(FileManager *file_manager, BufferSlice bytes,
}
PhotoSize get_photo_size(FileManager *file_manager, FileType file_type, int64 id, int64 access_hash,
DialogId owner_dialog_id, tl_object_ptr<telegram_api::PhotoSize> &&size_ptr, bool is_webp) {
std::string upload_file_reference, DialogId owner_dialog_id,
tl_object_ptr<telegram_api::PhotoSize> &&size_ptr, bool is_webp) {
tl_object_ptr<telegram_api::FileLocation> location_ptr;
string type;
@ -289,8 +292,8 @@ PhotoSize get_photo_size(FileManager *file_manager, FileType file_type, int64 id
break;
}
res.file_id = register_photo(file_manager, file_type, id, access_hash, std::move(location_ptr), owner_dialog_id,
res.size, is_webp);
res.file_id = register_photo(file_manager, file_type, id, access_hash, upload_file_reference, std::move(location_ptr),
owner_dialog_id, res.size, is_webp);
if (!content.empty()) {
file_manager->set_content(res.file_id, std::move(content));
@ -482,8 +485,9 @@ Photo get_photo(FileManager *file_manager, tl_object_ptr<telegram_api::photo> &&
// TODO use file_reference
for (auto &size_ptr : photo->sizes_) {
res.photos.push_back(get_photo_size(file_manager, FileType::Photo, photo->id_, photo->access_hash_, owner_dialog_id,
std::move(size_ptr), false));
res.photos.push_back(get_photo_size(file_manager, FileType::Photo, photo->id_, photo->access_hash_,
photo->file_reference_.as_slice().str(), owner_dialog_id, std::move(size_ptr),
false));
}
return res;

3
td/telegram/Photo.h
View File

@ -83,7 +83,8 @@ StringBuilder &operator<<(StringBuilder &string_builder, const DialogPhoto &dial
PhotoSize get_thumbnail_photo_size(FileManager *file_manager, BufferSlice bytes, DialogId owner_dialog_id, int32 width,
int32 height);
PhotoSize get_photo_size(FileManager *file_manager, FileType file_type, int64 id, int64 access_hash,
DialogId owner_dialog_id, tl_object_ptr<telegram_api::PhotoSize> &&size_ptr, bool is_webp);
std::string upload_file_reference, DialogId owner_dialog_id,
tl_object_ptr<telegram_api::PhotoSize> &&size_ptr, bool is_webp);
PhotoSize get_web_document_photo_size(FileManager *file_manager, FileType file_type, DialogId owner_dialog_id,
tl_object_ptr<telegram_api::WebDocument> web_document_ptr);
td_api::object_ptr<td_api::photoSize> get_photo_size_object(FileManager *file_manager, const PhotoSize *photo_size);

2
td/telegram/StickersManager.cpp
View File

@ -1069,7 +1069,7 @@ std::pair<int64, FileId> StickersManager::on_get_sticker_document(tl_object_ptr<
document->file_reference_.as_slice().str()),
FileLocationSource::FromServer, DialogId(), document->size_, 0, PSTRING() << document_id << ".webp");
PhotoSize thumbnail = get_photo_size(td_->file_manager_.get(), FileType::Thumbnail, 0, 0, DialogId(),
PhotoSize thumbnail = get_photo_size(td_->file_manager_.get(), FileType::Thumbnail, 0, 0, "", DialogId(),
std::move(document->thumb_), has_webp_thumbnail(sticker));
create_sticker(sticker_id, std::move(thumbnail), dimensions, from_message, std::move(sticker), nullptr);

9
td/telegram/WallpaperManager.cpp
View File

@ -97,10 +97,11 @@ void WallpaperManager::on_get_wallpapers(Result<vector<telegram_api::object_ptr<
switch (wallpaper_ptr->get_id()) {
case telegram_api::wallPaper::ID: {
auto wallpaper = move_tl_object_as<telegram_api::wallPaper>(wallpaper_ptr);
vector<PhotoSize> sizes = transform(
std::move(wallpaper->sizes_), [file_manager](tl_object_ptr<telegram_api::PhotoSize> &&photo_size) {
return get_photo_size(file_manager, FileType::Wallpaper, 0, 0, DialogId(), std::move(photo_size), false);
});
vector<PhotoSize> sizes = transform(std::move(wallpaper->sizes_),
[file_manager](tl_object_ptr<telegram_api::PhotoSize> &&photo_size) {
return get_photo_size(file_manager, FileType::Wallpaper, 0, 0, "",
DialogId(), std::move(photo_size), false);
});
return Wallpaper{wallpaper->id_, std::move(sizes), wallpaper->color_};
}
case telegram_api::wallPaperSolid::ID: {

4
td/telegram/files/FileDownloader.cpp
View File

@ -274,8 +274,8 @@ Status FileDownloader::check_net_query(NetQueryPtr &net_query) {
if (net_query->is_error()) {
auto error = net_query->move_as_error();
if (FileReferenceManager::is_file_reference_error(error)) {
error = Status::Error(error.code(),
PSLICE() << error.message() << "#BASE64" << base64_encode(remote_.get_file_reference()));
error = Status::Error(error.code(), PSLICE() << error.message() << "#BASE64"
<< base64_encode(remote_.get_download_file_reference()));
}
return error;
}

143
td/telegram/files/FileLocation.h
View File

@ -25,6 +25,93 @@
#include <tuple>
namespace td {
class FileReferenceView {
public:
static Slice invalid_file_reference() {
return "#";
}
static std::string create_one(td::Slice first) {
char second_length = char(255);
return PSTRING() << second_length << first;
}
static std::string create_two(td::Slice first, td::Slice second = {}) {
if (second.size() >= 255) {
LOG(ERROR) << "File reference is too big " << base64_encode(second);
second = invalid_file_reference();
}
char second_length = narrow_cast<unsigned char>(second.size());
return PSTRING() << second_length << first << second;
}
std::string create(td::Slice first, td::Slice second) const {
if (size() == 1) {
return create_one(first);
}
return create_two(first, second);
}
FileReferenceView(Slice data) {
if (data.empty()) {
size_ = 1;
return;
}
unsigned char second_size = data.ubegin()[0];
if (second_size == 255) {
first_ = data.substr(1);
second_ = data.substr(1);
size_ = 1;
} else {
if (second_size > data.size() - 1) {
size_ = 1;
return;
}
auto first_size = data.size() - 1 - second_size;
first_ = data.substr(1, first_size);
second_ = data.substr(1 + first_size);
size_ = 2;
}
}
size_t size() const {
return size_;
}
Slice first() const {
return first_;
}
Slice second() const {
return second_;
}
bool has_first() const {
return first() != invalid_file_reference();
}
bool has_second() const {
return second() != invalid_file_reference();
}
std::pair<std::string, bool> delete_file_reference(Slice bad_file_reference) const {
if (bad_file_reference == FileReferenceView::invalid_file_reference()) {
return {"", false};
}
auto first = this->first();
auto second = this->second();
bool changed = false;
if (first == bad_file_reference) {
first = invalid_file_reference();
changed = true;
}
if (second == bad_file_reference) {
second = invalid_file_reference();
changed = true;
}
if (!changed) {
return {"", false};
}
return {create(first, second), true};
}
private:
Slice first_;
Slice second_;
int size_;
};
struct EmptyRemoteFileLocation {
template <class StorerT>
@ -399,25 +486,31 @@ class FullRemoteFileLocation {
return 0;
}
}
static Slice invalid_file_reference() {
return "#";
}
bool delete_file_reference(Slice bad_file_reference) {
if (file_reference_ == invalid_file_reference()) {
return false;
auto res = FileReferenceView(file_reference_).delete_file_reference(bad_file_reference);
if (res.second) {
file_reference_ = res.first;
}
if (file_reference_ != bad_file_reference) {
return false;
}
file_reference_ = invalid_file_reference().str();
return true;
return res.second;
}
bool has_file_reference() const {
return file_reference_ != invalid_file_reference();
bool has_upload_file_reference() const {
return FileReferenceView(file_reference_).has_first();
}
const string &get_file_reference() const {
bool has_download_file_reference() const {
return FileReferenceView(file_reference_).has_second();
}
bool has_any_file_reference() const {
return has_upload_file_reference() || has_download_file_reference();
}
Slice get_raw_file_reference() const {
return file_reference_;
}
Slice get_upload_file_reference() const {
return FileReferenceView(file_reference_).first();
}
Slice get_download_file_reference() const {
return FileReferenceView(file_reference_).second();
}
string get_url() const {
if (is_web()) {
return web().url_;
@ -460,8 +553,9 @@ class FullRemoteFileLocation {
tl_object_ptr<telegram_api::InputFileLocation> as_input_file_location() const {
switch (location_type()) {
case LocationType::Photo:
return make_tl_object<telegram_api::inputFileLocation>(photo().volume_id_, photo().local_id_, photo().secret_,
BufferSlice(file_reference_));
return make_tl_object<telegram_api::inputFileLocation>(
photo().volume_id_, photo().local_id_, photo().secret_,
BufferSlice(FileReferenceView(file_reference_).second()));
case LocationType::Common:
if (is_encrypted_secret()) {
return make_tl_object<telegram_api::inputEncryptedFileLocation>(common().id_, common().access_hash_);
@ -490,7 +584,8 @@ class FullRemoteFileLocation {
#define as_input_photo() as_input_photo_impl(__FILE__, __LINE__)
tl_object_ptr<telegram_api::inputPhoto> as_input_photo_impl(const char *file, int line) const {
CHECK(is_photo()) << file << ' ' << line;
return make_tl_object<telegram_api::inputPhoto>(photo().id_, photo().access_hash_, BufferSlice(file_reference_));
return make_tl_object<telegram_api::inputPhoto>(photo().id_, photo().access_hash_,
BufferSlice(FileReferenceView(file_reference_).first()));
}
tl_object_ptr<telegram_api::inputEncryptedFile> as_input_encrypted_file() const {
@ -507,13 +602,14 @@ class FullRemoteFileLocation {
// TODO: this constructor is just for immediate unserialize
FullRemoteFileLocation() = default;
FullRemoteFileLocation(FileType file_type, int64 id, int64 access_hash, int32 local_id, int64 volume_id, int64 secret,
DcId dc_id, std::string file_reference)
DcId dc_id, std::string upload_file_reference, std::string download_file_reference)
: file_type_(file_type)
, dc_id_(dc_id)
, file_reference_(std::move(file_reference))
, file_reference_(FileReferenceView::create_two(upload_file_reference, download_file_reference))
, variant_(PhotoRemoteFileLocation{id, access_hash, volume_id, secret, local_id}) {
CHECK(is_photo());
if (file_reference_ == invalid_file_reference()) {
FileReferenceView view(file_reference_);
if (!view.has_first() || !view.has_second()) {
LOG(ERROR) << "Tried to register file with invalid file reference";
file_reference_.clear();
}
@ -524,7 +620,8 @@ class FullRemoteFileLocation {
, file_reference_(std::move(file_reference))
, variant_(CommonRemoteFileLocation{id, access_hash}) {
CHECK(is_common());
if (file_reference_ == invalid_file_reference()) {
FileReferenceView view(file_reference_);
if (!view.has_first() || !view.has_second()) {
LOG(ERROR) << "Tried to register file with invalid file reference";
file_reference_.clear();
}
@ -580,7 +677,7 @@ class FullRemoteFileLocation {
}
static const int32 KEY_MAGIC = 0x64374632;
};
}; // namespace td
inline StringBuilder &operator<<(StringBuilder &string_builder,
const FullRemoteFileLocation &full_remote_file_location) {
@ -659,9 +756,9 @@ class RemoteFileLocation {
explicit RemoteFileLocation(const PartialRemoteFileLocation &partial) : variant_(partial) {
}
RemoteFileLocation(FileType file_type, int64 id, int64 access_hash, int32 local_id, int64 volume_id, int64 secret,
DcId dc_id, std::string file_reference)
DcId dc_id, std::string upload_file_reference, std::string download_file_reference)
: variant_(FullRemoteFileLocation{file_type, id, access_hash, local_id, volume_id, secret, dc_id,
std::move(file_reference)}) {
std::move(upload_file_reference), std::move(download_file_reference)}) {
}
RemoteFileLocation(FileType file_type, int64 id, int64 access_hash, DcId dc_id, std::string file_reference)
: variant_(FullRemoteFileLocation{file_type, id, access_hash, dc_id, std::move(file_reference)}) {

34
td/telegram/files/FileManager.cpp
View File

@ -148,7 +148,7 @@ void FileNode::set_remote_location(const RemoteFileLocation &remote, FileLocatio
if (remote_ == remote) {
if (remote_.type() == RemoteFileLocation::Type::Full) {
if (remote_.full().get_access_hash() != remote.full().get_access_hash() ||
remote_.full().get_file_reference() != remote.full().get_file_reference()) {
remote_.full().get_raw_file_reference() != remote.full().get_raw_file_reference()) {
remote_ = remote;
remote_source_ = source;
on_pmc_changed();
@ -352,14 +352,24 @@ bool FileView::has_remote_location() const {
return node_->remote_.type() == RemoteFileLocation::Type::Full;
}
bool FileView::has_active_remote_location() const {
bool FileView::has_active_upload_remote_location() const {
if (!has_remote_location()) {
return false;
}
if (remote_location().is_encrypted_any()) {
return true;
}
return remote_location().has_file_reference();
return remote_location().has_upload_file_reference();
}
bool FileView::has_active_download_remote_location() const {
if (!has_remote_location()) {
return false;
}
if (remote_location().is_encrypted_any()) {
return true;
}
return remote_location().has_download_file_reference();
}
const FullRemoteFileLocation &FileView::remote_location() const {
@ -986,13 +996,13 @@ static int merge_choose_remote_location(const RemoteFileLocation &x, int8 x_sour
if (x.full().is_web() != y.full().is_web()) {
return x.full().is_web(); // prefer non-web
}
auto x_ref = x.full().has_file_reference();
auto y_ref = y.full().has_file_reference();
auto x_ref = x.full().has_any_file_reference();
auto y_ref = y.full().has_any_file_reference();
if (x_ref || y_ref) {
if (x_ref != y_ref) {
return !x_ref;
}
if (x.full().get_file_reference() != y.full().get_file_reference()) {
if (x.full().get_raw_file_reference() != y.full().get_raw_file_reference()) {
return x_source < y_source;
}
}
@ -1799,7 +1809,7 @@ void FileManager::run_download(FileNodePtr node) {
auto file_id = node->main_file_id_;
// If file reference is needed
if (!file_view.has_active_remote_location()) {
if (!file_view.has_active_download_remote_location()) {
VLOG(file_references) << "run_download: Do not have valid file_reference for file " << file_id;
QueryId id = queries_container_.create(Query{file_id, Query::DownloadWaitFileReferece});
node->download_id_ = id;
@ -1850,8 +1860,8 @@ void FileManager::resume_upload(FileId file_id, std::vector<int> bad_parts, std:
node->set_upload_pause(FileId());
}
FileView file_view(node);
if (file_view.has_active_remote_location() && file_view.get_type() != FileType::Thumbnail &&
file_view.get_type() != FileType::EncryptedThumbnail && file_view.get_type() != FileType::Photo) {
if (file_view.has_active_upload_remote_location() && file_view.get_type() != FileType::Thumbnail &&
file_view.get_type() != FileType::EncryptedThumbnail) {
LOG(INFO) << "File " << file_id << " is already uploaded";
if (callback) {
callback->on_upload_ok(file_id, nullptr);
@ -2085,9 +2095,9 @@ void FileManager::run_upload(FileNodePtr node, std::vector<int> bad_parts) {
}
CHECK(node->upload_id_ == 0);
if (file_view.has_remote_location() && !file_view.has_active_remote_location() &&
file_view.get_type() != FileType::Photo && file_view.get_type() != FileType::Thumbnail &&
file_view.get_type() != FileType::EncryptedThumbnail && !node->upload_was_update_file_reference_) {
if (file_view.has_remote_location() && !file_view.has_active_upload_remote_location() &&
file_view.get_type() != FileType::Thumbnail && file_view.get_type() != FileType::EncryptedThumbnail &&
!node->upload_was_update_file_reference_) {
QueryId id = queries_container_.create(Query{file_id, Query::UploadWaitFileReference});
node->upload_id_ = id;
node->upload_was_update_file_reference_ = true;

3
td/telegram/files/FileManager.h
View File

@ -212,7 +212,8 @@ class FileView {
bool has_local_location() const;
const FullLocalFileLocation &local_location() const;
bool has_remote_location() const;
bool has_active_remote_location() const;
bool has_active_upload_remote_location() const;
bool has_active_download_remote_location() const;
const FullRemoteFileLocation &remote_location() const;
bool has_generate_location() const;
const FullGenerateFileLocation &generate_location() const;