tdlight-team/tdlight
tdlight-team
/
tdlight
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement FileSource. 

GitOrigin-RevId: 2cbeab03d2ea1fa56d9f746c5c6c6a41e28d2795
This commit is contained in:
levlam 2019-01-14 21:46:04 +03:00
parent c0815f868c
commit 8851b9d066
7 changed files with 130 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
td/telegram/FileReferenceManager.cpp
View File

@ -6,53 +6,100 @@
//
#include "td/telegram/FileReferenceManager.h"
#include "td/telegram/AnimationsManager.h"
#include "td/telegram/ContactsManager.h"
#include "td/telegram/files/FileManager.h"
#include "td/telegram/Global.h"
#include "td/telegram/MessagesManager.h"
#include "td/telegram/WebPagesManager.h"
#include "td/actor/MultiPromise.h"
#include "td/utils/format.h"
#include "td/utils/overloaded.h"
#include "td/utils/Variant.h"
namespace td {
FileSourceId FileReferenceManager::create_file_source(FullMessageId full_message_id) {
auto it = from_full_message_id_.find(full_message_id);
if (it != from_full_message_id_.end()) {
int VERBOSITY_NAME(file_references) = VERBOSITY_NAME(WARNING);
/*
fileSourceMessage chat_id:int53 message_id:int53 = FileSource; // repaired with get_messages_from_server
fileSourceUserProfilePhoto user_id:int32 photo_id:int64 = FileSource; // repaired with photos.getUserPhotos
fileSourceBasicGroupPhoto basic_group_id:int32 = FileSource; // repaired with messages.getChats
fileSourceSupergroupPhoto supergroup_id:int32 = FileSource; // repaired with channels.getChannels
fileSourceWebPage url:string = FileSource; // repaired with messages.getWebPage
fileSourceWallpapers = FileSource; // repaired with account.getWallPapers
fileSourceSavedAnimations = FileSource; // repaired with messages.getSavedGifs
*/
FileSourceId FileReferenceManager::create_message_file_source(FullMessageId full_message_id) {
auto it = full_message_id_to_file_source_id_.find(full_message_id);
if (it != full_message_id_to_file_source_id_.end()) {
return it->second;
}
auto source_id = FileSourceId{++last_file_source_id_};
to_full_message_id_[source_id] = full_message_id;
from_full_message_id_[full_message_id] = source_id;
FileSourceMessage source{full_message_id};
file_sources_.emplace_back(source);
full_message_id_to_file_source_id_[full_message_id] = source_id;
return source_id;
}
void FileReferenceManager::update_file_reference(FileId file_id, std::vector<FileSourceId> sources, Promise<> promise) {
LOG(INFO) << "Trying to load valid file_reference from server: " << file_id << " " << format::as_array(sources);
void FileReferenceManager::update_file_reference(FileId file_id, vector<FileSourceId> file_source_ids,
Promise<> promise) {
VLOG(file_references) << "Trying to load valid file_reference from server: " << file_id << " " << file_source_ids;
MultiPromiseActorSafe mpas{"UpdateFileReferenceMultiPromiseActor"};
mpas.set_ignore_errors(true);
mpas.add_promise(std::move(promise));
auto lock = mpas.get_promise();
for (auto source_id : sources) {
auto it = to_full_message_id_.find(source_id);
for (auto source_id : file_source_ids) {
auto index = static_cast<size_t>(source_id.get()) - 1;
CHECK(index < file_sources_.size());
auto new_promise = PromiseCreator::lambda([promise = mpas.get_promise(), file_id, source_id,
file_manager = G()->file_manager()](Result<Unit> res) mutable {
if (res.is_error()) {
LOG(INFO) << "Invalid source id " << source_id << " " << res.error();
file_manager = G()->file_manager()](Result<Unit> result) mutable {
if (result.is_error() && result.error().code() != 429 && result.error().code() < 500 && !G()->close_flag()) {
VLOG(file_references) << "Invalid source id " << source_id << " " << result.error();
send_closure(file_manager, &FileManager::remove_file_source, file_id, source_id);
}
// NB: main promise must send closure to FileManager
// So the closure will be executed only after the bad source id is removed
promise.set_value(Unit());
});
if (it == to_full_message_id_.end()) {
new_promise.set_error(Status::Error("Unknown source id"));
continue;
}
std::vector<FullMessageId> message_ids = {it->second};
LOG(INFO) << source_id << ": load message from server " << it->second;
send_closure_later(G()->messages_manager(), &MessagesManager::get_messages_from_server, std::move(message_ids),
std::move(new_promise), nullptr);
file_sources_[index].visit(overloaded(
[&](const FileSourceMessage &source) {
send_closure_later(G()->messages_manager(), &MessagesManager::get_messages_from_server,
vector<FullMessageId>{source.full_message_id}, std::move(new_promise), nullptr);
},
[&](const FileSourceUserPhoto &source) {
// send_closure_later(G()->contacts_manager(), &ContactsManager::get_user_photo_from_server, source.user_id,
// source.photo_id, std::move(new_promise));
},
[&](const FileSourceChatPhoto &source) {
// send_closure_later(G()->contacts_manager(), &ContactsManager::get_chat_photo_from_server, source.chat_id,
// std::move(new_promise));
},
[&](const FileSourceChannelPhoto &source) {
// send_closure_later(G()->contacts_manager(), &ContactsManager::get_channel_photo_from_server,
// source.channel_id, std::move(new_promise));
},
[&](const FileSourceWallpapers &source) {
// send_closure_later(G()->wallpaper_manager(), &WallpaperManager::get_wallpapers_from_server,
// std::move(new_promise));
},
[&](const FileSourceWebPage &source) {
send_closure_later(G()->web_pages_manager(), &WebPagesManager::reload_web_page_by_url, source.url,
std::move(new_promise));
},
[&](const FileSourceSavedAnimations &source) {
/*
// TODO this is wrong, because we shouldn't pass animations hash to the call
// we also sometimes need to do two simultaneous calls one with and one without hash
send_closure_later(G()->animations_manager(), &AnimationsManager::reload_saved_animations,
true, std::move(new_promise));
*/
}));
}
lock.set_value(Unit());
}

46
td/telegram/FileReferenceManager.h
View File

@ -9,23 +9,57 @@
#include "td/actor/actor.h"
#include "td/actor/PromiseFuture.h"
#include "td/telegram/ChatId.h"
#include "td/telegram/ChannelId.h"
#include "td/telegram/files/FileManager.h"
#include "td/telegram/MessageId.h"
#include "td/telegram/UserId.h"
#include "td/utils/Variant.h"
#include <map>
#include <unordered_map>
namespace td {
extern int VERBOSITY_NAME(file_references);
class FileReferenceManager : public Actor {
public:
FileSourceId create_file_source(FullMessageId full_message_id);
void update_file_reference(FileId file_id, std::vector<FileSourceId> sources, Promise<> promise);
FileSourceId create_message_file_source(FullMessageId full_message_id);
void update_file_reference(FileId file_id, vector<FileSourceId> file_source_ids, Promise<> promise);
private:
td::int32 last_file_source_id_{0};
std::unordered_map<FileSourceId, FullMessageId, FileSourceIdHash> to_full_message_id_;
std::unordered_map<FullMessageId, FileSourceId, FullMessageIdHash> from_full_message_id_;
struct FileSourceMessage {
FullMessageId full_message_id;
};
struct FileSourceUserPhoto {
UserId user_id;
int64 photo_id;
};
struct FileSourceChatPhoto {
ChatId chat_id;
};
struct FileSourceChannelPhoto {
ChannelId channel_id;
};
struct FileSourceWallpapers {
// empty
};
struct FileSourceWebPage {
string url;
};
struct FileSourceSavedAnimations {
// empty
};
using FileSource = Variant<FileSourceMessage, FileSourceUserPhoto, FileSourceChatPhoto, FileSourceChannelPhoto,
FileSourceWallpapers, FileSourceWebPage, FileSourceSavedAnimations>;
vector<FileSource> file_sources_;
std::unordered_map<FullMessageId, FileSourceId, FullMessageIdHash> full_message_id_to_file_source_id_;
int32 last_file_source_id_{0};
};
} // namespace td

4
td/telegram/Logging.cpp
View File

@ -7,6 +7,7 @@
#include "td/telegram/Logging.h"
#include "td/telegram/ConfigManager.h"
#include "td/telegram/FileReferenceManager.h"
#include "td/telegram/files/FileGcWorker.h"
#include "td/telegram/files/FileManager.h"
#include "td/telegram/net/ConnectionCreator.h"
@ -40,7 +41,8 @@ static const std::map<Slice, int *> log_tags{
ADD_TAG(proxy), ADD_TAG(net_query), ADD_TAG(td_requests), ADD_TAG(dc),
ADD_TAG(files), ADD_TAG(mtproto), ADD_TAG(raw_mtproto), ADD_TAG(fd),
ADD_TAG(actor), ADD_TAG(buffer), ADD_TAG(sqlite), ADD_TAG(notifications),
ADD_TAG(get_difference), ADD_TAG(file_gc), ADD_TAG(config_recoverer), ADD_TAG(dns_resolver)};
ADD_TAG(get_difference), ADD_TAG(file_gc), ADD_TAG(config_recoverer), ADD_TAG(dns_resolver),
ADD_TAG(file_references)};
#undef ADD_TAG
Status Logging::set_current_stream(td_api::object_ptr<td_api::LogStream> stream) {

6
td/telegram/MessagesManager.cpp
View File

@ -13707,7 +13707,8 @@ void MessagesManager::add_message_file_sources(DialogId dialog_id, const Message
return;
}
auto file_source_id = td_->file_reference_manager_->create_file_source(FullMessageId(dialog_id, m->message_id));
auto file_source_id =
td_->file_reference_manager_->create_message_file_source(FullMessageId(dialog_id, m->message_id));
for (auto file_id : file_ids) {
td_->file_manager_->add_file_source(file_id, file_source_id);
}
@ -13723,7 +13724,8 @@ void MessagesManager::remove_message_file_sources(DialogId dialog_id, const Mess
return;
}
auto file_source_id = td_->file_reference_manager_->create_file_source(FullMessageId(dialog_id, m->message_id));
auto file_source_id =
td_->file_reference_manager_->create_message_file_source(FullMessageId(dialog_id, m->message_id));
for (auto file_id : file_ids) {
td_->file_manager_->remove_file_source(file_id, file_source_id);
}

2
td/telegram/NotificationManager.cpp
View File

@ -32,7 +32,7 @@
namespace td {
int VERBOSITY_NAME(notifications) = VERBOSITY_NAME(WARNING);
int VERBOSITY_NAME(notifications) = VERBOSITY_NAME(INFO);
NotificationManager::NotificationManager(Td *td, ActorShared<> parent) : td_(td), parent_(std::move(parent)) {
flush_pending_notifications_timeout_.set_callback(on_flush_pending_notifications_timeout_callback);

4
td/telegram/WebPagesManager.h
View File

@ -66,6 +66,8 @@ class WebPagesManager : public Actor {
WebPageId get_web_page_by_url(const string &url, Promise<Unit> &&promise);
void reload_web_page_by_url(const string &url, Promise<Unit> &&promise);
void on_get_web_page_preview_success(int64 request_id, const string &url,
tl_object_ptr<telegram_api::MessageMedia> &&message_media_ptr,
Promise<Unit> &&promise);
@ -210,8 +212,6 @@ class WebPagesManager : public Actor {
void load_web_page_by_url(const string &url, Promise<Unit> &&promise);
void reload_web_page_by_url(const string &url, Promise<Unit> &&promise);
void on_load_web_page_id_by_url_from_database(const string &url, string value, Promise<Unit> &&promise);
void on_load_web_page_by_url_from_database(WebPageId web_page_id, const string &url, Promise<Unit> &&promise,

17
td/telegram/files/FileManager.cpp
View File

@ -161,6 +161,7 @@ void FileNode::set_remote_location(const RemoteFileLocation &remote, FileLocatio
void FileNode::delete_file_reference(Slice file_reference) {
if (remote_.type() == RemoteFileLocation::Type::Full && remote_.full().delete_file_reference(file_reference)) {
VLOG(file_references) << "Delete file reference of file " << main_file_id_;
upload_may_update_file_reference_ = true;
download_may_update_file_reference_ = true;
on_pmc_changed();
@ -259,6 +260,8 @@ void FileNode::add_file_source(FileSourceId file_source_id) {
if (std::find(file_source_ids_.begin(), file_source_ids_.end(), file_source_id) != file_source_ids_.end()) {
return;
}
VLOG(file_references) << "Add " << file_source_id << " to file " << main_file_id_;
upload_may_update_file_reference_ = true;
download_may_update_file_reference_ = true;
file_source_ids_.push_back(file_source_id);
@ -269,6 +272,8 @@ void FileNode::remove_file_source(FileSourceId file_source_id) {
if (it == file_source_ids_.end()) {
return;
}
VLOG(file_references) << "Remove " << file_source_id << " from file " << main_file_id_;
file_source_ids_.erase(it);
}
@ -1760,7 +1765,7 @@ void FileManager::run_download(FileNodePtr node) {
// If file reference is needed
if (!file_view.has_active_remote_location()) {
LOG(INFO) << "run_download: Do not have valid file_reference for file " << file_id;
VLOG(file_references) << "run_download: Do not have valid file_reference for file " << file_id;
QueryId id = queries_container_.create(Query{file_id, Query::DownloadWaitFileReferece});
node->download_id_ = id;
if (node->file_source_ids_.empty()) {
@ -1782,7 +1787,8 @@ void FileManager::run_download(FileNodePtr node) {
} else {
error = res.move_as_error();
}
LOG(INFO) << "run_download: Got result from FileSourceManager for file " << file_id << ": " << error;
VLOG(file_references) << "run_download: Got result from FileSourceManager for file " << file_id
<< ": " << error;
send_closure(actor_id, &FileManager::on_error, id, std::move(error));
}));
return;
@ -1885,11 +1891,12 @@ bool FileManager::delete_partial_remote_location(FileId file_id) {
return true;
}
void FileManager::delete_file_reference(FileId file_id, std::string file_reference) {
LOG(INFO) << "Delete file reference to file " << file_id << " " << tag("reference", base64_encode(file_reference));
void FileManager::delete_file_reference(FileId file_id, string file_reference) {
VLOG(file_references) << "Delete file reference of file " << file_id << " "
<< tag("reference", base64_encode(file_reference));
auto node = get_sync_file_node(file_id);
if (!node) {
LOG(INFO) << "Wrong file id " << file_id;
LOG(ERROR) << "Wrong file id " << file_id;
return;
}
node->delete_file_reference(file_reference);