FileReference: minor improvements
GitOrigin-RevId: 22c605f90445d7d6aefefea16c834051e29b1fed
This commit is contained in:
parent
a75726d77a
commit
bbecdcda20
|
@ -20,52 +20,41 @@ FileSourceId FileReferenceManager::create_file_source(FullMessageId full_message
|
||||||
if (it != from_full_message_id_.end()) {
|
if (it != from_full_message_id_.end()) {
|
||||||
return it->second;
|
return it->second;
|
||||||
}
|
}
|
||||||
++last_file_source_id_;
|
auto source_id = FileSourceId{++last_file_source_id_};
|
||||||
to_full_message_id_[last_file_source_id_] = full_message_id;
|
to_full_message_id_[source_id] = full_message_id;
|
||||||
from_full_message_id_[full_message_id] = FileSourceId{last_file_source_id_};
|
from_full_message_id_[full_message_id] = source_id;
|
||||||
return FileSourceId{last_file_source_id_};
|
return source_id;
|
||||||
}
|
}
|
||||||
|
|
||||||
void FileReferenceManager::update_file_reference(FileId file_id, std::vector<FileSourceId> sources, Promise<> promise) {
|
void FileReferenceManager::update_file_reference(FileId file_id, std::vector<FileSourceId> sources, Promise<> promise) {
|
||||||
LOG(ERROR) << "update file reference: " << file_id << " " << format::as_array(sources);
|
LOG(INFO) << "Trying to load valid file_reference from server: " << file_id << " " << format::as_array(sources);
|
||||||
//if (td::Random::fast(0, 3) == 0) {
|
|
||||||
//promise.set_error(td::Status::Error("Error"));
|
|
||||||
//return;
|
|
||||||
//}
|
|
||||||
|
|
||||||
//if (td::Random::fast(0, 3) == 0) {
|
|
||||||
//promise.set_value(Unit());
|
|
||||||
//return;
|
|
||||||
//}
|
|
||||||
|
|
||||||
MultiPromiseActorSafe mpas{"UpdateFileReferenceMultiPromiseActor"};
|
MultiPromiseActorSafe mpas{"UpdateFileReferenceMultiPromiseActor"};
|
||||||
mpas.add_promise(std::move(promise));
|
mpas.add_promise(std::move(promise));
|
||||||
auto lock = mpas.get_promise();
|
auto lock = mpas.get_promise();
|
||||||
SCOPE_EXIT {
|
SCOPE_EXIT {
|
||||||
lock.set_value(Unit());
|
lock.set_value(Unit());
|
||||||
};
|
};
|
||||||
for (auto &source_id : sources) {
|
for (auto source_id : sources) {
|
||||||
auto it = to_full_message_id_.find(source_id);
|
auto it = to_full_message_id_.find(source_id);
|
||||||
if (it != to_full_message_id_.end()) {
|
auto new_promise = PromiseCreator::lambda([promise = mpas.get_promise(), file_id, source_id,
|
||||||
std::vector<FullMessageId> message_ids = {it->second};
|
file_manager = G()->file_manager()](Result<Unit> res) mutable {
|
||||||
auto new_promise = PromiseCreator::lambda([promise = mpas.get_promise(), file_id, source_id,
|
if (res.is_error()) {
|
||||||
file_manager = G()->file_manager()](Result<Unit> res) mutable {
|
LOG(INFO) << "Invalid source id " << source_id << " " << res.error();
|
||||||
if (res.is_error()) {
|
send_closure(file_manager, &FileManager::remove_file_source, file_id, source_id);
|
||||||
send_closure(file_manager, &FileManager::remove_file_source, file_id, source_id);
|
}
|
||||||
send_lambda(file_manager, [promise = std::move(promise)]() mutable { promise.set_value({}); });
|
// NB: main promise must send closure to FileManager
|
||||||
} else {
|
// So the closure will be executed only after the bad source id is removed
|
||||||
promise.set_value(Unit());
|
promise.set_value(Unit());
|
||||||
}
|
});
|
||||||
});
|
if (it == to_full_message_id_.end()) {
|
||||||
|
new_promise.set_error(Status::Error("Unkonwn source id"));
|
||||||
LOG(ERROR) << "Ask for " << it->second;
|
continue;
|
||||||
send_closure_later(G()->messages_manager(), &MessagesManager::get_messages_from_server, std::move(message_ids),
|
|
||||||
std::move(new_promise), nullptr);
|
|
||||||
} else {
|
|
||||||
LOG(ERROR) << "Invalid source id " << source_id << " " << file_id;
|
|
||||||
send_closure(G()->file_manager(), &FileManager::remove_file_source, file_id, source_id);
|
|
||||||
send_lambda(G()->file_manager(), [promise = mpas.get_promise()]() mutable { promise.set_value(Unit()); });
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::vector<FullMessageId> message_ids = {it->second};
|
||||||
|
LOG(INFO) << source_id << ": load message from server " << it->second;
|
||||||
|
send_closure_later(G()->messages_manager(), &MessagesManager::get_messages_from_server, std::move(message_ids),
|
||||||
|
std::move(new_promise), nullptr);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -24,7 +24,7 @@ class FileReferenceManager : public Actor {
|
||||||
|
|
||||||
private:
|
private:
|
||||||
td::int32 last_file_source_id_{0};
|
td::int32 last_file_source_id_{0};
|
||||||
std::map<FileSourceId, FullMessageId> to_full_message_id_;
|
std::unordered_map<FileSourceId, FullMessageId, FileSourceIdHash> to_full_message_id_;
|
||||||
std::unordered_map<FullMessageId, FileSourceId, FullMessageIdHash> from_full_message_id_;
|
std::unordered_map<FullMessageId, FileSourceId, FullMessageIdHash> from_full_message_id_;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -272,7 +272,8 @@ Status FileDownloader::check_net_query(NetQueryPtr &net_query) {
|
||||||
if (net_query->is_error()) {
|
if (net_query->is_error()) {
|
||||||
auto error = net_query->move_as_error();
|
auto error = net_query->move_as_error();
|
||||||
if (begins_with(error.message(), "FILE_REFERENCE_")) {
|
if (begins_with(error.message(), "FILE_REFERENCE_")) {
|
||||||
error = Status::Error(400, PSLICE() << "FILE_REFERENCE_BASE64" << base64_encode(remote_.get_file_reference()));
|
error = Status::Error(400, PSLICE()
|
||||||
|
<< "FILE_REFERENCE_EXPIRED_BASE64" << base64_encode(remote_.get_file_reference()));
|
||||||
}
|
}
|
||||||
return error;
|
return error;
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,7 +14,50 @@
|
||||||
|
|
||||||
namespace td {
|
namespace td {
|
||||||
|
|
||||||
using FileSourceId = int32;
|
class FileSourceId {
|
||||||
|
int32 id = 0;
|
||||||
|
|
||||||
|
public:
|
||||||
|
FileSourceId() = default;
|
||||||
|
|
||||||
|
explicit FileSourceId(int32 file_id) : id(file_id) {
|
||||||
|
}
|
||||||
|
template <class T1, typename = std::enable_if_t<std::is_convertible<T1, int32>::value>>
|
||||||
|
FileSourceId(T1 file_id) = delete;
|
||||||
|
|
||||||
|
bool empty() const {
|
||||||
|
return id <= 0;
|
||||||
|
}
|
||||||
|
bool is_valid() const {
|
||||||
|
return id > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int32 get() const {
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool operator<(const FileSourceId &other) const {
|
||||||
|
return id < other.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool operator==(const FileSourceId &other) const {
|
||||||
|
return id == other.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool operator!=(const FileSourceId &other) const {
|
||||||
|
return id != other.id;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
struct FileSourceIdHash {
|
||||||
|
std::size_t operator()(FileSourceId file_id) const {
|
||||||
|
return std::hash<int32>()(file_id.get());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
inline StringBuilder &operator<<(StringBuilder &string_builder, FileSourceId file_id) {
|
||||||
|
return string_builder << "FileSourceId(" << file_id.get() << ")";
|
||||||
|
}
|
||||||
|
|
||||||
class FileId {
|
class FileId {
|
||||||
int32 id = 0;
|
int32 id = 0;
|
||||||
|
|
|
@ -14,6 +14,7 @@
|
||||||
#include "td/telegram/files/FileEncryptionKey.h"
|
#include "td/telegram/files/FileEncryptionKey.h"
|
||||||
#include "td/telegram/net/DcId.h"
|
#include "td/telegram/net/DcId.h"
|
||||||
|
|
||||||
|
#include "td/utils/base64.h"
|
||||||
#include "td/utils/buffer.h"
|
#include "td/utils/buffer.h"
|
||||||
#include "td/utils/common.h"
|
#include "td/utils/common.h"
|
||||||
#include "td/utils/format.h"
|
#include "td/utils/format.h"
|
||||||
|
@ -549,6 +550,9 @@ class FullRemoteFileLocation {
|
||||||
file_reference_ = {};
|
file_reference_ = {};
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
bool has_file_reference() const {
|
||||||
|
return !file_reference_.empty();
|
||||||
|
}
|
||||||
string get_file_reference() const {
|
string get_file_reference() const {
|
||||||
return file_reference_;
|
return file_reference_;
|
||||||
}
|
}
|
||||||
|
@ -715,8 +719,7 @@ inline StringBuilder &operator<<(StringBuilder &string_builder,
|
||||||
string_builder << ", " << full_remote_file_location.get_dc_id();
|
string_builder << ", " << full_remote_file_location.get_dc_id();
|
||||||
}
|
}
|
||||||
if (!full_remote_file_location.file_reference_.empty()) {
|
if (!full_remote_file_location.file_reference_.empty()) {
|
||||||
string_builder << ", "
|
string_builder << ", " << tag("file_reference", base64_encode(full_remote_file_location.file_reference_));
|
||||||
<< tag("file_reference", format::as_hex_dump<0>(Slice(full_remote_file_location.file_reference_)));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
string_builder << ", location = ";
|
string_builder << ", location = ";
|
||||||
|
|
|
@ -362,7 +362,7 @@ bool FileView::has_active_remote_location() const {
|
||||||
if (remote_location().is_encrypted_any()) {
|
if (remote_location().is_encrypted_any()) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return !remote_location().get_file_reference().empty();
|
return remote_location().has_file_reference();
|
||||||
}
|
}
|
||||||
const FullRemoteFileLocation &FileView::remote_location() const {
|
const FullRemoteFileLocation &FileView::remote_location() const {
|
||||||
CHECK(has_remote_location());
|
CHECK(has_remote_location());
|
||||||
|
@ -983,8 +983,8 @@ static int merge_choose_remote_location(const RemoteFileLocation &x, int8 x_sour
|
||||||
if (x.full().is_web() != y.full().is_web()) {
|
if (x.full().is_web() != y.full().is_web()) {
|
||||||
return x.full().is_web(); // prefer non-web
|
return x.full().is_web(); // prefer non-web
|
||||||
}
|
}
|
||||||
auto x_ref = !x.full().get_file_reference().empty();
|
auto x_ref = x.full().has_file_reference();
|
||||||
auto y_ref = !y.full().get_file_reference().empty();
|
auto y_ref = !y.full().has_file_reference();
|
||||||
if (x_ref || y_ref) {
|
if (x_ref || y_ref) {
|
||||||
if (x_ref != y_ref) {
|
if (x_ref != y_ref) {
|
||||||
return !x_ref;
|
return !x_ref;
|
||||||
|
@ -1103,7 +1103,7 @@ void FileManager::cancel_generate(FileNodePtr node) {
|
||||||
}
|
}
|
||||||
|
|
||||||
Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) {
|
Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) {
|
||||||
LOG(ERROR) << x_file_id << " VS " << y_file_id;
|
LOG(DEBUG) << x_file_id << " VS " << y_file_id;
|
||||||
|
|
||||||
if (!x_file_id.is_valid()) {
|
if (!x_file_id.is_valid()) {
|
||||||
return Status::Error("First file_id is invalid");
|
return Status::Error("First file_id is invalid");
|
||||||
|
@ -1197,8 +1197,6 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
|
||||||
<< ", generate_i = " << generate_i << ", size_i = " << size_i << ", remote_name_i = " << remote_name_i
|
<< ", generate_i = " << generate_i << ", size_i = " << size_i << ", remote_name_i = " << remote_name_i
|
||||||
<< ", url_i = " << url_i << ", owner_i = " << owner_i << ", encryption_key_i = " << encryption_key_i
|
<< ", url_i = " << url_i << ", owner_i = " << owner_i << ", encryption_key_i = " << encryption_key_i
|
||||||
<< ", main_file_id_i = " << main_file_id_i;
|
<< ", main_file_id_i = " << main_file_id_i;
|
||||||
LOG(ERROR) << FileView(node).has_active_remote_location();
|
|
||||||
LOG(ERROR) << FileView(other_node).has_active_remote_location();
|
|
||||||
if (local_i == other_node_i) {
|
if (local_i == other_node_i) {
|
||||||
cancel_download(node);
|
cancel_download(node);
|
||||||
node->set_download_offset(other_node->download_offset_);
|
node->set_download_offset(other_node->download_offset_);
|
||||||
|
@ -1747,25 +1745,26 @@ void FileManager::run_download(FileNodePtr node) {
|
||||||
|
|
||||||
// If file reference is needed
|
// If file reference is needed
|
||||||
if (!file_view.has_active_remote_location()) {
|
if (!file_view.has_active_remote_location()) {
|
||||||
LOG(ERROR) << "run_download: no active location";
|
LOG(INFO) << "run_download: Do not have valid file_reference " << file_id;
|
||||||
QueryId id = queries_container_.create(Query{file_id, Query::Download});
|
QueryId id = queries_container_.create(Query{file_id, Query::Download});
|
||||||
node->download_id_ = id;
|
node->download_id_ = id;
|
||||||
if (node->file_source_ids_.empty()) {
|
if (node->file_source_ids_.empty()) {
|
||||||
on_error(id, Status::Error("Can't download file: no valid file refernce and no valid source id"));
|
on_error(id, Status::Error("Can't download file: have valid file reference and no valid source id"));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
send_closure(G()->file_reference_manager(), &FileReferenceManager::update_file_reference, file_id,
|
send_closure(
|
||||||
node->file_source_ids_, PromiseCreator::lambda([id, actor_id = actor_id(this)](Result<Unit> res) {
|
G()->file_reference_manager(), &FileReferenceManager::update_file_reference, file_id, node->file_source_ids_,
|
||||||
Status error;
|
PromiseCreator::lambda([id, actor_id = actor_id(this), file_id](Result<Unit> res) {
|
||||||
LOG(ERROR) << "run_download: update_file_reference finished";
|
Status error;
|
||||||
if (res.is_ok()) {
|
if (res.is_ok()) {
|
||||||
error = td::Status::Error("FILE_DOWNLOAD_RESTART_WITH_FILE_REFERENCE");
|
error = td::Status::Error("FILE_DOWNLOAD_RESTART_WITH_FILE_REFERENCE");
|
||||||
} else {
|
} else {
|
||||||
error = res.move_as_error();
|
error = res.move_as_error();
|
||||||
}
|
}
|
||||||
send_closure(actor_id, &FileManager::on_error, id, std::move(error));
|
LOG(INFO) << "run_download: Got result from FileSourceManager for file " << file_id << " : " << error;
|
||||||
}));
|
send_closure(actor_id, &FileManager::on_error, id, std::move(error));
|
||||||
|
}));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1867,7 +1866,7 @@ bool FileManager::delete_partial_remote_location(FileId file_id) {
|
||||||
}
|
}
|
||||||
|
|
||||||
void FileManager::delete_file_reference(FileId file_id, std::string file_reference) {
|
void FileManager::delete_file_reference(FileId file_id, std::string file_reference) {
|
||||||
LOG(ERROR) << "Delete file reference " << file_id << tag("reference", format::as_hex_dump<0>(Slice(file_reference)));
|
LOG(INFO) << "Delete file reference to file " << file_id << " " << tag("reference", base64_encode(file_reference));
|
||||||
auto node = get_sync_file_node(file_id);
|
auto node = get_sync_file_node(file_id);
|
||||||
if (!node) {
|
if (!node) {
|
||||||
LOG(INFO) << "Wrong file id " << file_id;
|
LOG(INFO) << "Wrong file id " << file_id;
|
||||||
|
@ -2714,7 +2713,6 @@ void FileManager::on_error_impl(FileNodePtr node, FileManager::Query::Type type,
|
||||||
SCOPE_EXIT {
|
SCOPE_EXIT {
|
||||||
try_flush_node(node, "on_error");
|
try_flush_node(node, "on_error");
|
||||||
};
|
};
|
||||||
//FIXME: should we apply error if !was_active?
|
|
||||||
if (status.code() != 1 && !G()->close_flag()) {
|
if (status.code() != 1 && !G()->close_flag()) {
|
||||||
LOG(WARNING) << "Failed to upload/download/generate file: " << status << ". Query type = " << type
|
LOG(WARNING) << "Failed to upload/download/generate file: " << status << ". Query type = " << type
|
||||||
<< ". File type is " << FileView(node).get_type();
|
<< ". File type is " << FileView(node).get_type();
|
||||||
|
@ -2758,13 +2756,15 @@ void FileManager::on_error_impl(FileNodePtr node, FileManager::Query::Type type,
|
||||||
}
|
}
|
||||||
if (begins_with(status.message(), "FILE_REFERENCE_")) {
|
if (begins_with(status.message(), "FILE_REFERENCE_")) {
|
||||||
string file_reference;
|
string file_reference;
|
||||||
Slice prefix = "FILE_REFERENCE_BASE64";
|
Slice prefix = "FILE_REFERENCE_EXPIRED_BASE64";
|
||||||
if (begins_with(status.message(), prefix)) {
|
if (begins_with(status.message(), prefix)) {
|
||||||
auto tmp = base64_decode(status.message().substr(prefix.size()));
|
auto tmp = base64_decode(status.message().substr(prefix.size()));
|
||||||
LOG_IF(WARNING, tmp.is_error()) << "Can't decode file reference from error " << status << " " << tmp.error();
|
LOG_IF(WARNING, tmp.is_error()) << "Can't decode file reference from error " << status << " " << tmp.error();
|
||||||
if (tmp.is_ok()) {
|
if (tmp.is_ok()) {
|
||||||
file_reference = tmp.move_as_ok();
|
file_reference = tmp.move_as_ok();
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
LOG(ERROR) << "Unexpected error, file_reference will be deleted just in case " << status;
|
||||||
}
|
}
|
||||||
CHECK(!node->file_ids_.empty());
|
CHECK(!node->file_ids_.empty());
|
||||||
delete_file_reference(node->file_ids_.back(), file_reference);
|
delete_file_reference(node->file_ids_.back(), file_reference);
|
||||||
|
@ -2779,12 +2779,12 @@ void FileManager::on_error_impl(FileNodePtr node, FileManager::Query::Type type,
|
||||||
if (begins_with(status.message(), "FILE_DOWNLOAD_RESTART")) {
|
if (begins_with(status.message(), "FILE_DOWNLOAD_RESTART")) {
|
||||||
if (ends_with(status.message(), "WITH_FILE_REFERENCE")) {
|
if (ends_with(status.message(), "WITH_FILE_REFERENCE")) {
|
||||||
if (FileView(node).has_active_remote_location()) {
|
if (FileView(node).has_active_remote_location()) {
|
||||||
LOG(ERROR) << "??????";
|
|
||||||
run_download(node);
|
run_download(node);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
LOG_IF(WARNING, !node->file_source_ids_.empty())
|
LOG_IF(WARNING, !node->file_source_ids_.empty())
|
||||||
<< "Got no active remote locations " << FileView(node).remote_location();
|
<< "Got no active remote locations, but have file_source_ids, download will be cancelled "
|
||||||
|
<< FileView(node).remote_location();
|
||||||
} else {
|
} else {
|
||||||
node->can_search_locally_ = false;
|
node->can_search_locally_ = false;
|
||||||
run_download(node);
|
run_download(node);
|
||||||
|
|
Reference in New Issue
Block a user