tdlight-team/tdlight
tdlight-team/tdlight
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Upload inported messages as small files.

Browse Source
This commit is contained in:
levlam 2021-01-22 18:13:38 +03:00
parent f307200ddc
commit bff723436a
4 changed files with 26 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
td/telegram/MessagesManager.cpp
View File

@ -26475,7 +26475,8 @@ void MessagesManager::upload_imported_messages(DialogId dialog_id, FileId file_i
file_id, td::make_unique<UploadedImportedMessagesInfo>(dialog_id, std::move(attached_file_ids), is_reupload, file_id, td::make_unique<UploadedImportedMessagesInfo>(dialog_id, std::move(attached_file_ids), is_reupload,
std::move(promise))); std::move(promise)));
// TODO use force_reupload if is_reupload // TODO use force_reupload if is_reupload
td_->file_manager_->resume_upload(file_id, std::move(bad_parts), upload_imported_messages_callback_, 1, 0); td_->file_manager_->resume_upload(file_id, std::move(bad_parts), upload_imported_messages_callback_, 1, 0, false,
true);
} }
void MessagesManager::start_import_messages(DialogId dialog_id, int64 import_id, vector<FileId> &&attached_file_ids, void MessagesManager::start_import_messages(DialogId dialog_id, int64 import_id, vector<FileId> &&attached_file_ids,
@ -26524,7 +26525,8 @@ void MessagesManager::upload_imported_message_attachment(DialogId dialog_id, int
file_id, file_id,
td::make_unique<UploadedImportedMessageAttachmentInfo>(dialog_id, import_id, is_reupload, std::move(promise))); td::make_unique<UploadedImportedMessageAttachmentInfo>(dialog_id, import_id, is_reupload, std::move(promise)));
// TODO use force_reupload if is_reupload // TODO use force_reupload if is_reupload
td_->file_manager_->resume_upload(file_id, std::move(bad_parts), upload_imported_message_attachment_callback_, 1, 0); td_->file_manager_->resume_upload(file_id, std::move(bad_parts), upload_imported_message_attachment_callback_, 1, 0,
false, true);
} }
void MessagesManager::on_imported_message_attachments_uploaded(int64 random_id, Result<Unit> &&result) { void MessagesManager::on_imported_message_attachments_uploaded(int64 random_id, Result<Unit> &&result) {

22
td/telegram/files/FileManager.cpp
View File

@ -1605,6 +1605,7 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
} }
node->need_load_from_pmc_ |= other_node->need_load_from_pmc_; node->need_load_from_pmc_ |= other_node->need_load_from_pmc_;
node->can_search_locally_ &= other_node->can_search_locally_; node->can_search_locally_ &= other_node->can_search_locally_;
node->upload_prefer_small_ |= other_node->upload_prefer_small_;
if (drop_last_successful_force_reupload_time) { if (drop_last_successful_force_reupload_time) {
node->last_successful_force_reupload_time_ = -1e10; node->last_successful_force_reupload_time_ = -1e10;
@ -2319,12 +2320,13 @@ void FileManager::run_download(FileNodePtr node, bool force_update_priority) {
class FileManager::ForceUploadActor : public Actor { class FileManager::ForceUploadActor : public Actor {
public: public:
ForceUploadActor(FileManager *file_manager, FileId file_id, std::shared_ptr<FileManager::UploadCallback> callback, ForceUploadActor(FileManager *file_manager, FileId file_id, std::shared_ptr<FileManager::UploadCallback> callback,
int32 new_priority, uint64 upload_order, ActorShared<> parent) int32 new_priority, uint64 upload_order, bool prefer_small, ActorShared<> parent)
: file_manager_(file_manager) : file_manager_(file_manager)
, file_id_(file_id) , file_id_(file_id)
, callback_(std::move(callback)) , callback_(std::move(callback))
, new_priority_(new_priority) , new_priority_(new_priority)
, upload_order_(upload_order) , upload_order_(upload_order)
, prefer_small_(prefer_small)
, parent_(std::move(parent)) { , parent_(std::move(parent)) {
} }
@ -2334,9 +2336,11 @@ class FileManager::ForceUploadActor : public Actor {
std::shared_ptr<FileManager::UploadCallback> callback_; std::shared_ptr<FileManager::UploadCallback> callback_;
int32 new_priority_; int32 new_priority_;
uint64 upload_order_; uint64 upload_order_;
bool prefer_small_;
ActorShared<> parent_; ActorShared<> parent_;
bool is_active_{false}; bool is_active_{false};
int attempt_{0}; int attempt_{0};
class UploadCallback : public FileManager::UploadCallback { class UploadCallback : public FileManager::UploadCallback {
public: public:
explicit UploadCallback(ActorId<ForceUploadActor> callback) : callback_(std::move(callback)) { explicit UploadCallback(ActorId<ForceUploadActor> callback) : callback_(std::move(callback)) {
@ -2433,7 +2437,7 @@ class FileManager::ForceUploadActor : public Actor {
is_active_ = true; is_active_ = true;
attempt_++; attempt_++;
send_closure(G()->file_manager(), &FileManager::resume_upload, file_id_, std::vector<int>(), create_callback(), send_closure(G()->file_manager(), &FileManager::resume_upload, file_id_, std::vector<int>(), create_callback(),
new_priority_, upload_order_, attempt_ == 2); new_priority_, upload_order_, attempt_ == 2, prefer_small_);
} }
void tear_down() override { void tear_down() override {
@ -2452,7 +2456,7 @@ void FileManager::on_force_reupload_success(FileId file_id) {
} }
void FileManager::resume_upload(FileId file_id, std::vector<int> bad_parts, std::shared_ptr<UploadCallback> callback, void FileManager::resume_upload(FileId file_id, std::vector<int> bad_parts, std::shared_ptr<UploadCallback> callback,
int32 new_priority, uint64 upload_order, bool force) { int32 new_priority, uint64 upload_order, bool force, bool prefer_small) {
auto node = get_sync_file_node(file_id); auto node = get_sync_file_node(file_id);
if (!node) { if (!node) {
LOG(INFO) << "File " << file_id << " not found"; LOG(INFO) << "File " << file_id << " not found";
@ -2472,7 +2476,7 @@ void FileManager::resume_upload(FileId file_id, std::vector<int> bad_parts, std:
} }
create_actor<ForceUploadActor>("ForceUploadActor", this, file_id, std::move(callback), new_priority, upload_order, create_actor<ForceUploadActor>("ForceUploadActor", this, file_id, std::move(callback), new_priority, upload_order,
context_->create_reference()) prefer_small, context_->create_reference())
.release(); .release();
return; return;
} }
@ -2481,6 +2485,9 @@ void FileManager::resume_upload(FileId file_id, std::vector<int> bad_parts, std:
if (force) { if (force) {
node->remote_.is_full_alive = false; node->remote_.is_full_alive = false;
} }
if (prefer_small) {
node->upload_prefer_small_ = true;
}
if (node->upload_pause_ == file_id) { if (node->upload_pause_ == file_id) {
node->set_upload_pause(FileId()); node->set_upload_pause(FileId());
} }
@ -2790,10 +2797,15 @@ void FileManager::run_upload(FileNodePtr node, std::vector<int> bad_parts) {
auto new_priority = narrow_cast<int8>(bad_parts.empty() ? -priority : priority); auto new_priority = narrow_cast<int8>(bad_parts.empty() ? -priority : priority);
td::remove_if(bad_parts, [](auto part_id) { return part_id < 0; }); td::remove_if(bad_parts, [](auto part_id) { return part_id < 0; });
auto expected_size = file_view.expected_size(true);
if (node->upload_prefer_small_ && (10 << 20) < expected_size && expected_size < (30 << 20)) {
expected_size = 10 << 20;
}
QueryId id = queries_container_.create(Query{file_id, Query::Type::Upload}); QueryId id = queries_container_.create(Query{file_id, Query::Type::Upload});
node->upload_id_ = id; node->upload_id_ = id;
send_closure(file_load_manager_, &FileLoadManager::upload, id, node->local_, node->remote_.partial_or_empty(), send_closure(file_load_manager_, &FileLoadManager::upload, id, node->local_, node->remote_.partial_or_empty(),
file_view.expected_size(true), node->encryption_key_, new_priority, std::move(bad_parts)); expected_size, node->encryption_key_, new_priority, std::move(bad_parts));
LOG(INFO) << "File " << file_id << " upload request has sent to FileLoadManager"; LOG(INFO) << "File " << file_id << " upload request has sent to FileLoadManager";
} }

5
td/telegram/files/FileManager.h
View File

@ -156,6 +156,7 @@ class FileNode {
double last_successful_force_reupload_time_ = -1e10; double last_successful_force_reupload_time_ = -1e10;
FileId upload_pause_; FileId upload_pause_;
int8 upload_priority_ = 0; int8 upload_priority_ = 0;
int8 download_priority_ = 0; int8 download_priority_ = 0;
int8 generate_priority_ = 0; int8 generate_priority_ = 0;
@ -183,6 +184,8 @@ class FileNode {
bool upload_was_update_file_reference_{false}; bool upload_was_update_file_reference_{false};
bool download_was_update_file_reference_{false}; bool download_was_update_file_reference_{false};
bool upload_prefer_small_{false};
void init_ready_size(); void init_ready_size();
void recalc_ready_prefix_size(int64 prefix_offset, int64 ready_prefix_size); void recalc_ready_prefix_size(int64 prefix_offset, int64 ready_prefix_size);
@ -444,7 +447,7 @@ class FileManager : public FileLoadManager::Callback {
int64 limit); int64 limit);
void upload(FileId file_id, std::shared_ptr<UploadCallback> callback, int32 new_priority, uint64 upload_order); void upload(FileId file_id, std::shared_ptr<UploadCallback> callback, int32 new_priority, uint64 upload_order);
void resume_upload(FileId file_id, std::vector<int> bad_parts, std::shared_ptr<UploadCallback> callback, void resume_upload(FileId file_id, std::vector<int> bad_parts, std::shared_ptr<UploadCallback> callback,
int32 new_priority, uint64 upload_order, bool force = false); int32 new_priority, uint64 upload_order, bool force = false, bool prefer_small = false);
void cancel_upload(FileId file_id); void cancel_upload(FileId file_id);
bool delete_partial_remote_location(FileId file_id); bool delete_partial_remote_location(FileId file_id);
void delete_file_reference(FileId file_id, std::string file_reference); void delete_file_reference(FileId file_id, std::string file_reference);

2
td/telegram/files/FileUploader.cpp
View File

@ -184,7 +184,7 @@ Result<FileLoader::PrefixInfo> FileUploader::on_update_local_location(const Loca
} }
local_size_ = local_size; local_size_ = local_size;
if (expected_size_ < local_size_) { if (expected_size_ < local_size_ && (expected_size_ != (10 << 20) || local_size_ >= (30 << 20))) {
expected_size_ = local_size_; expected_size_ = local_size_;
} }
local_is_ready_ = local_is_ready; local_is_ready_ = local_is_ready;