From c6544b681c6dc15dcfe18b711ff2ce67e0b5ffe9 Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Wed, 20 May 2020 19:50:27 +0200 Subject: [PATCH] Merger --- td/telegram/files/FileManager.cpp | 255 +++++++++++++++++++++++++++++- 1 file changed, 254 insertions(+), 1 deletion(-) diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index 45c506ad..8958daae 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -1370,7 +1370,260 @@ void FileManager::do_cancel_generate(FileNodePtr node) { } Result FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) { - return y_file_id; + LOG(DEBUG) << "Merge new file " << x_file_id << " and old file " << y_file_id; + + if (!x_file_id.is_valid()) { + return Status::Error("First file_id is invalid"); + } + FileNodePtr x_node = no_sync ? get_file_node(x_file_id) : get_sync_file_node(x_file_id); + if (!x_node) { + return Status::Error(PSLICE() << "Can't merge files. First id is invalid: " << x_file_id << " and " << y_file_id); + } + + if (!y_file_id.is_valid()) { + LOG(DEBUG) << "Old file is invalid"; + return x_node->main_file_id_; + } + FileNodePtr y_node = get_file_node(y_file_id); + if (!y_node) { + return Status::Error(PSLICE() << "Can't merge files. Second id is invalid: " << x_file_id << " and " << y_file_id); + } + + if (x_file_id == x_node->upload_pause_) { + x_node->set_upload_pause(FileId()); + } + if (x_node.get() == y_node.get()) { + LOG(DEBUG) << "Files are already merged"; + return x_node->main_file_id_; + } + if (y_file_id == y_node->upload_pause_) { + y_node->set_upload_pause(FileId()); + } + + if (x_node->remote_.full && y_node->remote_.full && !x_node->remote_.full.value().is_web() && + !y_node->remote_.full.value().is_web() && y_node->remote_.is_full_alive && + x_node->remote_.full_source == FileLocationSource::FromServer && + y_node->remote_.full_source == FileLocationSource::FromServer && + x_node->remote_.full.value().get_dc_id() != y_node->remote_.full.value().get_dc_id()) { + } + + bool drop_last_successful_force_reupload_time = x_node->last_successful_force_reupload_time_ <= 0 && + x_node->remote_.full && + x_node->remote_.full_source == FileLocationSource::FromServer; + + auto count_local = [](auto &node) { + return std::accumulate(node->file_ids_.begin(), node->file_ids_.end(), 0, + [](const auto &x, const auto &y) { return x + (y.get_remote() != 0); }); + }; + if (count_local(x_node) + count_local(y_node) > 100) { + } + + FileNodePtr nodes[] = {x_node, y_node, x_node}; + FileNodeId node_ids[] = {get_file_id_info(x_file_id)->node_id_, get_file_id_info(y_file_id)->node_id_}; + int trusted_by_source = merge_choose_file_source_location(x_node->remote_.full_source, y_node->remote_.full_source); + + int local_i = merge_choose_local_location(x_node->local_, y_node->local_); + int remote_i = merge_choose_remote_location(x_node->remote_, y_node->remote_); + int generate_i = merge_choose_generate_location(x_node->generate_, y_node->generate_); + int size_i = merge_choose_size(x_node->size_, y_node->size_); + int expected_size_i = merge_choose_expected_size(x_node->expected_size_, y_node->expected_size_); + int remote_name_i = merge_choose_name(x_node->remote_name_, y_node->remote_name_); + int url_i = merge_choose_name(x_node->url_, y_node->url_); + int owner_i = merge_choose_owner(x_node->owner_dialog_id_, y_node->owner_dialog_id_); + int encryption_key_i = merge_choose_encryption_key(x_node->encryption_key_, y_node->encryption_key_); + int main_file_id_i = merge_choose_main_file_id(x_node->main_file_id_, x_node->main_file_id_priority_, + y_node->main_file_id_, y_node->main_file_id_priority_); + + if (size_i == -1) { + return Status::Error(PSLICE() << "Can't merge files. Different size: " << x_node->size_ << " and " + << y_node->size_); + } + if (encryption_key_i == -1) { + if (nodes[remote_i]->remote_.full && nodes[local_i]->local_.type() != LocalFileLocation::Type::Partial) { + LOG(ERROR) << "Different encryption key in files, but lets choose same key as remote location"; + encryption_key_i = remote_i; + } else { + return Status::Error("Can't merge files. Different encryption keys"); + } + } + + // prefer more trusted source + if (remote_name_i == 2) { + remote_name_i = trusted_by_source; + } + if (url_i == 2) { + url_i = trusted_by_source; + } + if (expected_size_i == 2) { + expected_size_i = trusted_by_source; + } + + int node_i = + std::make_tuple(y_node->pmc_id_.is_valid(), x_node->pmc_id_, y_node->file_ids_.size(), main_file_id_i == 1) > + std::make_tuple(x_node->pmc_id_.is_valid(), y_node->pmc_id_, x_node->file_ids_.size(), main_file_id_i == 0); + + auto other_node_i = 1 - node_i; + FileNodePtr node = nodes[node_i]; + FileNodePtr other_node = nodes[other_node_i]; + auto file_view = FileView(node); + + LOG(DEBUG) << "Have x_node->pmc_id_ = " << x_node->pmc_id_.get() << ", y_node->pmc_id_ = " << y_node->pmc_id_.get() + << ", x_node_size = " << x_node->file_ids_.size() << ", y_node_size = " << y_node->file_ids_.size() + << ", node_i = " << node_i << ", local_i = " << local_i << ", remote_i = " << remote_i + << ", generate_i = " << generate_i << ", size_i = " << size_i << ", remote_name_i = " << remote_name_i + << ", url_i = " << url_i << ", owner_i = " << owner_i << ", encryption_key_i = " << encryption_key_i + << ", main_file_id_i = " << main_file_id_i << ", trusted_by_source = " << trusted_by_source + << ", x_source = " << x_node->remote_.full_source << ", y_source = " << y_node->remote_.full_source; + if (local_i == other_node_i) { + do_cancel_download(node); + node->set_download_offset(other_node->download_offset_); + node->set_local_location(other_node->local_, other_node->local_ready_size_, other_node->download_offset_, + other_node->local_ready_prefix_size_); + node->download_id_ = other_node->download_id_; + node->download_was_update_file_reference_ = other_node->download_was_update_file_reference_; + node->is_download_started_ |= other_node->is_download_started_; + node->set_download_priority(other_node->download_priority_); + other_node->download_id_ = 0; + other_node->download_was_update_file_reference_ = false; + other_node->is_download_started_ = false; + other_node->download_priority_ = 0; + other_node->download_offset_ = 0; + other_node->local_ready_prefix_size_ = 0; + + //do_cancel_generate(node); + //node->set_generate_location(std::move(other_node->generate_)); + //node->generate_id_ = other_node->generate_id_; + //node->set_generate_priority(other_node->generate_download_priority_, other_node->generate_upload_priority_); + //other_node->generate_id_ = 0; + //other_node->generate_was_update_ = false; + //other_node->generate_priority_ = 0; + //other_node->generate_download_priority_ = 0; + //other_node->generate_upload_priority_ = 0; + } else { + do_cancel_download(other_node); + //do_cancel_generate(other_node); + } + + if (remote_i == other_node_i) { + do_cancel_upload(node); + node->set_new_remote_location(std::move(other_node->remote_)); + node->upload_id_ = other_node->upload_id_; + node->upload_was_update_file_reference_ = other_node->upload_was_update_file_reference_; + node->set_upload_priority(other_node->upload_priority_); + node->set_upload_pause(other_node->upload_pause_); + other_node->upload_id_ = 0; + other_node->upload_was_update_file_reference_ = false; + other_node->upload_priority_ = 0; + other_node->set_upload_pause(FileId()); + } else { + do_cancel_upload(other_node); + } + + if (generate_i == other_node_i) { + do_cancel_generate(node); + node->set_generate_location(std::move(other_node->generate_)); + node->generate_id_ = other_node->generate_id_; + node->set_generate_priority(other_node->generate_download_priority_, other_node->generate_upload_priority_); + other_node->generate_id_ = 0; + other_node->generate_priority_ = 0; + other_node->generate_download_priority_ = 0; + other_node->generate_upload_priority_ = 0; + } else { + do_cancel_generate(other_node); + } + + if (size_i == other_node_i) { + node->set_size(other_node->size_); + } + + if (expected_size_i == other_node_i) { + node->set_expected_size(other_node->expected_size_); + } + + if (remote_name_i == other_node_i) { + node->set_remote_name(other_node->remote_name_); + } + + if (url_i == other_node_i) { + node->set_url(other_node->url_); + } + + if (owner_i == other_node_i) { + node->set_owner_dialog_id(other_node->owner_dialog_id_); + } + + if (encryption_key_i == other_node_i) { + node->set_encryption_key(other_node->encryption_key_); + nodes[node_i]->set_encryption_key(nodes[encryption_key_i]->encryption_key_); + } + node->need_load_from_pmc_ |= other_node->need_load_from_pmc_; + node->can_search_locally_ &= other_node->can_search_locally_; + + if (drop_last_successful_force_reupload_time) { + node->last_successful_force_reupload_time_ = -1e10; + } else if (other_node->last_successful_force_reupload_time_ > node->last_successful_force_reupload_time_) { + node->last_successful_force_reupload_time_ = other_node->last_successful_force_reupload_time_; + } + + if (main_file_id_i == other_node_i) { + context_->on_merge_files(other_node->main_file_id_, node->main_file_id_); + node->main_file_id_ = other_node->main_file_id_; + node->main_file_id_priority_ = other_node->main_file_id_priority_; + } else { + context_->on_merge_files(node->main_file_id_, other_node->main_file_id_); + } + + bool send_updates_flag = false; + auto other_pmc_id = other_node->pmc_id_; + node->file_ids_.insert(node->file_ids_.end(), other_node->file_ids_.begin(), other_node->file_ids_.end()); + + for (auto file_id : other_node->file_ids_) { + auto file_id_info = get_file_id_info(file_id); + LOG_CHECK(file_id_info->node_id_ == node_ids[other_node_i]) + << node_ids[node_i] << " " << node_ids[other_node_i] << " " << file_id << " " << file_id_info->node_id_; + file_id_info->node_id_ = node_ids[node_i]; + send_updates_flag |= file_id_info->send_updates_flag_; + } + other_node = {this}; + + if (send_updates_flag) { + // node might not changed, but other_node might changed, so we need to send update anyway + VLOG(update_file) << "File " << node->main_file_id_ << " has been merged"; + node->on_info_changed(); + } + + // Check is some download/upload queries are ready + for (auto file_id : vector(node->file_ids_)) { + auto *info = get_file_id_info(file_id); + if (info->download_priority_ != 0 && file_view.has_local_location()) { + info->download_priority_ = 0; + if (info->download_callback_) { + info->download_callback_->on_download_ok(file_id); + info->download_callback_.reset(); + } + } + if (info->upload_priority_ != 0 && file_view.has_active_upload_remote_location()) { + info->upload_priority_ = 0; + if (info->upload_callback_) { + info->upload_callback_->on_upload_ok(file_id, nullptr); + info->upload_callback_.reset(); + } + } + } + + file_nodes_[node_ids[other_node_i]] = nullptr; + + run_generate(node); + run_download(node); + run_upload(node, {}); + + if (other_pmc_id.is_valid()) { + // node might not changed, but we need to merge nodes in pmc anyway + node->on_pmc_changed(); + } + try_flush_node_full(node, node_i != remote_i, node_i != local_i, node_i != generate_i, other_pmc_id); + + return node->main_file_id_; } void FileManager::add_file_source(FileId file_id, FileSourceId file_source_id) {