andreacavalli
/
tdlib-fork
Archived
1
0
Fork 0
Code Pull Requests Releases Activity

remove data merging

This commit is contained in:
andrew (from workstation) 2020-05-18 15:17:49 +02:00
parent 53c1ef20ef
commit dff274f1a5
1 changed files with 1 additions and 255 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

256
td/telegram/files/FileManager.cpp
View File

@ -1360,260 +1360,7 @@ void FileManager::do_cancel_generate(FileNodePtr node) {
}
Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sync) {
LOG(DEBUG) << "Merge new file " << x_file_id << " and old file " << y_file_id;
if (!x_file_id.is_valid()) {
return Status::Error("First file_id is invalid");
}
FileNodePtr x_node = no_sync ? get_file_node(x_file_id) : get_sync_file_node(x_file_id);
if (!x_node) {
return Status::Error(PSLICE() << "Can't merge files. First id is invalid: " << x_file_id << " and " << y_file_id);
}
if (!y_file_id.is_valid()) {
LOG(DEBUG) << "Old file is invalid";
return x_node->main_file_id_;
}
FileNodePtr y_node = get_file_node(y_file_id);
if (!y_node) {
return Status::Error(PSLICE() << "Can't merge files. Second id is invalid: " << x_file_id << " and " << y_file_id);
}
if (x_file_id == x_node->upload_pause_) {
x_node->set_upload_pause(FileId());
}
if (x_node.get() == y_node.get()) {
LOG(DEBUG) << "Files are already merged";
return x_node->main_file_id_;
}
if (y_file_id == y_node->upload_pause_) {
y_node->set_upload_pause(FileId());
}
if (x_node->remote_.full && y_node->remote_.full && !x_node->remote_.full.value().is_web() &&
!y_node->remote_.full.value().is_web() && y_node->remote_.is_full_alive &&
x_node->remote_.full_source == FileLocationSource::FromServer &&
y_node->remote_.full_source == FileLocationSource::FromServer &&
x_node->remote_.full.value().get_dc_id() != y_node->remote_.full.value().get_dc_id()) {
}
bool drop_last_successful_force_reupload_time = x_node->last_successful_force_reupload_time_ <= 0 &&
x_node->remote_.full &&
x_node->remote_.full_source == FileLocationSource::FromServer;
auto count_local = [](auto &node) {
return std::accumulate(node->file_ids_.begin(), node->file_ids_.end(), 0,
[](const auto &x, const auto &y) { return x + (y.get_remote() != 0); });
};
if (count_local(x_node) + count_local(y_node) > 100) {
}
FileNodePtr nodes[] = {x_node, y_node, x_node};
FileNodeId node_ids[] = {get_file_id_info(x_file_id)->node_id_, get_file_id_info(y_file_id)->node_id_};
int trusted_by_source = merge_choose_file_source_location(x_node->remote_.full_source, y_node->remote_.full_source);
int local_i = merge_choose_local_location(x_node->local_, y_node->local_);
int remote_i = merge_choose_remote_location(x_node->remote_, y_node->remote_);
int generate_i = merge_choose_generate_location(x_node->generate_, y_node->generate_);
int size_i = merge_choose_size(x_node->size_, y_node->size_);
int expected_size_i = merge_choose_expected_size(x_node->expected_size_, y_node->expected_size_);
int remote_name_i = merge_choose_name(x_node->remote_name_, y_node->remote_name_);
int url_i = merge_choose_name(x_node->url_, y_node->url_);
int owner_i = merge_choose_owner(x_node->owner_dialog_id_, y_node->owner_dialog_id_);
int encryption_key_i = merge_choose_encryption_key(x_node->encryption_key_, y_node->encryption_key_);
int main_file_id_i = merge_choose_main_file_id(x_node->main_file_id_, x_node->main_file_id_priority_,
y_node->main_file_id_, y_node->main_file_id_priority_);
if (size_i == -1) {
return Status::Error(PSLICE() << "Can't merge files. Different size: " << x_node->size_ << " and "
<< y_node->size_);
}
if (encryption_key_i == -1) {
if (nodes[remote_i]->remote_.full && nodes[local_i]->local_.type() != LocalFileLocation::Type::Partial) {
LOG(ERROR) << "Different encryption key in files, but lets choose same key as remote location";
encryption_key_i = remote_i;
} else {
return Status::Error("Can't merge files. Different encryption keys");
}
}
// prefer more trusted source
if (remote_name_i == 2) {
remote_name_i = trusted_by_source;
}
if (url_i == 2) {
url_i = trusted_by_source;
}
if (expected_size_i == 2) {
expected_size_i = trusted_by_source;
}
int node_i =
std::make_tuple(y_node->pmc_id_.is_valid(), x_node->pmc_id_, y_node->file_ids_.size(), main_file_id_i == 1) >
std::make_tuple(x_node->pmc_id_.is_valid(), y_node->pmc_id_, x_node->file_ids_.size(), main_file_id_i == 0);
auto other_node_i = 1 - node_i;
FileNodePtr node = nodes[node_i];
FileNodePtr other_node = nodes[other_node_i];
auto file_view = FileView(node);
LOG(DEBUG) << "Have x_node->pmc_id_ = " << x_node->pmc_id_.get() << ", y_node->pmc_id_ = " << y_node->pmc_id_.get()
<< ", x_node_size = " << x_node->file_ids_.size() << ", y_node_size = " << y_node->file_ids_.size()
<< ", node_i = " << node_i << ", local_i = " << local_i << ", remote_i = " << remote_i
<< ", generate_i = " << generate_i << ", size_i = " << size_i << ", remote_name_i = " << remote_name_i
<< ", url_i = " << url_i << ", owner_i = " << owner_i << ", encryption_key_i = " << encryption_key_i
<< ", main_file_id_i = " << main_file_id_i << ", trusted_by_source = " << trusted_by_source
<< ", x_source = " << x_node->remote_.full_source << ", y_source = " << y_node->remote_.full_source;
if (local_i == other_node_i) {
do_cancel_download(node);
node->set_download_offset(other_node->download_offset_);
node->set_local_location(other_node->local_, other_node->local_ready_size_, other_node->download_offset_,
other_node->local_ready_prefix_size_);
node->download_id_ = other_node->download_id_;
node->download_was_update_file_reference_ = other_node->download_was_update_file_reference_;
node->is_download_started_ |= other_node->is_download_started_;
node->set_download_priority(other_node->download_priority_);
other_node->download_id_ = 0;
other_node->download_was_update_file_reference_ = false;
other_node->is_download_started_ = false;
other_node->download_priority_ = 0;
other_node->download_offset_ = 0;
other_node->local_ready_prefix_size_ = 0;
//do_cancel_generate(node);
//node->set_generate_location(std::move(other_node->generate_));
//node->generate_id_ = other_node->generate_id_;
//node->set_generate_priority(other_node->generate_download_priority_, other_node->generate_upload_priority_);
//other_node->generate_id_ = 0;
//other_node->generate_was_update_ = false;
//other_node->generate_priority_ = 0;
//other_node->generate_download_priority_ = 0;
//other_node->generate_upload_priority_ = 0;
} else {
do_cancel_download(other_node);
//do_cancel_generate(other_node);
}
if (remote_i == other_node_i) {
do_cancel_upload(node);
node->set_new_remote_location(std::move(other_node->remote_));
node->upload_id_ = other_node->upload_id_;
node->upload_was_update_file_reference_ = other_node->upload_was_update_file_reference_;
node->set_upload_priority(other_node->upload_priority_);
node->set_upload_pause(other_node->upload_pause_);
other_node->upload_id_ = 0;
other_node->upload_was_update_file_reference_ = false;
other_node->upload_priority_ = 0;
other_node->set_upload_pause(FileId());
} else {
do_cancel_upload(other_node);
}
if (generate_i == other_node_i) {
do_cancel_generate(node);
node->set_generate_location(std::move(other_node->generate_));
node->generate_id_ = other_node->generate_id_;
node->set_generate_priority(other_node->generate_download_priority_, other_node->generate_upload_priority_);
other_node->generate_id_ = 0;
other_node->generate_priority_ = 0;
other_node->generate_download_priority_ = 0;
other_node->generate_upload_priority_ = 0;
} else {
do_cancel_generate(other_node);
}
if (size_i == other_node_i) {
node->set_size(other_node->size_);
}
if (expected_size_i == other_node_i) {
node->set_expected_size(other_node->expected_size_);
}
if (remote_name_i == other_node_i) {
node->set_remote_name(other_node->remote_name_);
}
if (url_i == other_node_i) {
node->set_url(other_node->url_);
}
if (owner_i == other_node_i) {
node->set_owner_dialog_id(other_node->owner_dialog_id_);
}
if (encryption_key_i == other_node_i) {
node->set_encryption_key(other_node->encryption_key_);
nodes[node_i]->set_encryption_key(nodes[encryption_key_i]->encryption_key_);
}
node->need_load_from_pmc_ |= other_node->need_load_from_pmc_;
node->can_search_locally_ &= other_node->can_search_locally_;
if (drop_last_successful_force_reupload_time) {
node->last_successful_force_reupload_time_ = -1e10;
} else if (other_node->last_successful_force_reupload_time_ > node->last_successful_force_reupload_time_) {
node->last_successful_force_reupload_time_ = other_node->last_successful_force_reupload_time_;
}
if (main_file_id_i == other_node_i) {
context_->on_merge_files(other_node->main_file_id_, node->main_file_id_);
node->main_file_id_ = other_node->main_file_id_;
node->main_file_id_priority_ = other_node->main_file_id_priority_;
} else {
context_->on_merge_files(node->main_file_id_, other_node->main_file_id_);
}
bool send_updates_flag = false;
auto other_pmc_id = other_node->pmc_id_;
node->file_ids_.insert(node->file_ids_.end(), other_node->file_ids_.begin(), other_node->file_ids_.end());
for (auto file_id : other_node->file_ids_) {
auto file_id_info = get_file_id_info(file_id);
LOG_CHECK(file_id_info->node_id_ == node_ids[other_node_i])
<< node_ids[node_i] << " " << node_ids[other_node_i] << " " << file_id << " " << file_id_info->node_id_;
file_id_info->node_id_ = node_ids[node_i];
send_updates_flag |= file_id_info->send_updates_flag_;
}
other_node = {this};
if (send_updates_flag) {
// node might not changed, but other_node might changed, so we need to send update anyway
VLOG(update_file) << "File " << node->main_file_id_ << " has been merged";
node->on_info_changed();
}
// Check is some download/upload queries are ready
for (auto file_id : vector<FileId>(node->file_ids_)) {
auto *info = get_file_id_info(file_id);
if (info->download_priority_ != 0 && file_view.has_local_location()) {
info->download_priority_ = 0;
if (info->download_callback_) {
info->download_callback_->on_download_ok(file_id);
info->download_callback_.reset();
}
}
if (info->upload_priority_ != 0 && file_view.has_active_upload_remote_location()) {
info->upload_priority_ = 0;
if (info->upload_callback_) {
info->upload_callback_->on_upload_ok(file_id, nullptr);
info->upload_callback_.reset();
}
}
}
file_nodes_[node_ids[other_node_i]] = nullptr;
run_generate(node);
run_download(node);
run_upload(node, {});
if (other_pmc_id.is_valid()) {
// node might not changed, but we need to merge nodes in pmc anyway
node->on_pmc_changed();
}
try_flush_node_full(node, node_i != remote_i, node_i != local_i, node_i != generate_i, other_pmc_id);
return node->main_file_id_;
return Status::Error("stub");
}
void FileManager::add_file_source(FileId file_id, FileSourceId file_source_id) {
@ -3317,7 +3064,6 @@ void FileManager::on_download_ok(QueryId query_id, const FullLocalFileLocation &
if (is_new) {
context_->on_new_file(size, get_file_view(r_new_file_id.ok()).get_allocated_local_size(), 1);
}
LOG_STATUS(merge(r_new_file_id.ok(), file_id));
}
}