From 312ccd7cbda4acff59c76a9581012aba192c5a53 Mon Sep 17 00:00:00 2001 From: Arseny Smirnov Date: Tue, 6 Mar 2018 21:31:20 +0300 Subject: [PATCH] Store remote location with FileId GitOrigin-RevId: 43ff47088a5675df95427a610427a9908707392f --- td/telegram/files/FileDownloader.cpp | 4 ++- td/telegram/files/FileGenerateManager.cpp | 2 +- td/telegram/files/FileId.h | 9 +++++- td/telegram/files/FileManager.cpp | 38 ++++++++++++++++++++--- td/telegram/files/FileManager.h | 22 ++++++++++++- td/telegram/files/FileManager.hpp | 2 +- 6 files changed, 68 insertions(+), 9 deletions(-) diff --git a/td/telegram/files/FileDownloader.cpp b/td/telegram/files/FileDownloader.cpp index 34684981d..cd6a61a60 100644 --- a/td/telegram/files/FileDownloader.cpp +++ b/td/telegram/files/FileDownloader.cpp @@ -66,12 +66,14 @@ Result FileDownloader::init() { ready_part_count = partial.ready_part_count_; } } - if (false && search_file_ && fd_.empty() && size_ > 0 && size_ < 1000 * (1 << 20) && encryption_key_.empty() && + LOG(ERROR) << search_file_ << " " << size_; + if (search_file_ && fd_.empty() && size_ > 0 && size_ < 1000 * (1 << 20) && encryption_key_.empty() && !remote_.is_web()) { [&] { TRY_RESULT(path, search_file(get_files_dir(remote_.file_type_), name_, size_)); TRY_RESULT(fd, FileFd::open(path, FileFd::Read)); LOG(INFO) << "Check hash of local file " << path; + LOG(ERROR) << "Check hash of local file " << path; path_ = std::move(path); fd_ = std::move(fd); need_check_ = true; diff --git a/td/telegram/files/FileGenerateManager.cpp b/td/telegram/files/FileGenerateManager.cpp index de5d284e9..22472a248 100644 --- a/td/telegram/files/FileGenerateManager.cpp +++ b/td/telegram/files/FileGenerateManager.cpp @@ -223,7 +223,7 @@ void FileGenerateManager::generate_file(uint64 query_id, const FullGenerateFileL auto &query = it_flag.first->second; if (conversion.copy().truncate(file_id_query.size()) == file_id_query) { - auto file_id = FileId(to_integer(conversion.substr(file_id_query.size()))); + auto file_id = FileId(to_integer(conversion.substr(file_id_query.size())), 0); query.worker_ = create_actor("FileDownloadGenerateActor", generate_location.file_type_, file_id, std::move(callback), std::move(parent)); } else { diff --git a/td/telegram/files/FileId.h b/td/telegram/files/FileId.h index bf1024ac3..4bedce28e 100644 --- a/td/telegram/files/FileId.h +++ b/td/telegram/files/FileId.h @@ -16,11 +16,14 @@ namespace td { class FileId { int32 id = 0; + int32 remote_id = 0; public: FileId() = default; - explicit FileId(int32 file_id) : id(file_id) { + explicit FileId(int32 file_id, int32 remote_id) : id(file_id), remote_id(remote_id) { + } + explicit FileId(int32 file_id) : FileId(file_id, 0) { } template ::value>> FileId(T file_id) = delete; @@ -36,6 +39,10 @@ class FileId { return id; } + int32 get_remote() const { + return remote_id; + } + bool operator<(const FileId &other) const { return id < other.id; } diff --git a/td/telegram/files/FileManager.cpp b/td/telegram/files/FileManager.cpp index c29901d84..d4744a3f9 100644 --- a/td/telegram/files/FileManager.cpp +++ b/td/telegram/files/FileManager.cpp @@ -50,6 +50,10 @@ FileNode *FileNodePtr::get() const { return res; } +FullRemoteFileLocation *FileNodePtr::get_remote() const { + return file_manager_->get_remote(file_id_.get_remote()); +} + FileNode *FileNodePtr::get_unsafe() const { CHECK(file_manager_ != nullptr); return file_manager_->get_file_node_raw(file_id_); @@ -263,6 +267,10 @@ bool FileView::has_remote_location() const { } const FullRemoteFileLocation &FileView::remote_location() const { CHECK(has_remote_location()); + auto *remote = node_.get_remote(); + if (remote) { + return *remote; + } return node_->remote_.full(); } bool FileView::has_generate_location() const { @@ -733,8 +741,23 @@ Result FileManager::register_file(FileData data, FileLocationSource file } }; bool new_remote = false; + int32 remote_key = 0; if (file_view.has_remote_location()) { - new_remote = register_location(file_view.remote_location(), remote_location_to_file_id_); + RemoteInfo info{file_view.remote_location(), file_id, file_location_source}; + remote_key = remote_location_info_.add(info); + CHECK(remote_key < 1000); + auto &stored_info = remote_location_info_.get(remote_key); + if (stored_info.file_id == file_id) { + get_file_id_info(file_id)->pin_flag_ = true; + new_remote = true; + } else { + to_merge.push_back(stored_info.file_id); + if (stored_info.remote_ == file_view.remote_location() && + stored_info.remote_.get_access_hash() != file_view.remote_location().get_access_hash() && + file_location_source == FileLocationSource::FromServer) { + stored_info.remote_ = file_view.remote_location(); + } + } } bool new_local = false; if (file_view.has_local_location()) { @@ -760,7 +783,7 @@ Result FileManager::register_file(FileData data, FileLocationSource file try_flush_node(get_file_node(file_id)); auto main_file_id = get_file_node(file_id)->main_file_id_; try_forget_file_id(file_id); - return main_file_id; + return FileId(main_file_id.get(), remote_key); } // 0 -- choose x @@ -1930,7 +1953,7 @@ Result FileManager::get_input_file_id(FileType type, const tl_object_ptr owner_dialog_id, 0, get_by_hash); } case td_api::inputFileId::ID: { - FileId file_id(static_cast(file.get())->id_); + FileId file_id(static_cast(file.get())->id_, 0); if (!file_id.is_valid()) { return FileId(); } @@ -1977,7 +2000,7 @@ FileId FileManager::next_file_id() { empty_file_ids_.pop_back(); return res; } - FileId res(static_cast(file_id_info_.size())); + FileId res(static_cast(file_id_info_.size()), 0); // LOG(ERROR) << "NEXT file_id " << res; file_id_info_.push_back({}); return res; @@ -2304,6 +2327,13 @@ std::pair FileManager::finish_query(QueryId query_id) return std::make_pair(res, was_active); } +FullRemoteFileLocation *FileManager::get_remote(int32 key) { + if (key == 0) { + return nullptr; + } + return &remote_location_info_.get(key).remote_; +} + void FileManager::hangup() { file_db_.reset(); file_generate_manager_.reset(); diff --git a/td/telegram/files/FileManager.h b/td/telegram/files/FileManager.h index a1a9d1a6c..351ea0051 100644 --- a/td/telegram/files/FileManager.h +++ b/td/telegram/files/FileManager.h @@ -20,6 +20,7 @@ #include "td/utils/buffer.h" #include "td/utils/common.h" #include "td/utils/Container.h" +#include "td/utils/Enumerator.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" @@ -136,6 +137,7 @@ class FileNodePtr { FileNode *operator->() const; FileNode &operator*() const; FileNode *get() const; + FullRemoteFileLocation *get_remote() const; explicit operator bool() const; private: @@ -160,6 +162,9 @@ class ConstFileNodePtr { explicit operator bool() const { return bool(file_node_ptr_); } + const FullRemoteFileLocation *get_remote() const { + return file_node_ptr_.get_remote(); + } private: FileNodePtr file_node_ptr_; @@ -371,10 +376,23 @@ class FileManager : public FileLoadManager::Callback { FileIdInfo *get_file_id_info(FileId file_id); - std::map remote_location_to_file_id_; + struct RemoteInfo { + FullRemoteFileLocation remote_; + FileId file_id; + FileLocationSource source_; + bool operator==(const RemoteInfo &other) const { + return this->remote_ == other.remote_; + } + bool operator<(const RemoteInfo &other) const { + return this->remote_ < other.remote_; + } + }; + Enumerator remote_location_info_; + std::map local_location_to_file_id_; std::map generate_location_to_file_id_; std::map pmc_id_to_file_node_id_; + vector file_id_info_; vector empty_file_ids_; vector> file_nodes_; @@ -447,6 +465,8 @@ class FileManager : public FileLoadManager::Callback { std::pair finish_query(QueryId query_id); + FullRemoteFileLocation *get_remote(int32 key); + void hangup() override; void tear_down() override; diff --git a/td/telegram/files/FileManager.hpp b/td/telegram/files/FileManager.hpp index 69c820eac..28f481308 100644 --- a/td/telegram/files/FileManager.hpp +++ b/td/telegram/files/FileManager.hpp @@ -82,7 +82,7 @@ void FileManager::store_file(FileId file_id, StorerT &storer, int32 ttl) const { } else if (begins_with(generate_location.conversion_, "#file_id#")) { // It is not the best possible way to serialize file_id from_file_id = - FileId(to_integer(Slice(generate_location.conversion_).remove_prefix(Slice("#file_id#").size()))); + FileId(to_integer(Slice(generate_location.conversion_).remove_prefix(Slice("#file_id#").size())), 0); generate_location.conversion_ = "#_file_id#"; have_file_id = true; }