Store remote location with FileId

GitOrigin-RevId: 43ff47088a5675df95427a610427a9908707392f

td/telegram/files/FileDownloader.cpp

@@ -66,12 +66,14 @@ Result<FileLoader::FileInfo> FileDownloader::init() {
       ready_part_count = partial.ready_part_count_;
     }
   }
-  if (false && search_file_ && fd_.empty() && size_ > 0 && size_ < 1000 * (1 << 20) && encryption_key_.empty() &&
+  LOG(ERROR) << search_file_ << " " << size_;
+  if (search_file_ && fd_.empty() && size_ > 0 && size_ < 1000 * (1 << 20) && encryption_key_.empty() &&
       !remote_.is_web()) {
     [&] {
       TRY_RESULT(path, search_file(get_files_dir(remote_.file_type_), name_, size_));
       TRY_RESULT(fd, FileFd::open(path, FileFd::Read));
       LOG(INFO) << "Check hash of local file " << path;
+      LOG(ERROR) << "Check hash of local file " << path;
       path_ = std::move(path);
       fd_ = std::move(fd);
       need_check_ = true;

td/telegram/files/FileGenerateManager.cpp

@@ -223,7 +223,7 @@ void FileGenerateManager::generate_file(uint64 query_id, const FullGenerateFileL
 
   auto &query = it_flag.first->second;
   if (conversion.copy().truncate(file_id_query.size()) == file_id_query) {
-    auto file_id = FileId(to_integer<int32>(conversion.substr(file_id_query.size())));
+    auto file_id = FileId(to_integer<int32>(conversion.substr(file_id_query.size())), 0);
     query.worker_ = create_actor<FileDownloadGenerateActor>("FileDownloadGenerateActor", generate_location.file_type_,
                                                             file_id, std::move(callback), std::move(parent));
   } else {

td/telegram/files/FileId.h

@@ -16,11 +16,14 @@ namespace td {
 
 class FileId {
   int32 id = 0;
+  int32 remote_id = 0;
 
  public:
   FileId() = default;
 
-  explicit FileId(int32 file_id) : id(file_id) {
+  explicit FileId(int32 file_id, int32 remote_id) : id(file_id), remote_id(remote_id) {
+  }
+  explicit FileId(int32 file_id) : FileId(file_id, 0) {
   }
   template <class T, typename = std::enable_if_t<std::is_convertible<T, int32>::value>>
   FileId(T file_id) = delete;
@@ -36,6 +39,10 @@ class FileId {
     return id;
   }
 
+  int32 get_remote() const {
+    return remote_id;
+  }
+
   bool operator<(const FileId &other) const {
     return id < other.id;
   }

td/telegram/files/FileManager.cpp

@@ -50,6 +50,10 @@ FileNode *FileNodePtr::get() const {
   return res;
 }
 
+FullRemoteFileLocation *FileNodePtr::get_remote() const {
+  return file_manager_->get_remote(file_id_.get_remote());
+}
+
 FileNode *FileNodePtr::get_unsafe() const {
   CHECK(file_manager_ != nullptr);
   return file_manager_->get_file_node_raw(file_id_);
@@ -263,6 +267,10 @@ bool FileView::has_remote_location() const {
 }
 const FullRemoteFileLocation &FileView::remote_location() const {
   CHECK(has_remote_location());
+  auto *remote = node_.get_remote();
+  if (remote) {
+    return *remote;
+  }
   return node_->remote_.full();
 }
 bool FileView::has_generate_location() const {
@@ -733,8 +741,23 @@ Result<FileId> FileManager::register_file(FileData data, FileLocationSource file
     }
   };
   bool new_remote = false;
+  int32 remote_key = 0;
   if (file_view.has_remote_location()) {
-    new_remote = register_location(file_view.remote_location(), remote_location_to_file_id_);
+    RemoteInfo info{file_view.remote_location(), file_id, file_location_source};
+    remote_key = remote_location_info_.add(info);
+    CHECK(remote_key < 1000);
+    auto &stored_info = remote_location_info_.get(remote_key);
+    if (stored_info.file_id == file_id) {
+      get_file_id_info(file_id)->pin_flag_ = true;
+      new_remote = true;
+    } else {
+      to_merge.push_back(stored_info.file_id);
+      if (stored_info.remote_ == file_view.remote_location() &&
+          stored_info.remote_.get_access_hash() != file_view.remote_location().get_access_hash() &&
+          file_location_source == FileLocationSource::FromServer) {
+        stored_info.remote_ = file_view.remote_location();
+      }
+    }
   }
   bool new_local = false;
   if (file_view.has_local_location()) {
@@ -760,7 +783,7 @@ Result<FileId> FileManager::register_file(FileData data, FileLocationSource file
   try_flush_node(get_file_node(file_id));
   auto main_file_id = get_file_node(file_id)->main_file_id_;
   try_forget_file_id(file_id);
-  return main_file_id;
+  return FileId(main_file_id.get(), remote_key);
 }
 
 // 0 -- choose x
@@ -1930,7 +1953,7 @@ Result<FileId> FileManager::get_input_file_id(FileType type, const tl_object_ptr
                               owner_dialog_id, 0, get_by_hash);
       }
       case td_api::inputFileId::ID: {
-        FileId file_id(static_cast<const td_api::inputFileId *>(file.get())->id_);
+        FileId file_id(static_cast<const td_api::inputFileId *>(file.get())->id_, 0);
         if (!file_id.is_valid()) {
           return FileId();
         }
@@ -1977,7 +2000,7 @@ FileId FileManager::next_file_id() {
     empty_file_ids_.pop_back();
     return res;
   }
-  FileId res(static_cast<int32>(file_id_info_.size()));
+  FileId res(static_cast<int32>(file_id_info_.size()), 0);
   // LOG(ERROR) << "NEXT file_id " << res;
   file_id_info_.push_back({});
   return res;
@@ -2304,6 +2327,13 @@ std::pair<FileManager::Query, bool> FileManager::finish_query(QueryId query_id)
   return std::make_pair(res, was_active);
 }
 
+FullRemoteFileLocation *FileManager::get_remote(int32 key) {
+  if (key == 0) {
+    return nullptr;
+  }
+  return &remote_location_info_.get(key).remote_;
+}
+
 void FileManager::hangup() {
   file_db_.reset();
   file_generate_manager_.reset();

td/telegram/files/FileManager.h

@@ -20,6 +20,7 @@
 #include "td/utils/buffer.h"
 #include "td/utils/common.h"
 #include "td/utils/Container.h"
+#include "td/utils/Enumerator.h"
 #include "td/utils/Slice.h"
 #include "td/utils/Status.h"
 
@@ -136,6 +137,7 @@ class FileNodePtr {
   FileNode *operator->() const;
   FileNode &operator*() const;
   FileNode *get() const;
+  FullRemoteFileLocation *get_remote() const;
   explicit operator bool() const;
 
  private:
@@ -160,6 +162,9 @@ class ConstFileNodePtr {
   explicit operator bool() const {
     return bool(file_node_ptr_);
   }
+  const FullRemoteFileLocation *get_remote() const {
+    return file_node_ptr_.get_remote();
+  }
 
  private:
   FileNodePtr file_node_ptr_;
@@ -371,10 +376,23 @@ class FileManager : public FileLoadManager::Callback {
 
   FileIdInfo *get_file_id_info(FileId file_id);
 
-  std::map<FullRemoteFileLocation, FileId> remote_location_to_file_id_;
+  struct RemoteInfo {
+    FullRemoteFileLocation remote_;
+    FileId file_id;
+    FileLocationSource source_;
+    bool operator==(const RemoteInfo &other) const {
+      return this->remote_ == other.remote_;
+    }
+    bool operator<(const RemoteInfo &other) const {
+      return this->remote_ < other.remote_;
+    }
+  };
+  Enumerator<RemoteInfo> remote_location_info_;
+
   std::map<FullLocalFileLocation, FileId> local_location_to_file_id_;
   std::map<FullGenerateFileLocation, FileId> generate_location_to_file_id_;
   std::map<FileDbId, int32> pmc_id_to_file_node_id_;
+
   vector<FileIdInfo> file_id_info_;
   vector<FileId> empty_file_ids_;
   vector<std::unique_ptr<FileNode>> file_nodes_;
@@ -447,6 +465,8 @@ class FileManager : public FileLoadManager::Callback {
 
   std::pair<Query, bool> finish_query(QueryId query_id);
 
+  FullRemoteFileLocation *get_remote(int32 key);
+
   void hangup() override;
   void tear_down() override;
 

td/telegram/files/FileManager.hpp

@@ -82,7 +82,7 @@ void FileManager::store_file(FileId file_id, StorerT &storer, int32 ttl) const {
       } else if (begins_with(generate_location.conversion_, "#file_id#")) {
         // It is not the best possible way to serialize file_id
         from_file_id =
-            FileId(to_integer<int32>(Slice(generate_location.conversion_).remove_prefix(Slice("#file_id#").size())));
+            FileId(to_integer<int32>(Slice(generate_location.conversion_).remove_prefix(Slice("#file_id#").size())), 0);
         generate_location.conversion_ = "#_file_id#";
         have_file_id = true;
       }