From a1e93ebba5da31c99c868b66349a37da0829e79f Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 2 Mar 2020 11:59:47 +0300 Subject: [PATCH] Add optimizeStorage.return_deleted_file_statistics. GitOrigin-RevId: f7533781027891624017c605b1d3ac3e91da5086 --- td/generate/scheme/td_api.tl | 3 +- td/generate/scheme/td_api.tlo | Bin 167444 -> 167500 bytes td/telegram/StorageManager.cpp | 76 +++++++++++++++++------------ td/telegram/StorageManager.h | 9 ++-- td/telegram/Td.cpp | 3 +- td/telegram/cli.cpp | 10 ++-- td/telegram/files/FileGcWorker.cpp | 15 +++--- td/telegram/files/FileGcWorker.h | 7 ++- td/telegram/files/FileStats.cpp | 3 ++ 9 files changed, 78 insertions(+), 48 deletions(-) diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index 6bda440c..ebbca58e 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -4117,8 +4117,9 @@ getDatabaseStatistics = DatabaseStatistics; //@file_types If not empty, only files with the given type(s) are considered. By default, all types except thumbnails, profile photos, stickers and wallpapers are deleted //@chat_ids If not empty, only files from the given chats are considered. Use 0 as chat identifier to delete files not belonging to any chat (e.g., profile photos) //@exclude_chat_ids If not empty, files from the given chats are excluded. Use 0 as chat identifier to exclude all files not belonging to any chat (e.g., profile photos) +//@return_deleted_file_statistics Pass true, if deleted file statistics needs to be returned instead of the whole storage usage statistics. Affects only returned statistics //@chat_limit Same as in getStorageStatistics. Affects only returned statistics -optimizeStorage size:int53 ttl:int32 count:int32 immunity_delay:int32 file_types:vector chat_ids:vector exclude_chat_ids:vector chat_limit:int32 = StorageStatistics; +optimizeStorage size:int53 ttl:int32 count:int32 immunity_delay:int32 file_types:vector chat_ids:vector exclude_chat_ids:vector return_deleted_file_statistics:Bool chat_limit:int32 = StorageStatistics; //@description Sets the current network type. Can be called before authorization. Calling this method forces all network connections to reopen, mitigating the delay in switching between different networks, so it should be called whenever the network is changed, even if the network type remains the same. diff --git a/td/generate/scheme/td_api.tlo b/td/generate/scheme/td_api.tlo index f8711a96e7cf0fde311b768e064537d05d32239f..e1a37c5f047f1bacd6a8446ecbfbde2fc4f4ef6f 100644 GIT binary patch delta 107 zcmV-x0F?ifoC?gG3V?(Gv;ts09mkm3G7oQXbZKpAdSz2|Z*pO0WfjKq4B5sKObM3} zcL67pjKCO|kU|0=laTENm*GAF*B%~nWps6NZeL_&Y-Mz1WM5`!Y-L|_bYXO9b98BA NlYTWCx41t7Tuw2=D=`27 delta 76 zcmV-S0JHzhoC=hj3V?(Gv;ts09kH!%IuCDfbZKpAdSz2|Z*pO0WfjKq4B5sKObC|| icL67pjKCO^FpUM59e4p7lMp)vm*GAF*SGLL0$fgh3LQKE diff --git a/td/telegram/StorageManager.cpp b/td/telegram/StorageManager.cpp index fd8a79fd..96cffe83 100644 --- a/td/telegram/StorageManager.cpp +++ b/td/telegram/StorageManager.cpp @@ -73,7 +73,7 @@ void StorageManager::get_storage_stats(bool need_all_files, int32 dialog_limit, //TODO group same queries close_stats_worker(); } - if (!pending_run_gc_.empty()) { + if (!pending_run_gc_[0].empty() || !pending_run_gc_[1].empty()) { close_gc_worker(); } stats_dialog_limit_ = dialog_limit; @@ -107,12 +107,12 @@ void StorageManager::update_use_storage_optimizer() { schedule_next_gc(); } -void StorageManager::run_gc(FileGcParameters parameters, Promise promise) { +void StorageManager::run_gc(FileGcParameters parameters, bool return_deleted_file_statistics, + Promise promise) { if (is_closed_) { - promise.set_error(Status::Error(500, "Request aborted")); - return; + return promise.set_error(Status::Error(500, "Request aborted")); } - if (!pending_run_gc_.empty()) { + if (!pending_run_gc_[0].empty() || !pending_run_gc_[1].empty()) { close_gc_worker(); } @@ -126,7 +126,7 @@ void StorageManager::run_gc(FileGcParameters parameters, Promise prom })); //NB: get_storage_stats will cancel all gc queries, so promise needs to be added after the call - pending_run_gc_.emplace_back(std::move(promise)); + pending_run_gc_[return_deleted_file_statistics].push_back(std::move(promise)); } void StorageManager::on_file_stats(Result r_file_stats, uint32 generation) { @@ -141,6 +141,7 @@ void StorageManager::on_file_stats(Result r_file_stats, uint32 genera return; } + update_fast_stats(r_file_stats.ok()); send_stats(r_file_stats.move_as_ok(), stats_dialog_limit_, std::move(pending_storage_stats_)); } @@ -159,14 +160,14 @@ void StorageManager::on_all_files(FileGcParameters gc_parameters, Result r_file_stats) { - send_closure(actor_id, &StorageManager::on_gc_finished, dialog_limit, std::move(r_file_stats)); + send_closure(gc_worker_, &FileGcWorker::run_gc, std::move(gc_parameters), std::move(r_file_stats.ok_ref().all_files), + PromiseCreator::lambda([actor_id = actor_id(this), dialog_limit](Result r_file_gc_result) { + send_closure(actor_id, &StorageManager::on_gc_finished, dialog_limit, std::move(r_file_gc_result)); })); } @@ -212,19 +213,27 @@ void StorageManager::create_gc_worker() { } } -void StorageManager::on_gc_finished(int32 dialog_limit, Result r_file_stats) { - if (r_file_stats.is_error()) { - if (r_file_stats.error().code() != 500) { - LOG(ERROR) << "GC failed: " << r_file_stats.error(); +void StorageManager::on_gc_finished(int32 dialog_limit, Result r_file_gc_result) { + if (r_file_gc_result.is_error()) { + if (r_file_gc_result.error().code() != 500) { + LOG(ERROR) << "GC failed: " << r_file_gc_result.error(); } - auto promises = std::move(pending_run_gc_); + auto promises = std::move(pending_run_gc_[0]); + append(promises, std::move(pending_run_gc_[1])); + pending_run_gc_[0].clear(); + pending_run_gc_[1].clear(); for (auto &promise : promises) { - promise.set_error(r_file_stats.error().clone()); + promise.set_error(r_file_gc_result.error().clone()); } return; } - send_stats(r_file_stats.move_as_ok(), dialog_limit, std::move(pending_run_gc_)); + update_fast_stats(r_file_gc_result.ok().kept_file_stats_); + + auto kept_file_promises = std::move(pending_run_gc_[0]); + auto removed_file_promises = std::move(pending_run_gc_[1]); + send_stats(std::move(r_file_gc_result.ok_ref().kept_file_stats_), dialog_limit, std::move(kept_file_promises)); + send_stats(std::move(r_file_gc_result.ok_ref().removed_file_stats_), dialog_limit, std::move(removed_file_promises)); } void StorageManager::save_fast_stat() { @@ -239,21 +248,26 @@ void StorageManager::load_fast_stat() { LOG(INFO) << "Loaded fast storage statistics with " << fast_stat_.cnt << " files of total size " << fast_stat_.size; } -void StorageManager::send_stats(FileStats &&stats, int32 dialog_limit, std::vector> promises) { +void StorageManager::update_fast_stats(const FileStats &stats) { fast_stat_ = stats.get_total_nontemp_stat(); LOG(INFO) << "Recalculate fast storage statistics to " << fast_stat_.cnt << " files of total size " << fast_stat_.size; save_fast_stat(); +} + +void StorageManager::send_stats(FileStats &&stats, int32 dialog_limit, std::vector> &&promises) { + if (promises.empty()) { + return; + } stats.apply_dialog_limit(dialog_limit); - std::vector dialog_ids = stats.get_dialog_ids(); + auto dialog_ids = stats.get_dialog_ids(); - auto promise = - PromiseCreator::lambda([promises = std::move(promises), stats = std::move(stats)](Result) mutable { - for (auto &promise : promises) { - promise.set_value(FileStats(stats)); - } - }); + auto promise = PromiseCreator::lambda([promises = std::move(promises), stats = std::move(stats)](Unit) mutable { + for (auto &promise : promises) { + promise.set_value(FileStats(stats)); + } + }); send_closure(G()->messages_manager(), &MessagesManager::load_dialogs, std::move(dialog_ids), std::move(promise)); } @@ -282,8 +296,10 @@ void StorageManager::close_stats_worker() { } void StorageManager::close_gc_worker() { - auto promises = std::move(pending_run_gc_); - pending_run_gc_.clear(); + auto promises = std::move(pending_run_gc_[0]); + append(promises, std::move(pending_run_gc_[1])); + pending_run_gc_[0].clear(); + pending_run_gc_[1].clear(); for (auto &promise : promises) { promise.set_error(Status::Error(500, "Request aborted")); } @@ -338,14 +354,14 @@ void StorageManager::timeout_expired() { if (next_gc_at_ == 0) { return; } - if (!pending_run_gc_.empty() || !pending_storage_stats_.empty()) { + if (!pending_run_gc_[0].empty() || !pending_run_gc_[1].empty() || !pending_storage_stats_.empty()) { set_timeout_in(60); return; } next_gc_at_ = 0; - run_gc({}, PromiseCreator::lambda([actor_id = actor_id(this)](Result r_stats) { + run_gc({}, false, PromiseCreator::lambda([actor_id = actor_id(this)](Result r_stats) { if (!r_stats.is_error() || r_stats.error().code() != 500) { - // do not save gc timestamp is request was cancelled + // do not save gc timestamp if request was cancelled send_closure(actor_id, &StorageManager::save_last_gc_timestamp); } send_closure(actor_id, &StorageManager::schedule_next_gc); diff --git a/td/telegram/StorageManager.h b/td/telegram/StorageManager.h index c828adad..7af4086d 100644 --- a/td/telegram/StorageManager.h +++ b/td/telegram/StorageManager.h @@ -35,7 +35,7 @@ class StorageManager : public Actor { void get_storage_stats(bool need_all_files, int32 dialog_limit, Promise promise); void get_storage_stats_fast(Promise promise); void get_database_stats(Promise promise); - void run_gc(FileGcParameters parameters, Promise promise); + void run_gc(FileGcParameters parameters, bool return_deleted_file_statistics, Promise promise); void update_use_storage_optimizer(); void on_new_file(int64 size, int64 real_size, int32 cnt); @@ -63,7 +63,8 @@ class StorageManager : public Actor { void on_file_stats(Result r_file_stats, uint32 generation); void create_stats_worker(); - void send_stats(FileStats &&stats, int32 dialog_limit, std::vector> promises); + void update_fast_stats(const FileStats &stats); + void send_stats(FileStats &&stats, int32 dialog_limit, std::vector> &&promises); void save_fast_stat(); void load_fast_stat(); @@ -82,14 +83,14 @@ class StorageManager : public Actor { // Gc ActorOwn gc_worker_; - std::vector> pending_run_gc_; + std::vector> pending_run_gc_[2]; uint32 last_gc_timestamp_ = 0; double next_gc_at_ = 0; void on_all_files(FileGcParameters gc_parameters, Result r_file_stats); void create_gc_worker(); - void on_gc_finished(int32 dialog_limit, Result r_file_stats); + void on_gc_finished(int32 dialog_limit, Result r_file_gc_result); void close_stats_worker(); void close_gc_worker(); diff --git a/td/telegram/Td.cpp b/td/telegram/Td.cpp index fd97fafa..893a5b71 100644 --- a/td/telegram/Td.cpp +++ b/td/telegram/Td.cpp @@ -5021,7 +5021,8 @@ void Td::on_request(uint64 id, td_api::optimizeStorage &request) { promise.set_value(result.ok().as_td_api()); } }); - send_closure(storage_manager_, &StorageManager::run_gc, std::move(parameters), std::move(query_promise)); + send_closure(storage_manager_, &StorageManager::run_gc, std::move(parameters), + request.return_deleted_file_statistics_, std::move(query_promise)); } void Td::on_request(uint64 id, td_api::getNetworkStatistics &request) { diff --git a/td/telegram/cli.cpp b/td/telegram/cli.cpp index 192391ba..0765174a 100644 --- a/td/telegram/cli.cpp +++ b/td/telegram/cli.cpp @@ -432,7 +432,7 @@ class CliClient final : public Actor { static char get_delimiter(Slice str) { std::unordered_set chars; for (auto c : trim(str)) { - if (!is_alnum(c)) { + if (!is_alnum(c) && c != '-') { chars.insert(c); } } @@ -2237,7 +2237,7 @@ class CliClient final : public Actor { send_request(td_api::make_object()); } else if (op == "database") { send_request(td_api::make_object()); - } else if (op == "optimize_storage") { + } else if (op == "optimize_storage" || op == "optimize_storage_all") { string chat_ids; string exclude_chat_ids; string chat_ids_limit; @@ -2245,14 +2245,14 @@ class CliClient final : public Actor { std::tie(exclude_chat_ids, chat_ids_limit) = split(args); send_request(td_api::make_object( 10000000, -1, -1, 0, std::vector>(), as_chat_ids(chat_ids), - as_chat_ids(exclude_chat_ids), to_integer(chat_ids_limit))); + as_chat_ids(exclude_chat_ids), op == "optimize_storage", to_integer(chat_ids_limit))); } else if (op == "clean_storage_default") { send_request(td_api::make_object()); } else if (op == "clean_photos") { std::vector> types; types.push_back(td_api::make_object()); send_request(td_api::make_object(0, 0, 0, 0, std::move(types), as_chat_ids(""), - as_chat_ids(""), 20)); + as_chat_ids(""), true, 20)); } else if (op == "clean_storage") { std::vector> types; types.push_back(td_api::make_object()); @@ -2269,7 +2269,7 @@ class CliClient final : public Actor { types.push_back(td_api::make_object()); types.push_back(td_api::make_object()); send_request(td_api::make_object(0, -1, -1, 0, std::move(types), as_chat_ids(args), - as_chat_ids(""), 20)); + as_chat_ids(""), true, 20)); } else if (op == "network") { send_request(td_api::make_object()); } else if (op == "current_network") { diff --git a/td/telegram/files/FileGcWorker.cpp b/td/telegram/files/FileGcWorker.cpp index 1f12059a..93e8ea65 100644 --- a/td/telegram/files/FileGcWorker.cpp +++ b/td/telegram/files/FileGcWorker.cpp @@ -27,7 +27,7 @@ namespace td { int VERBOSITY_NAME(file_gc) = VERBOSITY_NAME(INFO); void FileGcWorker::run_gc(const FileGcParameters ¶meters, std::vector files, - Promise promise) { + Promise promise) { auto begin_time = Time::now(); VLOG(file_gc) << "Start files gc with " << parameters; // quite stupid implementations @@ -80,9 +80,11 @@ void FileGcWorker::run_gc(const FileGcParameters ¶meters, std::vectorfile_manager(), &FileManager::on_file_unlink, @@ -134,8 +136,8 @@ void FileGcWorker::run_gc(const FileGcParameters ¶meters, std::vector parameters.max_file_count) { remove_count = files.size() - parameters.max_file_count; @@ -183,6 +185,7 @@ void FileGcWorker::run_gc(const FileGcParameters ¶meters, std::vector parent, CancellationToken token) : parent_(std::move(parent)), token_(std::move(token)) { } - void run_gc(const FileGcParameters ¶meters, std::vector files, Promise promise); + void run_gc(const FileGcParameters ¶meters, std::vector files, Promise promise); private: ActorShared<> parent_; diff --git a/td/telegram/files/FileStats.cpp b/td/telegram/files/FileStats.cpp index 734a1c0f..e919363d 100644 --- a/td/telegram/files/FileStats.cpp +++ b/td/telegram/files/FileStats.cpp @@ -63,6 +63,7 @@ FileTypeStat get_nontemp_stat(const FileStats::StatByType &by_type) { } return stat; } + FileTypeStat FileStats::get_total_nontemp_stat() const { if (!split_by_owner_dialog_id) { return get_nontemp_stat(stat_by_type); @@ -75,6 +76,7 @@ FileTypeStat FileStats::get_total_nontemp_stat() const { } return stat; } + void FileStats::apply_dialog_limit(int32 limit) { if (limit == -1) { return; @@ -255,4 +257,5 @@ StringBuilder &operator<<(StringBuilder &sb, const FileStats &file_stats) { return sb; } + } // namespace td