tdlight-team/tdlight
tdlight-team/tdlight
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add optimizeStorage.return_deleted_file_statistics.

Browse Source 
GitOrigin-RevId: f7533781027891624017c605b1d3ac3e91da5086
This commit is contained in:
levlam 2020-03-02 11:59:47 +03:00
parent bd0621eeff
commit a1e93ebba5
9 changed files with 78 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
td/generate/scheme/td_api.tl
View File

@ -4117,8 +4117,9 @@ getDatabaseStatistics = DatabaseStatistics;
//@file_types If not empty, only files with the given type(s) are considered. By default, all types except thumbnails, profile photos, stickers and wallpapers are deleted
//@chat_ids If not empty, only files from the given chats are considered. Use 0 as chat identifier to delete files not belonging to any chat (e.g., profile photos)
//@exclude_chat_ids If not empty, files from the given chats are excluded. Use 0 as chat identifier to exclude all files not belonging to any chat (e.g., profile photos)
//@return_deleted_file_statistics Pass true, if deleted file statistics needs to be returned instead of the whole storage usage statistics. Affects only returned statistics
//@chat_limit Same as in getStorageStatistics. Affects only returned statistics
optimizeStorage size:int53 ttl:int32 count:int32 immunity_delay:int32 file_types:vector<FileType> chat_ids:vector<int53> exclude_chat_ids:vector<int53> chat_limit:int32 = StorageStatistics;
optimizeStorage size:int53 ttl:int32 count:int32 immunity_delay:int32 file_types:vector<FileType> chat_ids:vector<int53> exclude_chat_ids:vector<int53> return_deleted_file_statistics:Bool chat_limit:int32 = StorageStatistics;
//@description Sets the current network type. Can be called before authorization. Calling this method forces all network connections to reopen, mitigating the delay in switching between different networks, so it should be called whenever the network is changed, even if the network type remains the same.

BIN
td/generate/scheme/td_api.tlo
View File

Binary file not shown.

68
td/telegram/StorageManager.cpp
View File

@ -73,7 +73,7 @@ void StorageManager::get_storage_stats(bool need_all_files, int32 dialog_limit,
//TODO group same queries
close_stats_worker();
}
if (!pending_run_gc_.empty()) {
if (!pending_run_gc_[0].empty() || !pending_run_gc_[1].empty()) {
close_gc_worker();
}
stats_dialog_limit_ = dialog_limit;
@ -107,12 +107,12 @@ void StorageManager::update_use_storage_optimizer() {
schedule_next_gc();
}
void StorageManager::run_gc(FileGcParameters parameters, Promise<FileStats> promise) {
void StorageManager::run_gc(FileGcParameters parameters, bool return_deleted_file_statistics,
Promise<FileStats> promise) {
if (is_closed_) {
promise.set_error(Status::Error(500, "Request aborted"));
return;
return promise.set_error(Status::Error(500, "Request aborted"));
}
if (!pending_run_gc_.empty()) {
if (!pending_run_gc_[0].empty() || !pending_run_gc_[1].empty()) {
close_gc_worker();
}
@ -126,7 +126,7 @@ void StorageManager::run_gc(FileGcParameters parameters, Promise<FileStats> prom
}));
//NB: get_storage_stats will cancel all gc queries, so promise needs to be added after the call
pending_run_gc_.emplace_back(std::move(promise));
pending_run_gc_[return_deleted_file_statistics].push_back(std::move(promise));
}
void StorageManager::on_file_stats(Result<FileStats> r_file_stats, uint32 generation) {
@ -141,6 +141,7 @@ void StorageManager::on_file_stats(Result<FileStats> r_file_stats, uint32 genera
return;
}
update_fast_stats(r_file_stats.ok());
send_stats(r_file_stats.move_as_ok(), stats_dialog_limit_, std::move(pending_storage_stats_));
}
@ -159,14 +160,14 @@ void StorageManager::on_all_files(FileGcParameters gc_parameters, Result<FileSta
r_file_stats = Status::Error(500, "Request aborted");
}
if (r_file_stats.is_error()) {
return on_gc_finished(dialog_limit, std::move(r_file_stats));
return on_gc_finished(dialog_limit, r_file_stats.move_as_error());
}
create_gc_worker();
send_closure(gc_worker_, &FileGcWorker::run_gc, std::move(gc_parameters), r_file_stats.move_as_ok().all_files,
PromiseCreator::lambda([actor_id = actor_id(this), dialog_limit](Result<FileStats> r_file_stats) {
send_closure(actor_id, &StorageManager::on_gc_finished, dialog_limit, std::move(r_file_stats));
send_closure(gc_worker_, &FileGcWorker::run_gc, std::move(gc_parameters), std::move(r_file_stats.ok_ref().all_files),
PromiseCreator::lambda([actor_id = actor_id(this), dialog_limit](Result<FileGcResult> r_file_gc_result) {
send_closure(actor_id, &StorageManager::on_gc_finished, dialog_limit, std::move(r_file_gc_result));
}));
}
@ -212,19 +213,27 @@ void StorageManager::create_gc_worker() {
}
}
void StorageManager::on_gc_finished(int32 dialog_limit, Result<FileStats> r_file_stats) {
if (r_file_stats.is_error()) {
if (r_file_stats.error().code() != 500) {
LOG(ERROR) << "GC failed: " << r_file_stats.error();
void StorageManager::on_gc_finished(int32 dialog_limit, Result<FileGcResult> r_file_gc_result) {
if (r_file_gc_result.is_error()) {
if (r_file_gc_result.error().code() != 500) {
LOG(ERROR) << "GC failed: " << r_file_gc_result.error();
}
auto promises = std::move(pending_run_gc_);
auto promises = std::move(pending_run_gc_[0]);
append(promises, std::move(pending_run_gc_[1]));
pending_run_gc_[0].clear();
pending_run_gc_[1].clear();
for (auto &promise : promises) {
promise.set_error(r_file_stats.error().clone());
promise.set_error(r_file_gc_result.error().clone());
}
return;
}
send_stats(r_file_stats.move_as_ok(), dialog_limit, std::move(pending_run_gc_));
update_fast_stats(r_file_gc_result.ok().kept_file_stats_);
auto kept_file_promises = std::move(pending_run_gc_[0]);
auto removed_file_promises = std::move(pending_run_gc_[1]);
send_stats(std::move(r_file_gc_result.ok_ref().kept_file_stats_), dialog_limit, std::move(kept_file_promises));
send_stats(std::move(r_file_gc_result.ok_ref().removed_file_stats_), dialog_limit, std::move(removed_file_promises));
}
void StorageManager::save_fast_stat() {
@ -239,17 +248,22 @@ void StorageManager::load_fast_stat() {
LOG(INFO) << "Loaded fast storage statistics with " << fast_stat_.cnt << " files of total size " << fast_stat_.size;
}
void StorageManager::send_stats(FileStats &&stats, int32 dialog_limit, std::vector<Promise<FileStats>> promises) {
void StorageManager::update_fast_stats(const FileStats &stats) {
fast_stat_ = stats.get_total_nontemp_stat();
LOG(INFO) << "Recalculate fast storage statistics to " << fast_stat_.cnt << " files of total size "
<< fast_stat_.size;
save_fast_stat();
}
void StorageManager::send_stats(FileStats &&stats, int32 dialog_limit, std::vector<Promise<FileStats>> &&promises) {
if (promises.empty()) {
return;
}
stats.apply_dialog_limit(dialog_limit);
std::vector<DialogId> dialog_ids = stats.get_dialog_ids();
auto dialog_ids = stats.get_dialog_ids();
auto promise =
PromiseCreator::lambda([promises = std::move(promises), stats = std::move(stats)](Result<Unit>) mutable {
auto promise = PromiseCreator::lambda([promises = std::move(promises), stats = std::move(stats)](Unit) mutable {
for (auto &promise : promises) {
promise.set_value(FileStats(stats));
}
@ -282,8 +296,10 @@ void StorageManager::close_stats_worker() {
}
void StorageManager::close_gc_worker() {
auto promises = std::move(pending_run_gc_);
pending_run_gc_.clear();
auto promises = std::move(pending_run_gc_[0]);
append(promises, std::move(pending_run_gc_[1]));
pending_run_gc_[0].clear();
pending_run_gc_[1].clear();
for (auto &promise : promises) {
promise.set_error(Status::Error(500, "Request aborted"));
}
@ -338,14 +354,14 @@ void StorageManager::timeout_expired() {
if (next_gc_at_ == 0) {
return;
}
if (!pending_run_gc_.empty() || !pending_storage_stats_.empty()) {
if (!pending_run_gc_[0].empty() || !pending_run_gc_[1].empty() || !pending_storage_stats_.empty()) {
set_timeout_in(60);
return;
}
next_gc_at_ = 0;
run_gc({}, PromiseCreator::lambda([actor_id = actor_id(this)](Result<FileStats> r_stats) {
run_gc({}, false, PromiseCreator::lambda([actor_id = actor_id(this)](Result<FileStats> r_stats) {
if (!r_stats.is_error() || r_stats.error().code() != 500) {
// do not save gc timestamp is request was cancelled
// do not save gc timestamp if request was cancelled
send_closure(actor_id, &StorageManager::save_last_gc_timestamp);
}
send_closure(actor_id, &StorageManager::schedule_next_gc);

9
td/telegram/StorageManager.h
View File

@ -35,7 +35,7 @@ class StorageManager : public Actor {
void get_storage_stats(bool need_all_files, int32 dialog_limit, Promise<FileStats> promise);
void get_storage_stats_fast(Promise<FileStatsFast> promise);
void get_database_stats(Promise<DatabaseStats> promise);
void run_gc(FileGcParameters parameters, Promise<FileStats> promise);
void run_gc(FileGcParameters parameters, bool return_deleted_file_statistics, Promise<FileStats> promise);
void update_use_storage_optimizer();
void on_new_file(int64 size, int64 real_size, int32 cnt);
@ -63,7 +63,8 @@ class StorageManager : public Actor {
void on_file_stats(Result<FileStats> r_file_stats, uint32 generation);
void create_stats_worker();
void send_stats(FileStats &&stats, int32 dialog_limit, std::vector<Promise<FileStats>> promises);
void update_fast_stats(const FileStats &stats);
void send_stats(FileStats &&stats, int32 dialog_limit, std::vector<Promise<FileStats>> &&promises);
void save_fast_stat();
void load_fast_stat();
@ -82,14 +83,14 @@ class StorageManager : public Actor {
// Gc
ActorOwn<FileGcWorker> gc_worker_;
std::vector<Promise<FileStats>> pending_run_gc_;
std::vector<Promise<FileStats>> pending_run_gc_[2];
uint32 last_gc_timestamp_ = 0;
double next_gc_at_ = 0;
void on_all_files(FileGcParameters gc_parameters, Result<FileStats> r_file_stats);
void create_gc_worker();
void on_gc_finished(int32 dialog_limit, Result<FileStats> r_file_stats);
void on_gc_finished(int32 dialog_limit, Result<FileGcResult> r_file_gc_result);
void close_stats_worker();
void close_gc_worker();

3
td/telegram/Td.cpp
View File

@ -5021,7 +5021,8 @@ void Td::on_request(uint64 id, td_api::optimizeStorage &request) {
promise.set_value(result.ok().as_td_api());
}
});
send_closure(storage_manager_, &StorageManager::run_gc, std::move(parameters), std::move(query_promise));
send_closure(storage_manager_, &StorageManager::run_gc, std::move(parameters),
request.return_deleted_file_statistics_, std::move(query_promise));
}
void Td::on_request(uint64 id, td_api::getNetworkStatistics &request) {

10
td/telegram/cli.cpp
View File

@ -432,7 +432,7 @@ class CliClient final : public Actor {
static char get_delimiter(Slice str) {
std::unordered_set<char> chars;
for (auto c : trim(str)) {
if (!is_alnum(c)) {
if (!is_alnum(c) && c != '-') {
chars.insert(c);
}
}
@ -2237,7 +2237,7 @@ class CliClient final : public Actor {
send_request(td_api::make_object<td_api::getStorageStatisticsFast>());
} else if (op == "database") {
send_request(td_api::make_object<td_api::getDatabaseStatistics>());
} else if (op == "optimize_storage") {
} else if (op == "optimize_storage" || op == "optimize_storage_all") {
string chat_ids;
string exclude_chat_ids;
string chat_ids_limit;
@ -2245,14 +2245,14 @@ class CliClient final : public Actor {
std::tie(exclude_chat_ids, chat_ids_limit) = split(args);
send_request(td_api::make_object<td_api::optimizeStorage>(
10000000, -1, -1, 0, std::vector<td_api::object_ptr<td_api::FileType>>(), as_chat_ids(chat_ids),
as_chat_ids(exclude_chat_ids), to_integer<int32>(chat_ids_limit)));
as_chat_ids(exclude_chat_ids), op == "optimize_storage", to_integer<int32>(chat_ids_limit)));
} else if (op == "clean_storage_default") {
send_request(td_api::make_object<td_api::optimizeStorage>());
} else if (op == "clean_photos") {
std::vector<td_api::object_ptr<td_api::FileType>> types;
types.push_back(td_api::make_object<td_api::fileTypePhoto>());
send_request(td_api::make_object<td_api::optimizeStorage>(0, 0, 0, 0, std::move(types), as_chat_ids(""),
as_chat_ids(""), 20));
as_chat_ids(""), true, 20));
} else if (op == "clean_storage") {
std::vector<td_api::object_ptr<td_api::FileType>> types;
types.push_back(td_api::make_object<td_api::fileTypeThumbnail>());
@ -2269,7 +2269,7 @@ class CliClient final : public Actor {
types.push_back(td_api::make_object<td_api::fileTypeVideoNote>());
types.push_back(td_api::make_object<td_api::fileTypeSecure>());
send_request(td_api::make_object<td_api::optimizeStorage>(0, -1, -1, 0, std::move(types), as_chat_ids(args),
as_chat_ids(""), 20));
as_chat_ids(""), true, 20));
} else if (op == "network") {
send_request(td_api::make_object<td_api::getNetworkStatistics>());
} else if (op == "current_network") {

15
td/telegram/files/FileGcWorker.cpp
View File

@ -27,7 +27,7 @@ namespace td {
int VERBOSITY_NAME(file_gc) = VERBOSITY_NAME(INFO);
void FileGcWorker::run_gc(const FileGcParameters &parameters, std::vector<FullFileInfo> files,
Promise<FileStats> promise) {
Promise<FileGcResult> promise) {
auto begin_time = Time::now();
VLOG(file_gc) << "Start files gc with " << parameters;
// quite stupid implementations
@ -80,9 +80,11 @@ void FileGcWorker::run_gc(const FileGcParameters &parameters, std::vector<FullFi
}
FileStats new_stats;
new_stats.split_by_owner_dialog_id = parameters.dialog_limit != 0;
FileStats removed_stats;
removed_stats.split_by_owner_dialog_id = new_stats.split_by_owner_dialog_id = parameters.dialog_limit != 0;
auto do_remove_file = [](const FullFileInfo &info) {
auto do_remove_file = [&removed_stats](const FullFileInfo &info) {
removed_stats.add_copy(info);
auto status = unlink(info.path);
LOG_IF(WARNING, status.is_error()) << "Failed to unlink file \"" << info.path << "\" during files gc: " << status;
send_closure(G()->file_manager(), &FileManager::on_file_unlink,
@ -134,8 +136,8 @@ void FileGcWorker::run_gc(const FileGcParameters &parameters, std::vector<FullFi
// sort by max(atime, mtime)
std::sort(files.begin(), files.end(), [](const auto &a, const auto &b) { return a.atime_nsec < b.atime_nsec; });
// 1. Total memory must be less than max_memory
// 2. Total file count must be less than MAX_FILE_COUNT
// 1. Total size must be less than parameters.max_files_size
// 2. Total file count must be less than parameters.max_file_count
size_t remove_count = 0;
if (files.size() > parameters.max_file_count) {
remove_count = files.size() - parameters.max_file_count;
@ -183,6 +185,7 @@ void FileGcWorker::run_gc(const FileGcParameters &parameters, std::vector<FullFi
<< tag("owner_dialog_id_immunity", owner_dialog_id_ignored_cnt)
<< tag("exclude_owner_dialog_id_immunity", exclude_owner_dialog_id_ignored_cnt);
promise.set_value(std::move(new_stats));
promise.set_value({std::move(new_stats), std::move(removed_stats)});
}
} // namespace td

7
td/telegram/files/FileGcWorker.h
View File

@ -19,11 +19,16 @@ namespace td {
extern int VERBOSITY_NAME(file_gc);
struct FileGcResult {
FileStats kept_file_stats_;
FileStats removed_file_stats_;
};
class FileGcWorker : public Actor {
public:
FileGcWorker(ActorShared<> parent, CancellationToken token) : parent_(std::move(parent)), token_(std::move(token)) {
}
void run_gc(const FileGcParameters &parameters, std::vector<FullFileInfo> files, Promise<FileStats> promise);
void run_gc(const FileGcParameters &parameters, std::vector<FullFileInfo> files, Promise<FileGcResult> promise);
private:
ActorShared<> parent_;

3
td/telegram/files/FileStats.cpp
View File

@ -63,6 +63,7 @@ FileTypeStat get_nontemp_stat(const FileStats::StatByType &by_type) {
}
return stat;
}
FileTypeStat FileStats::get_total_nontemp_stat() const {
if (!split_by_owner_dialog_id) {
return get_nontemp_stat(stat_by_type);
@ -75,6 +76,7 @@ FileTypeStat FileStats::get_total_nontemp_stat() const {
}
return stat;
}
void FileStats::apply_dialog_limit(int32 limit) {
if (limit == -1) {
return;
@ -255,4 +257,5 @@ StringBuilder &operator<<(StringBuilder &sb, const FileStats &file_stats) {
return sb;
}
} // namespace td