tdlight-team/tdlight
tdlight-team/tdlight
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add VLOG file_gc.

Browse Source 
GitOrigin-RevId: 816f4f124006e29b0afba6a8e8838612faae4cf3
This commit is contained in:
levlam 2018-11-16 00:25:08 +03:00
parent bc54e5e335
commit bd8295c464
7 changed files with 52 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
td/telegram/Logging.cpp
View File

@ -6,6 +6,7 @@
//
#include "td/telegram/Logging.h"
#include "td/telegram/files/FileGcWorker.h"
#include "td/telegram/files/FileManager.h"
#include "td/telegram/net/ConnectionCreator.h"
#include "td/telegram/NotificationManager.h"
@ -33,11 +34,11 @@ static NullLog null_log;
#define ADD_TAG(tag) \
{ #tag, &VERBOSITY_NAME(tag) }
static const std::map<Slice, int *> log_tags{
ADD_TAG(td_init), ADD_TAG(update_file), ADD_TAG(connections), ADD_TAG(binlog),
ADD_TAG(proxy), ADD_TAG(net_query), ADD_TAG(td_requests), ADD_TAG(dc),
ADD_TAG(files), ADD_TAG(mtproto), ADD_TAG(raw_mtproto), ADD_TAG(fd),
ADD_TAG(actor), ADD_TAG(buffer), ADD_TAG(sqlite), ADD_TAG(notifications),
ADD_TAG(get_difference)};
ADD_TAG(td_init), ADD_TAG(update_file), ADD_TAG(connections), ADD_TAG(binlog),
ADD_TAG(proxy), ADD_TAG(net_query), ADD_TAG(td_requests), ADD_TAG(dc),
ADD_TAG(files), ADD_TAG(mtproto), ADD_TAG(raw_mtproto), ADD_TAG(fd),
ADD_TAG(actor), ADD_TAG(buffer), ADD_TAG(sqlite), ADD_TAG(notifications),
ADD_TAG(get_difference), ADD_TAG(file_gc)};
#undef ADD_TAG
Status Logging::set_current_stream(td_api::object_ptr<td_api::LogStream> stream) {

14
td/telegram/files/FileGcParameters.cpp
View File

@ -9,7 +9,10 @@
#include "td/telegram/ConfigShared.h"
#include "td/telegram/Global.h"
#include "td/utils/format.h"
namespace td {
FileGcParameters::FileGcParameters(int64 size, int32 ttl, int32 count, int32 immunity_delay,
vector<FileType> file_types, vector<DialogId> owner_dialog_ids,
vector<DialogId> exclude_owner_dialog_ids, int32 dialog_limit)
@ -29,4 +32,15 @@ FileGcParameters::FileGcParameters(int64 size, int32 ttl, int32 count, int32 imm
this->immunity_delay =
immunity_delay >= 0 ? immunity_delay : config.get_option_integer("storage_immunity_delay", 60 * 60);
}
StringBuilder &operator<<(StringBuilder &string_builder, const FileGcParameters &parameters) {
return string_builder << "FileGcParameters[" << tag("max_files_size", parameters.max_files_size)
<< tag("max_time_from_last_access", parameters.max_time_from_last_access)
<< tag("max_file_count", parameters.max_file_count)
<< tag("immunity_delay", parameters.immunity_delay) << tag("file_types", parameters.file_types)
<< tag("owner_dialog_ids", parameters.owner_dialog_ids)
<< tag("exclude_owner_dialog_ids", parameters.exclude_owner_dialog_ids)
<< tag("dialog_limit", parameters.dialog_limit) << ']';
}
} // namespace td

3
td/telegram/files/FileGcParameters.h
View File

@ -10,6 +10,7 @@
#include "td/telegram/files/FileLocation.h"
#include "td/utils/common.h"
#include "td/utils/StringBuilder.h"
namespace td {
@ -31,4 +32,6 @@ struct FileGcParameters {
int32 dialog_limit;
};
StringBuilder &operator<<(StringBuilder &string_builder, const FileGcParameters &parameters);
} // namespace td

22
td/telegram/files/FileGcWorker.cpp
View File

@ -20,6 +20,9 @@
#include <array>
namespace td {
int VERBOSITY_NAME(file_gc) = VERBOSITY_NAME(INFO);
void FileGcWorker::do_remove_file(const FullFileInfo &info) {
// LOG(WARNING) << "Gc remove file: " << tag("path", file) << tag("mtime", stat.mtime_nsec_ / 1000000000)
// << tag("atime", stat.atime_nsec_ / 1000000000);
@ -33,7 +36,7 @@ void FileGcWorker::do_remove_file(const FullFileInfo &info) {
void FileGcWorker::run_gc(const FileGcParameters &parameters, std::vector<FullFileInfo> files,
Promise<FileStats> promise) {
auto begin_time = Time::now();
LOG(INFO) << "Start files gc";
VLOG(file_gc) << "Start files gc with " << parameters;
// quite stupid implementations
// needs a lot of memory
// may write something more clever, but i will need at least 2 passes over the files
@ -160,14 +163,15 @@ void FileGcWorker::run_gc(const FileGcParameters &parameters, std::vector<FullFi
auto end_time = Time::now();
LOG(INFO) << "Finish files gc: " << tag("time", end_time - begin_time) << tag("total", file_cnt)
<< tag("removed", remove_by_atime_cnt + remove_by_count_cnt + remove_by_size_cnt)
<< tag("total_size", format::as_size(total_size))
<< tag("total_removed_size", format::as_size(total_removed_size)) << tag("by_atime", remove_by_atime_cnt)
<< tag("by_count", remove_by_count_cnt) << tag("by_size", remove_by_size_cnt)
<< tag("type_immunity", type_immunity_ignored_cnt) << tag("time_immunity", time_immunity_ignored_cnt)
<< tag("owner_dialog_id_immunity", owner_dialog_id_ignored_cnt)
<< tag("exclude_owner_dialog_id_immunity", exclude_owner_dialog_id_ignored_cnt);
VLOG(file_gc) << "Finish files gc: " << tag("time", end_time - begin_time) << tag("total", file_cnt)
<< tag("removed", remove_by_atime_cnt + remove_by_count_cnt + remove_by_size_cnt)
<< tag("total_size", format::as_size(total_size))
<< tag("total_removed_size", format::as_size(total_removed_size))
<< tag("by_atime", remove_by_atime_cnt) << tag("by_count", remove_by_count_cnt)
<< tag("by_size", remove_by_size_cnt) << tag("type_immunity", type_immunity_ignored_cnt)
<< tag("time_immunity", time_immunity_ignored_cnt)
<< tag("owner_dialog_id_immunity", owner_dialog_id_ignored_cnt)
<< tag("exclude_owner_dialog_id_immunity", exclude_owner_dialog_id_ignored_cnt);
promise.set_value(std::move(new_stats));
}

3
td/telegram/files/FileGcWorker.h
View File

@ -14,6 +14,9 @@
#include "td/telegram/files/FileStats.h"
namespace td {
extern int VERBOSITY_NAME(file_gc);
class FileGcWorker : public Actor {
public:
explicit FileGcWorker(ActorShared<> parent) : parent_(std::move(parent)) {

20
td/telegram/files/FileLocation.h
View File

@ -134,7 +134,15 @@ inline tl_object_ptr<td_api::FileType> as_td_api(FileType file_type) {
}
}
constexpr int32 file_type_size = static_cast<int32>(FileType::Size);
extern const char *file_type_name[file_type_size];
inline StringBuilder &operator<<(StringBuilder &string_builder, FileType file_type) {
return string_builder << file_type_name[static_cast<int32>(file_type)];
}
enum class FileDirType : int8 { Secure, Common };
inline FileDirType get_file_dir_type(FileType file_type) {
switch (file_type) {
case FileType::Thumbnail:
@ -152,9 +160,6 @@ inline FileDirType get_file_dir_type(FileType file_type) {
}
}
constexpr int32 file_type_size = static_cast<int32>(FileType::Size);
extern const char *file_type_name[file_type_size];
struct FileEncryptionKey {
enum class Type : int32 { None, Secret, Secure };
FileEncryptionKey() = default;
@ -786,7 +791,7 @@ class FullRemoteFileLocation {
inline StringBuilder &operator<<(StringBuilder &string_builder,
const FullRemoteFileLocation &full_remote_file_location) {
string_builder << "[" << file_type_name[static_cast<int32>(full_remote_file_location.file_type_)];
string_builder << "[" << full_remote_file_location.file_type_;
if (!full_remote_file_location.is_web()) {
string_builder << ", " << full_remote_file_location.get_dc_id();
}
@ -987,7 +992,7 @@ inline bool operator!=(const FullLocalFileLocation &lhs, const FullLocalFileLoca
}
inline StringBuilder &operator<<(StringBuilder &sb, const FullLocalFileLocation &location) {
return sb << "[" << file_type_name[static_cast<int32>(location.file_type_)] << "] at \"" << location.path_ << '"';
return sb << '[' << location.file_type_ << "] at \"" << location.path_ << '"';
}
class LocalFileLocation {
@ -1120,10 +1125,9 @@ inline bool operator!=(const FullGenerateFileLocation &lhs, const FullGenerateFi
inline StringBuilder &operator<<(StringBuilder &string_builder,
const FullGenerateFileLocation &full_generated_file_location) {
return string_builder << "["
<< tag("file_type", file_type_name[static_cast<int32>(full_generated_file_location.file_type_)])
return string_builder << '[' << tag("file_type", full_generated_file_location.file_type_)
<< tag("original_path", full_generated_file_location.original_path_)
<< tag("conversion", full_generated_file_location.conversion_) << "]";
<< tag("conversion", full_generated_file_location.conversion_) << ']';
}
class GenerateFileLocation {

2
td/telegram/files/FileManager.cpp
View File

@ -2450,7 +2450,7 @@ void FileManager::on_error_impl(FileNodePtr node, FileManager::Query::Type type,
};
if (status.code() != 1 && !G()->close_flag()) {
LOG(WARNING) << "Failed to upload/download/generate file: " << status << ". Query type = " << type
<< ". File type is " << file_type_name[static_cast<int32>(FileView(node).get_type())];
<< ". File type is " << FileView(node).get_type();
if (status.code() == 0) {
// Remove partial locations
if (node->local_.type() == LocalFileLocation::Type::Partial && status.message() != "FILE_UPLOAD_RESTART") {