2018-12-31 20:04:05 +01:00
|
|
|
//
|
2022-12-31 22:28:08 +01:00
|
|
|
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2023
|
2018-12-31 20:04:05 +01:00
|
|
|
//
|
|
|
|
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
|
|
|
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
|
|
|
//
|
|
|
|
#include "td/telegram/net/NetStatsManager.h"
|
|
|
|
|
|
|
|
#include "td/telegram/Global.h"
|
|
|
|
#include "td/telegram/logevent/LogEvent.h"
|
|
|
|
#include "td/telegram/StateManager.h"
|
2019-01-06 20:59:17 +01:00
|
|
|
#include "td/telegram/TdDb.h"
|
2018-04-09 20:06:37 +02:00
|
|
|
#include "td/telegram/Version.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
|
2021-01-01 13:59:53 +01:00
|
|
|
#include "td/utils/algorithm.h"
|
2019-02-12 22:26:36 +01:00
|
|
|
#include "td/utils/common.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
#include "td/utils/logging.h"
|
|
|
|
#include "td/utils/misc.h"
|
2021-05-17 14:21:11 +02:00
|
|
|
#include "td/utils/SliceBuilder.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
#include "td/utils/tl_helpers.h"
|
|
|
|
|
|
|
|
namespace td {
|
|
|
|
|
2019-02-21 18:54:20 +01:00
|
|
|
template <class StorerT>
|
|
|
|
static void store(const NetStatsData &net_stats, StorerT &storer) {
|
2018-12-31 20:04:05 +01:00
|
|
|
using ::td::store;
|
|
|
|
store(net_stats.read_size, storer);
|
|
|
|
store(net_stats.write_size, storer);
|
|
|
|
store(net_stats.count, storer);
|
|
|
|
store(net_stats.duration, storer);
|
|
|
|
}
|
|
|
|
|
2019-02-21 18:54:20 +01:00
|
|
|
template <class ParserT>
|
|
|
|
static void parse(NetStatsData &net_stats, ParserT &parser) {
|
2018-12-31 20:04:05 +01:00
|
|
|
using ::td::parse;
|
|
|
|
parse(net_stats.read_size, parser);
|
|
|
|
parse(net_stats.write_size, parser);
|
|
|
|
|
|
|
|
if (parser.version() >= static_cast<int32>(Version::NetStatsCountDuration)) {
|
|
|
|
parse(net_stats.count, parser);
|
|
|
|
parse(net_stats.duration, parser);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void NetStatsManager::init() {
|
2019-02-12 17:17:20 +01:00
|
|
|
LOG_CHECK(!empty()) << G()->close_flag();
|
2021-07-04 04:58:54 +02:00
|
|
|
class NetStatsInternalCallback final : public NetStats::Callback {
|
2018-12-31 20:04:05 +01:00
|
|
|
public:
|
|
|
|
NetStatsInternalCallback(ActorId<NetStatsManager> parent, size_t id) : parent_(std::move(parent)), id_(id) {
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
ActorId<NetStatsManager> parent_;
|
|
|
|
size_t id_;
|
2021-07-03 22:51:36 +02:00
|
|
|
void on_stats_updated() final {
|
2018-12-31 20:04:05 +01:00
|
|
|
send_closure(parent_, &NetStatsManager::on_stats_updated, id_);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2018-04-27 10:00:56 +02:00
|
|
|
for_each_stat([&](NetStatsInfo &stat, size_t id, CSlice name, FileType file_type) {
|
2020-06-22 07:43:00 +02:00
|
|
|
auto main_file_type = get_main_file_type(file_type);
|
|
|
|
id += static_cast<size_t>(main_file_type) - static_cast<size_t>(file_type);
|
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
stat.key = "net_stats_" + name.str();
|
2018-09-27 03:19:03 +02:00
|
|
|
stat.stats.set_callback(make_unique<NetStatsInternalCallback>(actor_id(this), id));
|
2018-12-31 20:04:05 +01:00
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
void NetStatsManager::get_network_stats(bool current, Promise<NetworkStats> promise) {
|
|
|
|
NetworkStats result;
|
|
|
|
|
|
|
|
result.since = current ? since_current_ : since_total_;
|
|
|
|
|
|
|
|
for_each_stat([&](NetStatsInfo &info, size_t id, CSlice name, FileType file_type) { update(info, false); });
|
|
|
|
|
|
|
|
for (size_t net_type_i = 0; net_type_i < net_type_size(); net_type_i++) {
|
|
|
|
auto net_type = NetType(net_type_i);
|
|
|
|
NetStatsData total;
|
|
|
|
NetStatsData total_files;
|
|
|
|
|
|
|
|
for_each_stat([&](NetStatsInfo &info, size_t id, CSlice name, FileType file_type) {
|
2021-11-10 19:09:28 +01:00
|
|
|
const auto &type_stats = info.stats_by_type[net_type_i];
|
2018-12-31 20:04:05 +01:00
|
|
|
auto stats = current ? type_stats.mem_stats : type_stats.mem_stats + type_stats.db_stats;
|
|
|
|
if (id == 0) {
|
|
|
|
} else if (id == 1) {
|
|
|
|
total = stats;
|
2020-06-16 04:10:16 +02:00
|
|
|
} else if (id == CALL_NET_STATS_ID) {
|
2018-12-31 20:04:05 +01:00
|
|
|
} else if (file_type != FileType::None) {
|
|
|
|
total_files = total_files + stats;
|
|
|
|
}
|
|
|
|
});
|
|
|
|
|
|
|
|
NetStatsData check;
|
|
|
|
for_each_stat([&](NetStatsInfo &info, size_t id, CSlice name, FileType file_type) {
|
|
|
|
if (id == 1) {
|
|
|
|
return;
|
|
|
|
}
|
2021-11-10 19:09:28 +01:00
|
|
|
const auto &type_stats = info.stats_by_type[net_type_i];
|
2018-12-31 20:04:05 +01:00
|
|
|
auto stats = current ? type_stats.mem_stats : type_stats.mem_stats + type_stats.db_stats;
|
|
|
|
|
|
|
|
NetworkStatsEntry entry;
|
|
|
|
entry.file_type = file_type;
|
|
|
|
entry.net_type = net_type;
|
|
|
|
entry.rx = stats.read_size;
|
|
|
|
entry.tx = stats.write_size;
|
|
|
|
entry.count = stats.count;
|
|
|
|
entry.duration = stats.duration;
|
|
|
|
if (id == 0) {
|
|
|
|
result.entries.push_back(std::move(entry));
|
2020-06-16 04:10:16 +02:00
|
|
|
} else if (id == CALL_NET_STATS_ID) {
|
2018-12-31 20:04:05 +01:00
|
|
|
entry.is_call = true;
|
|
|
|
result.entries.push_back(std::move(entry));
|
|
|
|
} else if (file_type != FileType::None) {
|
2020-06-22 07:43:00 +02:00
|
|
|
if (get_main_file_type(file_type) != file_type) {
|
2018-04-27 10:00:56 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
if (total_files.read_size != 0) {
|
|
|
|
entry.rx = static_cast<int64>(static_cast<double>(total.read_size) *
|
|
|
|
(static_cast<double>(entry.rx) / static_cast<double>(total_files.read_size)));
|
|
|
|
} else {
|
2020-06-22 01:27:58 +02:00
|
|
|
// entry.rx += total.read_size / MAX_FILE_TYPE;
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
if (total_files.write_size != 0) {
|
|
|
|
entry.tx = static_cast<int64>(static_cast<double>(total.write_size) *
|
|
|
|
(static_cast<double>(entry.tx) / static_cast<double>(total_files.write_size)));
|
|
|
|
} else {
|
2020-06-22 01:27:58 +02:00
|
|
|
// entry.tx += total.write_size / MAX_FILE_TYPE;
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
check.read_size += entry.rx;
|
|
|
|
check.write_size += entry.tx;
|
|
|
|
result.entries.push_back(std::move(entry));
|
|
|
|
}
|
|
|
|
});
|
|
|
|
// LOG(ERROR) << total.read_size << " " << check.read_size;
|
|
|
|
// LOG(ERROR) << total.write_size << " " << check.write_size;
|
|
|
|
}
|
|
|
|
|
|
|
|
promise.set_value(std::move(result));
|
|
|
|
}
|
|
|
|
|
|
|
|
void NetStatsManager::reset_network_stats() {
|
|
|
|
auto do_reset_network_stats = [&](auto &info) {
|
|
|
|
info.last_sync_stats = info.stats.get_stats();
|
|
|
|
for (size_t net_type_i = 0; net_type_i < net_type_size(); net_type_i++) {
|
|
|
|
auto net_type = NetType(net_type_i);
|
|
|
|
info.stats_by_type[net_type_i] = NetStatsInfo::TypeStats{};
|
2023-09-11 16:02:58 +02:00
|
|
|
auto key = PSTRING() << info.key << '#' << net_type_string(net_type);
|
2018-12-31 20:04:05 +01:00
|
|
|
G()->td_db()->get_binlog_pmc()->erase(key);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
for_each_stat([&](NetStatsInfo &info, size_t id, CSlice name, FileType) { do_reset_network_stats(info); });
|
|
|
|
|
|
|
|
auto unix_time = G()->unix_time();
|
|
|
|
since_total_ = unix_time;
|
|
|
|
since_current_ = unix_time;
|
|
|
|
G()->td_db()->get_binlog_pmc()->set("net_stats_since", to_string(unix_time));
|
|
|
|
}
|
|
|
|
|
|
|
|
void NetStatsManager::add_network_stats(const NetworkStatsEntry &entry) {
|
|
|
|
if (entry.is_call) {
|
|
|
|
return add_network_stats_impl(call_net_stats_, entry);
|
|
|
|
}
|
|
|
|
if (entry.file_type == FileType::None) {
|
|
|
|
return add_network_stats_impl(common_net_stats_, entry);
|
|
|
|
}
|
|
|
|
add_network_stats_impl(media_net_stats_, entry);
|
2021-10-19 17:11:16 +02:00
|
|
|
auto file_type_n = static_cast<size_t>(entry.file_type);
|
2020-06-22 01:27:58 +02:00
|
|
|
CHECK(file_type_n < static_cast<size_t>(MAX_FILE_TYPE));
|
2018-12-31 20:04:05 +01:00
|
|
|
add_network_stats_impl(files_stats_[file_type_n], entry);
|
|
|
|
}
|
|
|
|
|
|
|
|
void NetStatsManager::add_network_stats_impl(NetStatsInfo &info, const NetworkStatsEntry &entry) {
|
|
|
|
auto net_type_i = static_cast<size_t>(entry.net_type);
|
|
|
|
auto &data = info.stats_by_type[net_type_i].mem_stats;
|
|
|
|
|
|
|
|
if (data.read_size + entry.rx < data.read_size || data.write_size + entry.tx < data.write_size ||
|
|
|
|
data.count + entry.count < data.count) {
|
|
|
|
LOG(ERROR) << "Network stats overflow";
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
data.read_size += entry.rx;
|
|
|
|
data.write_size += entry.tx;
|
|
|
|
data.count += entry.count;
|
|
|
|
data.duration += entry.duration;
|
|
|
|
save_stats(info, entry.net_type);
|
|
|
|
}
|
|
|
|
|
|
|
|
void NetStatsManager::start_up() {
|
2018-04-27 10:00:56 +02:00
|
|
|
for_each_stat([&](NetStatsInfo &info, size_t id, CSlice name, FileType file_type) {
|
2020-06-22 07:43:00 +02:00
|
|
|
if (get_main_file_type(file_type) != file_type) {
|
2018-04-27 10:00:56 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
for (size_t net_type_i = 0; net_type_i < net_type_size(); net_type_i++) {
|
|
|
|
auto net_type = NetType(net_type_i);
|
2023-09-11 16:02:58 +02:00
|
|
|
auto key = PSTRING() << info.key << '#' << net_type_string(net_type);
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
auto value = G()->td_db()->get_binlog_pmc()->get(key);
|
|
|
|
if (value.empty()) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
log_event_parse(info.stats_by_type[net_type_i].db_stats, value).ensure();
|
|
|
|
}
|
|
|
|
});
|
|
|
|
auto unix_time = G()->unix_time();
|
|
|
|
since_total_ = 0;
|
|
|
|
since_current_ = unix_time;
|
|
|
|
auto since_str = G()->td_db()->get_binlog_pmc()->get("net_stats_since");
|
|
|
|
if (!since_str.empty()) {
|
|
|
|
auto since = to_integer<int32>(since_str);
|
2022-08-17 15:11:13 +02:00
|
|
|
auto authorization_date = G()->get_option_integer("authorization_date");
|
2018-12-31 20:04:05 +01:00
|
|
|
if (unix_time < since) {
|
|
|
|
since_total_ = unix_time;
|
|
|
|
G()->td_db()->get_binlog_pmc()->set("net_stats_since", to_string(since_total_));
|
2020-02-22 16:46:16 +01:00
|
|
|
} else if (since < authorization_date - 3600) {
|
2020-09-24 23:46:16 +02:00
|
|
|
since_total_ = narrow_cast<int32>(authorization_date);
|
2020-02-22 16:46:16 +01:00
|
|
|
G()->td_db()->get_binlog_pmc()->set("net_stats_since", to_string(since_total_));
|
2018-12-31 20:04:05 +01:00
|
|
|
} else {
|
|
|
|
since_total_ = since;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// approximate since_total by first run date for new users
|
|
|
|
since_total_ = unix_time;
|
|
|
|
G()->td_db()->get_binlog_pmc()->set("net_stats_since", to_string(since_total_));
|
|
|
|
}
|
|
|
|
|
2021-07-04 04:58:54 +02:00
|
|
|
class NetCallback final : public StateManager::Callback {
|
2018-12-31 20:04:05 +01:00
|
|
|
public:
|
|
|
|
explicit NetCallback(ActorId<NetStatsManager> net_stats_manager)
|
|
|
|
: net_stats_manager_(std::move(net_stats_manager)) {
|
|
|
|
}
|
2021-07-03 22:51:36 +02:00
|
|
|
bool on_network(NetType network_type, uint32 network_generation) final {
|
2018-12-31 20:04:05 +01:00
|
|
|
send_closure(net_stats_manager_, &NetStatsManager::on_net_type_updated, network_type);
|
|
|
|
return net_stats_manager_.is_alive();
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
ActorId<NetStatsManager> net_stats_manager_;
|
|
|
|
};
|
|
|
|
send_closure(G()->state_manager(), &StateManager::add_callback, make_unique<NetCallback>(actor_id(this)));
|
|
|
|
}
|
|
|
|
|
|
|
|
std::shared_ptr<NetStatsCallback> NetStatsManager::get_common_stats_callback() const {
|
|
|
|
return common_net_stats_.stats.get_callback();
|
|
|
|
}
|
|
|
|
|
|
|
|
std::shared_ptr<NetStatsCallback> NetStatsManager::get_media_stats_callback() const {
|
|
|
|
return media_net_stats_.stats.get_callback();
|
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<std::shared_ptr<NetStatsCallback>> NetStatsManager::get_file_stats_callbacks() const {
|
2018-04-27 10:00:56 +02:00
|
|
|
auto result = transform(files_stats_, [](auto &stat) { return stat.stats.get_callback(); });
|
2020-06-22 07:43:00 +02:00
|
|
|
for (int32 i = 0; i < MAX_FILE_TYPE; i++) {
|
2021-10-19 17:11:16 +02:00
|
|
|
auto main_file_type = static_cast<int32>(get_main_file_type(static_cast<FileType>(i)));
|
2020-06-22 07:43:00 +02:00
|
|
|
if (main_file_type != i) {
|
|
|
|
result[i] = result[main_file_type];
|
|
|
|
}
|
|
|
|
}
|
2018-04-27 10:00:56 +02:00
|
|
|
return result;
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
void NetStatsManager::update(NetStatsInfo &info, bool force_save) {
|
|
|
|
if (info.net_type == NetType::None) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
auto current_stats = info.stats.get_stats();
|
|
|
|
auto diff = current_stats - info.last_sync_stats;
|
|
|
|
|
|
|
|
auto net_type_i = static_cast<size_t>(info.net_type);
|
|
|
|
auto &type_stats = info.stats_by_type[net_type_i];
|
|
|
|
|
|
|
|
info.last_sync_stats = current_stats;
|
|
|
|
|
|
|
|
auto mem_stats = type_stats.mem_stats + diff;
|
|
|
|
type_stats.mem_stats = mem_stats;
|
|
|
|
type_stats.dirty_size += diff.read_size + diff.write_size;
|
|
|
|
|
|
|
|
if (type_stats.dirty_size < 1000 && !force_save) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
type_stats.dirty_size = 0;
|
|
|
|
|
|
|
|
save_stats(info, info.net_type);
|
|
|
|
}
|
|
|
|
|
|
|
|
void NetStatsManager::save_stats(NetStatsInfo &info, NetType net_type) {
|
2022-08-17 15:11:13 +02:00
|
|
|
if (G()->get_option_boolean("disable_persistent_network_statistics")) {
|
2020-07-13 15:56:31 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
auto net_type_i = static_cast<size_t>(net_type);
|
|
|
|
auto &type_stats = info.stats_by_type[net_type_i];
|
|
|
|
|
2023-09-11 16:02:58 +02:00
|
|
|
auto key = PSTRING() << info.key << '#' << net_type_string(info.net_type);
|
2018-12-31 20:04:05 +01:00
|
|
|
auto stats = type_stats.mem_stats + type_stats.db_stats;
|
2023-09-11 16:02:58 +02:00
|
|
|
// LOG(ERROR) << "SAVE " << key << ' ' << stats;
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
G()->td_db()->get_binlog_pmc()->set(key, log_event_store(stats).as_slice().str());
|
|
|
|
}
|
|
|
|
|
|
|
|
void NetStatsManager::info_loop(NetStatsInfo &info) {
|
|
|
|
if (info.net_type == NetType::None) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
auto mem_stats = info.stats.get_stats();
|
|
|
|
auto diff = mem_stats - info.last_sync_stats;
|
|
|
|
auto size = diff.read_size + diff.write_size;
|
|
|
|
if (size < 1000) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
update(info, false);
|
|
|
|
}
|
|
|
|
|
|
|
|
void NetStatsManager::on_stats_updated(size_t id) {
|
|
|
|
for_each_stat([&](NetStatsInfo &stat, size_t stat_id, CSlice name, FileType) {
|
|
|
|
if (stat_id == id) {
|
|
|
|
info_loop(stat);
|
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
void NetStatsManager::on_net_type_updated(NetType net_type) {
|
|
|
|
if (net_type == NetType::Unknown) {
|
|
|
|
net_type = NetType::None;
|
|
|
|
}
|
|
|
|
auto do_on_net_type_updated = [&](NetStatsInfo &info) { // g++ 4.9-6.2 crashes if (auto &info) is used
|
|
|
|
if (info.net_type == net_type) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
if (info.net_type != NetType::None) {
|
|
|
|
update(info, true);
|
|
|
|
}
|
|
|
|
info.net_type = net_type;
|
|
|
|
};
|
|
|
|
|
|
|
|
for_each_stat([&](NetStatsInfo &stat, size_t stat_id, CSlice name, FileType) { do_on_net_type_updated(stat); });
|
|
|
|
}
|
|
|
|
|
|
|
|
} // namespace td
|