Stats: json format (#22)

The work regarding #17.

With 374928c we could merge a working version where the json is still missing a few fields, to iterate from there.

It is available via the normal stats endpoint by calling /json as the path.
Everything else will have the old text style.
This commit is contained in:
Luckydonald 2021-01-24 18:09:53 +01:00 committed by GitHub
parent 9df6dd5fd3
commit de05b42272
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 444 additions and 69 deletions

View File

@ -9,6 +9,7 @@
#include "telegram-bot-api/Client.h"
#include "telegram-bot-api/ClientParameters.h"
#include "telegram-bot-api/WebhookActor.h"
#include "telegram-bot-api/StatsJson.h"
#include "td/telegram/ClientActor.h"
@ -204,14 +205,23 @@ bool ClientManager::check_flood_limits(PromisedQueryPtr &query, bool is_user_log
}
void ClientManager::get_stats(td::PromiseActor<td::BufferSlice> promise,
td::vector<std::pair<td::string, td::string>> args) {
td::vector<std::pair<td::string, td::string>> args,
bool as_json) {
if (close_flag_) {
promise.set_value(td::BufferSlice("Closing"));
return;
}
size_t buf_size = 1 << 14;
size_t buf_size = 1 << 18;
auto buf = td::StackAllocator::alloc(buf_size);
td::StringBuilder sb(buf.as_slice());
td::JsonBuilder jb(td::StringBuilder(buf.as_slice(), true), -1);
td::StringBuilder sb = std::move(jb.string_builder());
auto jb_root = jb.enter_object();
if (!as_json) {
jb_root.leave();
sb.clear();
} else {
jb_root("⚠️ WARNING", "The json representation is still a work in progress and will be changed later!");
}
td::Slice id_filter;
for (auto &arg : args) {
@ -251,43 +261,95 @@ void ClientManager::get_stats(td::PromiseActor<td::BufferSlice> promise,
top_bot_ids.emplace(static_cast<td::int64>(score * 1e9), id);
}
if(!as_json) {
sb << stat_.get_description() << "\n";
}
if (id_filter.empty()) {
if(as_json) {
jb_root("uptime", td::JsonFloat(now - parameters_->start_time_));
} else {
sb << "uptime\t" << now - parameters_->start_time_ << "\n";
}
if(as_json) {
jb_root("bot_count", td::JsonLong(clients_.size()));
} else {
sb << "bot_count\t" << clients_.size() << "\n";
}
if(as_json) {
jb_root("active_bot_count", td::JsonInt(active_bot_count));
} else {
sb << "active_bot_count\t" << active_bot_count << "\n";
}
auto r_mem_stat = td::mem_stat();
if (r_mem_stat.is_ok()) {
auto mem_stat = r_mem_stat.move_as_ok();
if(as_json) {
jb_root("memory", JsonStatsMem(mem_stat));
} else {
sb << "rss\t" << td::format::as_size(mem_stat.resident_size_) << "\n";
sb << "vm\t" << td::format::as_size(mem_stat.virtual_size_) << "\n";
sb << "rss_peak\t" << td::format::as_size(mem_stat.resident_size_peak_) << "\n";
sb << "vm_peak\t" << td::format::as_size(mem_stat.virtual_size_peak_) << "\n";
}
} else {
if(as_json) {
jb_root("memory", td::JsonNull());
} else {
LOG(INFO) << "Failed to get memory statistics: " << r_mem_stat.error();
}
}
ServerCpuStat::update(td::Time::now());
if(as_json) {
auto cpu_stats = ServerCpuStat::instance().as_json_ready_vector(td::Time::now());
jb_root("cpu", JsonStatsCpu(std::move(cpu_stats)));
} else {
auto cpu_stats = ServerCpuStat::instance().as_vector(td::Time::now());
for (auto &stat : cpu_stats) {
sb << stat.key_ << "\t" << stat.value_ << "\n";
}
}
if(as_json) {
jb_root("buffer_memory", JsonStatsSize(td::BufferAllocator::get_buffer_mem()));
jb_root("active_webhook_connections", td::JsonLong(WebhookActor::get_total_connections_count()));
jb_root("active_requests", td::JsonLong(parameters_->shared_data_->query_count_.load()));
jb_root("active_network_queries", td::JsonLong(td::get_pending_network_query_count(*parameters_->net_query_stats_)));
} else {
sb << "buffer_memory\t" << td::format::as_size(td::BufferAllocator::get_buffer_mem()) << "\n";
sb << "active_webhook_connections\t" << WebhookActor::get_total_connections_count() << "\n";
sb << "active_requests\t" << parameters_->shared_data_->query_count_.load() << "\n";
sb << "active_network_queries\t" << td::get_pending_network_query_count(*parameters_->net_query_stats_) << "\n";
}
if(as_json) {
} else {
auto stats = stat_.as_vector(now);
for (auto &stat : stats) {
sb << stat.key_ << "\t" << stat.value_ << "\n";
}
}
}
if(as_json) {
td::vector<JsonStatsBotAdvanced> bots;
for (std::pair<td::int64, td::uint64> top_bot_id : top_bot_ids) {
auto client_info = clients_.get(top_bot_id.second);
CHECK(client_info);
ServerBotInfo bot_info = client_info->client_->get_actor_unsafe()->get_bot_info();
auto stats = client_info->stat_.as_json_ready_vector(now);
JsonStatsBotAdvanced bot(
std::move(top_bot_id), std::move(bot_info), std::move(stats), parameters_->stats_hide_sensible_data_, now
);
bots.push_back(bot);
}
auto bot_count = bots.size();
jb_root("bots", JsonStatsBots(std::move(bots), bot_count > 100));
} else {
for (auto top_bot_id : top_bot_ids) {
auto *client_info = clients_.get(top_bot_id.second);
CHECK(client_info);
auto bot_info = client_info->client_->get_actor_unsafe()->get_bot_info();
sb << "\n";
sb << "id\t" << bot_info.id_ << "\n";
sb << "uptime\t" << now - bot_info.start_time_ << "\n";
@ -314,13 +376,17 @@ void ClientManager::get_stats(td::PromiseActor<td::BufferSlice> promise,
sb << stat.key_ << "/sec\t" << stat.value_ << "\n";
}
}
if (sb.is_error()) {
break;
}
}
}
if(as_json) {
jb_root.leave();
}
// ignore sb overflow
promise.set_value(td::BufferSlice(sb.as_cslice()));
promise.set_value(td::BufferSlice((as_json ? jb.string_builder() : sb).as_cslice()));
}
td::int64 ClientManager::get_tqueue_id(td::int64 user_id, bool is_test_dc) {

View File

@ -46,7 +46,7 @@ class ClientManager final : public td::Actor {
bool check_flood_limits(PromisedQueryPtr &query, bool is_user_login=false);
void get_stats(td::PromiseActor<td::BufferSlice> promise, td::vector<std::pair<td::string, td::string>> args);
void get_stats(td::PromiseActor<td::BufferSlice> promise, td::vector<std::pair<td::string, td::string>> args, bool as_json);
void close(td::Promise<td::Unit> &&promise);

View File

@ -23,7 +23,9 @@ void HttpStatConnection::handle(td::unique_ptr<td::HttpQuery> http_query,
init_promise_future(&promise, &future);
future.set_event(td::EventCreator::yield(actor_id()));
LOG(DEBUG) << "SEND";
send_closure(client_manager_, &ClientManager::get_stats, std::move(promise), http_query->get_args());
td::Parser url_path_parser(http_query->url_path_);
as_json_ = url_path_parser.try_skip("/json");
send_closure(client_manager_, &ClientManager::get_stats, std::move(promise), http_query->get_args(), as_json_);
result_ = std::move(future);
}
@ -37,7 +39,11 @@ void HttpStatConnection::wakeup() {
td::HttpHeaderCreator hc;
hc.init_status_line(200);
hc.set_keep_alive();
if (as_json_) {
hc.set_content_type("application/json");
} else {
hc.set_content_type("text/plain");
}
hc.set_content_size(content.size());
auto r_header = hc.finish();

View File

@ -27,6 +27,7 @@ class HttpStatConnection : public td::HttpInboundConnection::Callback {
void wakeup() override;
private:
bool as_json_;
td::FutureActor<td::BufferSlice> result_;
td::ActorId<ClientManager> client_manager_;
td::ActorOwn<td::HttpInboundConnection> connection_;

View File

@ -72,6 +72,21 @@ td::vector<StatItem> ServerCpuStat::as_vector(double now) {
return res;
}
td::vector<td::vector<StatItem>> ServerCpuStat::as_json_ready_vector(double now) {
std::lock_guard<std::mutex> guard(mutex_);
td::vector<td::vector<StatItem>> res;
auto first = stat_[0].get_stat(now).as_vector();
auto first_size = first.size();
res.push_back(first);
for (std::size_t i = 1; i < SIZE; i++) {
auto other = stat_[i].get_stat(now).as_vector();
CHECK(other.size() == first_size);
res.push_back(other);
}
return res;
}
constexpr int ServerCpuStat::DURATIONS[SIZE];
constexpr const char *ServerCpuStat::DESCR[SIZE];
@ -140,6 +155,18 @@ td::vector<StatItem> BotStatActor::as_vector(double now) {
return res;
}
td::vector<ServerBotStat> BotStatActor::as_json_ready_vector(double now) {
std::pair<ServerBotStat, double> first_sd = stat_[0].stat_duration(now);
first_sd.first.normalize(first_sd.second);
td::vector<ServerBotStat> res;
for (auto & single_stat : stat_) {
auto next_sd = single_stat.stat_duration(now);
next_sd.first.normalize(next_sd.second);
res.push_back(next_sd.first);
}
return res;
}
td::string BotStatActor::get_description() const {
td::string res = "DURATION";
for (auto &descr : DESCR) {
@ -148,6 +175,15 @@ td::string BotStatActor::get_description() const {
}
return res;
}
td::vector<td::string> BotStatActor::get_jsonable_description() const {
td::vector<td::string> strings;
strings.push_back("duration");
for (auto &descr : DESCR) {
strings.push_back(descr);
}
return strings;
}
bool BotStatActor::is_active(double now) const {
return last_activity_timestamp_ > now - 86400;

View File

@ -61,10 +61,11 @@ class ServerCpuStat {
td::string get_description() const;
td::vector<StatItem> as_vector(double now);
td::vector<td::vector<StatItem>> as_json_ready_vector(double now);
private:
static constexpr std::size_t SIZE = 4;
static constexpr const char *DESCR[SIZE] = {"inf", "5sec", "1min", "1hour"};
private:
static constexpr int DURATIONS[SIZE] = {0, 5, 60, 60 * 60};
std::mutex mutex_;
@ -179,15 +180,16 @@ class BotStatActor final : public td::Actor {
}
td::vector<StatItem> as_vector(double now);
td::vector<ServerBotStat> as_json_ready_vector(double now);
td::string get_description() const;
td::vector<td::string> get_jsonable_description() const;
bool is_active(double now) const;
private:
static constexpr std::size_t SIZE = 4;
static constexpr const char *DESCR[SIZE] = {"inf", "5sec", "1min", "1hour"};
static constexpr int DURATIONS[SIZE] = {0, 5, 60, 60 * 60};
static constexpr const char *DESCR[SIZE] = {"inf", "5sec", "1min", "1hour"};
private:
td::TimedStat<ServerBotStat> stat_[SIZE];
td::ActorId<BotStatActor> parent_;
double last_activity_timestamp_ = -1e9;

View File

@ -0,0 +1,264 @@
//
// Copyright Luckydonald (tdlight-telegram-bot-api+code@luckydonald.de) 2020
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "telegram-bot-api/Query.h"
#include "td/db/TQueue.h"
#include "td/net/HttpOutboundConnection.h"
#include "td/net/HttpQuery.h"
#include "td/net/SslStream.h"
#include "td/actor/actor.h"
#include "td/actor/PromiseFuture.h"
#include "td/utils/common.h"
#include "td/utils/Container.h"
#include "td/utils/FloodControlFast.h"
#include "td/utils/HttpUrl.h"
#include "td/utils/JsonBuilder.h"
#include "td/utils/List.h"
#include "td/utils/port/IPAddress.h"
#include "td/utils/port/SocketFd.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include "td/utils/VectorQueue.h"
#include "td/utils/utf8.h"
#include <atomic>
#include <functional>
#include <memory>
#include <set>
#include <tuple>
#include <unordered_map>
namespace telegram_bot_api {
class JsonStatsSize : public td::Jsonable {
public:
explicit JsonStatsSize(td::uint64 size) : size_(size) {
}
void store(td::JsonValueScope *scope) const {
auto object = scope->enter_object();
object("bytes", td::JsonLong(size_));
// Now basically td::format::as_size(...), but without need for an StringBuilder.
struct NamedValue {
const char *name;
td::uint64 value;
};
static constexpr NamedValue sizes[] = {{"B", 1}, {"KB", 1 << 10}, {"MB", 1 << 20}, {"GB", 1 << 30}};
static constexpr size_t sizes_n = sizeof(sizes) / sizeof(NamedValue);
size_t i = 0;
while (i + 1 < sizes_n && size_ > 10 * sizes[i + 1].value) {
i++;
}
object("human_readable", td::to_string(size_ / sizes[i].value) + sizes[i].name);
}
private:
const td::uint64 size_;
};
class JsonStatsMem : public td::Jsonable {
public:
explicit JsonStatsMem(const td::MemStat mem_stat) : mem_stat_(mem_stat) {
}
void store(td::JsonValueScope *scope) const {
auto object = scope->enter_object();
object("resident_size", JsonStatsSize(mem_stat_.resident_size_));
object("resident_size_peak", JsonStatsSize(mem_stat_.resident_size_peak_));
object("virtual_size", JsonStatsSize(mem_stat_.virtual_size_));
object("virtual_size_peak", JsonStatsSize(mem_stat_.virtual_size_peak_));
}
private:
const td::MemStat mem_stat_;
};
class JsonStatsCpuStat : public td::Jsonable {
public:
explicit JsonStatsCpuStat(const StatItem& inf, const StatItem& five_sec, const StatItem& one_min, const StatItem& one_hour) :
inf_(inf), five_sec_(five_sec), one_min_(one_min), one_hour_(one_hour) {
}
void store(td::JsonValueScope *scope) const {
auto object = scope->enter_object();
object(td::Slice(ServerCpuStat::DESCR[0]), td::JsonString(td::Slice(inf_.value_)));
object(td::Slice(ServerCpuStat::DESCR[1]), td::JsonString(td::Slice(five_sec_.value_)));
object(td::Slice(ServerCpuStat::DESCR[2]), td::JsonString(td::Slice(one_min_.value_)));
object(td::Slice(ServerCpuStat::DESCR[3]), td::JsonString(td::Slice(one_hour_.value_)));
}
private:
const StatItem& inf_;
const StatItem& five_sec_;
const StatItem& one_min_;
const StatItem& one_hour_;
};
class JsonStatsCpu : public td::Jsonable {
public:
explicit JsonStatsCpu(td::vector<td::vector<StatItem>> cpu_stats) : cpu_stats_(std::move(cpu_stats)) {
}
void store(td::JsonValueScope *scope) const {
auto object = scope->enter_object();
CHECK(cpu_stats_.size() == ServerCpuStat::SIZE);
CHECK(cpu_stats_[0].size() == 3);
CHECK(cpu_stats_[1].size() == 3);
CHECK(cpu_stats_[2].size() == 3);
object("total_cpu", JsonStatsCpuStat(cpu_stats_[0][0], cpu_stats_[1][0], cpu_stats_[2][0], cpu_stats_[3][0]));
object("user_cpu", JsonStatsCpuStat(cpu_stats_[0][1], cpu_stats_[1][1], cpu_stats_[2][1], cpu_stats_[3][1]));
object("system_cpu", JsonStatsCpuStat(cpu_stats_[0][2], cpu_stats_[1][2], cpu_stats_[2][2], cpu_stats_[3][2]));
}
private:
const td::vector<td::vector<StatItem>> cpu_stats_;
};
class JsonStatsBot : public td::Jsonable {
public:
explicit JsonStatsBot(std::pair<td::int64, td::uint64> score_id_pair) : score_id_pair_(std::move(score_id_pair)) {
}
void store(td::JsonValueScope *scope) const {
auto object = scope->enter_object();
object("score", td::JsonLong(score_id_pair_.first));
object("internal_id", td::JsonLong(score_id_pair_.second));
}
protected:
const std::pair<td::int64, td::uint64> score_id_pair_;
};
class JsonStatsBotStatDouble : public td::Jsonable {
public:
explicit JsonStatsBotStatDouble(const double inf, const double five_sec, const double one_min, const double one_hour) :
inf_(inf), five_sec_(five_sec), one_min_(one_min), one_hour_(one_hour) {
}
void store(td::JsonValueScope *scope) const {
auto object = scope->enter_object();
object(td::Slice(BotStatActor::DESCR[0]), td::JsonFloat(inf_));
object(td::Slice(BotStatActor::DESCR[1]), td::JsonFloat(five_sec_));
object(td::Slice(BotStatActor::DESCR[2]), td::JsonFloat(one_min_));
object(td::Slice(BotStatActor::DESCR[3]), td::JsonFloat(one_hour_));
}
private:
const double inf_;
const double five_sec_;
const double one_min_;
const double one_hour_;
};
class JsonStatsBotStatLong : public td::Jsonable {
public:
explicit JsonStatsBotStatLong(const td::int64 inf, const td::int64 five_sec, const td::int64 one_min, const td::int64 one_hour) :
inf_(inf), five_sec_(five_sec), one_min_(one_min), one_hour_(one_hour) {
}
void store(td::JsonValueScope *scope) const {
auto object = scope->enter_object();
object(td::Slice(BotStatActor::DESCR[0]), td::JsonLong(inf_));
object(td::Slice(BotStatActor::DESCR[1]), td::JsonLong(five_sec_));
object(td::Slice(BotStatActor::DESCR[2]), td::JsonLong(one_min_));
object(td::Slice(BotStatActor::DESCR[3]), td::JsonLong(one_hour_));
}
private:
const td::int64 inf_;
const td::int64 five_sec_;
const td::int64 one_min_;
const td::int64 one_hour_;
};
class JsonStatsBotStats : public td::Jsonable {
public:
explicit JsonStatsBotStats(td::vector<ServerBotStat> stats) : stats_(std::move(stats)) {
CHECK(BotStatActor::SIZE == 4 && "Check that we have 4 fields.");
}
void store(td::JsonValueScope *scope) const {
auto object = scope->enter_object();
object("request_count", JsonStatsBotStatDouble(stats_[0].request_count_, stats_[1].request_count_, stats_[2].request_count_, stats_[3].request_count_));
object("request_bytes", JsonStatsBotStatDouble(stats_[0].request_bytes_, stats_[1].request_bytes_, stats_[2].request_bytes_, stats_[3].request_bytes_));
object("request_file_count", JsonStatsBotStatDouble(stats_[0].request_file_count_, stats_[1].request_file_count_, stats_[2].request_file_count_, stats_[3].request_file_count_));
object("request_files_bytes", JsonStatsBotStatDouble(stats_[0].request_files_bytes_, stats_[1].request_files_bytes_, stats_[2].request_files_bytes_, stats_[3].request_files_bytes_));
object("request_files_max_bytes", JsonStatsBotStatLong(stats_[0].request_files_max_bytes_, stats_[1].request_files_max_bytes_, stats_[2].request_files_max_bytes_, stats_[3].request_files_max_bytes_));
object("response_count", JsonStatsBotStatDouble(stats_[0].response_count_, stats_[1].response_count_, stats_[2].response_count_, stats_[3].response_count_));
object("response_count_ok", JsonStatsBotStatDouble(stats_[0].response_count_ok_, stats_[1].response_count_ok_, stats_[2].response_count_ok_, stats_[3].response_count_ok_));
object("response_count_error", JsonStatsBotStatDouble(stats_[0].response_count_error_, stats_[1].response_count_error_, stats_[2].response_count_error_, stats_[3].response_count_error_));
object("response_bytes", JsonStatsBotStatDouble(stats_[0].response_bytes_, stats_[1].response_bytes_, stats_[2].response_bytes_, stats_[3].response_bytes_));
object("update_count", JsonStatsBotStatDouble(stats_[0].update_count_, stats_[1].update_count_, stats_[2].update_count_, stats_[3].update_count_));
}
protected:
const td::vector<ServerBotStat> stats_;
};
class JsonStatsBotAdvanced : public JsonStatsBot {
public:
explicit JsonStatsBotAdvanced(std::pair<td::int64, td::uint64> score_id_pair,
ServerBotInfo bot,
td::vector<ServerBotStat> stats,
const bool hide_sensible_data,
const double now)
: JsonStatsBot(std::move(score_id_pair)), bot_(std::move(bot)), stats_(std::move(stats)),
hide_sensible_data_(hide_sensible_data), now_(now) {
}
void store(td::JsonValueScope *scope) const {
auto object = scope->enter_object();
object("id", td::JsonLong(td::to_integer<td::int64>(bot_.id_)));
object("uptime", now_ - bot_.start_time_);
object("score", td::JsonLong(score_id_pair_.first));
object("internal_id", td::JsonLong(score_id_pair_.second));
if (!hide_sensible_data_) {
object("token", td::JsonString(bot_.token_));
}
object("username", bot_.username_);
td::CSlice url = bot_.webhook_;
object("webhook_set", td::JsonBool(!url.empty()));
if (!hide_sensible_data_) {
if (td::check_utf8(url)) {
object("webhook_url", url);
} else {
object("webhook_url", td::JsonRawString(url));
}
}
object("has_custom_certificate", td::JsonBool(bot_.has_webhook_certificate_));
object("head_update_id", td::JsonInt(bot_.head_update_id_));
object("tail_update_id", td::JsonInt(bot_.tail_update_id_));
object("pending_update_count", td::narrow_cast<td::int32>(bot_.pending_update_count_));
object("webhook_max_connections", td::JsonInt(bot_.webhook_max_connections_));
object("stats", JsonStatsBotStats(std::move(stats_)));
}
private:
ServerBotInfo bot_;
td::vector<ServerBotStat> stats_;
const bool hide_sensible_data_;
const double now_;
};
class JsonStatsBots : public td::Jsonable {
public:
JsonStatsBots(td::vector<JsonStatsBotAdvanced> bots, bool no_metadata)
: bots_(std::move(bots)), no_metadata_(no_metadata) {
}
void store(td::JsonValueScope *scope) const {
auto array = scope->enter_array();
for (const auto &bot : bots_) {
if (no_metadata_) {
array << static_cast<const JsonStatsBot &>(bot);
} else {
array << bot;
}
}
}
private:
const td::vector<JsonStatsBotAdvanced> bots_;
bool no_metadata_;
};
} // namespace telegram_bot_api