Fix crash

Rebase/merge upstream @ version 7.0

.clang-format

@@ -4,15 +4,36 @@ Language:        Cpp
 AccessModifierOffset: -1
 AlignAfterOpenBracket: Align
 AlignArrayOfStructures: None
-AlignConsecutiveMacros: None
-AlignConsecutiveAssignments: None
-AlignConsecutiveBitFields: None
-AlignConsecutiveDeclarations: None
+AlignConsecutiveAssignments:
+  Enabled:         false
+  AcrossEmptyLines: false
+  AcrossComments:  false
+  AlignCompound:   false
+  PadOperators:    true
+AlignConsecutiveBitFields:
+  Enabled:         false
+  AcrossEmptyLines: false
+  AcrossComments:  false
+  AlignCompound:   false
+  PadOperators:    false
+AlignConsecutiveDeclarations:
+  Enabled:         false
+  AcrossEmptyLines: false
+  AcrossComments:  false
+  AlignCompound:   false
+  PadOperators:    false
+AlignConsecutiveMacros:
+  Enabled:         false
+  AcrossEmptyLines: false
+  AcrossComments:  false
+  AlignCompound:   false
+  PadOperators:    false
 AlignEscapedNewlines: Left
 AlignOperands:   Align
-AlignTrailingComments: true
+AlignTrailingComments:
+  Kind:            Always
+  OverEmptyLines:  0
 AllowAllArgumentsOnNextLine: true
-AllowAllConstructorInitializersOnNextLine: true
 AllowAllParametersOfDeclarationOnNextLine: true
 AllowShortBlocksOnASingleLine: Never
 AllowShortCaseLabelsOnASingleLine: false
@@ -25,6 +46,8 @@ AlwaysBreakAfterDefinitionReturnType: None
 AlwaysBreakAfterReturnType: None
 AlwaysBreakBeforeMultilineStrings: true
 AlwaysBreakTemplateDeclarations: Yes
+# AttributeMacros:
+#   - __capability
 BinPackArguments: true
 BinPackParameters: true
 BitFieldColonSpacing: Both
@@ -33,12 +56,12 @@ BraceWrapping:
   AfterClass:      false
   AfterControlStatement: Never
   AfterEnum:       false
+  AfterExternBlock: false
   AfterFunction:   false
   AfterNamespace:  false
   AfterObjCDeclaration: false
   AfterStruct:     false
   AfterUnion:      false
-  AfterExternBlock: false
   BeforeCatch:     false
   BeforeElse:      false
   BeforeLambdaBody: false
@@ -47,24 +70,23 @@ BraceWrapping:
   SplitEmptyFunction: true
   SplitEmptyRecord: true
   SplitEmptyNamespace: true
+BreakAfterAttributes: Never
+# BreakAfterJavaFieldAnnotations: false
+BreakArrays:     true
 BreakBeforeBinaryOperators: None
 BreakBeforeBraces: Attach
-BreakBeforeConceptDeclarations: true
-BreakBeforeInheritanceComma: true # false
-BreakInheritanceList: BeforeComma # BeforeColon
+BreakBeforeConceptDeclarations: Always
+BreakBeforeInlineASMColon: OnlyMultiline
 BreakBeforeTernaryOperators: true
-BreakConstructorInitializersBeforeComma: true # false
 BreakConstructorInitializers: BeforeComma # BeforeColon
-# BreakAfterJavaFieldAnnotations: false
+BreakInheritanceList: BeforeComma # BeforeColon
 BreakStringLiterals: true
 ColumnLimit:     120 # 80
 CommentPragmas:  '^ IWYU pragma:'
 CompactNamespaces: false
-ConstructorInitializerAllOnOneLineOrOnePerLine: true
 ConstructorInitializerIndentWidth: 4
 ContinuationIndentWidth: 4
 Cpp11BracedListStyle: true
-DeriveLineEnding: true
 DerivePointerAlignment: true
 DisableFormat:   false
 EmptyLineAfterAccessModifier: Never
@@ -83,14 +105,21 @@ IndentCaseLabels: true
 IndentExternBlock: AfterExternBlock
 IndentGotoLabels: true
 IndentPPDirectives: None
-IndentRequires:  false
+IndentRequiresClause: true
 IndentWidth:     2
 IndentWrappedFunctionNames: false
+InsertBraces:    false
+InsertNewlineAtEOF: false
 # InsertTrailingCommas: None
+IntegerLiteralSeparator:
+  Binary:          0
+  Decimal:         0
+  Hex:             0
 # JavaScriptQuotes: Leave
 # JavaScriptWrapImports: true
 KeepEmptyLinesAtTheStartOfBlocks: false
 LambdaBodyIndentation: Signature
+LineEnding:      DeriveLF
 MacroBlockBegin: ''
 MacroBlockEnd:   ''
 MaxEmptyLinesToKeep: 1
@@ -100,23 +129,31 @@ NamespaceIndentation: None
 # ObjCBreakBeforeNestedBlockParam: true
 # ObjCSpaceAfterProperty: false
 # ObjCSpaceBeforeProtocolList: true
+PackConstructorInitializers: NextLine
 PenaltyBreakAssignment: 2
 PenaltyBreakBeforeFirstCallParameter: 1
 PenaltyBreakComment: 300
 PenaltyBreakFirstLessLess: 120
+PenaltyBreakOpenParenthesis: 0
 PenaltyBreakString: 1000
 PenaltyBreakTemplateDeclaration: 10
 PenaltyExcessCharacter: 1000000
 PenaltyIndentedWhitespace: 0
 PenaltyReturnTypeOnItsOwnLine: 200
-PointerAlignment: Right
+PointerAlignment: Right # Left
 PPIndentWidth:   -1
+QualifierAlignment: Leave
 ReferenceAlignment: Pointer
 ReflowComments:  false # true
+RemoveBracesLLVM: false
+RemoveSemicolon: false
+RequiresClausePosition: OwnLine
+RequiresExpressionIndentation: OuterScope
+SeparateDefinitionBlocks: Leave
 ShortNamespaceLines: 0 # 1
 SortIncludes: CaseInsensitive # CaseSensitive
 # SortJavaStaticImport: Before
-SortUsingDeclarations: false # true
+SortUsingDeclarations: Lexicographic # LexicographicNumeric
 SpaceAfterCStyleCast: false
 SpaceAfterLogicalNot: false
 SpaceAfterTemplateKeyword: true
@@ -127,6 +164,16 @@ SpaceBeforeCpp11BracedList: false
 SpaceBeforeCtorInitializerColon: true
 SpaceBeforeInheritanceColon: true
 SpaceBeforeParens: ControlStatements
+SpaceBeforeParensOptions:
+  AfterControlStatements: true
+  AfterForeachMacros: true
+  AfterFunctionDefinitionName: false
+  AfterFunctionDeclarationName: false
+  AfterIfMacros:   true
+  AfterOverloadedOperator: false
+  AfterRequiresInClause: false
+  AfterRequiresInExpression: false
+  BeforeNonEmptyParentheses: false
 SpaceBeforeRangeBasedForLoopColon: true
 SpaceBeforeSquareBrackets: false
 SpaceInEmptyBlock: false
@@ -143,6 +190,5 @@ SpacesInParentheses: false
 SpacesInSquareBrackets: false
 Standard:        Auto
 TabWidth:        100 # 8
-UseCRLF:         false
 UseTab:          Never
 ...

.github/workflows/docker.yml

@@ -180,7 +180,7 @@ jobs:
           --amend ${{ env.IMAGE_TAG }}:${{ env.HASH_VERSION }}-linuxarmv7 \
           --amend ${{ env.IMAGE_TAG }}:${{ env.HASH_VERSION }}-linuxarm64 \
           --amend ${{ env.IMAGE_TAG }}:${{ env.HASH_VERSION }}-linuxppc64le
-          docker manifest push ${{ env.IMAGE_TAG }}:${{ env.HASH_VERSION }}
+          #docker manifest push ${{ env.IMAGE_TAG }}:${{ env.HASH_VERSION }}
 
           # Tag images as VERSION (like 'latest')
           docker tag ${{ env.IMAGE_TAG }}:${{ env.HASH_VERSION }}-linux386 ${{ env.IMAGE_TAG }}:${{ env.VERSION }}-linux386
@@ -197,7 +197,7 @@ jobs:
           --amend ${{ env.IMAGE_TAG }}:${{ env.VERSION }}-linuxarmv7 \
           --amend ${{ env.IMAGE_TAG }}:${{ env.VERSION }}-linuxarm64 \
           --amend ${{ env.IMAGE_TAG }}:${{ env.VERSION }}-linuxppc64le
-          docker manifest push ${{ env.IMAGE_TAG }}:${{ env.VERSION }}
+          #docker manifest push ${{ env.IMAGE_TAG }}:${{ env.VERSION }}
 
           # -- Push to Docker Hub
           docker tag ${{ env.IMAGE_TAG }}:${{ env.HASH_VERSION }}-linux386 ${{ env.IMAGE_TAG_DH }}:${{ env.VERSION }}-linux386

CMakeLists.txt

@@ -6,7 +6,7 @@ if (POLICY CMP0065)
   cmake_policy(SET CMP0065 NEW)
 endif()
 
-project(TelegramBotApi VERSION 6.5 LANGUAGES CXX)
+project(TelegramBotApi VERSION 7.0 LANGUAGES CXX)
 
 if (POLICY CMP0069)
   option(TELEGRAM_BOT_API_ENABLE_LTO "Use \"ON\" to enable Link Time Optimization.")
@@ -73,6 +73,9 @@ if (CLANG OR GCC)
     elseif (APPLE)
       set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-no_pie")
     endif()
+    include(AddCXXCompilerFlag)
+    add_cxx_compiler_flag("-static-libstdc++")
+    add_cxx_compiler_flag("-static-libgcc")
   endif()
 endif()
 

README.md

@@ -54,17 +54,6 @@ Get the member list of a supergroup or channel
 
 ###### Returns `ChatMember`
 
-##### Method `deleteMessages`
-Delete all the messages with message_id in range between `start` and `end`.  
-The `start` parameter MUST be less than the `end` parameter  
-Both `start` and `end` must be positive non zero numbers  
-The method will always return `true` as a result, even if the messages cannot be deleted  
-This method does not work on private chat or normal groups
-It is not suggested to delete more than 200 messages per call
-
-**NOTE**  
-The maximum number of messages to be deleted in a single batch is determined by the `max-batch-operations` parameter and is 10000 by default
-
 ###### Parameters
 - `chat_id` Chat id
 - `start` First message id to delete
@@ -122,6 +111,17 @@ _For Docker containers, `$TELEGRAM_VERBOSITY` can be set._
 ##### Method `getChat`
 The command `getChat` will also try to resolve the username online, if it can't be found locally
 
+##### Method `deleteMessages`
+The command `deleteMessages` can also delete all the messages with message_id in range between `start` and `end`.  
+The `start` parameter MUST be less than the `end` parameter  
+Both `start` and `end` must be positive non-zero numbers  
+The method will always return `true` as a result, even if the messages cannot be deleted  
+This method does not work on private chat or normal groups
+It is not suggested to delete more than 200 messages per call
+
+**NOTE**  
+The maximum number of messages to be deleted in a single batch is determined by the `max-batch-operations` parameter and is 10000 by default
+
 ##### Object `Message`
 The `Message` object now has two new fields:
 - `views`: how many views has the message (usually the views are shown only for channel messages)

td

@@ -1 +1 @@
-Subproject commit 3179d35694a28267a0b6273fc9b5bdce3b6b1235
+Subproject commit 27c3eaeb4964bd5f18d8488e354abde1a4383e49

tdlight-api-openapi.yaml

@@ -547,16 +547,23 @@ paths:
   /deleteMessages:
     post:
       tags:
-        - added
+        - modified
       description: |-
-        Delete all the messages with message_id in range between start and end.
-        The start parameter MUST be less than the end parameter
-        Both start and end must be positive non zero numbers
-        The method will always return true as a result, even if the messages cannot be deleted
-        This method does not work on private chat or normal groups It is not suggested to delete more than 200 messages per call.
+        Use this method to delete multiple messages simultaneously.
+        This method can delete a set of message ids, or a range of message ids.
+        
+        If you specify "message_ids", this method tries to delete the specified set of ids:
+          If some of the specified messages can't be found, they are skipped.
+          Returns True on success.
 
-        *NOTE*
-        The maximum number of messages to be deleted in a single batch is determined by the max-batch-operations parameter and is 10000 by default.
+        If you specify "start" and "end", this method deletes all the messages with message_id in range between start and end:
+          The start parameter MUST be less than the end parameter
+          Both start and end must be positive non zero numbers
+          The method will always return true as a result, even if the messages cannot be deleted
+          This method does not work on private chat or normal groups It is not suggested to delete more than 200 messages per call.
+
+          *NOTE*
+          The maximum number of messages to be deleted in a single batch is determined by the max-batch-operations parameter and is 10000 by default.
       requestBody:
         content:
           application/x-www-form-urlencoded:
@@ -568,6 +575,10 @@ paths:
                   anyOf:
                     - type: integer
                     - type: string
+                message_ids:
+                  type: array
+                  items:
+                    type: integer
                 start:
                   description: First message id to delete
                   type: integer
@@ -576,8 +587,6 @@ paths:
                   type: integer
               required:
                 - chat_id
-                - start
-                - end
           multipart/form-data:
             schema:
               type: object
@@ -587,6 +596,10 @@ paths:
                   anyOf:
                     - type: integer
                     - type: string
+                message_ids:
+                  type: array
+                  items:
+                    type: integer
                 start:
                   description: First message id to delete
                   type: integer
@@ -595,8 +608,6 @@ paths:
                   type: integer
               required:
                 - chat_id
-                - start
-                - end
           application/json:
             schema:
               type: object
@@ -606,6 +617,10 @@ paths:
                   anyOf:
                     - type: integer
                     - type: string
+                message_ids:
+                  type: array
+                  items:
+                    type: integer
                 start:
                   description: First message id to delete
                   type: integer
@@ -614,12 +629,10 @@ paths:
                   type: integer
               required:
                 - chat_id
-                - start
-                - end
         required: true
       responses:
         '200':
-          description: ''
+          description: 'Request was successful, the result is returned.'
           content:
             application/json:
               schema:

telegram-bot-api/ClientManager.cpp

@@ -177,13 +177,7 @@ void ClientManager::user_login(PromisedQueryPtr query) {
 }
 
 bool ClientManager::check_flood_limits(PromisedQueryPtr &query, bool is_user_login) {
-  td::string ip_address;
-  if (query->peer_address().is_valid() && !query->peer_address().is_reserved()) {  // external connection
-    ip_address = query->peer_address().get_ip_str().str();
-  } else {
-    // invalid peer address or connection from the local network
-    ip_address = query->get_header("x-real-ip").str();
-  }
+  td::string ip_address = query->get_peer_ip_address();
   if (!ip_address.empty()) {
     td::IPAddress tmp;
     tmp.init_host_port(ip_address, 0).ignore();
@@ -192,7 +186,7 @@ bool ClientManager::check_flood_limits(PromisedQueryPtr &query, bool is_user_log
       ip_address = tmp.get_ip_str().str();
     }
   }
-  LOG(DEBUG) << "Receive incoming query for new bot " << query->token() << " from " << query->peer_address();
+  LOG(DEBUG) << "Receive incoming query for new bot " << query->token() << " from " << ip_address;
   if (!ip_address.empty()) {
     LOG(DEBUG) << "Check Client creation flood control for IP address " << ip_address;
     if (is_user_login) {
@@ -302,7 +296,7 @@ void ClientManager::get_stats(td::Promise<td::BufferSlice> promise,
   auto top_clients = get_top_clients(50, id_filter);
 
   if(!as_json) {
-    sb << stat_.get_description() << '\n';
+    sb << BotStatActor::get_description() << '\n';
   }
   if (id_filter.empty()) {
     if(as_json) {
@@ -454,9 +448,6 @@ td::int64 ClientManager::get_tqueue_id(td::int64 user_id, bool is_test_dc) {
 }
 
 void ClientManager::start_up() {
-  //NB: the same scheduler as for database in Td
-  auto scheduler_id = 1;
-
   // init tqueue
   {
     auto load_start_time = td::Time::now();
@@ -484,7 +475,8 @@ void ClientManager::start_up() {
       }
     }
 
-    auto concurrent_binlog = std::make_shared<td::ConcurrentBinlog>(std::move(binlog), scheduler_id);
+    auto concurrent_binlog =
+        std::make_shared<td::ConcurrentBinlog>(std::move(binlog), SharedData::get_binlog_scheduler_id());
     auto concurrent_tqueue_binlog = td::make_unique<td::TQueueBinlog<td::BinlogInterface>>();
     concurrent_tqueue_binlog->set_binlog(std::move(concurrent_binlog));
     tqueue->set_callback(std::move(concurrent_tqueue_binlog));
@@ -499,18 +491,18 @@ void ClientManager::start_up() {
   // init webhook_db and user_db
   auto concurrent_webhook_db = td::make_unique<td::BinlogKeyValue<td::ConcurrentBinlog>>();
   auto status = concurrent_webhook_db->init(parameters_->working_directory_ + "webhooks_db.binlog", td::DbKey::empty(),
-                                            scheduler_id);
+                                            SharedData::get_binlog_scheduler_id());
   LOG_IF(FATAL, status.is_error()) << "Can't open webhooks_db.binlog " << status;
   parameters_->shared_data_->webhook_db_ = std::move(concurrent_webhook_db);
 
   auto concurrent_user_db = td::make_unique<td::BinlogKeyValue<td::ConcurrentBinlog>>();
-  status = concurrent_user_db->init(parameters_->working_directory_ + "user_db.binlog", td::DbKey::empty(), scheduler_id);
+  status = concurrent_user_db->init(parameters_->working_directory_ + "user_db.binlog", td::DbKey::empty(), SharedData::get_binlog_scheduler_id());
   LOG_IF(FATAL, status.is_error()) << "Can't open user_db.binlog " << status.error();
   parameters_->shared_data_->user_db_ = std::move(concurrent_user_db);
 
   auto &webhook_db = *parameters_->shared_data_->webhook_db_;
   auto &user_db = *parameters_->shared_data_->user_db_;
-  for (auto key_value : webhook_db.get_all()) {
+  for (const auto &key_value : webhook_db.get_all()) {
     if (!token_range_(td::to_integer<td::uint64>(key_value.first))) {
       LOG(WARNING) << "DROP WEBHOOK: " << key_value.first << " ---> " << key_value.second;
       webhook_db.erase(key_value.first);
@@ -522,8 +514,8 @@ void ClientManager::start_up() {
   }
 
   // launch watchdog
-  watchdog_id_ = td::create_actor_on_scheduler<Watchdog>(
-      "ManagerWatchdog", td::Scheduler::instance()->sched_count() - 3, td::this_thread::get_id(), WATCHDOG_TIMEOUT);
+  watchdog_id_ = td::create_actor_on_scheduler<Watchdog>("ManagerWatchdog", SharedData::get_watchdog_scheduler_id(),
+                                                         td::this_thread::get_id(), WATCHDOG_TIMEOUT);
   set_timeout_in(600.0);
 }
 
@@ -690,7 +682,7 @@ void ClientManager::raw_event(const td::Event::Raw &event) {
 
 void ClientManager::timeout_expired() {
   send_closure(watchdog_id_, &Watchdog::kick);
-  set_timeout_in(WATCHDOG_TIMEOUT / 2);
+  set_timeout_in(WATCHDOG_TIMEOUT / 10);
 
   double now = td::Time::now();
   if (now > next_tqueue_gc_time_) {

telegram-bot-api/ClientParameters.h

@@ -54,15 +54,54 @@ struct SharedData {
     return static_cast<td::int32>(result);
   }
 
-  static td::int32 get_database_scheduler_id() {
-    // the same scheduler as for database in Td
-    return 1;
-  }
-
   static td::int32 get_file_gc_scheduler_id() {
     // the same scheduler as for file GC in Td
     return 2;
   }
+
+  static td::int32 get_client_scheduler_id() {
+    // the thread for ClientManager and all Clients
+    return 4;
+  }
+
+  static td::int32 get_watchdog_scheduler_id() {
+    // the thread for watchdogs
+    return 5;
+  }
+
+  static td::int32 get_slow_incoming_http_scheduler_id() {
+    // the thread for slow incoming HTTP connections
+    return 6;
+  }
+
+  static td::int32 get_slow_outgoing_http_scheduler_id() {
+    // the thread for slow outgoing HTTP connections
+    return 7;
+  }
+
+  static td::int32 get_dns_resolver_scheduler_id() {
+    // the thread for DNS resolving
+    return 8;
+  }
+
+  static td::int32 get_binlog_scheduler_id() {
+    // the thread for TQueue and webhook binlogs
+    return 9;
+  }
+
+  static td::int32 get_webhook_certificate_scheduler_id() {
+    // the thread for webhook certificate processing
+    return 10;
+  }
+
+  static td::int32 get_statistics_thread_id() {
+    // the thread for CPU usage updating
+    return 11;
+  }
+
+  static td::int32 get_thread_count() {
+    return 12;
+  }
 };
 
 struct ClientParameters {
@@ -72,7 +111,7 @@ struct ClientParameters {
   bool local_mode_ = false;
   bool allow_http_ = false;
   bool use_relative_path_ = false;
-  bool no_file_limit_ = true;
+  bool no_file_limit_ = false;
   bool allow_users_ = false;
   bool allow_users_registration_ = false;
   bool stats_hide_sensible_data_ = false;

telegram-bot-api/HttpServer.h

@@ -6,12 +6,15 @@
 //
 #pragma once
 
+#include "telegram-bot-api/ClientParameters.h"
+
 #include "td/net/HttpInboundConnection.h"
 #include "td/net/TcpListener.h"
 
 #include "td/actor/actor.h"
 
 #include "td/utils/BufferedFd.h"
+#include "td/utils/common.h"
 #include "td/utils/FloodControlFast.h"
 #include "td/utils/format.h"
 #include "td/utils/logging.h"
@@ -47,7 +50,7 @@ class HttpServer final : public td::TcpListener::Callback {
       return;
     }
     flood_control_.add_event(now);
-    LOG(INFO) << "Create tcp listener " << td::tag("address", ip_address_) << td::tag("port", port_);
+    LOG(INFO) << "Create TCP listener " << td::tag("address", ip_address_) << td::tag("port", port_);
     listener_ = td::create_actor<td::TcpListener>(
         PSLICE() << "TcpListener" << td::tag("address", ip_address_) << td::tag("port", port_), port_,
         actor_shared(this, 1), ip_address_);
@@ -60,13 +63,8 @@ class HttpServer final : public td::TcpListener::Callback {
   }
 
   void accept(td::SocketFd fd) final {
-    auto scheduler_count = td::Scheduler::instance()->sched_count();
-    auto scheduler_id = scheduler_count - 1;
-    if (scheduler_id > 0) {
-      scheduler_id--;
-    }
     td::create_actor<td::HttpInboundConnection>("HttpInboundConnection", td::BufferedFd<td::SocketFd>(std::move(fd)), 0,
-                                                20, 500, creator_(), scheduler_id)
+                                                50, 500, creator_(), SharedData::get_slow_incoming_http_scheduler_id())
         .release();
   }
 

telegram-bot-api/Query.cpp

@@ -26,10 +26,10 @@ td::FlatHashMap<td::string, td::unique_ptr<td::VirtuallyJsonable>> empty_paramet
 Query::Query(td::vector<td::BufferSlice> &&container, td::Slice token, bool is_user, bool is_test_dc, td::MutableSlice method,
              td::vector<std::pair<td::MutableSlice, td::MutableSlice>> &&args,
              td::vector<std::pair<td::MutableSlice, td::MutableSlice>> &&headers, td::vector<td::HttpFile> &&files,
-             std::shared_ptr<SharedData> shared_data, const td::IPAddress &peer_address, bool is_internal)
+             std::shared_ptr<SharedData> shared_data, const td::IPAddress &peer_ip_address, bool is_internal)
     : state_(State::Query)
     , shared_data_(shared_data)
-    , peer_address_(peer_address)
+    , peer_ip_address_(peer_ip_address)
     , container_(std::move(container))
     , token_(token)
     , is_user_(is_user)
@@ -44,7 +44,7 @@ Query::Query(td::vector<td::BufferSlice> &&container, td::Slice token, bool is_u
   }
   td::to_lower_inplace(method_);
   start_timestamp_ = td::Time::now();
-  LOG(INFO) << "QUERY: create " << td::tag("ptr", this) << *this;
+  LOG(INFO) << "Query " << this << ": " << *this;
   if (shared_data_) {
     shared_data_->query_count_.fetch_add(1, std::memory_order_relaxed);
     if (method_ != "getupdates") {
@@ -54,6 +54,15 @@ Query::Query(td::vector<td::BufferSlice> &&container, td::Slice token, bool is_u
   }
 }
 
+td::string Query::get_peer_ip_address() const {
+  if (peer_ip_address_.is_valid() && !peer_ip_address_.is_reserved()) {  // external connection
+    return peer_ip_address_.get_ip_str().str();
+  } else {
+    // invalid peer IP address or connection from the local network
+    return get_header("x-real-ip").str();
+  }
+}
+
 td::int64 Query::query_size() const {
   return std::accumulate(
       container_.begin(), container_.end(), td::int64{0},
@@ -77,7 +86,7 @@ void Query::set_stat_actor(td::ActorId<BotStatActor> stat_actor) {
 
 void Query::set_ok(td::BufferSlice result) {
   CHECK(state_ == State::Query);
-  LOG(INFO) << "QUERY: got ok " << td::tag("ptr", this) << td::tag("text", result.as_slice());
+  LOG(INFO) << "Query " << this << ": " << td::tag("method", method_) << td::tag("text", result.as_slice());
   answer_ = std::move(result);
   state_ = State::OK;
   http_status_code_ = 200;
@@ -85,7 +94,7 @@ void Query::set_ok(td::BufferSlice result) {
 }
 
 void Query::set_error(int http_status_code, td::BufferSlice result) {
-  LOG(INFO) << "QUERY: got error " << td::tag("ptr", this) << td::tag("code", http_status_code)
+  LOG(INFO) << "Query " << this << ": " << td::tag("method", method_) << td::tag("code", http_status_code)
             << td::tag("text", result.as_slice());
   CHECK(state_ == State::Query);
   answer_ = std::move(result);
@@ -107,9 +116,25 @@ td::StringBuilder &operator<<(td::StringBuilder &sb, const Query &query) {
   auto padded_time =
       td::lpad(PSTRING() << td::format::as_time(td::Time::now_cached() - query.start_timestamp()), 10, ' ');
   sb << "[bot" << td::rpad(query.token().str(), 46, ' ') << "][time:" << padded_time << ']'
-     << td::tag("method", td::lpad(query.method().str(), 20, ' '));
+     << td::tag("method", td::lpad(query.method().str(), 25, ' '));
   if (!query.args().empty()) {
-    sb << td::oneline(PSLICE() << query.args());
+    sb << '{';
+    for (const auto &arg : query.args()) {
+      sb << '[';
+      if (arg.first.size() > 128) {
+        sb << '<' << arg.first.size() << '>' << td::oneline(arg.first.substr(0, 128)) << "...";
+      } else {
+        sb << td::oneline(arg.first);
+      }
+      sb << ':';
+      if (arg.second.size() > 4096) {
+        sb << '<' << arg.second.size() << '>' << td::oneline(arg.second.substr(0, 4096)) << "...";
+      } else {
+        sb << td::oneline(arg.second);
+      }
+      sb << ']';
+    }
+    sb << '}';
   }
   if (!query.files().empty()) {
     sb << query.files();

telegram-bot-api/Query.h

@@ -38,44 +38,53 @@ class Query final : public td::ListNode {
   td::Slice token() const {
     return token_;
   }
+  
   bool is_user() const {
     return is_user_;
   }
+
   bool is_test_dc() const {
     return is_test_dc_;
   }
+
   td::Slice method() const {
     return method_;
   }
+
   bool has_arg(td::Slice key) const {
     auto it = std::find_if(args_.begin(), args_.end(),
                            [&key](const std::pair<td::MutableSlice, td::MutableSlice> &s) { return s.first == key; });
     return it != args_.end();
   }
+
   td::MutableSlice arg(td::Slice key) const {
     auto it = std::find_if(args_.begin(), args_.end(),
                            [&key](const std::pair<td::MutableSlice, td::MutableSlice> &s) { return s.first == key; });
     return it == args_.end() ? td::MutableSlice() : it->second;
   }
+
   const td::vector<std::pair<td::MutableSlice, td::MutableSlice>> &args() const {
     return args_;
   }
+
   td::Slice get_header(td::Slice key) const {
     auto it = std::find_if(headers_.begin(), headers_.end(),
                            [&key](const std::pair<td::MutableSlice, td::MutableSlice> &s) { return s.first == key; });
     return it == headers_.end() ? td::Slice() : it->second;
   }
+
   const td::HttpFile *file(td::Slice key) const {
     auto it = std::find_if(files_.begin(), files_.end(), [&key](const td::HttpFile &f) { return f.field_name == key; });
     return it == files_.end() ? nullptr : &*it;
   }
+
   const td::vector<td::HttpFile> &files() const {
     return files_;
   }
 
-  const td::IPAddress &peer_address() const {
-    return peer_address_;
-  }
+  td::int64 files_size() const;
+
+  td::string get_peer_ip_address() const;
 
   td::BufferSlice &answer() {
     return answer_;
@@ -106,7 +115,7 @@ class Query final : public td::ListNode {
   Query(td::vector<td::BufferSlice> &&container, td::Slice token, bool is_user, bool is_test_dc, td::MutableSlice method,
         td::vector<std::pair<td::MutableSlice, td::MutableSlice>> &&args,
         td::vector<std::pair<td::MutableSlice, td::MutableSlice>> &&headers, td::vector<td::HttpFile> &&files,
-        std::shared_ptr<SharedData> shared_data, const td::IPAddress &peer_address, bool is_internal);
+        std::shared_ptr<SharedData> shared_data, const td::IPAddress &peer_ip_address, bool is_internal);
   Query(const Query &) = delete;
   Query &operator=(const Query &) = delete;
   Query(Query &&) = delete;
@@ -132,7 +141,7 @@ class Query final : public td::ListNode {
   State state_;
   std::shared_ptr<SharedData> shared_data_;
   double start_timestamp_;
-  td::IPAddress peer_address_;
+  td::IPAddress peer_ip_address_;
   td::ActorId<BotStatActor> stat_actor_;
 
   // request
@@ -158,8 +167,6 @@ class Query final : public td::ListNode {
 
   td::int64 query_size() const;
 
-  td::int64 files_size() const;
-
   td::int64 files_max_size() const;
 
   void send_request_stat() const;

telegram-bot-api/Stats.cpp

@@ -7,6 +7,7 @@
 #include "telegram-bot-api/Stats.h"
 
 #include "td/utils/common.h"
+#include "td/utils/logging.h"
 #include "td/utils/port/thread.h"
 #include "td/utils/SliceBuilder.h"
 #include "td/utils/StringBuilder.h"
@@ -19,17 +20,24 @@ ServerCpuStat::ServerCpuStat() {
   }
 }
 
-void ServerCpuStat::add_event(const td::CpuStat &cpu_stat, double now) {
-  std::lock_guard<std::mutex> guard(mutex_);
-  for (auto &stat : stat_) {
-    stat.add_event(cpu_stat, now);
+void ServerCpuStat::update(double now) {
+  auto r_cpu_stat = td::cpu_stat();
+  if (r_cpu_stat.is_error()) {
+    return;
   }
+
+  auto &cpu_stat = instance();
+  std::lock_guard<std::mutex> guard(cpu_stat.mutex_);
+  for (auto &stat : cpu_stat.stat_) {
+    stat.add_event(r_cpu_stat.ok(), now);
+  }
+  LOG(WARNING) << "CPU usage: " << cpu_stat.stat_[1].get_stat(now).as_vector()[0].value_;
 }
 
-td::string ServerCpuStat::get_description() const {
+td::string ServerCpuStat::get_description() {
   td::string res = "DURATION";
   for (auto &descr : DESCR) {
-    res += "\t";
+    res += '\t';
     res += descr;
   }
   return res;
@@ -37,7 +45,7 @@ td::string ServerCpuStat::get_description() const {
 
 static td::string to_percentage(td::uint64 ticks, td::uint64 total_ticks) {
   static double multiplier = 100.0 * (td::thread::hardware_concurrency() ? td::thread::hardware_concurrency() : 1);
-  return PSTRING() << (static_cast<double>(ticks) / static_cast<double>(total_ticks) * multiplier) << "%";
+  return PSTRING() << (static_cast<double>(ticks) / static_cast<double>(total_ticks) * multiplier) << '%';
 }
 
 td::vector<StatItem> CpuStat::as_vector() const {
@@ -167,7 +175,7 @@ td::vector<ServerBotStat> BotStatActor::as_json_ready_vector(double now) {
   return res;
 }
 
-td::string BotStatActor::get_description() const {
+td::string BotStatActor::get_description() {
   td::string res = "DURATION";
   for (auto &descr : DESCR) {
     res += "\t";
@@ -187,13 +195,18 @@ td::vector<td::string> BotStatActor::get_jsonable_description() const {
 
 double BotStatActor::get_score(double now) {
   auto minute_stat = stat_[2].stat_duration(now);
-  double result = minute_stat.first.request_count_ + minute_stat.first.update_count_;
+  double minute_score = minute_stat.first.request_count_ + minute_stat.first.update_count_;
   if (minute_stat.second != 0) {
-    result /= minute_stat.second;
+    minute_score /= minute_stat.second;
   }
-  result += td::max(static_cast<double>(get_active_request_count() - 10), static_cast<double>(0));
-  result += static_cast<double>(get_active_file_upload_bytes()) * 1e-8;
-  return result;
+  auto all_time_stat = stat_[0].stat_duration(now);
+  double all_time_score = 0.01 * (all_time_stat.first.request_count_ + all_time_stat.first.update_count_);
+  if (all_time_stat.second != 0) {
+    all_time_score /= all_time_stat.second;
+  }
+  auto active_request_score = static_cast<double>(td::max(get_active_request_count() - 10, static_cast<td::int64>(0)));
+  auto active_file_upload_score = static_cast<double>(get_active_file_upload_bytes()) * 1e-8;
+  return minute_score + all_time_score + active_request_score + active_file_upload_score;
 }
 
 double BotStatActor::get_minute_update_count(double now) {

telegram-bot-api/Stats.h

@@ -9,7 +9,6 @@
 #include "td/actor/actor.h"
 
 #include "td/utils/common.h"
-#include "td/utils/logging.h"
 #include "td/utils/port/Stat.h"
 #include "td/utils/Time.h"
 #include "td/utils/TimedStat.h"
@@ -49,16 +48,10 @@ class ServerCpuStat {
     static ServerCpuStat stat;
     return stat;
   }
-  static void update(double now) {
-    auto r_event = td::cpu_stat();
-    if (r_event.is_error()) {
-      return;
-    }
-    instance().add_event(r_event.ok(), now);
-    LOG(WARNING) << "CPU usage: " << instance().stat_[1].get_stat(now).as_vector()[0].value_;
-  }
 
-  td::string get_description() const;
+  static void update(double now);
+
+  static td::string get_description();
 
   td::vector<StatItem> as_vector(double now);
   td::vector<td::vector<StatItem>> as_json_ready_vector(double now);
@@ -72,8 +65,6 @@ class ServerCpuStat {
   td::TimedStat<CpuStat> stat_[SIZE];
 
   ServerCpuStat();
-
-  void add_event(const td::CpuStat &stat, double now);
 };
 
 class ServerBotInfo {
@@ -157,7 +148,7 @@ class BotStatActor final : public td::Actor {
   }
 
   BotStatActor(const BotStatActor &) = delete;
-  BotStatActor &operator=(const BotStatActor &other) = delete;
+  BotStatActor &operator=(const BotStatActor &) = delete;
   BotStatActor(BotStatActor &&) = default;
   BotStatActor &operator=(BotStatActor &&other) noexcept {
     if (!empty()) {
@@ -185,8 +176,8 @@ class BotStatActor final : public td::Actor {
   td::vector<StatItem> as_vector(double now);
   td::vector<ServerBotStat> as_json_ready_vector(double now);
 
-  td::string get_description() const;
   td::vector<td::string> get_jsonable_description() const;
+  static td::string get_description();
 
   double get_score(double now);
 

telegram-bot-api/Watchdog.cpp

@@ -13,7 +13,7 @@ namespace telegram_bot_api {
 
 void Watchdog::kick() {
   auto now = td::Time::now();
-  if (now >= last_kick_time_ + timeout_ && last_kick_time_ > 0) {
+  if (now >= last_kick_time_ + timeout_ && last_kick_time_ > 0 && GET_VERBOSITY_LEVEL() >= VERBOSITY_NAME(ERROR)) {
     LOG(ERROR) << get_name() << " timeout expired after " << now - last_kick_time_ << " seconds";
     td::thread::send_real_time_signal(main_thread_id_, 2);
   }

telegram-bot-api/WebhookActor.cpp

@@ -47,10 +47,8 @@ WebhookActor::WebhookActor(td::ActorShared<Callback> callback, td::int64 tqueue_
     , fix_ip_address_(fix_ip_address)
     , from_db_flag_(from_db_flag)
     , max_connections_(max_connections)
-    , secret_token_(std::move(secret_token))
-    , slow_scheduler_id_(td::Scheduler::instance()->sched_count() - 2) {
+    , secret_token_(std::move(secret_token)) {
   CHECK(max_connections_ > 0);
-  CHECK(slow_scheduler_id_ > 0);
 
   if (!cached_ip_address.empty()) {
     auto r_ip_address = td::IPAddress::get_ip_address(cached_ip_address);
@@ -73,7 +71,7 @@ WebhookActor::WebhookActor(td::ActorShared<Callback> callback, td::int64 tqueue_
 
 WebhookActor::~WebhookActor() {
   td::Scheduler::instance()->destroy_on_scheduler(SharedData::get_file_gc_scheduler_id(), update_map_, queue_updates_,
-                                                  queues_);
+                                                  queues_, ssl_ctx_);
 }
 
 void WebhookActor::relax_wakeup_at(double wakeup_at, const char *source) {
@@ -207,7 +205,7 @@ td::Status WebhookActor::create_webhook_error(td::Slice error_message, td::Statu
     on_webhook_error(error_message);
   }
   on_error(std::move(result));
-  return std::move(error);
+  return error;
 }
 
 td::Result<td::SslStream> WebhookActor::create_ssl_stream() {
@@ -229,8 +227,9 @@ td::Status WebhookActor::create_connection(td::BufferedFd<td::SocketFd> fd) {
   auto id = connections_.create(Connection());
   auto *conn = connections_.get(id);
   conn->actor_id_ = td::create_actor<td::HttpOutboundConnection>(
-      PSLICE() << "Connect:" << id, std::move(fd), std::move(ssl_stream), 0, 20, 60,
-      td::ActorShared<td::HttpOutboundConnection::Callback>(actor_id(this), id), slow_scheduler_id_);
+      PSLICE() << "Connect:" << id, std::move(fd), std::move(ssl_stream), 0, 50, 60,
+      td::ActorShared<td::HttpOutboundConnection::Callback>(actor_id(this), id),
+      SharedData::get_slow_outgoing_http_scheduler_id());
   conn->ip_generation_ = ip_generation_;
   conn->event_id_ = {};
   conn->id_ = id;
@@ -399,7 +398,7 @@ void WebhookActor::load_updates() {
     CHECK(update.id.is_valid());
     auto &dest_ptr = update_map_[update.id];
     if (dest_ptr != nullptr) {
-      LOG(ERROR) << "Receive duplicated event " << update.id << " from TQueue";
+      LOG(ERROR) << "Receive duplicate event " << update.id << " from TQueue";
       continue;
     }
     dest_ptr = td::make_unique<Update>();
@@ -623,10 +622,12 @@ void WebhookActor::handle(td::unique_ptr<td::HttpQuery> response) {
         if (!method.empty() && method != "deletewebhook" && method != "setwebhook" && method != "close" &&
             method != "logout" && !td::begins_with(method, "get")) {
           VLOG(webhook) << "Receive request " << method << " in response to webhook";
-          auto query = td::make_unique<Query>(std::move(response->container_), td::MutableSlice(), false, false,
-                                              td::MutableSlice(), std::move(response->args_),
-                                              std::move(response->headers_), std::move(response->files_),
-                                              parameters_->shared_data_, response->peer_address_, false);
+          response->container_.emplace_back(PSLICE() << (tqueue_id_ & ((static_cast<td::int64>(1) << 54) - 1)));
+          auto token = response->container_.back().as_slice();
+          auto query = td::make_unique<Query>(
+              std::move(response->container_), token, tqueue_id_ >= (static_cast<td::int64>(1) << 54), false,
+              td::MutableSlice(), std::move(response->args_), std::move(response->headers_),
+              std::move(response->files_), parameters_->shared_data_, response->peer_address_, false);
           auto promised_query = PromisedQueryPtr(query.release(), PromiseDeleter(td::Promise<td::unique_ptr<Query>>()));
           send_closure(callback_, &Callback::send, std::move(promised_query));
         }
@@ -690,7 +691,12 @@ void WebhookActor::handle(td::unique_ptr<td::HttpQuery> response) {
 void WebhookActor::start_up() {
   max_loaded_updates_ = max_connections_ * 2;
 
-  next_ip_address_resolve_time_ = last_success_time_ = td::Time::now() - 3600;
+  last_success_time_ = td::Time::now() - 2 * IP_ADDRESS_CACHE_TIME;
+  if (from_db_flag_) {
+    next_ip_address_resolve_time_ = td::Time::now() + td::Random::fast(0, IP_ADDRESS_CACHE_TIME);
+  } else {
+    next_ip_address_resolve_time_ = last_success_time_;
+  }
 
   active_new_connection_flood_.add_limit(0.5, 10);
 
@@ -723,11 +729,12 @@ void WebhookActor::start_up() {
 
   if (url_.protocol_ != td::HttpUrl::Protocol::Http && !stop_flag_) {
     // asynchronously create SSL context
-    td::Scheduler::instance()->run_on_scheduler(
-        SharedData::get_database_scheduler_id(), [actor_id = actor_id(this), cert_path = cert_path_](td::Unit) mutable {
-          send_closure(actor_id, &WebhookActor::on_ssl_context_created,
-                       td::SslCtx::create(cert_path, td::SslCtx::VerifyPeer::On));
-        });
+    td::Scheduler::instance()->run_on_scheduler(SharedData::get_webhook_certificate_scheduler_id(),
+                                                [actor_id = actor_id(this), cert_path = cert_path_](td::Unit) mutable {
+                                                  send_closure(
+                                                      actor_id, &WebhookActor::on_ssl_context_created,
+                                                      td::SslCtx::create(cert_path, td::SslCtx::VerifyPeer::On));
+                                                });
   }
 
   yield();

telegram-bot-api/WebhookActor.h

@@ -177,7 +177,6 @@ class WebhookActor final : public td::HttpOutboundConnection::Callback {
   double last_success_time_ = 0;
   double wakeup_at_ = 0;
   bool last_update_was_successful_ = true;
-  td::int32 slow_scheduler_id_ = -1;
 
   void relax_wakeup_at(double wakeup_at, const char *source);
 

telegram-bot-api/telegram-bot-api.cpp

@@ -165,7 +165,7 @@ int main(int argc, char *argv[]) {
   auto start_time = td::Time::now();
   auto shared_data = std::make_shared<SharedData>();
   auto parameters = std::make_unique<ClientParameters>();
-  parameters->version_ = "6.5";
+  parameters->version_ = "7.0";
   parameters->shared_data_ = shared_data;
   parameters->start_time_ = start_time;
   auto net_query_stats = td::create_net_query_stats();
@@ -476,27 +476,22 @@ int main(int argc, char *argv[]) {
   //              << (td::GitInfo::is_dirty() ? "(dirty)" : "") << " started";
   LOG(WARNING) << "TDLight Bot API " << parameters->version_ << " server started";
 
-  // +3 threads for Td
-  // one thread for ClientManager and all Clients
-  // one thread for watchdogs
-  // one thread for slow HTTP connections
-  // one thread for DNS resolving
-  const int thread_count = 7;
-  td::ConcurrentScheduler sched(thread_count, cpu_affinity);
+  td::ConcurrentScheduler sched(SharedData::get_thread_count() - 1, cpu_affinity);
 
   td::GetHostByNameActor::Options get_host_by_name_options;
-  get_host_by_name_options.scheduler_id = thread_count;
+  get_host_by_name_options.scheduler_id = SharedData::get_dns_resolver_scheduler_id();
   parameters->get_host_by_name_actor_id_ =
       sched.create_actor_unsafe<td::GetHostByNameActor>(0, "GetHostByName", std::move(get_host_by_name_options))
           .release();
 
-  auto client_manager =
-      sched.create_actor_unsafe<ClientManager>(thread_count - 3, "ClientManager", std::move(parameters), token_range)
-          .release();
+  auto client_manager = sched
+                            .create_actor_unsafe<ClientManager>(SharedData::get_client_scheduler_id(), "ClientManager",
+                                                                std::move(parameters), token_range)
+                            .release();
 
   sched
       .create_actor_unsafe<HttpServer>(
-          thread_count - 3, "HttpServer", http_ip_address, http_port,
+          SharedData::get_client_scheduler_id(), "HttpServer", http_ip_address, http_port,
           [client_manager, shared_data] {
             return td::ActorOwn<td::HttpInboundConnection::Callback>(
                 td::create_actor<HttpConnection>("HttpConnection", client_manager, shared_data));
@@ -506,7 +501,7 @@ int main(int argc, char *argv[]) {
   if (http_stat_port != 0) {
     sched
         .create_actor_unsafe<HttpServer>(
-            thread_count - 3, "HttpStatsServer", http_stat_ip_address, http_stat_port,
+            SharedData::get_client_scheduler_id(), "HttpStatsServer", http_stat_ip_address, http_stat_port,
             [client_manager] {
               return td::ActorOwn<td::HttpInboundConnection::Callback>(
                   td::create_actor<HttpStatConnection>("HttpStatConnection", client_manager));
@@ -515,8 +510,8 @@ int main(int argc, char *argv[]) {
   }
 
   constexpr double WATCHDOG_TIMEOUT = 0.25;
-  auto watchdog_id =
-      sched.create_actor_unsafe<Watchdog>(thread_count - 2, "Watchdog", td::this_thread::get_id(), WATCHDOG_TIMEOUT);
+  auto watchdog_id = sched.create_actor_unsafe<Watchdog>(SharedData::get_watchdog_scheduler_id(), "Watchdog",
+                                                         td::this_thread::get_id(), WATCHDOG_TIMEOUT);
 
   sched.start();
 
@@ -578,13 +573,15 @@ int main(int argc, char *argv[]) {
         next_cron_time = now;
       }
       next_cron_time += 1.0;
-      ServerCpuStat::update(now);
+      auto guard = sched.get_main_guard();
+      td::Scheduler::instance()->run_on_scheduler(SharedData::get_statistics_thread_id(),
+                                                  [](td::Unit) { ServerCpuStat::update(td::Time::now()); });
     }
 
     if (now >= start_time + 600) {
       auto guard = sched.get_main_guard();
       send_closure(watchdog_id, &Watchdog::kick);
-      next_watchdog_kick_time = now + WATCHDOG_TIMEOUT / 2;
+      next_watchdog_kick_time = now + WATCHDOG_TIMEOUT / 10;
     }
 
     if (!need_dump_statistics.test_and_set() || now > last_dump_time + 300.0) {