fix sigfault in "DESTROY NULL file_id_info_"

GitOrigin-RevId: 5ff0dc048c53abbc27d931b39374738ba9e25531

CHANGELOG.md

@@ -716,7 +716,7 @@ Changes in 1.2.0:
 * Added method `searchInstalledStickerSets` to search by title and name for installed sticker sets.
 * Added methods for handling connected websites: `getConnectedWebsites`, `disconnectWebsite` and
   `disconnectAllWebsites`.
-* Added method `getCountryCode`, which uses current user IP to identify their country.
+* Added method `getCountryCode`, which uses current user IP address to identify their country.
 * Added option `t_me_url`.
 * Fixed `BlackBerry` spelling in `deviceTokenBlackBerryPush`.
 * Fixed return type of `getChatMessageByDate` method, which is `Message` and not `Messages`.

CMakeLists.txt

@@ -356,7 +356,6 @@ set(TL_DOTNET_SCHEME_SOURCE
 
 set(TDLIB_SOURCE
   td/mtproto/AuthData.cpp
-  td/mtproto/crypto.cpp
   td/mtproto/DhHandshake.cpp
   td/mtproto/Handshake.cpp
   td/mtproto/HandshakeActor.cpp
@@ -367,6 +366,7 @@ set(TDLIB_SOURCE
   td/mtproto/PingConnection.cpp
   td/mtproto/ProxySecret.cpp
   td/mtproto/RawConnection.cpp
+  td/mtproto/RSA.cpp
   td/mtproto/SessionConnection.cpp
   td/mtproto/TcpTransport.cpp
   td/mtproto/TlsInit.cpp
@@ -494,7 +494,6 @@ set(TDLIB_SOURCE
 
   td/mtproto/AuthData.h
   td/mtproto/AuthKey.h
-  td/mtproto/crypto.h
   td/mtproto/CryptoStorer.h
   td/mtproto/DhHandshake.h
   td/mtproto/Handshake.h
@@ -511,6 +510,7 @@ set(TDLIB_SOURCE
   td/mtproto/ProxySecret.h
   td/mtproto/Query.h
   td/mtproto/RawConnection.h
+  td/mtproto/RSA.h
   td/mtproto/SessionConnection.h
   td/mtproto/TcpTransport.h
   td/mtproto/TlsInit.h

README.md

@@ -52,7 +52,6 @@ for a list of all available `TDLib` [methods](https://core.telegram.org/tdlib/do
 * gperf (build only)
 * CMake (3.0.2+, build only)
 * PHP (optional, for documentation generation)
-* Doxygen (optional, for documentation generation)
 
 <a name="building"></a>
 ## Building

benchmark/wget.cpp

@@ -14,21 +14,22 @@
 #include "td/utils/logging.h"
 #include "td/utils/Status.h"
 
-#include <string>
-
 int main(int argc, char *argv[]) {
   SET_VERBOSITY_LEVEL(VERBOSITY_NAME(DEBUG));
   td::VERBOSITY_NAME(fd) = VERBOSITY_NAME(INFO);
 
-  std::string url = (argc > 1 ? argv[1] : "https://telegram.org");
+  td::string url = (argc > 1 ? argv[1] : "https://telegram.org");
   auto timeout = 10;
   auto ttl = 3;
-  auto prefer_ipv6 = (argc > 2 && std::string(argv[2]) == "-6");
+  auto prefer_ipv6 = (argc > 2 && td::string(argv[2]) == "-6");
   auto scheduler = td::make_unique<td::ConcurrentScheduler>();
   scheduler->init(0);
   scheduler
       ->create_actor_unsafe<td::Wget>(0, "Client",
                                       td::PromiseCreator::lambda([](td::Result<td::unique_ptr<td::HttpQuery>> res) {
+                                        if (res.is_error()) {
+                                          LOG(FATAL) << res.error();
+                                        }
                                         LOG(ERROR) << *res.ok();
                                         td::Scheduler::instance()->finish();
                                       }),

example/README.md

@@ -162,9 +162,14 @@ See [erl-tdlib](https://github.com/lattenwald/erl-tdlib) for an example of TDLib
 <a name="php"></a>
 ## Using TDLib in PHP projects
 
-TDLib can be used from the PHP programming language by wrapping its functionality in a PHP extension.
+If you use modern PHP >= 7.4, you can use TDLib via a PHP FFI extension. For example, take a look at [ffi-tdlib](https://github.com/aurimasniekis/php-ffi-tdlib) - an FFI-based TDLib wrapper.
+
+See also [tdlib-schema](https://github.com/aurimasniekis/php-tdlib-schema) - a generator for TDLib API classes.
+
+For older PHP versions you can use TDLib by wrapping its functionality in a PHP extension.
 
 See [phptdlib](https://github.com/yaroslavche/phptdlib), [tdlib](https://github.com/aurimasniekis/php-ext-tdlib) or [PIF-TDPony](https://github.com/danog/pif-tdpony) for examples of such extensions which provide access to TDLib from PHP.
+
 See [tdlib-bundle](https://github.com/yaroslavche/tdlib-bundle) – a Symfony bundle based on [phptdlib](https://github.com/yaroslavche/phptdlib).
 
 <a name="lua"></a>

sqlite/CMakeLists.txt

@@ -11,7 +11,6 @@ endif()
 
 set(SQLITE_SOURCE
   sqlite/sqlite3.c
-
   sqlite/sqlite3.h
   sqlite/sqlite3ext.h
   sqlite/sqlite3session.h
@@ -21,15 +20,14 @@ add_library(tdsqlite STATIC ${SQLITE_SOURCE})
 target_include_directories(tdsqlite PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
 target_include_directories(tdsqlite SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR})
 target_link_libraries(tdsqlite PRIVATE ${OPENSSL_CRYPTO_LIBRARY} ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES})
+
 target_compile_definitions(tdsqlite PRIVATE
-  -DSQLITE_DEFAULT_MEMSTATUS=0
-  -DSQLITE_OMIT_LOAD_EXTENSION
-  -DSQLITE_OMIT_DECLTYPE
-  -DSQLITE_OMIT_PROGRESS_CALLBACK
-  #-DSQLITE_OMIT_DEPRECATED  # SQLCipher uses deprecated sqlite3_profile
-  #-DSQLITE_OMIT_SHARED_CACHE
+        -DSQLITE_MAX_MMAP_SIZE=50331648
+        -DSQLITE_MAX_MEMORY=50331648
+        -DSQLITE_ENABLE_SORTER_REFERENCES
+        -DSQLITE_DIRECT_OVERFLOW_READ
+        -DSQLITE_ENABLE_MEMORY_MANAGEMENT
 )
-target_compile_definitions(tdsqlite PRIVATE -DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=2 -DSQLITE_ENABLE_FTS5 -DSQLITE_DISABLE_LFS)
 
 if (NOT WIN32)
   target_compile_definitions(tdsqlite PRIVATE -DHAVE_USLEEP -DNDEBUG=1)
@@ -44,6 +42,9 @@ if (GCC OR CLANG)
   if (CLANG)
     target_compile_options(tdsqlite PRIVATE -Wno-parentheses-equality -Wno-unused-value)
   endif()
+  if (GCC AND NOT (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 10.0))
+    target_compile_options(tdsqlite PRIVATE -Wno-return-local-addr -Wno-stringop-overflow)
+  endif()
 elseif (MSVC)
   target_compile_options(tdsqlite PRIVATE /wd4996)
 endif()

sqlite/sqlite/sqlite3ext.h

@@ -134,7 +134,7 @@ struct sqlite3_api_routines {
   int  (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*,
                          const char*,const char*),void*);
   void  (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*));
-  char * (*snprintf)(int,char*,const char*,...);
+  char * (*xsnprintf)(int,char*,const char*,...);
   int  (*step)(sqlite3_stmt*);
   int  (*table_column_metadata)(sqlite3*,const char*,const char*,const char*,
                                 char const**,char const**,int*,int*,int*);
@@ -246,7 +246,7 @@ struct sqlite3_api_routines {
   int (*uri_boolean)(const char*,const char*,int);
   sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64);
   const char *(*uri_parameter)(const char*,const char*);
-  char *(*vsnprintf)(int,char*,const char*,va_list);
+  char *(*xvsnprintf)(int,char*,const char*,va_list);
   int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*);
   /* Version 3.8.7 and later */
   int (*auto_extension)(void(*)(void));
@@ -282,6 +282,49 @@ struct sqlite3_api_routines {
   /* Version 3.14.0 and later */
   int (*trace_v2)(sqlite3*,unsigned,int(*)(unsigned,void*,void*,void*),void*);
   char *(*expanded_sql)(sqlite3_stmt*);
+  /* Version 3.18.0 and later */
+  void (*set_last_insert_rowid)(sqlite3*,sqlite3_int64);
+  /* Version 3.20.0 and later */
+  int (*prepare_v3)(sqlite3*,const char*,int,unsigned int,
+                    sqlite3_stmt**,const char**);
+  int (*prepare16_v3)(sqlite3*,const void*,int,unsigned int,
+                      sqlite3_stmt**,const void**);
+  int (*bind_pointer)(sqlite3_stmt*,int,void*,const char*,void(*)(void*));
+  void (*result_pointer)(sqlite3_context*,void*,const char*,void(*)(void*));
+  void *(*value_pointer)(sqlite3_value*,const char*);
+  int (*vtab_nochange)(sqlite3_context*);
+  int (*value_nochange)(sqlite3_value*);
+  const char *(*vtab_collation)(sqlite3_index_info*,int);
+  /* Version 3.24.0 and later */
+  int (*keyword_count)(void);
+  int (*keyword_name)(int,const char**,int*);
+  int (*keyword_check)(const char*,int);
+  sqlite3_str *(*str_new)(sqlite3*);
+  char *(*str_finish)(sqlite3_str*);
+  void (*str_appendf)(sqlite3_str*, const char *zFormat, ...);
+  void (*str_vappendf)(sqlite3_str*, const char *zFormat, va_list);
+  void (*str_append)(sqlite3_str*, const char *zIn, int N);
+  void (*str_appendall)(sqlite3_str*, const char *zIn);
+  void (*str_appendchar)(sqlite3_str*, int N, char C);
+  void (*str_reset)(sqlite3_str*);
+  int (*str_errcode)(sqlite3_str*);
+  int (*str_length)(sqlite3_str*);
+  char *(*str_value)(sqlite3_str*);
+  /* Version 3.25.0 and later */
+  int (*create_window_function)(sqlite3*,const char*,int,int,void*,
+                            void (*xStep)(sqlite3_context*,int,sqlite3_value**),
+                            void (*xFinal)(sqlite3_context*),
+                            void (*xValue)(sqlite3_context*),
+                            void (*xInv)(sqlite3_context*,int,sqlite3_value**),
+                            void(*xDestroy)(void*));
+  /* Version 3.26.0 and later */
+  const char *(*normalized_sql)(sqlite3_stmt*);
+  /* Version 3.28.0 and later */
+  int (*stmt_isexplain)(sqlite3_stmt*);
+  int (*value_frombind)(sqlite3_value*);
+  /* Version 3.30.0 and later */
+  int (*drop_modules)(sqlite3*,const char**);
+  sqlite3_int64 (*hard_heap_limit64)(sqlite3_int64);
 };
 
 /*
@@ -408,7 +451,7 @@ typedef int (*sqlite3_loadext_entry)(
 #define sqlite3_rollback_hook          sqlite3_api->rollback_hook
 #define sqlite3_set_authorizer         sqlite3_api->set_authorizer
 #define sqlite3_set_auxdata            sqlite3_api->set_auxdata
-#define sqlite3_snprintf               sqlite3_api->snprintf
+#define sqlite3_snprintf               sqlite3_api->xsnprintf
 #define sqlite3_step                   sqlite3_api->step
 #define sqlite3_table_column_metadata  sqlite3_api->table_column_metadata
 #define sqlite3_thread_cleanup         sqlite3_api->thread_cleanup
@@ -432,7 +475,7 @@ typedef int (*sqlite3_loadext_entry)(
 #define sqlite3_value_text16le         sqlite3_api->value_text16le
 #define sqlite3_value_type             sqlite3_api->value_type
 #define sqlite3_vmprintf               sqlite3_api->vmprintf
-#define sqlite3_vsnprintf              sqlite3_api->vsnprintf
+#define sqlite3_vsnprintf              sqlite3_api->xvsnprintf
 #define sqlite3_overload_function      sqlite3_api->overload_function
 #define sqlite3_prepare_v2             sqlite3_api->prepare_v2
 #define sqlite3_prepare16_v2           sqlite3_api->prepare16_v2
@@ -508,7 +551,7 @@ typedef int (*sqlite3_loadext_entry)(
 #define sqlite3_uri_boolean            sqlite3_api->uri_boolean
 #define sqlite3_uri_int64              sqlite3_api->uri_int64
 #define sqlite3_uri_parameter          sqlite3_api->uri_parameter
-#define sqlite3_uri_vsnprintf          sqlite3_api->vsnprintf
+#define sqlite3_uri_vsnprintf          sqlite3_api->xvsnprintf
 #define sqlite3_wal_checkpoint_v2      sqlite3_api->wal_checkpoint_v2
 /* Version 3.8.7 and later */
 #define sqlite3_auto_extension         sqlite3_api->auto_extension
@@ -540,6 +583,43 @@ typedef int (*sqlite3_loadext_entry)(
 /* Version 3.14.0 and later */
 #define sqlite3_trace_v2               sqlite3_api->trace_v2
 #define sqlite3_expanded_sql           sqlite3_api->expanded_sql
+/* Version 3.18.0 and later */
+#define sqlite3_set_last_insert_rowid  sqlite3_api->set_last_insert_rowid
+/* Version 3.20.0 and later */
+#define sqlite3_prepare_v3             sqlite3_api->prepare_v3
+#define sqlite3_prepare16_v3           sqlite3_api->prepare16_v3
+#define sqlite3_bind_pointer           sqlite3_api->bind_pointer
+#define sqlite3_result_pointer         sqlite3_api->result_pointer
+#define sqlite3_value_pointer          sqlite3_api->value_pointer
+/* Version 3.22.0 and later */
+#define sqlite3_vtab_nochange          sqlite3_api->vtab_nochange
+#define sqlite3_value_nochange         sqlite3_api->value_nochange
+#define sqlite3_vtab_collation         sqlite3_api->vtab_collation
+/* Version 3.24.0 and later */
+#define sqlite3_keyword_count          sqlite3_api->keyword_count
+#define sqlite3_keyword_name           sqlite3_api->keyword_name
+#define sqlite3_keyword_check          sqlite3_api->keyword_check
+#define sqlite3_str_new                sqlite3_api->str_new
+#define sqlite3_str_finish             sqlite3_api->str_finish
+#define sqlite3_str_appendf            sqlite3_api->str_appendf
+#define sqlite3_str_vappendf           sqlite3_api->str_vappendf
+#define sqlite3_str_append             sqlite3_api->str_append
+#define sqlite3_str_appendall          sqlite3_api->str_appendall
+#define sqlite3_str_appendchar         sqlite3_api->str_appendchar
+#define sqlite3_str_reset              sqlite3_api->str_reset
+#define sqlite3_str_errcode            sqlite3_api->str_errcode
+#define sqlite3_str_length             sqlite3_api->str_length
+#define sqlite3_str_value              sqlite3_api->str_value
+/* Version 3.25.0 and later */
+#define sqlite3_create_window_function sqlite3_api->create_window_function
+/* Version 3.26.0 and later */
+#define sqlite3_normalized_sql         sqlite3_api->normalized_sql
+/* Version 3.28.0 and later */
+#define sqlite3_stmt_isexplain         sqlite3_api->isexplain
+#define sqlite3_value_frombind         sqlite3_api->frombind
+/* Version 3.30.0 and later */
+#define sqlite3_drop_modules           sqlite3_api->drop_modules
+#define sqlite3_hard_heap_limit64      sqlite3_api->hard_heap_limit64
 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
 
 #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)

sqlite/sqlite/sqlite3session.h

@@ -1,3 +1,4 @@
+
 #if !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION)
 #define __SQLITESESSION_H_ 1
 
@@ -12,16 +13,23 @@ extern "C" {
 
 /*
 ** CAPI3REF: Session Object Handle
+**
+** An instance of this object is a [session] that can be used to
+** record changes to a database.
 */
 typedef struct sqlite3_session sqlite3_session;
 
 /*
 ** CAPI3REF: Changeset Iterator Handle
+**
+** An instance of this object acts as a cursor for iterating
+** over the elements of a [changeset] or [patchset].
 */
 typedef struct sqlite3_changeset_iter sqlite3_changeset_iter;
 
 /*
 ** CAPI3REF: Create A New Session Object
+** CONSTRUCTOR: sqlite3_session
 **
 ** Create a new session object attached to database handle db. If successful,
 ** a pointer to the new object is written to *ppSession and SQLITE_OK is
@@ -58,6 +66,7 @@ int sqlite3session_create(
 
 /*
 ** CAPI3REF: Delete A Session Object
+** DESTRUCTOR: sqlite3_session
 **
 ** Delete a session object previously allocated using 
 ** [sqlite3session_create()]. Once a session object has been deleted, the
@@ -73,6 +82,7 @@ void sqlite3session_delete(sqlite3_session *pSession);
 
 /*
 ** CAPI3REF: Enable Or Disable A Session Object
+** METHOD: sqlite3_session
 **
 ** Enable or disable the recording of changes by a session object. When
 ** enabled, a session object records changes made to the database. When
@@ -92,6 +102,7 @@ int sqlite3session_enable(sqlite3_session *pSession, int bEnable);
 
 /*
 ** CAPI3REF: Set Or Clear the Indirect Change Flag
+** METHOD: sqlite3_session
 **
 ** Each change recorded by a session object is marked as either direct or
 ** indirect. A change is marked as indirect if either:
@@ -121,6 +132,7 @@ int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect);
 
 /*
 ** CAPI3REF: Attach A Table To A Session Object
+** METHOD: sqlite3_session
 **
 ** If argument zTab is not NULL, then it is the name of a table to attach
 ** to the session object passed as the first argument. All subsequent changes 
@@ -146,6 +158,35 @@ int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect);
 **
 ** SQLITE_OK is returned if the call completes without error. Or, if an error 
 ** occurs, an SQLite error code (e.g. SQLITE_NOMEM) is returned.
+**
+** <h3>Special sqlite_stat1 Handling</h3>
+**
+** As of SQLite version 3.22.0, the "sqlite_stat1" table is an exception to 
+** some of the rules above. In SQLite, the schema of sqlite_stat1 is:
+**  <pre>
+**  &nbsp;     CREATE TABLE sqlite_stat1(tbl,idx,stat)  
+**  </pre>
+**
+** Even though sqlite_stat1 does not have a PRIMARY KEY, changes are 
+** recorded for it as if the PRIMARY KEY is (tbl,idx). Additionally, changes 
+** are recorded for rows for which (idx IS NULL) is true. However, for such
+** rows a zero-length blob (SQL value X'') is stored in the changeset or
+** patchset instead of a NULL value. This allows such changesets to be
+** manipulated by legacy implementations of sqlite3changeset_invert(),
+** concat() and similar.
+**
+** The sqlite3changeset_apply() function automatically converts the 
+** zero-length blob back to a NULL value when updating the sqlite_stat1
+** table. However, if the application calls sqlite3changeset_new(),
+** sqlite3changeset_old() or sqlite3changeset_conflict on a changeset 
+** iterator directly (including on a changeset iterator passed to a
+** conflict-handler callback) then the X'' value is returned. The application
+** must translate X'' to NULL itself if required.
+**
+** Legacy (older than 3.22.0) versions of the sessions module cannot capture
+** changes made to the sqlite_stat1 table. Legacy versions of the
+** sqlite3changeset_apply() function silently ignore any modifications to the
+** sqlite_stat1 table that are part of a changeset or patchset.
 */
 int sqlite3session_attach(
   sqlite3_session *pSession,      /* Session object */
@@ -154,6 +195,7 @@ int sqlite3session_attach(
 
 /*
 ** CAPI3REF: Set a table filter on a Session Object.
+** METHOD: sqlite3_session
 **
 ** The second argument (xFilter) is the "filter callback". For changes to rows 
 ** in tables that are not attached to the Session object, the filter is called
@@ -172,6 +214,7 @@ void sqlite3session_table_filter(
 
 /*
 ** CAPI3REF: Generate A Changeset From A Session Object
+** METHOD: sqlite3_session
 **
 ** Obtain a changeset containing changes to the tables attached to the 
 ** session object passed as the first argument. If successful, 
@@ -281,7 +324,8 @@ int sqlite3session_changeset(
 );
 
 /*
-** CAPI3REF: Load The Difference Between Tables Into A Session 
+** CAPI3REF: Load The Difference Between Tables Into A Session
+** METHOD: sqlite3_session
 **
 ** If it is not already attached to the session object passed as the first
 ** argument, this function attaches table zTbl in the same manner as the
@@ -318,7 +362,8 @@ int sqlite3session_changeset(
 **     the from-table, a DELETE record is added to the session object.
 **
 **   <li> For each row (primary key) that exists in both tables, but features 
-**     different in each, an UPDATE record is added to the session.
+**     different non-PK values in each, an UPDATE record is added to the
+**     session.  
 ** </ul>
 **
 ** To clarify, if this function is called and then a changeset constructed
@@ -345,6 +390,7 @@ int sqlite3session_diff(
 
 /*
 ** CAPI3REF: Generate A Patchset From A Session Object
+** METHOD: sqlite3_session
 **
 ** The differences between a patchset and a changeset are that:
 **
@@ -373,8 +419,8 @@ int sqlite3session_diff(
 */
 int sqlite3session_patchset(
   sqlite3_session *pSession,      /* Session object */
-  int *pnPatchset,                /* OUT: Size of buffer at *ppChangeset */
-  void **ppPatchset               /* OUT: Buffer containing changeset */
+  int *pnPatchset,                /* OUT: Size of buffer at *ppPatchset */
+  void **ppPatchset               /* OUT: Buffer containing patchset */
 );
 
 /*
@@ -396,6 +442,7 @@ int sqlite3session_isempty(sqlite3_session *pSession);
 
 /*
 ** CAPI3REF: Create An Iterator To Traverse A Changeset 
+** CONSTRUCTOR: sqlite3_changeset_iter
 **
 ** Create an iterator used to iterate through the contents of a changeset.
 ** If successful, *pp is set to point to the iterator handle and SQLITE_OK
@@ -426,16 +473,43 @@ int sqlite3session_isempty(sqlite3_session *pSession);
 ** consecutively. There is no chance that the iterator will visit a change 
 ** the applies to table X, then one for table Y, and then later on visit 
 ** another change for table X.
+**
+** The behavior of sqlite3changeset_start_v2() and its streaming equivalent
+** may be modified by passing a combination of
+** [SQLITE_CHANGESETSTART_INVERT | supported flags] as the 4th parameter.
+**
+** Note that the sqlite3changeset_start_v2() API is still <b>experimental</b>
+** and therefore subject to change.
 */
 int sqlite3changeset_start(
   sqlite3_changeset_iter **pp,    /* OUT: New changeset iterator handle */
   int nChangeset,                 /* Size of changeset blob in bytes */
   void *pChangeset                /* Pointer to blob containing changeset */
 );
+int sqlite3changeset_start_v2(
+  sqlite3_changeset_iter **pp,    /* OUT: New changeset iterator handle */
+  int nChangeset,                 /* Size of changeset blob in bytes */
+  void *pChangeset,               /* Pointer to blob containing changeset */
+  int flags                       /* SESSION_CHANGESETSTART_* flags */
+);
+
+/*
+** CAPI3REF: Flags for sqlite3changeset_start_v2
+**
+** The following flags may passed via the 4th parameter to
+** [sqlite3changeset_start_v2] and [sqlite3changeset_start_v2_strm]:
+**
+** <dt>SQLITE_CHANGESETAPPLY_INVERT <dd>
+**   Invert the changeset while iterating through it. This is equivalent to
+**   inverting a changeset using sqlite3changeset_invert() before applying it.
+**   It is an error to specify this flag with a patchset.
+*/
+#define SQLITE_CHANGESETSTART_INVERT        0x0002
 
 
 /*
 ** CAPI3REF: Advance A Changeset Iterator
+** METHOD: sqlite3_changeset_iter
 **
 ** This function may only be used with iterators created by function
 ** [sqlite3changeset_start()]. If it is called on an iterator passed to
@@ -460,6 +534,7 @@ int sqlite3changeset_next(sqlite3_changeset_iter *pIter);
 
 /*
 ** CAPI3REF: Obtain The Current Operation From A Changeset Iterator
+** METHOD: sqlite3_changeset_iter
 **
 ** The pIter argument passed to this function may either be an iterator
 ** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator
@@ -473,7 +548,7 @@ int sqlite3changeset_next(sqlite3_changeset_iter *pIter);
 ** sqlite3changeset_next() is called on the iterator or until the 
 ** conflict-handler function returns. If pnCol is not NULL, then *pnCol is 
 ** set to the number of columns in the table affected by the change. If
-** pbIncorrect is not NULL, then *pbIndirect is set to true (1) if the change
+** pbIndirect is not NULL, then *pbIndirect is set to true (1) if the change
 ** is an indirect change, or false (0) otherwise. See the documentation for
 ** [sqlite3session_indirect()] for a description of direct and indirect
 ** changes. Finally, if pOp is not NULL, then *pOp is set to one of 
@@ -494,6 +569,7 @@ int sqlite3changeset_op(
 
 /*
 ** CAPI3REF: Obtain The Primary Key Definition Of A Table
+** METHOD: sqlite3_changeset_iter
 **
 ** For each modified table, a changeset includes the following:
 **
@@ -525,6 +601,7 @@ int sqlite3changeset_pk(
 
 /*
 ** CAPI3REF: Obtain old.* Values From A Changeset Iterator
+** METHOD: sqlite3_changeset_iter
 **
 ** The pIter argument passed to this function may either be an iterator
 ** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator
@@ -555,6 +632,7 @@ int sqlite3changeset_old(
 
 /*
 ** CAPI3REF: Obtain new.* Values From A Changeset Iterator
+** METHOD: sqlite3_changeset_iter
 **
 ** The pIter argument passed to this function may either be an iterator
 ** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator
@@ -588,6 +666,7 @@ int sqlite3changeset_new(
 
 /*
 ** CAPI3REF: Obtain Conflicting Row Values From A Changeset Iterator
+** METHOD: sqlite3_changeset_iter
 **
 ** This function should only be used with iterator objects passed to a
 ** conflict-handler callback by [sqlite3changeset_apply()] with either
@@ -615,6 +694,7 @@ int sqlite3changeset_conflict(
 
 /*
 ** CAPI3REF: Determine The Number Of Foreign Key Constraint Violations
+** METHOD: sqlite3_changeset_iter
 **
 ** This function may only be called with an iterator passed to an
 ** SQLITE_CHANGESET_FOREIGN_KEY conflict handler callback. In this case
@@ -631,6 +711,7 @@ int sqlite3changeset_fk_conflicts(
 
 /*
 ** CAPI3REF: Finalize A Changeset Iterator
+** METHOD: sqlite3_changeset_iter
 **
 ** This function is used to finalize an iterator allocated with
 ** [sqlite3changeset_start()].
@@ -647,6 +728,7 @@ int sqlite3changeset_fk_conflicts(
 ** to that error is returned by this function. Otherwise, SQLITE_OK is
 ** returned. This is to allow the following pattern (pseudo-code):
 **
+** <pre>
 **   sqlite3changeset_start();
 **   while( SQLITE_ROW==sqlite3changeset_next() ){
 **     // Do something with change.
@@ -655,6 +737,7 @@ int sqlite3changeset_fk_conflicts(
 **   if( rc!=SQLITE_OK ){
 **     // An error has occurred 
 **   }
+** </pre>
 */
 int sqlite3changeset_finalize(sqlite3_changeset_iter *pIter);
 
@@ -702,6 +785,7 @@ int sqlite3changeset_invert(
 ** sqlite3_changegroup object. Calling it produces similar results as the
 ** following code fragment:
 **
+** <pre>
 **   sqlite3_changegroup *pGrp;
 **   rc = sqlite3_changegroup_new(&pGrp);
 **   if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nA, pA);
@@ -712,6 +796,7 @@ int sqlite3changeset_invert(
 **     *ppOut = 0;
 **     *pnOut = 0;
 **   }
+** </pre>
 **
 ** Refer to the sqlite3_changegroup documentation below for details.
 */
@@ -727,11 +812,15 @@ int sqlite3changeset_concat(
 
 /*
 ** CAPI3REF: Changegroup Handle
+**
+** A changegroup is an object used to combine two or more 
+** [changesets] or [patchsets]
 */
 typedef struct sqlite3_changegroup sqlite3_changegroup;
 
 /*
 ** CAPI3REF: Create A New Changegroup Object
+** CONSTRUCTOR: sqlite3_changegroup
 **
 ** An sqlite3_changegroup object is used to combine two or more changesets
 ** (or patchsets) into a single changeset (or patchset). A single changegroup
@@ -769,6 +858,7 @@ int sqlite3changegroup_new(sqlite3_changegroup **pp);
 
 /*
 ** CAPI3REF: Add A Changeset To A Changegroup
+** METHOD: sqlite3_changegroup
 **
 ** Add all changes within the changeset (or patchset) in buffer pData (size
 ** nData bytes) to the changegroup. 
@@ -846,6 +936,7 @@ int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData);
 
 /*
 ** CAPI3REF: Obtain A Composite Changeset From A Changegroup
+** METHOD: sqlite3_changegroup
 **
 ** Obtain a buffer containing a changeset (or patchset) representing the
 ** current contents of the changegroup. If the inputs to the changegroup
@@ -876,25 +967,25 @@ int sqlite3changegroup_output(
 
 /*
 ** CAPI3REF: Delete A Changegroup Object
+** DESTRUCTOR: sqlite3_changegroup
 */
 void sqlite3changegroup_delete(sqlite3_changegroup*);
 
 /*
 ** CAPI3REF: Apply A Changeset To A Database
 **
-** Apply a changeset to a database. This function attempts to update the
-** "main" database attached to handle db with the changes found in the
-** changeset passed via the second and third arguments.
+** Apply a changeset or patchset to a database. These functions attempt to
+** update the "main" database attached to handle db with the changes found in
+** the changeset passed via the second and third arguments. 
 **
-** The fourth argument (xFilter) passed to this function is the "filter
+** The fourth argument (xFilter) passed to these functions is the "filter
 ** callback". If it is not NULL, then for each table affected by at least one
 ** change in the changeset, the filter callback is invoked with
 ** the table name as the second argument, and a copy of the context pointer
-** passed as the sixth argument to this function as the first. If the "filter
-** callback" returns zero, then no attempt is made to apply any changes to 
-** the table. Otherwise, if the return value is non-zero or the xFilter
-** argument to this function is NULL, all changes related to the table are
-** attempted.
+** passed as the sixth argument as the first. If the "filter callback"
+** returns zero, then no attempt is made to apply any changes to the table.
+** Otherwise, if the return value is non-zero or the xFilter argument to
+** is NULL, all changes related to the table are attempted.
 **
 ** For each table that is not excluded by the filter callback, this function 
 ** tests that the target database contains a compatible table. A table is 
@@ -903,7 +994,7 @@ void sqlite3changegroup_delete(sqlite3_changegroup*);
 ** <ul>
 **   <li> The table has the same name as the name recorded in the 
 **        changeset, and
-**   <li> The table has the same number of columns as recorded in the 
+**   <li> The table has at least as many columns as recorded in the 
 **        changeset, and
 **   <li> The table has primary key columns in the same position as 
 **        recorded in the changeset.
@@ -939,7 +1030,7 @@ void sqlite3changegroup_delete(sqlite3_changegroup*);
 **
 ** <dl>
 ** <dt>DELETE Changes<dd>
-**   For each DELETE change, this function checks if the target database 
+**   For each DELETE change, the function checks if the target database 
 **   contains a row with the same primary key value (or values) as the 
 **   original row values stored in the changeset. If it does, and the values 
 **   stored in all non-primary key columns also match the values stored in 
@@ -948,7 +1039,11 @@ void sqlite3changegroup_delete(sqlite3_changegroup*);
 **   If a row with matching primary key values is found, but one or more of
 **   the non-primary key fields contains a value different from the original
 **   row value stored in the changeset, the conflict-handler function is
-**   invoked with [SQLITE_CHANGESET_DATA] as the second argument.
+**   invoked with [SQLITE_CHANGESET_DATA] as the second argument. If the
+**   database table has more columns than are recorded in the changeset,
+**   only the values of those non-primary key fields are compared against
+**   the current database contents - any trailing database table columns
+**   are ignored.
 **
 **   If no row with matching primary key values is found in the database,
 **   the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND]
@@ -963,7 +1058,9 @@ void sqlite3changegroup_delete(sqlite3_changegroup*);
 **
 ** <dt>INSERT Changes<dd>
 **   For each INSERT change, an attempt is made to insert the new row into
-**   the database.
+**   the database. If the changeset row contains fewer fields than the
+**   database table, the trailing fields are populated with their default
+**   values.
 **
 **   If the attempt to insert the row fails because the database already 
 **   contains a row with the same primary key values, the conflict handler
@@ -978,16 +1075,16 @@ void sqlite3changegroup_delete(sqlite3_changegroup*);
 **   [SQLITE_CHANGESET_REPLACE].
 **
 ** <dt>UPDATE Changes<dd>
-**   For each UPDATE change, this function checks if the target database 
+**   For each UPDATE change, the function checks if the target database 
 **   contains a row with the same primary key value (or values) as the 
 **   original row values stored in the changeset. If it does, and the values 
-**   stored in all non-primary key columns also match the values stored in 
-**   the changeset the row is updated within the target database.
+**   stored in all modified non-primary key columns also match the values
+**   stored in the changeset the row is updated within the target database.
 **
 **   If a row with matching primary key values is found, but one or more of
-**   the non-primary key fields contains a value different from an original
-**   row value stored in the changeset, the conflict-handler function is
-**   invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since
+**   the modified non-primary key fields contains a value different from an
+**   original row value stored in the changeset, the conflict-handler function
+**   is invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since
 **   UPDATE changes only contain values for non-primary key fields that are
 **   to be modified, only those fields need to match the original values to
 **   avoid the SQLITE_CHANGESET_DATA conflict-handler callback.
@@ -1009,11 +1106,28 @@ void sqlite3changegroup_delete(sqlite3_changegroup*);
 ** This can be used to further customize the applications conflict
 ** resolution strategy.
 **
-** All changes made by this function are enclosed in a savepoint transaction.
+** All changes made by these functions are enclosed in a savepoint transaction.
 ** If any other error (aside from a constraint failure when attempting to
 ** write to the target database) occurs, then the savepoint transaction is
 ** rolled back, restoring the target database to its original state, and an 
 ** SQLite error code returned.
+**
+** If the output parameters (ppRebase) and (pnRebase) are non-NULL and
+** the input is a changeset (not a patchset), then sqlite3changeset_apply_v2()
+** may set (*ppRebase) to point to a "rebase" that may be used with the 
+** sqlite3_rebaser APIs buffer before returning. In this case (*pnRebase)
+** is set to the size of the buffer in bytes. It is the responsibility of the
+** caller to eventually free any such buffer using sqlite3_free(). The buffer
+** is only allocated and populated if one or more conflicts were encountered
+** while applying the patchset. See comments surrounding the sqlite3_rebaser
+** APIs for further details.
+**
+** The behavior of sqlite3changeset_apply_v2() and its streaming equivalent
+** may be modified by passing a combination of
+** [SQLITE_CHANGESETAPPLY_NOSAVEPOINT | supported flags] as the 9th parameter.
+**
+** Note that the sqlite3changeset_apply_v2() API is still <b>experimental</b>
+** and therefore subject to change.
 */
 int sqlite3changeset_apply(
   sqlite3 *db,                    /* Apply change to "main" db of this handle */
@@ -1030,6 +1144,47 @@ int sqlite3changeset_apply(
   ),
   void *pCtx                      /* First argument passed to xConflict */
 );
+int sqlite3changeset_apply_v2(
+  sqlite3 *db,                    /* Apply change to "main" db of this handle */
+  int nChangeset,                 /* Size of changeset in bytes */
+  void *pChangeset,               /* Changeset blob */
+  int(*xFilter)(
+    void *pCtx,                   /* Copy of sixth arg to _apply() */
+    const char *zTab              /* Table name */
+  ),
+  int(*xConflict)(
+    void *pCtx,                   /* Copy of sixth arg to _apply() */
+    int eConflict,                /* DATA, MISSING, CONFLICT, CONSTRAINT */
+    sqlite3_changeset_iter *p     /* Handle describing change and conflict */
+  ),
+  void *pCtx,                     /* First argument passed to xConflict */
+  void **ppRebase, int *pnRebase, /* OUT: Rebase data */
+  int flags                       /* SESSION_CHANGESETAPPLY_* flags */
+);
+
+/*
+** CAPI3REF: Flags for sqlite3changeset_apply_v2
+**
+** The following flags may passed via the 9th parameter to
+** [sqlite3changeset_apply_v2] and [sqlite3changeset_apply_v2_strm]:
+**
+** <dl>
+** <dt>SQLITE_CHANGESETAPPLY_NOSAVEPOINT <dd>
+**   Usually, the sessions module encloses all operations performed by
+**   a single call to apply_v2() or apply_v2_strm() in a [SAVEPOINT]. The
+**   SAVEPOINT is committed if the changeset or patchset is successfully
+**   applied, or rolled back if an error occurs. Specifying this flag
+**   causes the sessions module to omit this savepoint. In this case, if the
+**   caller has an open transaction or savepoint when apply_v2() is called, 
+**   it may revert the partially applied changeset by rolling it back.
+**
+** <dt>SQLITE_CHANGESETAPPLY_INVERT <dd>
+**   Invert the changeset before applying it. This is equivalent to inverting
+**   a changeset using sqlite3changeset_invert() before applying it. It is
+**   an error to specify this flag with a patchset.
+*/
+#define SQLITE_CHANGESETAPPLY_NOSAVEPOINT   0x0001
+#define SQLITE_CHANGESETAPPLY_INVERT        0x0002
 
 /* 
 ** CAPI3REF: Constants Passed To The Conflict Handler
@@ -1127,6 +1282,161 @@ int sqlite3changeset_apply(
 #define SQLITE_CHANGESET_REPLACE    1
 #define SQLITE_CHANGESET_ABORT      2
 
+/* 
+** CAPI3REF: Rebasing changesets
+** EXPERIMENTAL
+**
+** Suppose there is a site hosting a database in state S0. And that
+** modifications are made that move that database to state S1 and a
+** changeset recorded (the "local" changeset). Then, a changeset based
+** on S0 is received from another site (the "remote" changeset) and 
+** applied to the database. The database is then in state 
+** (S1+"remote"), where the exact state depends on any conflict
+** resolution decisions (OMIT or REPLACE) made while applying "remote".
+** Rebasing a changeset is to update it to take those conflict 
+** resolution decisions into account, so that the same conflicts
+** do not have to be resolved elsewhere in the network. 
+**
+** For example, if both the local and remote changesets contain an
+** INSERT of the same key on "CREATE TABLE t1(a PRIMARY KEY, b)":
+**
+**   local:  INSERT INTO t1 VALUES(1, 'v1');
+**   remote: INSERT INTO t1 VALUES(1, 'v2');
+**
+** and the conflict resolution is REPLACE, then the INSERT change is
+** removed from the local changeset (it was overridden). Or, if the
+** conflict resolution was "OMIT", then the local changeset is modified
+** to instead contain:
+**
+**           UPDATE t1 SET b = 'v2' WHERE a=1;
+**
+** Changes within the local changeset are rebased as follows:
+**
+** <dl>
+** <dt>Local INSERT<dd>
+**   This may only conflict with a remote INSERT. If the conflict 
+**   resolution was OMIT, then add an UPDATE change to the rebased
+**   changeset. Or, if the conflict resolution was REPLACE, add
+**   nothing to the rebased changeset.
+**
+** <dt>Local DELETE<dd>
+**   This may conflict with a remote UPDATE or DELETE. In both cases the
+**   only possible resolution is OMIT. If the remote operation was a
+**   DELETE, then add no change to the rebased changeset. If the remote
+**   operation was an UPDATE, then the old.* fields of change are updated
+**   to reflect the new.* values in the UPDATE.
+**
+** <dt>Local UPDATE<dd>
+**   This may conflict with a remote UPDATE or DELETE. If it conflicts
+**   with a DELETE, and the conflict resolution was OMIT, then the update
+**   is changed into an INSERT. Any undefined values in the new.* record
+**   from the update change are filled in using the old.* values from
+**   the conflicting DELETE. Or, if the conflict resolution was REPLACE,
+**   the UPDATE change is simply omitted from the rebased changeset.
+**
+**   If conflict is with a remote UPDATE and the resolution is OMIT, then
+**   the old.* values are rebased using the new.* values in the remote
+**   change. Or, if the resolution is REPLACE, then the change is copied
+**   into the rebased changeset with updates to columns also updated by
+**   the conflicting remote UPDATE removed. If this means no columns would 
+**   be updated, the change is omitted.
+** </dl>
+**
+** A local change may be rebased against multiple remote changes 
+** simultaneously. If a single key is modified by multiple remote 
+** changesets, they are combined as follows before the local changeset
+** is rebased:
+**
+** <ul>
+**    <li> If there has been one or more REPLACE resolutions on a
+**         key, it is rebased according to a REPLACE.
+**
+**    <li> If there have been no REPLACE resolutions on a key, then
+**         the local changeset is rebased according to the most recent
+**         of the OMIT resolutions.
+** </ul>
+**
+** Note that conflict resolutions from multiple remote changesets are 
+** combined on a per-field basis, not per-row. This means that in the 
+** case of multiple remote UPDATE operations, some fields of a single 
+** local change may be rebased for REPLACE while others are rebased for 
+** OMIT.
+**
+** In order to rebase a local changeset, the remote changeset must first
+** be applied to the local database using sqlite3changeset_apply_v2() and
+** the buffer of rebase information captured. Then:
+**
+** <ol>
+**   <li> An sqlite3_rebaser object is created by calling 
+**        sqlite3rebaser_create().
+**   <li> The new object is configured with the rebase buffer obtained from
+**        sqlite3changeset_apply_v2() by calling sqlite3rebaser_configure().
+**        If the local changeset is to be rebased against multiple remote
+**        changesets, then sqlite3rebaser_configure() should be called
+**        multiple times, in the same order that the multiple
+**        sqlite3changeset_apply_v2() calls were made.
+**   <li> Each local changeset is rebased by calling sqlite3rebaser_rebase().
+**   <li> The sqlite3_rebaser object is deleted by calling
+**        sqlite3rebaser_delete().
+** </ol>
+*/
+typedef struct sqlite3_rebaser sqlite3_rebaser;
+
+/*
+** CAPI3REF: Create a changeset rebaser object.
+** EXPERIMENTAL
+**
+** Allocate a new changeset rebaser object. If successful, set (*ppNew) to
+** point to the new object and return SQLITE_OK. Otherwise, if an error
+** occurs, return an SQLite error code (e.g. SQLITE_NOMEM) and set (*ppNew) 
+** to NULL. 
+*/
+int sqlite3rebaser_create(sqlite3_rebaser **ppNew);
+
+/*
+** CAPI3REF: Configure a changeset rebaser object.
+** EXPERIMENTAL
+**
+** Configure the changeset rebaser object to rebase changesets according
+** to the conflict resolutions described by buffer pRebase (size nRebase
+** bytes), which must have been obtained from a previous call to
+** sqlite3changeset_apply_v2().
+*/
+int sqlite3rebaser_configure(
+  sqlite3_rebaser*, 
+  int nRebase, const void *pRebase
+); 
+
+/*
+** CAPI3REF: Rebase a changeset
+** EXPERIMENTAL
+**
+** Argument pIn must point to a buffer containing a changeset nIn bytes
+** in size. This function allocates and populates a buffer with a copy
+** of the changeset rebased rebased according to the configuration of the
+** rebaser object passed as the first argument. If successful, (*ppOut)
+** is set to point to the new buffer containing the rebased changeset and 
+** (*pnOut) to its size in bytes and SQLITE_OK returned. It is the
+** responsibility of the caller to eventually free the new buffer using
+** sqlite3_free(). Otherwise, if an error occurs, (*ppOut) and (*pnOut)
+** are set to zero and an SQLite error code returned.
+*/
+int sqlite3rebaser_rebase(
+  sqlite3_rebaser*,
+  int nIn, const void *pIn, 
+  int *pnOut, void **ppOut 
+);
+
+/*
+** CAPI3REF: Delete a changeset rebaser object.
+** EXPERIMENTAL
+**
+** Delete the changeset rebaser object and all associated resources. There
+** should be one call to this function for each successful invocation
+** of sqlite3rebaser_create().
+*/
+void sqlite3rebaser_delete(sqlite3_rebaser *p); 
+
 /*
 ** CAPI3REF: Streaming Versions of API functions.
 **
@@ -1135,12 +1445,13 @@ int sqlite3changeset_apply(
 **
 ** <table border=1 style="margin-left:8ex;margin-right:8ex">
 **   <tr><th>Streaming function<th>Non-streaming equivalent</th>
-**   <tr><td>sqlite3changeset_apply_str<td>[sqlite3changeset_apply] 
-**   <tr><td>sqlite3changeset_concat_str<td>[sqlite3changeset_concat] 
-**   <tr><td>sqlite3changeset_invert_str<td>[sqlite3changeset_invert] 
-**   <tr><td>sqlite3changeset_start_str<td>[sqlite3changeset_start] 
-**   <tr><td>sqlite3session_changeset_str<td>[sqlite3session_changeset] 
-**   <tr><td>sqlite3session_patchset_str<td>[sqlite3session_patchset] 
+**   <tr><td>sqlite3changeset_apply_strm<td>[sqlite3changeset_apply] 
+**   <tr><td>sqlite3changeset_apply_strm_v2<td>[sqlite3changeset_apply_v2] 
+**   <tr><td>sqlite3changeset_concat_strm<td>[sqlite3changeset_concat] 
+**   <tr><td>sqlite3changeset_invert_strm<td>[sqlite3changeset_invert] 
+**   <tr><td>sqlite3changeset_start_strm<td>[sqlite3changeset_start] 
+**   <tr><td>sqlite3session_changeset_strm<td>[sqlite3session_changeset] 
+**   <tr><td>sqlite3session_patchset_strm<td>[sqlite3session_patchset] 
 ** </table>
 **
 ** Non-streaming functions that accept changesets (or patchsets) as input
@@ -1231,6 +1542,23 @@ int sqlite3changeset_apply_strm(
   ),
   void *pCtx                      /* First argument passed to xConflict */
 );
+int sqlite3changeset_apply_v2_strm(
+  sqlite3 *db,                    /* Apply change to "main" db of this handle */
+  int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */
+  void *pIn,                                          /* First arg for xInput */
+  int(*xFilter)(
+    void *pCtx,                   /* Copy of sixth arg to _apply() */
+    const char *zTab              /* Table name */
+  ),
+  int(*xConflict)(
+    void *pCtx,                   /* Copy of sixth arg to _apply() */
+    int eConflict,                /* DATA, MISSING, CONFLICT, CONSTRAINT */
+    sqlite3_changeset_iter *p     /* Handle describing change and conflict */
+  ),
+  void *pCtx,                     /* First argument passed to xConflict */
+  void **ppRebase, int *pnRebase,
+  int flags
+);
 int sqlite3changeset_concat_strm(
   int (*xInputA)(void *pIn, void *pData, int *pnData),
   void *pInA,
@@ -1250,6 +1578,12 @@ int sqlite3changeset_start_strm(
   int (*xInput)(void *pIn, void *pData, int *pnData),
   void *pIn
 );
+int sqlite3changeset_start_v2_strm(
+  sqlite3_changeset_iter **pp,
+  int (*xInput)(void *pIn, void *pData, int *pnData),
+  void *pIn,
+  int flags
+);
 int sqlite3session_changeset_strm(
   sqlite3_session *pSession,
   int (*xOutput)(void *pOut, const void *pData, int nData),
@@ -1268,7 +1602,53 @@ int sqlite3changegroup_output_strm(sqlite3_changegroup*,
     int (*xOutput)(void *pOut, const void *pData, int nData), 
     void *pOut
 );
+int sqlite3rebaser_rebase_strm(
+  sqlite3_rebaser *pRebaser,
+  int (*xInput)(void *pIn, void *pData, int *pnData),
+  void *pIn,
+  int (*xOutput)(void *pOut, const void *pData, int nData),
+  void *pOut
+);
 
+/*
+** CAPI3REF: Configure global parameters
+**
+** The sqlite3session_config() interface is used to make global configuration
+** changes to the sessions module in order to tune it to the specific needs 
+** of the application.
+**
+** The sqlite3session_config() interface is not threadsafe. If it is invoked
+** while any other thread is inside any other sessions method then the
+** results are undefined. Furthermore, if it is invoked after any sessions
+** related objects have been created, the results are also undefined. 
+**
+** The first argument to the sqlite3session_config() function must be one
+** of the SQLITE_SESSION_CONFIG_XXX constants defined below. The 
+** interpretation of the (void*) value passed as the second parameter and
+** the effect of calling this function depends on the value of the first
+** parameter.
+**
+** <dl>
+** <dt>SQLITE_SESSION_CONFIG_STRMSIZE<dd>
+**    By default, the sessions module streaming interfaces attempt to input
+**    and output data in approximately 1 KiB chunks. This operand may be used
+**    to set and query the value of this configuration setting. The pointer
+**    passed as the second argument must point to a value of type (int).
+**    If this value is greater than 0, it is used as the new streaming data
+**    chunk size for both input and output. Before returning, the (int) value
+**    pointed to by pArg is set to the final value of the streaming interface
+**    chunk size.
+** </dl>
+**
+** This function returns SQLITE_OK if successful, or an SQLite error code
+** otherwise.
+*/
+int sqlite3session_config(int op, void *pArg);
+
+/*
+** CAPI3REF: Values for sqlite3session_config().
+*/
+#define SQLITE_SESSION_CONFIG_STRMSIZE 1
 
 /*
 ** Make sure we can call this stuff from C++.

td/generate/scheme/td_api.tl

@@ -1857,17 +1857,14 @@ httpUrl url:string = HttpUrl;
 
 //@class InputInlineQueryResult @description Represents a single result of an inline query; for bots only
 
-//@description Represents a link to an animated GIF @id Unique identifier of the query result @title Title of the query result @thumbnail_url URL of the static result thumbnail (JPEG or GIF), if it exists
-//@gif_url The URL of the GIF-file (file size must not exceed 1MB) @gif_duration Duration of the GIF, in seconds @gif_width Width of the GIF @gif_height Height of the GIF
+//@description Represents a link to an animated GIF or an animated (i.e. without sound) H.264/MPEG-4 AVC video
+//@id Unique identifier of the query result @title Title of the query result
+//@thumbnail_url URL of the result thumbnail (JPEG, GIF, or MPEG4), if it exists @thumbnail_mime_type MIME type of the video thumbnail. If non-empty, must be one of "image/jpeg", "image/gif" and "video/mp4"
+//@video_url The URL of the video file (file size must not exceed 1MB) @video_mime_type MIME type of the video file. Must be one of "image/gif" and "video/mp4"
+//@video_duration Duration of the video, in seconds @video_width Width of the video @video_height Height of the video
 //@reply_markup The message reply markup. Must be of type replyMarkupInlineKeyboard or null
 //@input_message_content The content of the message to be sent. Must be one of the following types: InputMessageText, InputMessageAnimation, InputMessageLocation, InputMessageVenue or InputMessageContact
-inputInlineQueryResultAnimatedGif id:string title:string thumbnail_url:string gif_url:string gif_duration:int32 gif_width:int32 gif_height:int32 reply_markup:ReplyMarkup input_message_content:InputMessageContent = InputInlineQueryResult;
-
-//@description Represents a link to an animated (i.e. without sound) H.264/MPEG-4 AVC video @id Unique identifier of the query result @title Title of the result @thumbnail_url URL of the static result thumbnail (JPEG or GIF), if it exists
-//@mpeg4_url The URL of the MPEG4-file (file size must not exceed 1MB) @mpeg4_duration Duration of the video, in seconds @mpeg4_width Width of the video @mpeg4_height Height of the video
-//@reply_markup The message reply markup. Must be of type replyMarkupInlineKeyboard or null
-//@input_message_content The content of the message to be sent. Must be one of the following types: InputMessageText, InputMessageAnimation, InputMessageLocation, InputMessageVenue or InputMessageContact
-inputInlineQueryResultAnimatedMpeg4 id:string title:string thumbnail_url:string mpeg4_url:string mpeg4_duration:int32 mpeg4_width:int32 mpeg4_height:int32 reply_markup:ReplyMarkup input_message_content:InputMessageContent = InputInlineQueryResult;
+inputInlineQueryResultAnimation id:string title:string thumbnail_url:string thumbnail_mime_type:string video_url:string video_mime_type:string video_duration:int32 video_width:int32 video_height:int32 reply_markup:ReplyMarkup input_message_content:InputMessageContent = InputInlineQueryResult;
 
 //@description Represents a link to an article or web page @id Unique identifier of the query result @url URL of the result, if it exists @hide_url True, if the URL must be not shown @title Title of the result
 //@param_description A short description of the result @thumbnail_url URL of the result thumbnail, if it exists @thumbnail_width Thumbnail width, if known @thumbnail_height Thumbnail height, if known
@@ -4386,7 +4383,7 @@ answerCustomQuery custom_query_id:int64 data:string = Ok;
 setAlarm seconds:double = Ok;
 
 
-//@description Uses current user IP to found their country. Returns two-letter ISO 3166-1 alpha-2 country code. Can be called before authorization
+//@description Uses current user IP address to found their country. Returns two-letter ISO 3166-1 alpha-2 country code. Can be called before authorization
 getCountryCode = Text;
 
 //@description Returns the default text for invitation messages to be used as a placeholder when the current user invites friends to Telegram

td/mtproto/Handshake.cpp

@@ -6,7 +6,6 @@
 //
 #include "td/mtproto/Handshake.h"
 
-#include "td/mtproto/crypto.h"
 #include "td/mtproto/KDF.h"
 #include "td/mtproto/utils.h"
 

td/mtproto/Handshake.h

@@ -7,8 +7,8 @@
 #pragma once
 
 #include "td/mtproto/AuthKey.h"
-#include "td/mtproto/crypto.h"
 #include "td/mtproto/DhHandshake.h"
+#include "td/mtproto/RSA.h"
 
 #include "td/utils/buffer.h"
 #include "td/utils/Slice.h"

td/mtproto/RSA.cpp

@@ -4,7 +4,7 @@
 // Distributed under the Boost Software License, Version 1.0. (See accompanying
 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 //
-#include "td/mtproto/crypto.h"
+#include "td/mtproto/RSA.h"
 
 #include "td/mtproto/mtproto_api.h"
 
@@ -25,7 +25,6 @@
 
 namespace td {
 
-/*** RSA ***/
 RSA::RSA(BigNum n, BigNum e) : n_(std::move(n)), e_(std::move(e)) {
 }
 

td/mtproto/RSA.h

@@ -15,7 +15,6 @@
 
 namespace td {
 
-/*** RSA ***/
 class RSA {
  public:
   RSA clone() const;
@@ -33,7 +32,6 @@ class RSA {
   BigNum e_;
 };
 
-/*** PublicRsaKeyInterface ***/
 class PublicRsaKeyInterface {
  public:
   virtual ~PublicRsaKeyInterface() = default;

td/telegram/AnimationsManager.cpp

@@ -78,7 +78,9 @@ class SaveGifQuery : public Td::ResultHandler {
   }
 
   void send(FileId file_id, tl_object_ptr<telegram_api::inputDocument> &&input_document, bool unsave) {
-    CHECK(input_document != nullptr);
+    if (input_document == nullptr) {
+        return;
+    }
     CHECK(file_id.is_valid());
     file_id_ = file_id;
     file_reference_ = input_document->file_reference_.as_slice().str();
@@ -145,7 +147,9 @@ void AnimationsManager::tear_down() {
 
 int32 AnimationsManager::get_animation_duration(FileId file_id) const {
   auto it = animations_.find(file_id);
-  CHECK(it != animations_.end());
+  if (it == animations_.end() || it->second == nullptr) {
+      return 0;
+  }
   return it->second->duration;
 }
 
@@ -155,6 +159,9 @@ tl_object_ptr<td_api::animation> AnimationsManager::get_animation_object(FileId
   }
 
   auto &animation = animations_[file_id];
+  if (animation == nullptr) {
+      return nullptr;
+  }
   LOG_CHECK(animation != nullptr) << source << " " << file_id << " "
                                   << static_cast<int32>(td_->file_manager_->get_file_view(file_id).get_type());
   // TODO can we make that function const?
@@ -217,8 +224,8 @@ FileId AnimationsManager::on_get_animation(unique_ptr<Animation> new_animation,
 
 const AnimationsManager::Animation *AnimationsManager::get_animation(FileId file_id) const {
   auto animation = animations_.find(file_id);
-  if (animation == animations_.end()) {
-    return nullptr;
+  if (animation == animations_.end() || animation->second == nullptr) {
+    return make_unique<Animation>().get();
   }
 
   CHECK(animation->second->file_id == file_id);
@@ -227,13 +234,17 @@ const AnimationsManager::Animation *AnimationsManager::get_animation(FileId file
 
 FileId AnimationsManager::get_animation_thumbnail_file_id(FileId file_id) const {
   auto animation = get_animation(file_id);
-  CHECK(animation != nullptr);
+  if (animation == nullptr) {
+      return FileId();
+  }
   return animation->thumbnail.file_id;
 }
 
 void AnimationsManager::delete_animation_thumbnail(FileId file_id) {
   auto &animation = animations_[file_id];
-  CHECK(animation != nullptr);
+  if (animation == nullptr) {
+      return;
+  }
   animation->thumbnail = PhotoSize();
 }
 
@@ -263,7 +274,7 @@ bool AnimationsManager::merge_animations(FileId new_id, FileId old_id, bool can_
   }
 
   auto new_it = animations_.find(new_id);
-  if (new_it == animations_.end()) {
+  if (new_it == animations_.end() || new_it->second == nullptr) {
     auto &old = animations_[old_id];
     old->is_changed = true;
     if (!can_delete_old) {
@@ -317,7 +328,9 @@ tl_object_ptr<telegram_api::InputMedia> AnimationsManager::get_input_media(
 
   if (input_file != nullptr) {
     const Animation *animation = get_animation(file_id);
-    CHECK(animation != nullptr);
+    if (animation == nullptr) {
+        return nullptr;
+    }
 
     vector<tl_object_ptr<telegram_api::DocumentAttribute>> attributes;
     if (!animation->file_name.empty()) {
@@ -354,7 +367,9 @@ SecretInputMedia AnimationsManager::get_secret_input_media(FileId animation_file
                                                            const string &caption, BufferSlice thumbnail,
                                                            int32 layer) const {
   auto *animation = get_animation(animation_file_id);
-  CHECK(animation != nullptr);
+  if (animation == nullptr) {
+      return SecretInputMedia{};
+  }
   auto file_view = td_->file_manager_->get_file_view(animation_file_id);
   auto &encryption_key = file_view.encryption_key();
   if (!file_view.is_encrypted_secret() || encryption_key.empty()) {
@@ -501,6 +516,9 @@ void AnimationsManager::load_saved_animations(Promise<Unit> &&promise) {
 }
 
 void AnimationsManager::on_load_saved_animations_from_database(const string &value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (value.empty()) {
     LOG(INFO) << "Saved animations aren't found in database";
     reload_saved_animations(true);
@@ -602,7 +620,9 @@ int32 AnimationsManager::get_saved_animations_hash(const char *source) const {
   numbers.reserve(saved_animation_ids_.size() * 2);
   for (auto animation_id : saved_animation_ids_) {
     auto animation = get_animation(animation_id);
-    CHECK(animation != nullptr);
+    if (animation == nullptr) {
+        continue;
+    }
     auto file_view = td_->file_manager_->get_file_view(animation_id);
     CHECK(file_view.has_remote_location());
     if (!file_view.remote_location().is_document()) {
@@ -805,7 +825,9 @@ FileSourceId AnimationsManager::get_saved_animations_file_source_id() {
 
 string AnimationsManager::get_animation_search_text(FileId file_id) const {
   auto animation = get_animation(file_id);
-  CHECK(animation != nullptr);
+  if (animation == nullptr) {
+      return "";
+  }
   return animation->file_name;
 }
 
@@ -824,5 +846,9 @@ void AnimationsManager::get_current_state(vector<td_api::object_ptr<td_api::Upda
     updates.push_back(get_update_saved_animations_object());
   }
 }
+void AnimationsManager::memory_cleanup() {
+  animations_.clear();
+  animations_.rehash(0);
+}
 
 }  // namespace td

td/telegram/AnimationsManager.h

@@ -31,6 +31,8 @@ class AnimationsManager : public Actor {
  public:
   AnimationsManager(Td *td, ActorShared<> parent);
 
+  void memory_cleanup();
+
   int32 get_animation_duration(FileId file_id) const;
 
   tl_object_ptr<td_api::animation> get_animation_object(FileId file_id, const char *source);

td/telegram/AnimationsManager.hpp

@@ -20,7 +20,9 @@ namespace td {
 template <class StorerT>
 void AnimationsManager::store_animation(FileId file_id, StorerT &storer) const {
   auto it = animations_.find(file_id);
-  CHECK(it != animations_.end());
+  if (it == animations_.end() || it->second == nullptr) {
+      return;
+  }
   const Animation *animation = it->second.get();
   store(animation->duration, storer);
   store(animation->dimensions, storer);

td/telegram/AudiosManager.cpp

@@ -24,7 +24,9 @@ AudiosManager::AudiosManager(Td *td) : td_(td) {
 
 int32 AudiosManager::get_audio_duration(FileId file_id) const {
   auto it = audios_.find(file_id);
-  CHECK(it != audios_.end());
+  if (it == audios_.end() || it->second == nullptr) {
+      return 0;
+  }
   return it->second->duration;
 }
 
@@ -34,7 +36,9 @@ tl_object_ptr<td_api::audio> AudiosManager::get_audio_object(FileId file_id) {
   }
 
   auto &audio = audios_[file_id];
-  CHECK(audio != nullptr);
+  if (audio == nullptr) {
+      return nullptr;
+  }
   audio->is_changed = false;
   return make_tl_object<td_api::audio>(audio->duration, audio->title, audio->performer, audio->file_name,
                                        audio->mime_type, get_minithumbnail_object(audio->minithumbnail),
@@ -89,8 +93,8 @@ FileId AudiosManager::on_get_audio(unique_ptr<Audio> new_audio, bool replace) {
 
 const AudiosManager::Audio *AudiosManager::get_audio(FileId file_id) const {
   auto audio = audios_.find(file_id);
-  if (audio == audios_.end()) {
-    return nullptr;
+  if (audio == audios_.end() || audio->second == nullptr) {
+    return make_unique<Audio>().get();
   }
 
   CHECK(audio->second->file_id == file_id);
@@ -122,7 +126,7 @@ bool AudiosManager::merge_audios(FileId new_id, FileId old_id, bool can_delete_o
   }
 
   auto new_it = audios_.find(new_id);
-  if (new_it == audios_.end()) {
+  if (new_it == audios_.end() || new_it->second == nullptr) {
     auto &old = audios_[old_id];
     old->is_changed = true;
     if (!can_delete_old) {
@@ -153,19 +157,25 @@ bool AudiosManager::merge_audios(FileId new_id, FileId old_id, bool can_delete_o
 
 string AudiosManager::get_audio_search_text(FileId file_id) const {
   auto audio = get_audio(file_id);
-  CHECK(audio != nullptr);
+  if (audio == nullptr) {
+      return "";
+  }
   return PSTRING() << audio->file_name << " " << audio->title << " " << audio->performer;
 }
 
 FileId AudiosManager::get_audio_thumbnail_file_id(FileId file_id) const {
   auto audio = get_audio(file_id);
-  CHECK(audio != nullptr);
+  if (audio == nullptr) {
+      return FileId();
+  }
   return audio->thumbnail.file_id;
 }
 
 void AudiosManager::delete_audio_thumbnail(FileId file_id) {
   auto &audio = audios_[file_id];
-  CHECK(audio != nullptr);
+  if (audio == nullptr) {
+      return;
+  }
   audio->thumbnail = PhotoSize();
 }
 
@@ -187,7 +197,9 @@ SecretInputMedia AudiosManager::get_secret_input_media(FileId audio_file_id,
                                                        tl_object_ptr<telegram_api::InputEncryptedFile> input_file,
                                                        const string &caption, BufferSlice thumbnail) const {
   auto *audio = get_audio(audio_file_id);
-  CHECK(audio != nullptr);
+  if (audio == nullptr) {
+      return SecretInputMedia{};
+  }
   auto file_view = td_->file_manager_->get_file_view(audio_file_id);
   auto &encryption_key = file_view.encryption_key();
   if (!file_view.is_encrypted_secret() || encryption_key.empty()) {
@@ -234,7 +246,9 @@ tl_object_ptr<telegram_api::InputMedia> AudiosManager::get_input_media(
 
   if (input_file != nullptr) {
     const Audio *audio = get_audio(file_id);
-    CHECK(audio != nullptr);
+    if (audio == nullptr) {
+        return nullptr;
+    }
 
     vector<tl_object_ptr<telegram_api::DocumentAttribute>> attributes;
     attributes.push_back(make_tl_object<telegram_api::documentAttributeAudio>(
@@ -260,5 +274,9 @@ tl_object_ptr<telegram_api::InputMedia> AudiosManager::get_input_media(
 
   return nullptr;
 }
+void AudiosManager::memory_cleanup() {
+  audios_.clear();
+  audios_.rehash(0);
+}
 
 }  // namespace td

td/telegram/AudiosManager.h

@@ -26,6 +26,8 @@ class AudiosManager {
  public:
   explicit AudiosManager(Td *td);
 
+  void memory_cleanup();
+
   int32 get_audio_duration(FileId file_id) const;
 
   tl_object_ptr<td_api::audio> get_audio_object(FileId file_id);

td/telegram/AudiosManager.hpp

@@ -20,7 +20,9 @@ namespace td {
 template <class StorerT>
 void AudiosManager::store_audio(FileId file_id, StorerT &storer) const {
   auto it = audios_.find(file_id);
-  CHECK(it != audios_.end());
+  if (it == audios_.end() || it->second == nullptr) {
+      return;
+  }
   const Audio *audio = it->second.get();
   store(audio->file_name, storer);
   store(audio->mime_type, storer);

td/telegram/AutoDownloadSettings.cpp

@@ -116,13 +116,13 @@ class SaveAutoDownloadSettingsQuery : public Td::ResultHandler {
 AutoDownloadSettings get_auto_download_settings(const td_api::object_ptr<td_api::autoDownloadSettings> &settings) {
   CHECK(settings != nullptr);
   AutoDownloadSettings result;
-  result.max_photo_file_size = settings->max_photo_file_size_;
-  result.max_video_file_size = settings->max_video_file_size_;
-  result.max_other_file_size = settings->max_other_file_size_;
+  result.max_photo_file_size = -1;
+  result.max_video_file_size = -1;
+  result.max_other_file_size = -1;
   result.video_upload_bitrate = settings->video_upload_bitrate_;
-  result.is_enabled = settings->is_auto_download_enabled_;
-  result.preload_large_videos = settings->preload_large_videos_;
-  result.preload_next_audio = settings->preload_next_audio_;
+  result.is_enabled = false;
+  result.preload_large_videos = false;
+  result.preload_next_audio = false;
   result.use_less_data_for_calls = settings->use_less_data_for_calls_;
   return result;
 }

td/telegram/BackgroundManager.cpp

@@ -137,7 +137,9 @@ class UploadBackgroundQuery : public Td::ResultHandler {
 
   void send(FileId file_id, tl_object_ptr<telegram_api::InputFile> &&input_file, const BackgroundType &type,
             bool for_dark_theme) {
-    CHECK(input_file != nullptr);
+    if (input_file == nullptr) {
+        return;
+    }
     file_id_ = file_id;
     type_ = type;
     for_dark_theme_ = for_dark_theme;
@@ -473,6 +475,10 @@ BackgroundId BackgroundManager::search_background(const string &name, Promise<Un
 }
 
 void BackgroundManager::on_load_background_from_database(string name, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   auto promises_it = being_loaded_from_database_backgrounds_.find(name);
   CHECK(promises_it != being_loaded_from_database_backgrounds_.end());
   auto promises = std::move(promises_it->second);
@@ -690,7 +696,9 @@ void BackgroundManager::on_upload_background_file(FileId file_id, tl_object_ptr<
   LOG(INFO) << "Background file " << file_id << " has been uploaded";
 
   auto it = being_uploaded_files_.find(file_id);
-  CHECK(it != being_uploaded_files_.end());
+  if (it == being_uploaded_files_.end()) {
+      return;
+  }
 
   auto type = it->second.type;
   auto for_dark_theme = it->second.for_dark_theme;
@@ -711,7 +719,9 @@ void BackgroundManager::on_upload_background_file_error(FileId file_id, Status s
   CHECK(status.is_error());
 
   auto it = being_uploaded_files_.find(file_id);
-  CHECK(it != being_uploaded_files_.end());
+  if (it == being_uploaded_files_.end()) {
+      return;
+  }
 
   auto promise = std::move(it->second.promise);
 

td/telegram/BackgroundType.cpp

@@ -136,7 +136,7 @@ StringBuilder &operator<<(StringBuilder &string_builder, const BackgroundType &t
 
 Result<BackgroundType> get_background_type(const td_api::BackgroundType *type) {
   if (type == nullptr) {
-    return Status::Error(400, "Type must not be empty");
+    return Status::Error(400, "Type must be non-empty");
   }
 
   BackgroundType result;
@@ -149,7 +149,7 @@ Result<BackgroundType> get_background_type(const td_api::BackgroundType *type) {
     case td_api::backgroundTypePattern::ID: {
       auto pattern = static_cast<const td_api::backgroundTypePattern *>(type);
       if (pattern->fill_ == nullptr) {
-        return Status::Error(400, "Fill info must not be empty");
+        return Status::Error(400, "Fill info must be non-empty");
       }
       result = BackgroundType(pattern->is_moving_, get_background_fill(pattern->fill_.get()), pattern->intensity_);
       break;
@@ -157,7 +157,7 @@ Result<BackgroundType> get_background_type(const td_api::BackgroundType *type) {
     case td_api::backgroundTypeFill::ID: {
       auto fill = static_cast<const td_api::backgroundTypeFill *>(type);
       if (fill->fill_ == nullptr) {
-        return Status::Error(400, "Fill info must not be empty");
+        return Status::Error(400, "Fill info must be non-empty");
       }
       result = BackgroundType(get_background_fill(fill->fill_.get()));
       break;

td/telegram/CallbackQueriesManager.cpp

@@ -44,7 +44,9 @@ class GetBotCallbackAnswerQuery : public Td::ResultHandler {
     message_id_ = message_id;
 
     auto input_peer = td->messages_manager_->get_input_peer(dialog_id, AccessRights::Read);
-    CHECK(input_peer != nullptr);
+    if (input_peer == nullptr) {
+        return;
+    }
 
     int32 flags = 0;
     BufferSlice data;

td/telegram/ConfigManager.cpp

@@ -27,8 +27,8 @@
 
 #include "td/mtproto/AuthData.h"
 #include "td/mtproto/AuthKey.h"
-#include "td/mtproto/crypto.h"
 #include "td/mtproto/RawConnection.h"
+#include "td/mtproto/RSA.h"
 #include "td/mtproto/TransportType.h"
 
 #include "td/net/HttpQuery.h"
@@ -1426,7 +1426,6 @@ void ConfigManager::process_app_config(tl_object_ptr<telegram_api::JSONValue> &c
   }
 
   if (!dice_emojis.empty()) {
-    shared_config.set_option_string("dice_emojis", implode(dice_emojis, '\x01'));
     vector<string> dice_success_values(dice_emojis.size());
     for (auto &it : dice_emoji_success_value) {
       if (dice_emoji_index.find(it.first) == dice_emoji_index.end()) {
@@ -1436,6 +1435,7 @@ void ConfigManager::process_app_config(tl_object_ptr<telegram_api::JSONValue> &c
       dice_success_values[dice_emoji_index[it.first]] = it.second;
     }
     shared_config.set_option_string("dice_success_values", implode(dice_success_values, ','));
+    shared_config.set_option_string("dice_emojis", implode(dice_emojis, '\x01'));
   }
 }
 

td/telegram/ContactsManager.cpp

@@ -4463,7 +4463,9 @@ void ContactsManager::on_get_blocked_users_result(int32 offset, int32 limit, int
                                                   vector<tl_object_ptr<telegram_api::contactBlocked>> &&blocked_users) {
   LOG(INFO) << "Receive " << blocked_users.size() << " blocked users out of " << total_count;
   auto it = found_blocked_users_.find(random_id);
-  CHECK(it != found_blocked_users_.end());
+  if (it == found_blocked_users_.end()) {
+      return;
+  }
 
   auto &result = it->second.second;
   CHECK(result.empty());
@@ -4481,13 +4483,17 @@ void ContactsManager::on_get_blocked_users_result(int32 offset, int32 limit, int
 
 void ContactsManager::on_failed_get_blocked_users(int64 random_id) {
   auto it = found_blocked_users_.find(random_id);
-  CHECK(it != found_blocked_users_.end());
+  if (it == found_blocked_users_.end()) {
+      return;
+  }
   found_blocked_users_.erase(it);
 }
 
 tl_object_ptr<td_api::users> ContactsManager::get_blocked_users_object(int64 random_id) {
   auto it = found_blocked_users_.find(random_id);
-  CHECK(it != found_blocked_users_.end());
+  if (it == found_blocked_users_.end()) {
+      return nullptr;
+  }
   auto result = get_users_object(it->second.first, it->second.second);
   found_blocked_users_.erase(it);
   return result;
@@ -4718,6 +4724,10 @@ void ContactsManager::load_imported_contacts(Promise<Unit> &&promise) {
 }
 
 void ContactsManager::on_load_imported_contacts_from_database(string value) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   CHECK(!are_imported_contacts_loaded_);
   if (need_clear_imported_contacts_) {
     need_clear_imported_contacts_ = false;
@@ -6673,6 +6683,9 @@ void ContactsManager::on_deleted_contacts(const vector<UserId> &deleted_contact_
 }
 
 void ContactsManager::save_next_contacts_sync_date() {
+  if (G()->close_flag()) {
+    return;
+  }
   if (!G()->parameters().use_chat_info_db) {
     return;
   }
@@ -6769,6 +6782,9 @@ void ContactsManager::on_get_contacts_failed(Status error) {
 }
 
 void ContactsManager::on_load_contacts_from_database(string value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (value.empty()) {
     reload_contacts(true);
     return;
@@ -7161,6 +7177,10 @@ void ContactsManager::save_user_to_database_impl(User *u, UserId user_id, string
 }
 
 void ContactsManager::on_save_user_to_database(UserId user_id, bool success) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   User *u = get_user(user_id);
   CHECK(u != nullptr);
   LOG_CHECK(u->is_being_saved) << user_id << " " << u->is_saved << " " << u->is_status_saved << " "
@@ -7213,6 +7233,10 @@ void ContactsManager::load_user_from_database_impl(UserId user_id, Promise<Unit>
 }
 
 void ContactsManager::on_load_user_from_database(UserId user_id, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   if (!loaded_from_database_users_.insert(user_id).second) {
     return;
   }
@@ -7427,6 +7451,10 @@ void ContactsManager::save_chat_to_database_impl(Chat *c, ChatId chat_id, string
 }
 
 void ContactsManager::on_save_chat_to_database(ChatId chat_id, bool success) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   Chat *c = get_chat(chat_id);
   CHECK(c != nullptr);
   CHECK(c->is_being_saved);
@@ -7473,6 +7501,10 @@ void ContactsManager::load_chat_from_database_impl(ChatId chat_id, Promise<Unit>
 }
 
 void ContactsManager::on_load_chat_from_database(ChatId chat_id, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   if (!loaded_from_database_chats_.insert(chat_id).second) {
     return;
   }
@@ -7656,6 +7688,10 @@ void ContactsManager::save_channel_to_database_impl(Channel *c, ChannelId channe
 }
 
 void ContactsManager::on_save_channel_to_database(ChannelId channel_id, bool success) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   Channel *c = get_channel(channel_id);
   CHECK(c != nullptr);
   CHECK(c->is_being_saved);
@@ -7702,6 +7738,10 @@ void ContactsManager::load_channel_from_database_impl(ChannelId channel_id, Prom
 }
 
 void ContactsManager::on_load_channel_from_database(ChannelId channel_id, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   if (!loaded_from_database_channels_.insert(channel_id).second) {
     return;
   }
@@ -7888,6 +7928,10 @@ void ContactsManager::save_secret_chat_to_database_impl(SecretChat *c, SecretCha
 }
 
 void ContactsManager::on_save_secret_chat_to_database(SecretChatId secret_chat_id, bool success) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   SecretChat *c = get_secret_chat(secret_chat_id);
   CHECK(c != nullptr);
   CHECK(c->is_being_saved);
@@ -7935,6 +7979,10 @@ void ContactsManager::load_secret_chat_from_database_impl(SecretChatId secret_ch
 }
 
 void ContactsManager::on_load_secret_chat_from_database(SecretChatId secret_chat_id, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   if (!loaded_from_database_secret_chats_.insert(secret_chat_id).second) {
     return;
   }
@@ -8850,7 +8898,7 @@ void ContactsManager::on_get_user_photos(UserId user_id, int32 offset, int32 lim
     total_count = min_total_count;
   }
   LOG_IF(ERROR, limit < photo_count) << "Requested not more than " << limit << " photos, but " << photo_count
-                                     << " returned";
+                                     << " received";
 
   User *u = get_user(user_id);
   if (u == nullptr) {
@@ -8920,11 +8968,8 @@ void ContactsManager::on_get_user_photos(UserId user_id, int32 offset, int32 lim
   }
 
   auto known_photo_count = narrow_cast<int32>(user_photos->photos.size());
-  CHECK(user_photos->count >= known_photo_count);
   if (user_photos->offset + known_photo_count > user_photos->count) {
-    LOG(ERROR) << "Fix total photo count of " << user_id << " from " << user_photos->count << " to "
-               << user_photos->offset << " + " << known_photo_count;
-    user_photos->count = user_photos->offset + known_photo_count;
+    user_photos->photos.resize(user_photos->count - user_photos->offset);
   }
 }
 
@@ -9355,6 +9400,8 @@ void ContactsManager::on_update_user_photo(User *u, UserId user_id,
 void ContactsManager::do_update_user_photo(User *u, UserId user_id,
                                            tl_object_ptr<telegram_api::UserProfilePhoto> &&photo, const char *source) {
   u->is_photo_inited = true;
+  LOG_IF(INFO, u->access_hash == -1) << "Update profile photo of " << user_id << " without access hash from "
+                                      << source;
   ProfilePhoto new_photo = get_profile_photo(td_->file_manager_.get(), user_id, u->access_hash, std::move(photo));
 
   if (new_photo != u->photo) {
@@ -9704,8 +9751,9 @@ void ContactsManager::update_user_online_member_count(User *u) {
       case DialogType::Chat: {
         auto chat_id = dialog_id.get_chat_id();
         auto chat_full = get_chat_full(chat_id);
-        CHECK(chat_full != nullptr);
-        update_chat_online_member_count(chat_full, chat_id, false);
+        if (chat_full != nullptr) {
+          update_chat_online_member_count(chat_full, chat_id, false);
+        }
         break;
       }
       case DialogType::Channel: {
@@ -12728,7 +12776,7 @@ void ContactsManager::load_dialog_administrators(DialogId dialog_id, Promise<Uni
 
 void ContactsManager::on_load_dialog_administrators_from_database(DialogId dialog_id, string value,
                                                                   Promise<Unit> &&promise) {
-  if (value.empty()) {
+  if (value.empty() || G()->close_flag()) {
     promise.set_value(Unit());
     return;
   }
@@ -13396,7 +13444,9 @@ tl_object_ptr<td_api::supergroupFullInfo> ContactsManager::get_supergroup_full_i
 
 tl_object_ptr<td_api::supergroupFullInfo> ContactsManager::get_supergroup_full_info_object(
     const ChannelFull *channel_full) const {
-  CHECK(channel_full != nullptr);
+  if (channel_full == nullptr) {
+    return nullptr;
+  }
   double slow_mode_delay_expires_in = 0;
   if (channel_full->slow_mode_next_send_date != 0) {
     slow_mode_delay_expires_in = max(channel_full->slow_mode_next_send_date - G()->server_time(), 1e-3);
@@ -13598,4 +13648,52 @@ void ContactsManager::get_current_state(vector<td_api::object_ptr<td_api::Update
   }
 }
 
+void ContactsManager::memory_cleanup() {
+    users_full_.clear();
+    users_full_.rehash(0);
+    bot_infos_.clear();
+    bot_infos_.rehash(0);
+    user_photos_.clear();
+    user_photos_.rehash(0);
+    user_profile_photo_file_source_ids_.clear();
+    user_profile_photo_file_source_ids_.rehash(0);
+    chat_photo_file_source_ids_.clear();
+    chat_photo_file_source_ids_.rehash(0);
+    channels_full_.clear();
+    channels_full_.rehash(0);
+    channel_photo_file_source_ids_.clear();
+    channel_photo_file_source_ids_.rehash(0);
+    secret_chats_.clear();
+    secret_chats_.rehash(0);
+    secret_chats_with_user_.clear();
+    secret_chats_with_user_.rehash(0);
+    chat_invite_links_.clear();
+    chat_invite_links_.rehash(0);
+    channel_invite_links_.clear();
+    channel_invite_links_.rehash(0);
+    invite_link_infos_.clear();
+    invite_link_infos_.rehash(0);
+    load_user_from_database_queries_.clear();
+    load_user_from_database_queries_.rehash(0);
+    load_chat_from_database_queries_.clear();
+    load_chat_from_database_queries_.rehash(0);
+    load_channel_from_database_queries_.clear();
+    load_channel_from_database_queries_.rehash(0);
+    load_secret_chat_from_database_queries_.clear();
+    load_secret_chat_from_database_queries_.rehash(0);
+    dialog_administrators_.clear();
+    dialog_administrators_.rehash(0);
+    uploaded_profile_photos_.clear();
+    uploaded_profile_photos_.rehash(0);
+    imported_contacts_.clear();
+    imported_contacts_.rehash(0);
+    received_channel_participant_.clear();
+    received_channel_participant_.rehash(0);
+    received_channel_participants_.clear();
+    received_channel_participants_.rehash(0);
+    cached_channel_participants_.clear();
+    cached_channel_participants_.rehash(0);
+    linked_channel_ids_.clear();
+    linked_channel_ids_.rehash(0);
+}
 }  // namespace td

td/telegram/ContactsManager.h

@@ -77,6 +77,8 @@ class ContactsManager : public Actor {
 
   static UserId load_my_id();
 
+  void memory_cleanup();
+
   static UserId get_user_id(const tl_object_ptr<telegram_api::User> &user);
   static ChatId get_chat_id(const tl_object_ptr<telegram_api::Chat> &chat);
   static ChannelId get_channel_id(const tl_object_ptr<telegram_api::Chat> &chat);

td/telegram/DialogDb.cpp

@@ -5,289 +5,71 @@
 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 //
 #include "td/telegram/DialogDb.h"
-
-#include "td/telegram/Version.h"
-
 #include "td/actor/actor.h"
 #include "td/actor/SchedulerLocalStorage.h"
 
 #include "td/db/SqliteConnectionSafe.h"
 #include "td/db/SqliteDb.h"
-#include "td/db/SqliteKeyValue.h"
 #include "td/db/SqliteStatement.h"
 
-#include "td/utils/format.h"
-#include "td/utils/logging.h"
 #include "td/utils/ScopeGuard.h"
 #include "td/utils/Time.h"
 
 namespace td {
 // NB: must happen inside a transaction
 Status init_dialog_db(SqliteDb &db, int32 version, bool &was_created) {
-  LOG(INFO) << "Init dialog database " << tag("version", version);
-  was_created = false;
-
-  // Check if database exists
-  TRY_RESULT(has_table, db.has_table("dialogs"));
-  if (!has_table) {
-    version = 0;
-  }
-
-  if (version < static_cast<int32>(DbVersion::DialogDbCreated) || version > current_db_version()) {
-    TRY_STATUS(drop_dialog_db(db, version));
-    version = 0;
-  }
-
-  auto create_notification_group_table = [&db] {
-    return db.exec(
-        "CREATE TABLE IF NOT EXISTS notification_groups (notification_group_id INT4 PRIMARY KEY, dialog_id "
-        "INT8, last_notification_date INT4)");
-  };
-
-  auto create_last_notification_date_index = [&db] {
-    return db.exec(
-        "CREATE INDEX IF NOT EXISTS notification_group_by_last_notification_date ON notification_groups "
-        "(last_notification_date, dialog_id, notification_group_id) WHERE last_notification_date IS NOT NULL");
-  };
-
-  auto add_dialogs_in_folder_index = [&db] {
-    return db.exec(
-        "CREATE INDEX IF NOT EXISTS dialog_in_folder_by_dialog_order ON dialogs (folder_id, dialog_order, dialog_id) "
-        "WHERE folder_id IS NOT NULL");
-  };
-
-  if (version == 0) {
-    LOG(INFO) << "Create new dialog database";
-    was_created = true;
-    TRY_STATUS(
-        db.exec("CREATE TABLE IF NOT EXISTS dialogs (dialog_id INT8 PRIMARY KEY, dialog_order INT8, data BLOB, "
-                "folder_id INT4)"));
-    TRY_STATUS(create_notification_group_table());
-    TRY_STATUS(create_last_notification_date_index());
-    TRY_STATUS(add_dialogs_in_folder_index());
-    version = current_db_version();
-  }
-  if (version < static_cast<int32>(DbVersion::AddNotificationsSupport)) {
-    TRY_STATUS(create_notification_group_table());
-    TRY_STATUS(create_last_notification_date_index());
-  }
-  if (version < static_cast<int32>(DbVersion::AddFolders)) {
-    TRY_STATUS(db.exec("DROP INDEX IF EXISTS dialog_by_dialog_order"));
-    TRY_STATUS(db.exec("ALTER TABLE dialogs ADD COLUMN folder_id INT4"));
-    TRY_STATUS(add_dialogs_in_folder_index());
-    TRY_STATUS(db.exec("UPDATE dialogs SET folder_id = 0 WHERE dialog_id < -1500000000000 AND dialog_order > 0"));
-  }
-
   return Status::OK();
 }
 
 // NB: must happen inside a transaction
 Status drop_dialog_db(SqliteDb &db, int version) {
-  if (version < static_cast<int32>(DbVersion::DialogDbCreated)) {
-    LOG(WARNING) << "Drop old pmc dialog_db";
-    SqliteKeyValue kv;
-    kv.init_with_connection(db.clone(), "common").ensure();
-    kv.erase_by_prefix("di");
-  }
-
-  LOG(WARNING) << "Drop dialog_db " << tag("version", version) << tag("current_db_version", current_db_version());
-  auto status = db.exec("DROP TABLE IF EXISTS dialogs");
-  TRY_STATUS(db.exec("DROP TABLE IF EXISTS notification_groups"));
-  return status;
+  return Status::OK();
 }
 
 class DialogDbImpl : public DialogDbSyncInterface {
  public:
-  explicit DialogDbImpl(SqliteDb db) : db_(std::move(db)) {
+  explicit DialogDbImpl(SqliteDb db) {
     init().ensure();
   }
 
   Status init() {
-    TRY_RESULT_ASSIGN(add_dialog_stmt_, db_.get_statement("INSERT OR REPLACE INTO dialogs VALUES(?1, ?2, ?3, ?4)"));
-    TRY_RESULT_ASSIGN(add_notification_group_stmt_,
-                      db_.get_statement("INSERT OR REPLACE INTO notification_groups VALUES(?1, ?2, ?3)"));
-    TRY_RESULT_ASSIGN(delete_notification_group_stmt_,
-                      db_.get_statement("DELETE FROM notification_groups WHERE notification_group_id = ?1"));
-    TRY_RESULT_ASSIGN(get_dialog_stmt_, db_.get_statement("SELECT data FROM dialogs WHERE dialog_id = ?1"));
-    TRY_RESULT_ASSIGN(
-        get_dialogs_stmt_,
-        db_.get_statement("SELECT data, dialog_id, dialog_order FROM dialogs WHERE "
-                          "folder_id == ?1 AND (dialog_order < ?2 OR (dialog_order = ?2 AND dialog_id < ?3)) ORDER "
-                          "BY dialog_order DESC, dialog_id DESC LIMIT ?4"));
-    TRY_RESULT_ASSIGN(
-        get_notification_groups_by_last_notification_date_stmt_,
-        db_.get_statement("SELECT notification_group_id, dialog_id, last_notification_date FROM notification_groups "
-                          "WHERE last_notification_date < ?1 OR (last_notification_date = ?1 "
-                          "AND (dialog_id < ?2 OR (dialog_id = ?2 AND notification_group_id < ?3))) ORDER BY "
-                          "last_notification_date DESC, dialog_id DESC LIMIT ?4"));
-    //                          "WHERE (last_notification_date, dialog_id, notification_group_id) < (?1, ?2, ?3) ORDER BY "
-    //                          "last_notification_date DESC, dialog_id DESC, notification_group_id DESC LIMIT ?4"));
-    TRY_RESULT_ASSIGN(
-        get_notification_group_stmt_,
-        db_.get_statement(
-            "SELECT dialog_id, last_notification_date FROM notification_groups WHERE notification_group_id = ?1"));
-    TRY_RESULT_ASSIGN(
-        get_secret_chat_count_stmt_,
-        db_.get_statement(
-            "SELECT COUNT(*) FROM dialogs WHERE folder_id = ?1 AND dialog_order > 0 AND dialog_id < -1500000000000"));
-
-    // LOG(ERROR) << get_dialog_stmt_.explain().ok();
-    // LOG(ERROR) << get_dialogs_stmt_.explain().ok();
-    // LOG(ERROR) << get_notification_groups_by_last_notification_date_stmt_.explain().ok();
-    // LOG(ERROR) << get_notification_group_stmt_.explain().ok();
-    // LOG(FATAL) << "EXPLAINED";
-
     return Status::OK();
   }
 
   Status add_dialog(DialogId dialog_id, FolderId folder_id, int64 order, BufferSlice data,
                     vector<NotificationGroupKey> notification_groups) override {
-    SCOPE_EXIT {
-      add_dialog_stmt_.reset();
-    };
-    add_dialog_stmt_.bind_int64(1, dialog_id.get()).ensure();
-    add_dialog_stmt_.bind_int64(2, order).ensure();
-    add_dialog_stmt_.bind_blob(3, data.as_slice()).ensure();
-    if (order > 0) {
-      add_dialog_stmt_.bind_int32(4, folder_id.get()).ensure();
-    } else {
-      add_dialog_stmt_.bind_null(4).ensure();
-    }
-
-    TRY_STATUS(add_dialog_stmt_.step());
-
-    for (auto &to_add : notification_groups) {
-      if (to_add.dialog_id.is_valid()) {
-        SCOPE_EXIT {
-          add_notification_group_stmt_.reset();
-        };
-        add_notification_group_stmt_.bind_int32(1, to_add.group_id.get()).ensure();
-        add_notification_group_stmt_.bind_int64(2, to_add.dialog_id.get()).ensure();
-        if (to_add.last_notification_date != 0) {
-          add_notification_group_stmt_.bind_int32(3, to_add.last_notification_date).ensure();
-        } else {
-          add_notification_group_stmt_.bind_null(3).ensure();
-        }
-        TRY_STATUS(add_notification_group_stmt_.step());
-      } else {
-        SCOPE_EXIT {
-          delete_notification_group_stmt_.reset();
-        };
-        delete_notification_group_stmt_.bind_int32(1, to_add.group_id.get()).ensure();
-        TRY_STATUS(delete_notification_group_stmt_.step());
-      }
-    }
     return Status::OK();
   }
 
   Result<BufferSlice> get_dialog(DialogId dialog_id) override {
-    SCOPE_EXIT {
-      get_dialog_stmt_.reset();
-    };
-
-    get_dialog_stmt_.bind_int64(1, dialog_id.get()).ensure();
-    TRY_STATUS(get_dialog_stmt_.step());
-    if (!get_dialog_stmt_.has_row()) {
-      return Status::Error("Not found");
-    }
-    return BufferSlice(get_dialog_stmt_.view_blob(0));
+    return Status::Error("Not found");
   }
 
   Result<NotificationGroupKey> get_notification_group(NotificationGroupId notification_group_id) override {
-    SCOPE_EXIT {
-      get_notification_group_stmt_.reset();
-    };
-    get_notification_group_stmt_.bind_int32(1, notification_group_id.get()).ensure();
-    TRY_STATUS(get_notification_group_stmt_.step());
-    if (!get_notification_group_stmt_.has_row()) {
-      return Status::Error("Not found");
-    }
-    return NotificationGroupKey(notification_group_id, DialogId(get_notification_group_stmt_.view_int64(0)),
-                                get_last_notification_date(get_notification_group_stmt_, 1));
+    return Status::Error("Not found");
   }
 
   Result<int32> get_secret_chat_count(FolderId folder_id) override {
-    SCOPE_EXIT {
-      get_secret_chat_count_stmt_.reset();
-    };
-    get_secret_chat_count_stmt_.bind_int32(1, folder_id.get()).ensure();
-    TRY_STATUS(get_secret_chat_count_stmt_.step());
-    CHECK(get_secret_chat_count_stmt_.has_row());
-    return get_secret_chat_count_stmt_.view_int32(0);
+    return 0;
   }
 
   Result<DialogDbGetDialogsResult> get_dialogs(FolderId folder_id, int64 order, DialogId dialog_id,
                                                int32 limit) override {
-    SCOPE_EXIT {
-      get_dialogs_stmt_.reset();
-    };
-
-    get_dialogs_stmt_.bind_int32(1, folder_id.get()).ensure();
-    get_dialogs_stmt_.bind_int64(2, order).ensure();
-    get_dialogs_stmt_.bind_int64(3, dialog_id.get()).ensure();
-    get_dialogs_stmt_.bind_int32(4, limit).ensure();
-
     DialogDbGetDialogsResult result;
-    TRY_STATUS(get_dialogs_stmt_.step());
-    while (get_dialogs_stmt_.has_row()) {
-      BufferSlice data(get_dialogs_stmt_.view_blob(0));
-      result.next_dialog_id = DialogId(get_dialogs_stmt_.view_int64(1));
-      result.next_order = get_dialogs_stmt_.view_int64(2);
-      LOG(INFO) << "Load " << result.next_dialog_id << " with order " << result.next_order;
-      result.dialogs.emplace_back(std::move(data));
-      TRY_STATUS(get_dialogs_stmt_.step());
-    }
-
     return std::move(result);
   }
 
   Result<vector<NotificationGroupKey>> get_notification_groups_by_last_notification_date(
       NotificationGroupKey notification_group_key, int32 limit) override {
-    auto &stmt = get_notification_groups_by_last_notification_date_stmt_;
-    SCOPE_EXIT {
-      stmt.reset();
-    };
-
-    stmt.bind_int32(1, notification_group_key.last_notification_date).ensure();
-    stmt.bind_int64(2, notification_group_key.dialog_id.get()).ensure();
-    stmt.bind_int32(3, notification_group_key.group_id.get()).ensure();
-    stmt.bind_int32(4, limit).ensure();
-
     vector<NotificationGroupKey> notification_groups;
-    TRY_STATUS(stmt.step());
-    while (stmt.has_row()) {
-      notification_groups.emplace_back(NotificationGroupId(stmt.view_int32(0)), DialogId(stmt.view_int64(1)),
-                                       get_last_notification_date(stmt, 2));
-      TRY_STATUS(stmt.step());
-    }
-
     return std::move(notification_groups);
   }
 
   Status begin_transaction() override {
-    return db_.begin_transaction();
+    return Status::OK();
   }
   Status commit_transaction() override {
-    return db_.commit_transaction();
-  }
-
- private:
-  SqliteDb db_;
-
-  SqliteStatement add_dialog_stmt_;
-  SqliteStatement add_notification_group_stmt_;
-  SqliteStatement delete_notification_group_stmt_;
-  SqliteStatement get_dialog_stmt_;
-  SqliteStatement get_dialogs_stmt_;
-  SqliteStatement get_notification_groups_by_last_notification_date_stmt_;
-  SqliteStatement get_notification_group_stmt_;
-  SqliteStatement get_secret_chat_count_stmt_;
-
-  static int32 get_last_notification_date(SqliteStatement &stmt, int id) {
-    if (stmt.view_datatype(id) == SqliteStatement::Datatype::Null) {
-      return 0;
-    }
-    return stmt.view_int32(id);
+    return Status::OK();
   }
 };
 
@@ -431,28 +213,12 @@ class DialogDbAsync : public DialogDbAsyncInterface {
     }
 
     void add_read_query() {
-      do_flush();
     }
 
     void do_flush() {
-      if (pending_writes_.empty()) {
-        return;
-      }
-      sync_db_->begin_transaction().ensure();
-      for (auto &query : pending_writes_) {
-        query.set_value(Unit());
-      }
-      sync_db_->commit_transaction().ensure();
-      pending_writes_.clear();
-      for (auto &p : pending_write_results_) {
-        p.first.set_result(std::move(p.second));
-      }
-      pending_write_results_.clear();
-      cancel_timeout();
     }
 
     void timeout_expired() override {
-      do_flush();
     }
 
     void start_up() override {

td/telegram/Document.hpp

@@ -33,7 +33,9 @@ namespace td {
 template <class StorerT>
 void store(const Document &document, StorerT &storer) {
   Td *td = storer.context()->td().get_actor_unsafe();
-  CHECK(td != nullptr);
+  if (td == nullptr) {
+      return;
+  }
 
   store(document.type, storer);
   switch (document.type) {
@@ -67,7 +69,9 @@ void store(const Document &document, StorerT &storer) {
 template <class ParserT>
 void parse(Document &document, ParserT &parser) {
   Td *td = parser.context()->td().get_actor_unsafe();
-  CHECK(td != nullptr);
+  if (td == nullptr) {
+      return;
+  }
 
   parse(document.type, parser);
   switch (document.type) {

td/telegram/DocumentsManager.cpp

@@ -55,6 +55,9 @@ tl_object_ptr<td_api::document> DocumentsManager::get_document_object(FileId fil
 
   LOG(INFO) << "Return document " << file_id << " object";
   auto &document = documents_[file_id];
+  if (document == nullptr) {
+      return nullptr;
+  }
   LOG_CHECK(document != nullptr) << tag("file_id", file_id);
   document->is_changed = false;
   return make_tl_object<td_api::document>(document->file_name, document->mime_type,
@@ -135,7 +138,7 @@ Document DocumentsManager::on_get_document(RemoteDocument remote_document, Dialo
   FileType file_type = FileType::Document;
   Slice default_extension;
   bool supports_streaming = false;
-  bool has_webp_thumbnail = false;
+  PhotoFormat thumbnail_format = PhotoFormat::Jpeg;
   if (type_attributes == 1 || default_document_type != Document::Type::General) {  // not a general document
     if (animated != nullptr || default_document_type == Document::Type::Animation) {
       document_type = Document::Type::Animation;
@@ -163,7 +166,9 @@ Document DocumentsManager::on_get_document(RemoteDocument remote_document, Dialo
       default_extension = Slice("webp");
       owner_dialog_id = DialogId();
       file_name.clear();
-      has_webp_thumbnail = td_->stickers_manager_->has_webp_thumbnail(sticker);
+      if (td_->stickers_manager_->has_webp_thumbnail(sticker) && remote_document.secret_file == nullptr) {
+        thumbnail_format = PhotoFormat::Webp;
+      }
     } else if (video != nullptr || default_document_type == Document::Type::Video ||
                default_document_type == Document::Type::VideoNote) {
       bool is_video_note = default_document_type == Document::Type::VideoNote;
@@ -190,7 +195,6 @@ Document DocumentsManager::on_get_document(RemoteDocument remote_document, Dialo
                  << ", has_stickers = " << has_stickers;
   }
 
-  bool has_png_thumbnail = false;
   if (is_background) {
     if (document_type != Document::Type::General) {
       LOG(ERROR) << "Receive background of type " << document_type;
@@ -199,7 +203,7 @@ Document DocumentsManager::on_get_document(RemoteDocument remote_document, Dialo
     file_type = FileType::Background;
     if (is_pattern) {
       default_extension = Slice("png");
-      has_png_thumbnail = true;
+      thumbnail_format = PhotoFormat::Png;
     } else {
       default_extension = Slice("jpg");
     }
@@ -258,9 +262,9 @@ Document DocumentsManager::on_get_document(RemoteDocument remote_document, Dialo
 
     if (document_type != Document::Type::VoiceNote) {
       for (auto &thumb : document->thumbs_) {
-        auto photo_size = get_photo_size(td_->file_manager_.get(), {FileType::Thumbnail, 0}, id, access_hash,
-                                         file_reference, DcId::create(dc_id), owner_dialog_id, std::move(thumb),
-                                         has_webp_thumbnail, has_png_thumbnail);
+        auto photo_size =
+            get_photo_size(td_->file_manager_.get(), {FileType::Thumbnail, 0}, id, access_hash, file_reference,
+                           DcId::create(dc_id), owner_dialog_id, std::move(thumb), thumbnail_format);
         if (photo_size.get_offset() == 0) {
           thumbnail = std::move(photo_size.get<0>());
         } else {
@@ -492,11 +496,13 @@ void DocumentsManager::create_document(FileId file_id, string minithumbnail, Pho
 
 const DocumentsManager::GeneralDocument *DocumentsManager::get_document(FileId file_id) const {
   auto document = documents_.find(file_id);
-  if (document == documents_.end()) {
-    return nullptr;
+
+  if (document == documents_.end() ||
+      document->second == nullptr ||
+      document->second->file_id != file_id) {
+    return make_unique<GeneralDocument>().get();
   }
 
-  CHECK(document->second->file_id == file_id);
   return document->second.get();
 }
 
@@ -526,7 +532,9 @@ SecretInputMedia DocumentsManager::get_secret_input_media(FileId document_file_i
                                                           tl_object_ptr<telegram_api::InputEncryptedFile> input_file,
                                                           const string &caption, BufferSlice thumbnail) const {
   const GeneralDocument *document = get_document(document_file_id);
-  CHECK(document != nullptr);
+  if (document == nullptr) {
+      return SecretInputMedia{};
+  }
   auto file_view = td_->file_manager_->get_file_view(document_file_id);
   auto &encryption_key = file_view.encryption_key();
   if (!file_view.is_encrypted_secret() || encryption_key.empty()) {
@@ -569,7 +577,9 @@ tl_object_ptr<telegram_api::InputMedia> DocumentsManager::get_input_media(
 
   if (input_file != nullptr) {
     const GeneralDocument *document = get_document(file_id);
-    CHECK(document != nullptr);
+    if (document == nullptr) {
+        return nullptr;
+    }
 
     vector<tl_object_ptr<telegram_api::DocumentAttribute>> attributes;
     if (document->file_name.size()) {
@@ -591,13 +601,17 @@ tl_object_ptr<telegram_api::InputMedia> DocumentsManager::get_input_media(
 
 FileId DocumentsManager::get_document_thumbnail_file_id(FileId file_id) const {
   auto document = get_document(file_id);
-  CHECK(document != nullptr);
+  if (document == nullptr) {
+      return FileId();
+  }
   return document->thumbnail.file_id;
 }
 
 void DocumentsManager::delete_document_thumbnail(FileId file_id) {
   auto &document = documents_[file_id];
-  CHECK(document != nullptr);
+  if (document == nullptr) {
+      return;
+  }
   document->thumbnail = PhotoSize();
 }
 
@@ -626,7 +640,7 @@ bool DocumentsManager::merge_documents(FileId new_id, FileId old_id, bool can_de
   }
 
   auto new_it = documents_.find(new_id);
-  if (new_it == documents_.end()) {
+  if (new_it == documents_.end() || new_it->second == nullptr) {
     auto &old = documents_[old_id];
     old->is_changed = true;
     if (!can_delete_old) {
@@ -652,6 +666,11 @@ bool DocumentsManager::merge_documents(FileId new_id, FileId old_id, bool can_de
   return true;
 }
 
+void DocumentsManager::memory_cleanup() {
+  documents_.clear();
+  documents_.rehash(0);
+}
+
 string DocumentsManager::get_document_search_text(FileId file_id) const {
   auto document = get_document(file_id);
   CHECK(document);

td/telegram/DocumentsManager.h

@@ -76,6 +76,8 @@ class DocumentsManager {
 
   tl_object_ptr<td_api::document> get_document_object(FileId file_id);
 
+  void memory_cleanup();
+
   Document on_get_document(RemoteDocument remote_document, DialogId owner_dialog_id,
                            MultiPromiseActor *load_data_multipromise_ptr = nullptr,
                            Document::Type default_document_type = Document::Type::General, bool is_background = false,

td/telegram/DocumentsManager.hpp

@@ -15,14 +15,21 @@
 #include "td/utils/logging.h"
 #include "td/utils/tl_helpers.h"
 
+#include "td/telegram/ConfigShared.h"
+
 namespace td {
 
 template <class StorerT>
 void DocumentsManager::store_document(FileId file_id, StorerT &storer) const {
-  LOG(DEBUG) << "Store document " << file_id;
+  //LOG(DEBUG) << "Store document " << file_id;
   auto it = documents_.find(file_id);
-  CHECK(it != documents_.end());
+  if (it == documents_.end() || it->second == nullptr) {
+    return;
+  }
   const GeneralDocument *document = it->second.get();
+  if (document == nullptr) {
+      return;
+  }
   store(document->file_name, storer);
   store(document->mime_type, storer);
   store(document->minithumbnail, storer);
@@ -33,13 +40,32 @@ void DocumentsManager::store_document(FileId file_id, StorerT &storer) const {
 template <class ParserT>
 FileId DocumentsManager::parse_document(ParserT &parser) {
   auto document = make_unique<GeneralDocument>();
-  parse(document->file_name, parser);
+
+  string tmp_filename;
+  parse(tmp_filename, parser);
+
   parse(document->mime_type, parser);
-  if (parser.version() >= static_cast<int32>(Version::SupportMinithumbnails)) {
-    parse(document->minithumbnail, parser);
+
+  if ( G()->shared_config().get_option_boolean("disable_document_filenames") && (
+        document->mime_type.rfind("image/") == 0 ||
+        document->mime_type.rfind("video/") == 0 ||
+        document->mime_type.rfind("audio/") == 0)) {
+      document->file_name = "0";
+  } else {
+      document->file_name = tmp_filename;
   }
+
+  if (parser.version() >= static_cast<int32>(Version::SupportMinithumbnails)) {
+      string tmp_minithumbnail;
+      parse(tmp_minithumbnail, parser);
+      if (!G()->shared_config().get_option_boolean("disable_minithumbnails")) {
+          document->minithumbnail = tmp_minithumbnail;
+      }
+  }
+
   parse(document->thumbnail, parser);
   parse(document->file_id, parser);
+
   LOG(DEBUG) << "Parsed document " << document->file_id;
   if (parser.get_error() != nullptr || !document->file_id.is_valid()) {
     return FileId();

td/telegram/FileReferenceManager.cpp

@@ -364,4 +364,11 @@ void FileReferenceManager::reload_photo(PhotoSizeSource source, Promise<Unit> pr
   }
 }
 
+void FileReferenceManager::memory_cleanup(FileId file_id) {
+  auto &node = nodes_[file_id];
+  node.query.reset();
+  node.file_source_ids.reset_position();
+  nodes_.erase(file_id);
+}
+
 }  // namespace td

td/telegram/FileReferenceManager.h

@@ -38,6 +38,8 @@ class FileReferenceManager : public Actor {
   static bool is_file_reference_error(const Status &error);
   static size_t get_file_reference_error_pos(const Status &error);
 
+  void memory_cleanup(FileId file_id);
+
   FileSourceId create_message_file_source(FullMessageId full_message_id);
   FileSourceId create_user_photo_file_source(UserId user_id, int64 photo_id);
   FileSourceId create_chat_photo_file_source(ChatId chat_id);

td/telegram/HashtagHints.cpp

@@ -86,6 +86,10 @@ void HashtagHints::hashtag_used_impl(const string &hashtag) {
 }
 
 void HashtagHints::from_db(Result<string> data, bool dummy) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   sync_with_db_ = true;
   if (data.is_error() || data.ok().empty()) {
     return;

td/telegram/InlineQueriesManager.cpp

@@ -341,7 +341,7 @@ void InlineQueriesManager::answer_inline_query(int64 inline_query_id, bool is_pe
   bool force_vertical = false;
   for (auto &input_result : input_results) {
     if (input_result == nullptr) {
-      return promise.set_error(Status::Error(400, "Inline query result must not be empty"));
+      return promise.set_error(Status::Error(400, "Inline query result must be non-empty"));
     }
 
     string id;
@@ -350,6 +350,7 @@ void InlineQueriesManager::answer_inline_query(int64 inline_query_id, bool is_pe
     string title;
     string description;
     string thumbnail_url;
+    string thumbnail_type = "image/jpeg";
     string content_url;
     string content_type;
     int32 thumbnail_width = 0;
@@ -361,42 +362,28 @@ void InlineQueriesManager::answer_inline_query(int64 inline_query_id, bool is_pe
     FileType file_type = FileType::Temp;
     Result<tl_object_ptr<telegram_api::InputBotInlineMessage>> r_inline_message = Status::Error(500, "Uninited");
     switch (input_result->get_id()) {
-      case td_api::inputInlineQueryResultAnimatedGif::ID: {
-        auto animated_gif = move_tl_object_as<td_api::inputInlineQueryResultAnimatedGif>(input_result);
+      case td_api::inputInlineQueryResultAnimation::ID: {
+        auto animation = move_tl_object_as<td_api::inputInlineQueryResultAnimation>(input_result);
         type = "gif";
-        id = std::move(animated_gif->id_);
-        title = std::move(animated_gif->title_);
-        thumbnail_url = std::move(animated_gif->thumbnail_url_);
-        content_url = std::move(animated_gif->gif_url_);
-        content_type = "image/gif";
-        // duration = animated_gif->gif_duration_;
-        width = animated_gif->gif_width_;
-        height = animated_gif->gif_height_;
+        id = std::move(animation->id_);
+        title = std::move(animation->title_);
+        thumbnail_url = std::move(animation->thumbnail_url_);
+        if (!animation->thumbnail_mime_type_.empty()) {
+          thumbnail_type = std::move(animation->thumbnail_mime_type_);
+        }
+        content_url = std::move(animation->video_url_);
+        content_type = std::move(animation->video_mime_type_);
+        if (content_type != "image/gif" && content_type != "video/mp4") {
+          return promise.set_error(Status::Error(400, "Wrong animation MIME type specified"));
+        }
+        duration = animation->video_duration_;
+        width = animation->video_width_;
+        height = animation->video_height_;
         is_gallery = true;
 
         file_type = FileType::Animation;
-        r_inline_message =
-            get_inline_message(std::move(animated_gif->input_message_content_), std::move(animated_gif->reply_markup_),
-                               td_api::inputMessageAnimation::ID);
-        break;
-      }
-      case td_api::inputInlineQueryResultAnimatedMpeg4::ID: {
-        auto animated_mpeg4 = move_tl_object_as<td_api::inputInlineQueryResultAnimatedMpeg4>(input_result);
-        type = "gif";
-        id = std::move(animated_mpeg4->id_);
-        title = std::move(animated_mpeg4->title_);
-        thumbnail_url = std::move(animated_mpeg4->thumbnail_url_);
-        content_url = std::move(animated_mpeg4->mpeg4_url_);
-        content_type = "video/mp4";
-        duration = animated_mpeg4->mpeg4_duration_;
-        width = animated_mpeg4->mpeg4_width_;
-        height = animated_mpeg4->mpeg4_height_;
-        is_gallery = true;
-
-        file_type = FileType::Animation;
-        r_inline_message =
-            get_inline_message(std::move(animated_mpeg4->input_message_content_),
-                               std::move(animated_mpeg4->reply_markup_), td_api::inputMessageAnimation::ID);
+        r_inline_message = get_inline_message(std::move(animation->input_message_content_),
+                                              std::move(animation->reply_markup_), td_api::inputMessageAnimation::ID);
         break;
       }
       case td_api::inputInlineQueryResultArticle::ID: {
@@ -701,7 +688,8 @@ void InlineQueriesManager::answer_inline_query(int64 inline_query_id, bool is_pe
         attributes.push_back(
             make_tl_object<telegram_api::documentAttributeImageSize>(thumbnail_width, thumbnail_height));
       }
-      thumbnail = make_tl_object<telegram_api::inputWebDocument>(thumbnail_url, 0, "image/jpeg", std::move(attributes));
+      thumbnail =
+          make_tl_object<telegram_api::inputWebDocument>(thumbnail_url, 0, thumbnail_type, std::move(attributes));
     }
     tl_object_ptr<telegram_api::inputWebDocument> content;
     if (!content_url.empty() || !content_type.empty()) {

td/telegram/LanguagePackManager.cpp

@@ -1579,7 +1579,7 @@ Result<LanguagePackManager::LanguageInfo> LanguagePackManager::get_language_info
 Result<LanguagePackManager::LanguageInfo> LanguagePackManager::get_language_info(
     td_api::languagePackInfo *language_pack_info) {
   if (language_pack_info == nullptr) {
-    return Status::Error(400, "Language pack info must not be empty");
+    return Status::Error(400, "Language pack info must be non-empty");
   }
 
   if (!clean_input_string(language_pack_info->id_)) {

td/telegram/Logging.cpp

@@ -47,7 +47,7 @@ static const std::map<Slice, int *> log_tags{
 
 Status Logging::set_current_stream(td_api::object_ptr<td_api::LogStream> stream) {
   if (stream == nullptr) {
-    return Status::Error("Log stream must not be empty");
+    return Status::Error("Log stream must be non-empty");
   }
 
   std::lock_guard<std::mutex> lock(logging_mutex);

td/telegram/MessageContent.cpp

@@ -663,7 +663,7 @@ class MessageDice : public MessageContent {
     if (dice_value < 0) {
       return false;
     }
-    if (emoji == "DEFAULT_EMOJI" || emoji == "🎯") {
+    if (emoji == DEFAULT_EMOJI || emoji == "🎯") {
       return dice_value <= 6;
     }
     return dice_value <= 1000;
@@ -1778,7 +1778,7 @@ static Result<InputMessageContent> create_input_message_content(
       int32 correct_option_id = -1;
       FormattedText explanation;
       if (input_poll->type_ == nullptr) {
-        return Status::Error(400, "Poll type must not be empty");
+        return Status::Error(400, "Poll type must be non-empty");
       }
       switch (input_poll->type_->get_id()) {
         case td_api::pollTypeRegular::ID: {

td/telegram/MessageEntity.cpp

@@ -422,7 +422,7 @@ static vector<Slice> match_bank_card_numbers(Slice str) {
   const unsigned char *end = str.uend();
   const unsigned char *ptr = begin;
 
-  // '/(?<=^|[^+_\pL\d-])[\d -]{13,}([^_\pL\d-]|$)/'
+  // '/(?<=^|[^+_\pL\d-.,])[\d -]{13,}([^_\pL\d-]|$)/'
 
   while (true) {
     while (ptr != end && !is_digit(*ptr)) {
@@ -435,7 +435,7 @@ static vector<Slice> match_bank_card_numbers(Slice str) {
       uint32 prev;
       next_utf8_unsafe(prev_utf8_unsafe(ptr), &prev, "match_bank_card_numbers");
 
-      if (prev == '+' || prev == '-' || prev == '_' ||
+      if (prev == '.' || prev == ',' || prev == '+' || prev == '-' || prev == '_' ||
           get_unicode_simple_category(prev) == UnicodeSimpleCategory::Letter) {
         while (ptr != end && (is_digit(*ptr) || *ptr == ' ' || *ptr == '-')) {
           ptr++;
@@ -1965,10 +1965,12 @@ static vector<Slice> find_text_url_entities_v3(Slice text) {
 }
 
 // entities must be valid for the text
-static FormattedText parse_text_url_entities_v3(Slice text, vector<MessageEntity> entities) {
+static FormattedText parse_text_url_entities_v3(Slice text, const vector<MessageEntity> &entities) {
   // continuous entities can't intersect TextUrl entities,
   // so try to find new TextUrl entities only between the predetermined continuous entities
 
+  Slice debug_initial_text = text;
+
   FormattedText result;
   int32 result_text_utf16_length = 0;
   vector<MessageEntity> part_entities;
@@ -2109,7 +2111,7 @@ static FormattedText parse_text_url_entities_v3(Slice text, vector<MessageEntity
         splittable_entities.clear();
       } else {
         CHECK(pos == splittable_entities.size() - 1);
-        CHECK(!text.empty());
+        LOG_CHECK(!text.empty()) << '"' << debug_initial_text << "\" " << entities;
         splittable_entities[0] = std::move(splittable_entities.back());
         splittable_entities.resize(1);
       }
@@ -2118,7 +2120,7 @@ static FormattedText parse_text_url_entities_v3(Slice text, vector<MessageEntity
     part_begin = part_end;
   };
 
-  for (auto &entity : entities) {
+  for (const auto &entity : entities) {
     if (is_splittable_entity(entity.type)) {
       auto index = get_splittable_entity_type_index(entity.type);
       part_splittable_entities[index].push_back(entity);
@@ -2207,7 +2209,7 @@ static FormattedText parse_markdown_v3_without_pre(Slice text, vector<MessageEnt
 
   FormattedText parsed_text_url_text;
   if (text.find('[') != string::npos) {
-    parsed_text_url_text = parse_text_url_entities_v3(text, std::move(entities));
+    parsed_text_url_text = parse_text_url_entities_v3(text, entities);
     text = parsed_text_url_text.text;
     entities = std::move(parsed_text_url_text.entities);
   }
@@ -3543,6 +3545,25 @@ static std::pair<size_t, int32> remove_invalid_entities(const string &text, vect
       break;
     }
 
+    if (!nested_entities_stack.empty() && nested_entities_stack.back()->offset == utf16_offset &&
+        (text[pos] == '\n' || text[pos] == ' ')) {
+      // entities was fixed, so there can't be more than one splittable entity of each type, one blockquote and
+      // one continuous entity for the given offset
+      for (size_t i = nested_entities_stack.size(); i > 0; i--) {
+        auto *entity = nested_entities_stack[i - 1];
+        if (entity->offset != utf16_offset || entity->type == MessageEntity::Type::TextUrl ||
+            entity->type == MessageEntity::Type::MentionName || is_pre_entity(entity->type)) {
+          break;
+        }
+        entity->offset++;
+        entity->length--;
+        if (entity->length == 0) {
+          CHECK(i == nested_entities_stack.size());
+          nested_entities_stack.pop_back();
+        }
+      }
+    }
+
     auto c = static_cast<unsigned char>(text[pos]);
     switch (c) {
       case '\n':

td/telegram/MessagesDb.cpp

@@ -53,44 +53,44 @@ Status init_messages_db(SqliteDb &db, int32 version) {
   }
 
   auto add_media_indices = [&db](int begin, int end) {
-    for (int i = begin; i < end; i++) {
-      TRY_STATUS(db.exec(PSLICE() << "CREATE INDEX IF NOT EXISTS message_index_" << i
-                                  << " ON messages (dialog_id, message_id) WHERE (index_mask & " << (1 << i)
-                                  << ") != 0"));
-    }
+    // for (int i = begin; i < end; i++) {
+    //   TRY_STATUS(db.exec(PSLICE() << "CREATE INDEX IF NOT EXISTS message_index_" << i
+    //                               << " ON messages (dialog_id, message_id) WHERE (index_mask & " << (1 << i)
+    //                               << ") != 0"));
+    // }
     return Status::OK();
   };
 
   auto add_fts = [&db] {
-    TRY_STATUS(
-        db.exec("CREATE INDEX IF NOT EXISTS message_by_search_id ON messages "
-                "(search_id) WHERE search_id IS NOT NULL"));
+    // TRY_STATUS(
+    //     db.exec("CREATE INDEX IF NOT EXISTS message_by_search_id ON messages "
+    //             "(search_id) WHERE search_id IS NOT NULL"));
 
-    TRY_STATUS(
-        db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(text, content='messages', "
-                "content_rowid='search_id', tokenize = \"unicode61 remove_diacritics 0 tokenchars '\a'\")"));
-    TRY_STATUS(db.exec(
-        "CREATE TRIGGER IF NOT EXISTS trigger_fts_delete BEFORE DELETE ON messages WHEN OLD.search_id IS NOT NULL"
-        " BEGIN INSERT INTO messages_fts(messages_fts, rowid, text) VALUES(\'delete\', OLD.search_id, OLD.text); END"));
-    TRY_STATUS(db.exec(
-        "CREATE TRIGGER IF NOT EXISTS trigger_fts_insert AFTER INSERT ON messages WHEN NEW.search_id IS NOT NULL"
-        " BEGIN INSERT INTO messages_fts(rowid, text) VALUES(NEW.search_id, NEW.text); END"));
-    //TRY_STATUS(db.exec(
-    //"CREATE TRIGGER IF NOT EXISTS trigger_fts_update AFTER UPDATE ON messages WHEN NEW.search_id IS NOT NULL OR "
-    //"OLD.search_id IS NOT NULL"
-    //" BEGIN "
-    //"INSERT INTO messages_fts(messages_fts, rowid, text) VALUES(\'delete\', OLD.search_id, OLD.text); "
-    //"INSERT INTO messages_fts(rowid, text) VALUES(NEW.search_id, NEW.text); "
-    //" END"));
+    // TRY_STATUS(
+    //     db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(text, content='messages', "
+    //             "content_rowid='search_id', tokenize = \"unicode61 remove_diacritics 0 tokenchars '\a'\")"));
+    // TRY_STATUS(db.exec(
+    //     "CREATE TRIGGER IF NOT EXISTS trigger_fts_delete BEFORE DELETE ON messages WHEN OLD.search_id IS NOT NULL"
+    //     " BEGIN INSERT INTO messages_fts(messages_fts, rowid, text) VALUES(\'delete\', OLD.search_id, OLD.text); END"));
+    // TRY_STATUS(db.exec(
+    //     "CREATE TRIGGER IF NOT EXISTS trigger_fts_insert AFTER INSERT ON messages WHEN NEW.search_id IS NOT NULL"
+    //     " BEGIN INSERT INTO messages_fts(rowid, text) VALUES(NEW.search_id, NEW.text); END"));
+    // //TRY_STATUS(db.exec(
+    // //"CREATE TRIGGER IF NOT EXISTS trigger_fts_update AFTER UPDATE ON messages WHEN NEW.search_id IS NOT NULL OR "
+    // //"OLD.search_id IS NOT NULL"
+    // //" BEGIN "
+    // //"INSERT INTO messages_fts(messages_fts, rowid, text) VALUES(\'delete\', OLD.search_id, OLD.text); "
+    // //"INSERT INTO messages_fts(rowid, text) VALUES(NEW.search_id, NEW.text); "
+    // //" END"));
 
     return Status::OK();
   };
-  auto add_call_index = [&db] {
-    for (int i = static_cast<int>(SearchMessagesFilter::Call) - 1;
-         i < static_cast<int>(SearchMessagesFilter::MissedCall); i++) {
-      TRY_STATUS(db.exec(PSLICE() << "CREATE INDEX IF NOT EXISTS full_message_index_" << i
-                                  << " ON messages (unique_message_id) WHERE (index_mask & " << (1 << i) << ") != 0"));
-    }
+  auto add_call_index = [&db]() {
+    // for (int i = static_cast<int>(SearchMessagesFilter::Call) - 1;
+    //      i < static_cast<int>(SearchMessagesFilter::MissedCall); i++) {
+    //   TRY_STATUS(db.exec(PSLICE() << "CREATE INDEX IF NOT EXISTS full_message_index_" << i
+    //                               << " ON messages (unique_message_id) WHERE (index_mask & " << (1 << i) << ") != 0"));
+    // }
     return Status::OK();
   };
   auto add_notification_id_index = [&db] {
@@ -114,7 +114,7 @@ Status init_messages_db(SqliteDb &db, int32 version) {
     TRY_STATUS(
         db.exec("CREATE TABLE IF NOT EXISTS messages (dialog_id INT8, message_id INT8, "
                 "unique_message_id INT4, sender_user_id INT4, random_id INT8, data BLOB, "
-                "ttl_expires_at INT4, index_mask INT4, search_id INT8, text STRING, notification_id INT4, PRIMARY KEY "
+                "ttl_expires_at INT4, index_mask INT4, search_id INT8, text STRING, notification_id INT4, seqno INT32, PRIMARY KEY "
                 "(dialog_id, message_id))"));
 
     TRY_STATUS(
@@ -180,56 +180,60 @@ class MessagesDbImpl : public MessagesDbSyncInterface {
   }
 
   Status init() {
+      seqno_ = 0;
+      db_memory_ = SqliteDb::open_with_key(":memory:", DbKey::empty()).move_as_ok();
+      TRY_STATUS(init_messages_db(db_memory_, db_.user_version().move_as_ok()));
+
     TRY_RESULT_ASSIGN(
         add_message_stmt_,
-        db_.get_statement("INSERT OR REPLACE INTO messages VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"));
+        db_memory_.get_statement("INSERT OR REPLACE INTO messages VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)"));
     TRY_RESULT_ASSIGN(delete_message_stmt_,
-                      db_.get_statement("DELETE FROM messages WHERE dialog_id = ?1 AND message_id = ?2"));
+                      db_memory_.get_statement("DELETE FROM messages WHERE dialog_id = ?1 AND message_id = ?2"));
     TRY_RESULT_ASSIGN(delete_all_dialog_messages_stmt_,
-                      db_.get_statement("DELETE FROM messages WHERE dialog_id = ?1 AND message_id <= ?2"));
+                      db_memory_.get_statement("DELETE FROM messages WHERE dialog_id = ?1 AND message_id <= ?2"));
     TRY_RESULT_ASSIGN(delete_dialog_messages_from_user_stmt_,
-                      db_.get_statement("DELETE FROM messages WHERE dialog_id = ?1 AND sender_user_id == ?2"));
+                      db_memory_.get_statement("DELETE FROM messages WHERE dialog_id = ?1 AND sender_user_id == ?2"));
 
     TRY_RESULT_ASSIGN(get_message_stmt_,
-                      db_.get_statement("SELECT data FROM messages WHERE dialog_id = ?1 AND message_id = ?2"));
+                      db_memory_.get_statement("SELECT data FROM messages WHERE dialog_id = ?1 AND message_id = ?2"));
     TRY_RESULT_ASSIGN(get_message_by_random_id_stmt_,
-                      db_.get_statement("SELECT data FROM messages WHERE dialog_id = ?1 AND random_id = ?2"));
+                      db_memory_.get_statement("SELECT data FROM messages WHERE dialog_id = ?1 AND random_id = ?2"));
     TRY_RESULT_ASSIGN(get_message_by_unique_message_id_stmt_,
-                      db_.get_statement("SELECT dialog_id, data FROM messages WHERE unique_message_id = ?1"));
+                      db_memory_.get_statement("SELECT dialog_id, data FROM messages WHERE unique_message_id = ?1"));
 
     TRY_RESULT_ASSIGN(
         get_expiring_messages_stmt_,
-        db_.get_statement("SELECT dialog_id, data FROM messages WHERE ?1 < ttl_expires_at AND ttl_expires_at <= ?2"));
+        db_memory_.get_statement("SELECT dialog_id, data FROM messages WHERE ?1 < ttl_expires_at AND ttl_expires_at <= ?2"));
     TRY_RESULT_ASSIGN(get_expiring_messages_helper_stmt_,
-                      db_.get_statement("SELECT MAX(ttl_expires_at), COUNT(*) FROM (SELECT ttl_expires_at FROM "
+                      db_memory_.get_statement("SELECT MAX(ttl_expires_at), COUNT(*) FROM (SELECT ttl_expires_at FROM "
                                         "messages WHERE ?1 < ttl_expires_at LIMIT ?2) AS T"));
 
     TRY_RESULT_ASSIGN(get_messages_stmt_.asc_stmt_,
-                      db_.get_statement("SELECT data, message_id FROM messages WHERE dialog_id = ?1 AND message_id > "
+                      db_memory_.get_statement("SELECT data, message_id FROM messages WHERE dialog_id = ?1 AND message_id > "
                                         "?2 ORDER BY message_id ASC LIMIT ?3"));
     TRY_RESULT_ASSIGN(get_messages_stmt_.desc_stmt_,
-                      db_.get_statement("SELECT data, message_id FROM messages WHERE dialog_id = ?1 AND message_id < "
+                      db_memory_.get_statement("SELECT data, message_id FROM messages WHERE dialog_id = ?1 AND message_id < "
                                         "?2 ORDER BY message_id DESC LIMIT ?3"));
     TRY_RESULT_ASSIGN(get_scheduled_messages_stmt_,
-                      db_.get_statement("SELECT data, message_id FROM scheduled_messages WHERE dialog_id = ?1 AND "
+                      db_memory_.get_statement("SELECT data, message_id FROM scheduled_messages WHERE dialog_id = ?1 AND "
                                         "message_id < ?2 ORDER BY message_id DESC LIMIT ?3"));
     TRY_RESULT_ASSIGN(get_messages_from_notification_id_stmt_,
-                      db_.get_statement("SELECT data, message_id FROM messages WHERE dialog_id = ?1 AND "
+                      db_memory_.get_statement("SELECT data, message_id FROM messages WHERE dialog_id = ?1 AND "
                                         "notification_id < ?2 ORDER BY notification_id DESC LIMIT ?3"));
-    TRY_RESULT_ASSIGN(
-        get_messages_fts_stmt_,
-        db_.get_statement(
-            "SELECT dialog_id, data, search_id FROM messages WHERE search_id IN (SELECT rowid FROM messages_fts WHERE "
-            "messages_fts MATCH ?1 AND rowid < ?2 ORDER BY rowid DESC LIMIT ?3) ORDER BY search_id DESC"));
+    // TRY_RESULT_ASSIGN(
+    //     get_messages_fts_stmt_,
+    //     db_memory_.get_statement(
+    //         "SELECT dialog_id, data, search_id FROM messages WHERE search_id IN (SELECT rowid FROM messages_fts WHERE "
+    //         "messages_fts MATCH ?1 AND rowid < ?2 ORDER BY rowid DESC LIMIT ?3) ORDER BY search_id DESC"));
 
     for (int32 i = 0; i < MESSAGES_DB_INDEX_COUNT; i++) {
       TRY_RESULT_ASSIGN(get_messages_from_index_stmts_[i].desc_stmt_,
-                        db_.get_statement(PSLICE() << "SELECT data, message_id FROM messages WHERE dialog_id = ?1 "
+                        db_memory_.get_statement(PSLICE() << "SELECT data, message_id FROM messages WHERE dialog_id = ?1 "
                                                       "AND message_id < ?2 AND (index_mask & "
                                                    << (1 << i) << ") != 0 ORDER BY message_id DESC LIMIT ?3"));
 
       TRY_RESULT_ASSIGN(get_messages_from_index_stmts_[i].asc_stmt_,
-                        db_.get_statement(PSLICE() << "SELECT data, message_id FROM messages WHERE dialog_id = ?1 "
+                        db_memory_.get_statement(PSLICE() << "SELECT data, message_id FROM messages WHERE dialog_id = ?1 "
                                                       "AND message_id > ?2 AND (index_mask & "
                                                    << (1 << i) << ") != 0 ORDER BY message_id ASC LIMIT ?3"));
 
@@ -241,24 +245,24 @@ class MessagesDbImpl : public MessagesDbSyncInterface {
          i < static_cast<int>(SearchMessagesFilter::MissedCall); i++, pos++) {
       TRY_RESULT_ASSIGN(
           get_calls_stmts_[pos],
-          db_.get_statement(
+          db_memory_.get_statement(
               PSLICE() << "SELECT dialog_id, data FROM messages WHERE unique_message_id < ?1 AND (index_mask & "
                        << (1 << i) << ") != 0 ORDER BY unique_message_id DESC LIMIT ?2"));
     }
 
     TRY_RESULT_ASSIGN(add_scheduled_message_stmt_,
-                      db_.get_statement("INSERT OR REPLACE INTO scheduled_messages VALUES(?1, ?2, ?3, ?4)"));
+                      db_memory_.get_statement("INSERT OR REPLACE INTO scheduled_messages VALUES(?1, ?2, ?3, ?4)"));
     TRY_RESULT_ASSIGN(
         get_scheduled_message_stmt_,
-        db_.get_statement("SELECT data FROM scheduled_messages WHERE dialog_id = ?1 AND message_id = ?2"));
+        db_memory_.get_statement("SELECT data FROM scheduled_messages WHERE dialog_id = ?1 AND message_id = ?2"));
     TRY_RESULT_ASSIGN(
         get_scheduled_server_message_stmt_,
-        db_.get_statement("SELECT data FROM scheduled_messages WHERE dialog_id = ?1 AND server_message_id = ?2"));
+        db_memory_.get_statement("SELECT data FROM scheduled_messages WHERE dialog_id = ?1 AND server_message_id = ?2"));
     TRY_RESULT_ASSIGN(delete_scheduled_message_stmt_,
-                      db_.get_statement("DELETE FROM scheduled_messages WHERE dialog_id = ?1 AND message_id = ?2"));
+                      db_memory_.get_statement("DELETE FROM scheduled_messages WHERE dialog_id = ?1 AND message_id = ?2"));
     TRY_RESULT_ASSIGN(
         delete_scheduled_server_message_stmt_,
-        db_.get_statement("DELETE FROM scheduled_messages WHERE dialog_id = ?1 AND server_message_id = ?2"));
+        db_memory_.get_statement("DELETE FROM scheduled_messages WHERE dialog_id = ?1 AND server_message_id = ?2"));
 
     // LOG(ERROR) << get_message_stmt_.explain().ok();
     // LOG(ERROR) << get_messages_from_notification_id_stmt.explain().ok();
@@ -284,6 +288,16 @@ class MessagesDbImpl : public MessagesDbSyncInterface {
     SCOPE_EXIT {
       add_message_stmt_.reset();
     };
+
+    seqno_++;
+
+    if (seqno_ % 8128 == 0) {
+        TRY_STATUS(db_memory_.exec("DELETE FROM messages WHERE seqno < " + to_string(seqno_ - 7168)));
+        TRY_STATUS(db_.exec("PRAGMA shrink_memory"));
+    }
+
+    add_message_stmt_.bind_int32(12, seqno_).ensure();
+
     add_message_stmt_.bind_int64(1, dialog_id.get()).ensure();
     add_message_stmt_.bind_int64(2, message_id.get()).ensure();
 
@@ -664,56 +678,57 @@ class MessagesDbImpl : public MessagesDbSyncInterface {
   }
 
   Result<MessagesDbFtsResult> get_messages_fts(MessagesDbFtsQuery query) override {
-    SCOPE_EXIT {
-      get_messages_fts_stmt_.reset();
-    };
+    // SCOPE_EXIT {
+    //   get_messages_fts_stmt_.reset();
+    // };
 
-    LOG(INFO) << tag("query", query.query) << query.dialog_id << tag("index_mask", query.index_mask)
-              << tag("from_search_id", query.from_search_id) << tag("limit", query.limit);
-    string words = prepare_query(query.query);
-    LOG(INFO) << tag("from", query.query) << tag("to", words);
+    // LOG(INFO) << tag("query", query.query) << query.dialog_id << tag("index_mask", query.index_mask)
+    //           << tag("from_search_id", query.from_search_id) << tag("limit", query.limit);
+    // string words = prepare_query(query.query);
+    // LOG(INFO) << tag("from", query.query) << tag("to", words);
 
-    // dialog_id kludge
-    if (query.dialog_id.is_valid()) {
-      words += PSTRING() << " \"\a" << query.dialog_id.get() << "\"";
-    }
+    // // dialog_id kludge
+    // if (query.dialog_id.is_valid()) {
+    //   words += PSTRING() << " \"\a" << query.dialog_id.get() << "\"";
+    // }
 
-    // index_mask kludge
-    if (query.index_mask != 0) {
-      int index_i = -1;
-      for (int i = 0; i < MESSAGES_DB_INDEX_COUNT; i++) {
-        if (query.index_mask == (1 << i)) {
-          index_i = i;
-          break;
-        }
-      }
-      if (index_i == -1) {
-        return Status::Error("Union of index types is not supported");
-      }
-      words += PSTRING() << " \"\a\a" << index_i << "\"";
-    }
+    // // index_mask kludge
+    // if (query.index_mask != 0) {
+    //   int index_i = -1;
+    //   for (int i = 0; i < MESSAGES_DB_INDEX_COUNT; i++) {
+    //     if (query.index_mask == (1 << i)) {
+    //       index_i = i;
+    //       break;
+    //     }
+    //   }
+    //   if (index_i == -1) {
+    //     return Status::Error("Union of index types is not supported");
+    //   }
+    //   words += PSTRING() << " \"\a\a" << index_i << "\"";
+    // }
 
-    auto &stmt = get_messages_fts_stmt_;
-    stmt.bind_string(1, words).ensure();
-    if (query.from_search_id == 0) {
-      query.from_search_id = std::numeric_limits<int64>::max();
-    }
-    stmt.bind_int64(2, query.from_search_id).ensure();
-    stmt.bind_int32(3, query.limit).ensure();
-    MessagesDbFtsResult result;
-    auto status = stmt.step();
-    if (status.is_error()) {
-      LOG(ERROR) << status;
-      return std::move(result);
-    }
-    while (stmt.has_row()) {
-      auto dialog_id = stmt.view_int64(0);
-      auto data_slice = stmt.view_blob(1);
-      auto search_id = stmt.view_int64(2);
-      result.next_search_id = search_id;
-      result.messages.push_back(MessagesDbMessage{DialogId(dialog_id), BufferSlice(data_slice)});
-      stmt.step().ensure();
-    }
+    // auto &stmt = get_messages_fts_stmt_;
+    // stmt.bind_string(1, words).ensure();
+    // if (query.from_search_id == 0) {
+    //   query.from_search_id = std::numeric_limits<int64>::max();
+    // }
+    // stmt.bind_int64(2, query.from_search_id).ensure();
+    // stmt.bind_int32(3, query.limit).ensure();
+    // MessagesDbFtsResult result;
+    // auto status = stmt.step();
+    // if (status.is_error()) {
+    //   LOG(ERROR) << status;
+    //   return std::move(result);
+    // }
+    // while (stmt.has_row()) {
+    //   auto dialog_id = stmt.view_int64(0);
+    //   auto data_slice = stmt.view_blob(1);
+    //   auto search_id = stmt.view_int64(2);
+    //   result.next_search_id = search_id;
+    //   result.messages.push_back(MessagesDbMessage{DialogId(dialog_id), BufferSlice(data_slice)});
+    //   stmt.step().ensure();
+    // }
+	MessagesDbFtsResult result;
     return std::move(result);
   }
 
@@ -786,6 +801,9 @@ class MessagesDbImpl : public MessagesDbSyncInterface {
 
  private:
   SqliteDb db_;
+  SqliteDb db_memory_;
+
+  int32_t seqno_;
 
   SqliteStatement add_message_stmt_;
 
@@ -810,7 +828,7 @@ class MessagesDbImpl : public MessagesDbSyncInterface {
   std::array<GetMessagesStmt, MESSAGES_DB_INDEX_COUNT> get_messages_from_index_stmts_;
   std::array<SqliteStatement, 2> get_calls_stmts_;
 
-  SqliteStatement get_messages_fts_stmt_;
+//  SqliteStatement get_messages_fts_stmt_;
 
   SqliteStatement add_scheduled_message_stmt_;
   SqliteStatement get_scheduled_message_stmt_;

td/telegram/MessagesManager.cpp

@@ -293,7 +293,9 @@ class GetChannelMessagesQuery : public Td::ResultHandler {
   void send(ChannelId channel_id, tl_object_ptr<telegram_api::InputChannel> &&input_channel,
             vector<tl_object_ptr<telegram_api::InputMessage>> &&message_ids) {
     channel_id_ = channel_id;
-    CHECK(input_channel != nullptr);
+    if (input_channel == nullptr) {
+        return;
+    }
     send_query(G()->net_query_creator().create(
         telegram_api::channels_getMessages(std::move(input_channel), std::move(message_ids))));
   }
@@ -433,7 +435,9 @@ class ExportChannelMessageLinkQuery : public Td::ResultHandler {
     for_group_ = for_group;
     ignore_result_ = ignore_result;
     auto input_channel = td->contacts_manager_->get_input_channel(channel_id);
-    CHECK(input_channel != nullptr);
+    if (input_channel == nullptr) {
+        return;
+    }
     send_query(G()->net_query_creator().create(telegram_api::channels_exportMessageLink(
         std::move(input_channel), message_id.get_server_message_id().get(), for_group)));
   }
@@ -572,7 +576,9 @@ class GetCommonDialogsQuery : public Td::ResultHandler {
     LOG(INFO) << "Get common dialogs with " << user_id << " from " << offset_chat_id << " with limit " << limit;
 
     auto input_user = td->contacts_manager_->get_input_user(user_id);
-    CHECK(input_user != nullptr);
+    if (input_user == nullptr) {
+        return;
+    }
 
     send_query(G()->net_query_creator().create(
         telegram_api::messages_getCommonChats(std::move(input_user), offset_chat_id, limit)));
@@ -696,7 +702,9 @@ class EditDialogPhotoQuery : public Td::ResultHandler {
   }
 
   void send(DialogId dialog_id, FileId file_id, tl_object_ptr<telegram_api::InputChatPhoto> &&input_chat_photo) {
-    CHECK(input_chat_photo != nullptr);
+    if (input_chat_photo == nullptr) {
+        return;
+    }
     file_id_ = file_id;
     was_uploaded_ = FileManager::extract_was_uploaded(input_chat_photo);
     file_reference_ = FileManager::extract_file_reference(input_chat_photo);
@@ -710,7 +718,9 @@ class EditDialogPhotoQuery : public Td::ResultHandler {
       case DialogType::Channel: {
         auto channel_id = dialog_id.get_channel_id();
         auto input_channel = td->contacts_manager_->get_input_channel(channel_id);
-        CHECK(input_channel != nullptr);
+        if (input_channel == nullptr) {
+            return;
+        }
         send_query(G()->net_query_creator().create(
             telegram_api::channels_editPhoto(std::move(input_channel), std::move(input_chat_photo))));
         break;
@@ -2714,7 +2724,9 @@ class GetGameHighScoresQuery : public Td::ResultHandler {
     random_id_ = random_id;
 
     auto input_peer = td->messages_manager_->get_input_peer(dialog_id, AccessRights::Read);
-    CHECK(input_peer != nullptr);
+    if (input_peer == nullptr) {
+        return;
+    }
 
     CHECK(input_user != nullptr);
     send_query(G()->net_query_creator().create(telegram_api::messages_getGameHighScores(
@@ -2898,7 +2910,9 @@ class SendScreenshotNotificationQuery : public Td::ResultHandler {
     dialog_id_ = dialog_id;
 
     auto input_peer = td->messages_manager_->get_input_peer(dialog_id, AccessRights::Write);
-    CHECK(input_peer != nullptr);
+    if (input_peer == nullptr) {
+        return;
+    }
 
     auto query = G()->net_query_creator().create(
         telegram_api::messages_sendScreenshotNotification(std::move(input_peer), 0, random_id));
@@ -3056,7 +3070,9 @@ class DeleteChannelMessagesQuery : public Td::ResultHandler {
 
       query_count_++;
       auto input_channel = td->contacts_manager_->get_input_channel(channel_id);
-      CHECK(input_channel != nullptr);
+      if (input_channel == nullptr) {
+          return;
+      }
       send_query(G()->net_query_creator().create(
           telegram_api::channels_deleteMessages(std::move(input_channel), std::move(slice))));
     }
@@ -4930,8 +4946,9 @@ void MessagesManager::save_dialog_to_database(DialogId dialog_id) {
   };
   add_group_key(d->message_notification_group);
   add_group_key(d->mention_notification_group);
+  auto fixed_folder_id = d->folder_id == FolderId::archive() ? FolderId::archive() : FolderId::main();
   G()->td_db()->get_dialog_db_async()->add_dialog(
-      dialog_id, d->folder_id, d->is_folder_id_inited ? d->order : 0, get_dialog_database_value(d),
+      dialog_id, fixed_folder_id, d->is_folder_id_inited ? d->order : 0, get_dialog_database_value(d),
       std::move(changed_group_keys), PromiseCreator::lambda([dialog_id, can_reuse_notification_group](Result<> result) {
         send_closure(G()->messages_manager(), &MessagesManager::on_save_dialog_to_database, dialog_id,
                      can_reuse_notification_group, result.is_ok());
@@ -4941,7 +4958,7 @@ void MessagesManager::save_dialog_to_database(DialogId dialog_id) {
 void MessagesManager::on_save_dialog_to_database(DialogId dialog_id, bool can_reuse_notification_group, bool success) {
   LOG(INFO) << "Successfully saved " << dialog_id << " to database";
 
-  if (success && can_reuse_notification_group) {
+  if (success && can_reuse_notification_group && !G()->close_flag()) {
     auto d = get_dialog(dialog_id);
     CHECK(d != nullptr);
     try_reuse_notification_group(d->message_notification_group);
@@ -7042,7 +7059,7 @@ void MessagesManager::report_dialog(DialogId dialog_id, const tl_object_ptr<td_a
   }
 
   if (reason == nullptr) {
-    return promise.set_error(Status::Error(3, "Reason must not be empty"));
+    return promise.set_error(Status::Error(3, "Reason must be non-empty"));
   }
 
   Dialog *user_d = d;
@@ -7869,7 +7886,7 @@ void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_
   CHECK(offset < 0 || from_the_end);
   CHECK(!from_message_id.is_scheduled());
 
-  // it is likely that there is no more history messages on the server
+  // it is likely that there are no more history messages on the server
   bool have_full_history = from_the_end && narrow_cast<int32>(messages.size()) < limit;
   Dialog *d = get_dialog(dialog_id);
 
@@ -7924,7 +7941,7 @@ void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_
     MessageId first_received_message_id = get_message_id(messages.back(), false);
     if (first_received_message_id >= from_message_id && d->first_database_message_id.is_valid() &&
         first_received_message_id >= d->first_database_message_id) {
-      // it is likely that there is no more history messages on the server
+      // it is likely that there are no more history messages on the server
       have_full_history = true;
     }
   }
@@ -7942,6 +7959,46 @@ void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_
     prev_have_full_history = d->have_full_history;
   }
 
+  if (from_the_end && d != nullptr) {
+    auto last_server_message_id = get_message_id(messages[0], false);
+    // delete all server messages with ID > last_server_message_id
+    vector<MessageId> message_ids;
+    find_newer_messages(d->messages.get(), last_server_message_id, message_ids);
+    if (!message_ids.empty()) {
+      bool need_update_dialog_pos = false;
+      vector<int64> deleted_message_ids;
+      for (auto message_id : message_ids) {
+        CHECK(message_id > last_server_message_id);
+        if (message_id.is_server()) {
+          auto message = delete_message(d, message_id, true, &need_update_dialog_pos, "on_get_gistory 1");
+          if (message != nullptr) {
+            deleted_message_ids.push_back(message->message_id.get());
+          }
+        }
+      }
+      if (need_update_dialog_pos) {
+        send_update_chat_last_message(d, "on_get_gistory 2");
+      }
+
+      if (!deleted_message_ids.empty()) {
+        send_update_delete_messages(dialog_id, std::move(deleted_message_ids), true, false);
+
+        message_ids.clear();
+        find_newer_messages(d->messages.get(), last_server_message_id, message_ids);
+      }
+
+      // connect all messages with ID > last_server_message_id
+      for (size_t i = 0; i + 1 < message_ids.size(); i++) {
+        auto m = get_message(d, message_ids[i]);
+        if (m == nullptr) { continue; }
+        if (!m->have_next) {
+          m->have_next = true;
+          attach_message_to_next(d, message_ids[i], "on_get_history 3");
+        }
+      }
+    }
+  }
+
   for (auto &message : messages) {
     if (!have_next && from_the_end && d != nullptr && get_message_id(message, false) < d->last_message_id) {
       // last message in the dialog should be attached to the next message if there is some
@@ -8153,18 +8210,20 @@ void MessagesManager::on_get_dialog_messages_search_result(DialogId dialog_id, c
 
     MessageId first_added_message_id;
     if (messages.empty()) {
-      // messages may be empty because there is no more messages or they can't be found due to global limit
-      // anyway pretend that there is no more messages
+      // messages may be empty because there are no more messages or they can't be found due to global limit
+      // anyway pretend that there are no more messages
       first_added_message_id = MessageId::min();
     }
 
     auto &result = it->second.second;
     CHECK(result.empty());
+    int32 added_message_count = 0;
     for (auto &message : messages) {
       auto new_full_message_id =
           on_get_message(std::move(message), false, false, false, false, false, "search call messages");
       if (new_full_message_id != FullMessageId()) {
         result.push_back(new_full_message_id);
+        added_message_count++;
       }
 
       auto message_id = new_full_message_id.get_message_id();
@@ -8172,6 +8231,11 @@ void MessagesManager::on_get_dialog_messages_search_result(DialogId dialog_id, c
         first_added_message_id = message_id;
       }
     }
+    if (total_count < added_message_count) {
+      LOG(ERROR) << "Receive total_count = " << total_count << ", but added " << added_message_count
+                 << " messages out of " << messages.size();
+      total_count = added_message_count;
+    }
     if (G()->parameters().use_message_db) {
       bool update_state = false;
 
@@ -8208,7 +8272,7 @@ void MessagesManager::on_get_dialog_messages_search_result(DialogId dialog_id, c
   CHECK(result.empty());
   MessageId first_added_message_id;
   if (messages.empty()) {
-    // messages may be empty because there is no more messages or they can't be found due to global limit
+    // messages may be empty because there are no more messages or they can't be found due to global limit
     // anyway pretend that there is no more messages
     first_added_message_id = MessageId::min();
   }
@@ -9050,18 +9114,18 @@ void MessagesManager::find_old_messages(const Message *m, MessageId max_message_
   }
 }
 
-void MessagesManager::find_new_messages(const Message *m, MessageId min_message_id, vector<MessageId> &message_ids) {
+void MessagesManager::find_newer_messages(const Message *m, MessageId min_message_id, vector<MessageId> &message_ids) {
   if (m == nullptr) {
     return;
   }
 
   if (m->message_id > min_message_id) {
-    find_new_messages(m->left.get(), min_message_id, message_ids);
+    find_newer_messages(m->left.get(), min_message_id, message_ids);
 
     message_ids.push_back(m->message_id);
   }
 
-  find_new_messages(m->right.get(), min_message_id, message_ids);
+  find_newer_messages(m->right.get(), min_message_id, message_ids);
 }
 
 void MessagesManager::find_unloadable_messages(const Dialog *d, int32 unload_before_date, const Message *m,
@@ -9813,6 +9877,10 @@ void MessagesManager::repair_secret_chat_total_count(FolderId folder_id) {
 }
 
 void MessagesManager::on_get_secret_chat_total_count(FolderId folder_id, int32 total_count) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   CHECK(!td_->auth_manager_->is_bot());
   auto &list = get_dialog_list(folder_id);
   CHECK(total_count >= 0);
@@ -10777,6 +10845,10 @@ void MessagesManager::ttl_db_loop(double server_now) {
 
 void MessagesManager::ttl_db_on_result(Result<std::pair<std::vector<std::pair<DialogId, BufferSlice>>, int32>> r_result,
                                        bool dummy) {
+  if (G()->close_flag()) {
+    return;
+  }
+
   auto result = r_result.move_as_ok();
   ttl_db_has_query_ = false;
   ttl_db_expires_from_ = ttl_db_expires_till_;
@@ -11571,6 +11643,7 @@ FullMessageId MessagesManager::on_get_message(MessageInfo &&message_info, bool f
     }
 
     if (message_id.is_scheduled()) {
+      CHECK(message_id.is_scheduled_server());
       auto dialog_it = update_scheduled_message_ids_.find(dialog_id);
       CHECK(dialog_it != update_scheduled_message_ids_.end());
       dialog_it->second.erase(message_id.get_scheduled_server_message_id());
@@ -11741,7 +11814,7 @@ void MessagesManager::set_dialog_last_new_message_id(Dialog *d, MessageId last_n
     invalidate_message_indexes(d);
 
     vector<MessageId> to_delete_message_ids;
-    find_new_messages(d->messages.get(), last_new_message_id, to_delete_message_ids);
+    find_newer_messages(d->messages.get(), last_new_message_id, to_delete_message_ids);
     td::remove_if(to_delete_message_ids, [](MessageId message_id) { return message_id.is_yet_unsent(); });
     if (!to_delete_message_ids.empty()) {
       LOG(WARNING) << "Delete " << format::as_array(to_delete_message_ids) << " because of received last new "
@@ -13307,6 +13380,9 @@ void MessagesManager::load_dialog_list_from_database(FolderId folder_id, int32 l
 
 void MessagesManager::on_get_dialogs_from_database(FolderId folder_id, int32 limit, DialogDbGetDialogsResult &&dialogs,
                                                    Promise<Unit> &&promise) {
+  if (G()->close_flag()) {
+    return promise.set_error(Status::Error(500, "Request aborted"));
+  }
   CHECK(!td_->auth_manager_->is_bot());
   auto &list = get_dialog_list(folder_id);
   LOG(INFO) << "Receive " << dialogs.dialogs.size() << " from expected " << limit << " chats in " << folder_id
@@ -13959,7 +14035,7 @@ void MessagesManager::get_messages_from_server(vector<FullMessageId> &&message_i
     auto dialog_id = full_message_id.get_dialog_id();
     auto message_id = full_message_id.get_message_id();
     if (!message_id.is_valid() || !message_id.is_server()) {
-      if (message_id.is_valid_scheduled()) {
+      if (message_id.is_valid_scheduled() && message_id.is_scheduled_server()) {
         scheduled_message_ids[dialog_id].push_back(message_id.get_scheduled_server_message_id().get());
       }
       continue;
@@ -15555,11 +15631,15 @@ td_api::object_ptr<td_api::ChatActionBar> MessagesManager::get_chat_action_bar_o
   return nullptr;
 }
 
-td_api::object_ptr<td_api::chat> MessagesManager::get_chat_object(const Dialog *d) const {
+td_api::object_ptr<td_api::chat> MessagesManager::get_chat_object(const Dialog *d, int64 real_order) const {
   CHECK(d != nullptr);
 
   auto chat_source = is_dialog_sponsored(d) ? sponsored_dialog_source_.get_chat_source_object() : nullptr;
 
+  if (real_order == DEFAULT_ORDER) {
+    real_order = d->order;
+  }
+
   bool can_delete_for_self = false;
   bool can_delete_for_all_users = false;
   if (chat_source != nullptr) {
@@ -15572,7 +15652,7 @@ td_api::object_ptr<td_api::chat> MessagesManager::get_chat_object(const Dialog *
         // can't delete
         break;
     }
-  } else if (!td_->auth_manager_->is_bot() && d->order != DEFAULT_ORDER) {
+  } else if (!td_->auth_manager_->is_bot() && real_order != DEFAULT_ORDER) {
     switch (d->dialog_id.get_type()) {
       case DialogType::User:
         can_delete_for_self = true;
@@ -16576,6 +16656,9 @@ vector<FullMessageId> MessagesManager::get_active_live_location_messages(Promise
 }
 
 void MessagesManager::on_load_active_live_location_full_message_ids_from_database(string value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (value.empty()) {
     LOG(INFO) << "Active live location messages aren't found in the database";
     on_load_active_live_location_messages_finished();
@@ -16866,6 +16949,9 @@ void MessagesManager::on_search_dialog_messages_db_result(int64 random_id, Dialo
                                                           SearchMessagesFilter filter_type, int32 offset, int32 limit,
                                                           Result<std::vector<BufferSlice>> r_messages,
                                                           Promise<> promise) {
+  if (G()->close_flag()) {
+    return promise.set_error(Status::Error(500, "Request aborted"));
+  }
   if (r_messages.is_error()) {
     LOG(ERROR) << r_messages.error();
     if (first_db_message_id != MessageId::min() && dialog_id.get_type() != DialogType::SecretChat &&
@@ -16903,8 +16989,9 @@ void MessagesManager::on_search_dialog_messages_db_result(int64 random_id, Dialo
   int32 result_size = narrow_cast<int32>(res.size());
   bool from_the_end =
       from_message_id == MessageId::max() || (offset < 0 && (result_size == 0 || res[0] < from_message_id));
-  if (message_count < result_size || (message_count > result_size && from_the_end &&
-                                      first_db_message_id == MessageId::min() && result_size < limit + offset)) {
+  if ((message_count != -1 && message_count < result_size) ||
+      (message_count > result_size && from_the_end && first_db_message_id == MessageId::min() &&
+       result_size < limit + offset)) {
     LOG(INFO) << "Fix found message count in " << dialog_id << " from " << message_count << " to " << result_size;
     message_count = result_size;
     if (filter_type == SearchMessagesFilter::UnreadMention) {
@@ -16981,7 +17068,10 @@ std::pair<int64, vector<FullMessageId>> MessagesManager::offline_search_messages
 }
 
 void MessagesManager::on_messages_db_fts_result(Result<MessagesDbFtsResult> result, int64 random_id,
-                                                Promise<> &&promise) {
+                                                Promise<Unit> &&promise) {
+  if (G()->close_flag()) {
+    result = Status::Error(500, "Request aborted");
+  }
   if (result.is_error()) {
     found_fts_messages_.erase(random_id);
     return promise.set_error(result.move_as_error());
@@ -17010,6 +17100,9 @@ void MessagesManager::on_messages_db_calls_result(Result<MessagesDbCallsResult>
                                                   MessageId first_db_message_id, SearchMessagesFilter filter,
                                                   Promise<> &&promise) {
   CHECK(!first_db_message_id.is_scheduled());
+  if (G()->close_flag()) {
+    result = Status::Error(500, "Request aborted");
+  }
   if (result.is_error()) {
     found_call_messages_.erase(random_id);
     return promise.set_error(result.move_as_error());
@@ -17161,6 +17254,9 @@ MessageId MessagesManager::find_message_by_date(const Message *m, int32 date) {
 
 void MessagesManager::on_get_dialog_message_by_date_from_database(DialogId dialog_id, int32 date, int64 random_id,
                                                                   Result<BufferSlice> result, Promise<Unit> promise) {
+  if (G()->close_flag()) {
+    return promise.set_error(Status::Error(500, "Request aborted"));
+  }
   Dialog *d = get_dialog(dialog_id);
   CHECK(d != nullptr);
   if (result.is_ok()) {
@@ -17392,6 +17488,10 @@ void MessagesManager::on_get_history_from_database(DialogId dialog_id, MessageId
   CHECK(offset < 0 || from_the_end);
   CHECK(!from_message_id.is_scheduled());
 
+  if (G()->close_flag()) {
+    return promise.set_error(Status::Error(500, "Request aborted"));
+  }
+
   if (!have_input_peer(dialog_id, AccessRights::Read)) {
     LOG(WARNING) << "Ignore result of get_history_from_database in " << dialog_id;
     promise.set_value(Unit());
@@ -17576,13 +17676,13 @@ void MessagesManager::on_get_history_from_database(DialogId dialog_id, MessageId
 void MessagesManager::get_history_from_the_end(DialogId dialog_id, bool from_database, bool only_local,
                                                Promise<Unit> &&promise) {
   CHECK(dialog_id.is_valid());
+  if (G()->close_flag()) {
+    return promise.set_error(Status::Error(500, "Request aborted"));
+  }
   if (!have_input_peer(dialog_id, AccessRights::Read)) {
     // can't get history in dialogs without read access
     return promise.set_value(Unit());
   }
-  if (G()->close_flag()) {
-    return promise.set_error(Status::Error(500, "Request aborted"));
-  }
   int32 limit = MAX_GET_HISTORY;
   if (from_database && G()->parameters().use_message_db) {
     if (!promise) {
@@ -17783,8 +17883,20 @@ void MessagesManager::load_dialog_scheduled_messages(DialogId dialog_id, bool fr
 }
 
 void MessagesManager::on_get_scheduled_messages_from_database(DialogId dialog_id, vector<BufferSlice> &&messages) {
+  if (G()->close_flag()) {
+    auto it = load_scheduled_messages_from_database_queries_.find(dialog_id);
+    if (it == load_scheduled_messages_from_database_queries_.end()) { return; }
+    if (it->second.empty()) { return; }
+    auto promises = std::move(it->second);
+    load_scheduled_messages_from_database_queries_.erase(it);
+
+    for (auto &promise : promises) {
+      promise.set_error(Status::Error(500, "Request aborted"));
+    }
+    return;
+  }
   auto d = get_dialog(dialog_id);
-  CHECK(d != nullptr);
+  if (d == nullptr) { return; }
   d->has_loaded_scheduled_messages_from_database = true;
 
   LOG(INFO) << "Receive " << messages.size() << " scheduled messages from database in " << dialog_id;
@@ -17954,12 +18066,13 @@ tl_object_ptr<td_api::message> MessagesManager::get_message_object(DialogId dial
   auto live_location_date = m->is_failed_to_send ? 0 : m->date;
   auto date = is_scheduled ? 0 : m->date;
   auto edit_date = m->hide_edit_date ? 0 : m->edit_date;
+  auto views = m->message_id.is_scheduled() || (m->message_id.is_local() && m->forward_info == nullptr) ? 0 : m->views;
   return make_tl_object<td_api::message>(
       m->message_id.get(), td_->contacts_manager_->get_user_id_object(m->sender_user_id, "sender_user_id"),
       dialog_id.get(), std::move(sending_state), std::move(scheduling_state), is_outgoing, can_be_edited,
       can_be_forwarded, can_delete_for_self, can_delete_for_all_users, m->is_channel_post, contains_unread_mention,
       date, edit_date, get_message_forward_info_object(m->forward_info), reply_to_message_id, ttl, ttl_expires_in,
-      td_->contacts_manager_->get_user_id_object(m->via_bot_user_id, "via_bot_user_id"), m->author_signature, m->views,
+      td_->contacts_manager_->get_user_id_object(m->via_bot_user_id, "via_bot_user_id"), m->author_signature, views,
       media_album_id, get_restriction_reason_description(m->restriction_reasons),
       get_message_content_object(m->content.get(), td_, live_location_date, m->is_content_secret),
       get_reply_markup_object(m->reply_markup));
@@ -22196,6 +22309,9 @@ void MessagesManager::on_get_message_notifications_from_database(DialogId dialog
                                                                  NotificationId initial_from_notification_id,
                                                                  int32 limit, Result<vector<BufferSlice>> result,
                                                                  Promise<vector<Notification>> promise) {
+  if (G()->close_flag()) {
+    result = Status::Error(500, "Request aborted");
+  }
   if (result.is_error()) {
     return promise.set_error(result.move_as_error());
   }
@@ -22382,7 +22498,7 @@ void MessagesManager::remove_message_notifications_by_message_ids(DialogId dialo
 
 void MessagesManager::do_remove_message_notification(DialogId dialog_id, bool from_mentions,
                                                      NotificationId notification_id, vector<BufferSlice> result) {
-  if (result.empty()) {
+  if (result.empty() || G()->close_flag()) {
     return;
   }
   CHECK(result.size() == 1);
@@ -22874,10 +22990,10 @@ void MessagesManager::send_update_delete_messages(DialogId dialog_id, vector<int
       make_tl_object<td_api::updateDeleteMessages>(dialog_id.get(), std::move(message_ids), is_permanent, from_cache));
 }
 
-void MessagesManager::send_update_new_chat(Dialog *d) {
+void MessagesManager::send_update_new_chat(Dialog *d, int64 real_order) {
   CHECK(d != nullptr);
   CHECK(d->messages == nullptr);
-  auto chat_object = get_chat_object(d);
+  auto chat_object = get_chat_object(d, real_order);
   bool has_action_bar = chat_object->action_bar_ != nullptr;
   d->last_sent_has_scheduled_messages = chat_object->has_scheduled_messages_;
   send_closure(G()->td(), &Td::send_update, make_tl_object<td_api::updateNewChat>(std::move(chat_object)));
@@ -23713,6 +23829,9 @@ void MessagesManager::fail_send_message(FullMessageId full_message_id, int error
   } else {
     CHECK(message->message_id.is_valid());
   }
+  if (message->forward_info == nullptr && message->views == 1) {
+    message->views = 0;
+  }
   message->is_failed_to_send = true;
   message->send_error_code = error_code;
   message->send_error_message = error_message;
@@ -24041,6 +24160,9 @@ void MessagesManager::set_dialog_has_scheduled_server_messages(Dialog *d, bool h
 
 void MessagesManager::set_dialog_has_scheduled_database_messages(DialogId dialog_id,
                                                                  bool has_scheduled_database_messages) {
+  if (G()->close_flag()) {
+    return;
+  }
   return set_dialog_has_scheduled_database_messages_impl(get_dialog(dialog_id), has_scheduled_database_messages);
 }
 
@@ -24641,7 +24763,7 @@ bool MessagesManager::is_dialog_action_unneded(DialogId dialog_id) const {
 void MessagesManager::send_dialog_action(DialogId dialog_id, const tl_object_ptr<td_api::ChatAction> &action,
                                          Promise<Unit> &&promise) {
   if (action == nullptr) {
-    return promise.set_error(Status::Error(5, "Action must not be empty"));
+    return promise.set_error(Status::Error(5, "Action must be non-empty"));
   }
 
   if (!have_dialog_force(dialog_id)) {
@@ -25203,7 +25325,7 @@ void MessagesManager::set_dialog_permissions(DialogId dialog_id,
   }
 
   if (permissions == nullptr) {
-    return promise.set_error(Status::Error(3, "New permissions must not be empty"));
+    return promise.set_error(Status::Error(3, "New permissions must be non-empty"));
   }
 
   switch (dialog_id.get_type()) {
@@ -26125,8 +26247,10 @@ MessagesManager::Message *MessagesManager::on_get_message_from_database(DialogId
       return nullptr;
     }
 
-    // can succeed in private and group chats
-    get_message_from_server({dialog_id, m->message_id}, Auto());
+    if (m->message_id.is_valid() && m->message_id.is_any_server() &&
+        (dialog_id.get_type() == DialogType::User || dialog_id.get_type() == DialogType::Chat)) {
+      get_message_from_server({dialog_id, m->message_id}, Auto());
+    }
 
     force_create_dialog(dialog_id, source);
     d = get_dialog_force(dialog_id);
@@ -26357,7 +26481,7 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, uniq
     return nullptr;
   }
 
-  if (*need_update && message_id <= d->last_new_message_id) {
+  if (*need_update && message_id <= d->last_new_message_id && !td_->auth_manager_->is_bot()) {
     *need_update = false;
   }
 
@@ -26469,7 +26593,7 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, uniq
     // in get_message_notification_group_force
     get_dialog_notification_group_id(d->dialog_id, get_notification_group_info(d, message.get()));
   }
-  if (*need_update || (!d->last_new_message_id.is_valid() && !message_id.is_yet_unsent() && !message->from_database)) {
+  if (*need_update || (!d->last_new_message_id.is_valid() && !message_id.is_yet_unsent())) {
     auto pinned_message_id = get_message_content_pinned_message_id(message->content.get());
     if (pinned_message_id.is_valid() && pinned_message_id < message_id &&
         have_message_force({dialog_id, pinned_message_id}, "preload pinned message")) {
@@ -26477,7 +26601,7 @@ MessagesManager::Message *MessagesManager::add_message_to_dialog(Dialog *d, uniq
     }
 
     if (d->pinned_message_notification_message_id.is_valid() &&
-        d->pinned_message_notification_message_id < message_id &&
+        d->pinned_message_notification_message_id != message_id &&
         have_message_force({dialog_id, d->pinned_message_notification_message_id},
                            "preload previously pinned message")) {
       LOG(INFO) << "Preloaded previously pinned " << d->pinned_message_notification_message_id << " from database";
@@ -27066,7 +27190,7 @@ void MessagesManager::delete_all_dialog_messages_from_database(Dialog *d, Messag
   }
   if (d->pinned_message_notification_message_id.is_valid() &&
       d->pinned_message_notification_message_id <= max_message_id) {
-    remove_dialog_pinned_message_notification(d, "delete_all_dialog_messages_from_database");
+    remove_dialog_pinned_message_notification(d, source);
   }
   remove_message_dialog_notifications(d, max_message_id, false, source);
   remove_message_dialog_notifications(d, max_message_id, true, source);
@@ -27141,47 +27265,11 @@ void MessagesManager::delete_message_files(DialogId dialog_id, const Message *m)
 }
 
 bool MessagesManager::need_delete_file(FullMessageId full_message_id, FileId file_id) const {
-  if (being_readded_message_id_ == full_message_id) {
-    return false;
-  }
-
-  auto main_file_id = td_->file_manager_->get_file_view(file_id).file_id();
-  auto full_message_ids = td_->file_reference_manager_->get_some_message_file_sources(main_file_id);
-  LOG(INFO) << "Receive " << full_message_ids << " as sources for file " << main_file_id << "/" << file_id << " from "
-            << full_message_id;
-  for (auto other_full_messsage_id : full_message_ids) {
-    if (other_full_messsage_id != full_message_id) {
-      return false;
-    }
-  }
-
-  return true;
+  return false;
 }
 
 bool MessagesManager::need_delete_message_files(DialogId dialog_id, const Message *m) const {
-  if (m == nullptr) {
-    return false;
-  }
-
-  auto dialog_type = dialog_id.get_type();
-  if (!m->message_id.is_scheduled() && !m->message_id.is_server() && dialog_type != DialogType::SecretChat) {
-    return false;
-  }
-  if (being_readded_message_id_ == FullMessageId{dialog_id, m->message_id}) {
-    return false;
-  }
-
-  if (m->forward_info != nullptr && m->forward_info->from_dialog_id.is_valid() &&
-      m->forward_info->from_message_id.is_valid()) {
-    // this function must not try to load the message, because it can be called from
-    // do_delete_message or add_scheduled_message_to_dialog
-    const Message *old_m = get_message({m->forward_info->from_dialog_id, m->forward_info->from_message_id});
-    if (old_m != nullptr && get_message_file_ids(old_m) == get_message_file_ids(m)) {
-      return false;
-    }
-  }
-
-  return true;
+  return false;
 }
 
 void MessagesManager::delete_message_from_database(Dialog *d, MessageId message_id, const Message *m,
@@ -28015,7 +28103,7 @@ MessagesManager::Dialog *MessagesManager::add_new_dialog(unique_ptr<Dialog> &&d,
 
   fix_dialog_action_bar(dialog);
 
-  send_update_new_chat(dialog);
+  send_update_new_chat(dialog, order);
 
   fix_new_dialog(dialog, std::move(last_database_message), last_database_message_id, order, last_clear_history_date,
                  last_clear_history_message_id, is_loaded_from_database);
@@ -28245,8 +28333,6 @@ void MessagesManager::fix_new_dialog(Dialog *d, unique_ptr<Message> &&last_datab
       UNREACHABLE();
   }
 
-  update_dialogs_hints(d);
-
   if (d->delete_last_message_date != 0) {
     if (d->last_message_id.is_valid()) {
       LOG(ERROR) << "Last " << d->deleted_last_message_id << " in " << dialog_id << " was deleted at "
@@ -28260,19 +28346,6 @@ void MessagesManager::fix_new_dialog(Dialog *d, unique_ptr<Message> &&last_datab
     }
   }
 
-  if (need_get_history && !td_->auth_manager_->is_bot() && have_input_peer(dialog_id, AccessRights::Read) &&
-      (d->order != DEFAULT_ORDER || is_dialog_sponsored(d))) {
-    get_history_from_the_end(dialog_id, true, false, Auto());
-  }
-
-  if (d->need_repair_server_unread_count && need_unread_counter(d->order)) {
-    CHECK(dialog_type != DialogType::SecretChat);
-    repair_server_unread_count(dialog_id, d->server_unread_count);
-  }
-  if (d->need_repair_channel_server_unread_count) {
-    repair_channel_server_unread_count(d);
-  }
-
   if (!G()->parameters().use_message_db) {
     d->has_loaded_scheduled_messages_from_database = true;
   }
@@ -28283,6 +28356,19 @@ void MessagesManager::fix_new_dialog(Dialog *d, unique_ptr<Message> &&last_datab
     LOG(ERROR) << dialog_id << " has order " << d->order << " instead of saved to database order " << order;
   }
 
+  // must be after update_dialog_pos, because uses d->order
+  if (need_get_history && !td_->auth_manager_->is_bot() && dialog_id != being_added_dialog_id_ &&
+      have_input_peer(dialog_id, AccessRights::Read) && (d->order != DEFAULT_ORDER || is_dialog_sponsored(d))) {
+    get_history_from_the_end(dialog_id, true, false, Auto());
+  }
+  if (d->need_repair_server_unread_count && need_unread_counter(d->order)) {
+    CHECK(dialog_type != DialogType::SecretChat);
+    repair_server_unread_count(dialog_id, d->server_unread_count);
+  }
+  if (d->need_repair_channel_server_unread_count) {
+    repair_channel_server_unread_count(d);
+  }
+
   LOG(INFO) << "Loaded " << dialog_id << " with last new " << d->last_new_message_id << ", first database "
             << d->first_database_message_id << ", last database " << d->last_database_message_id << ", last "
             << d->last_message_id << " with order " << d->order << " and pinned order " << d->pinned_order;
@@ -28527,7 +28613,7 @@ void MessagesManager::update_dialog_pos(Dialog *d, const char *source, bool need
       }
     }
     if (new_order == DEFAULT_ORDER && !d->is_empty) {
-      LOG(INFO) << "There is no known messages in the chat, just leave it where it is";
+      LOG(INFO) << "There are no known messages in the chat, just leave it where it is";
       new_order = d->order;
     }
   }
@@ -29951,6 +30037,7 @@ void MessagesManager::on_binlog_events(vector<BinlogEvent> &&events) {
         }
 
         for (auto message_id : log_event.message_ids_) {
+          CHECK(message_id.is_scheduled_server());
           d->deleted_scheduled_server_message_ids.insert(message_id.get_scheduled_server_message_id());
         }
 

td/telegram/MessagesManager.h

@@ -1680,7 +1680,7 @@ class MessagesManager : public Actor {
 
   static void find_old_messages(const Message *m, MessageId max_message_id, vector<MessageId> &message_ids);
 
-  static void find_new_messages(const Message *m, MessageId min_message_id, vector<MessageId> &message_ids);
+  static void find_newer_messages(const Message *m, MessageId min_message_id, vector<MessageId> &message_ids);
 
   void find_unloadable_messages(const Dialog *d, int32 unload_before_date, const Message *m,
                                 vector<MessageId> &message_ids, int32 &left_to_unload) const;
@@ -1910,7 +1910,7 @@ class MessagesManager : public Actor {
   void send_update_delete_messages(DialogId dialog_id, vector<int64> &&message_ids, bool is_permanent,
                                    bool from_cache) const;
 
-  void send_update_new_chat(Dialog *d);
+  void send_update_new_chat(Dialog *d, int64 real_order);
 
   void send_update_chat_draft_message(const Dialog *d);
 
@@ -2092,7 +2092,7 @@ class MessagesManager : public Actor {
 
   td_api::object_ptr<td_api::ChatActionBar> get_chat_action_bar_object(const Dialog *d) const;
 
-  td_api::object_ptr<td_api::chat> get_chat_object(const Dialog *d) const;
+  td_api::object_ptr<td_api::chat> get_chat_object(const Dialog *d, int64 real_order = DEFAULT_ORDER) const;
 
   bool have_dialog_info(DialogId dialog_id) const;
   bool have_dialog_info_force(DialogId dialog_id) const;
@@ -2225,7 +2225,7 @@ class MessagesManager : public Actor {
   void on_messages_db_fts_result(Result<MessagesDbFtsResult> result, int64 random_id, Promise<> &&promise);
 
   void on_messages_db_calls_result(Result<MessagesDbCallsResult> result, int64 random_id, MessageId first_db_message_id,
-                                   SearchMessagesFilter filter, Promise<> &&promise);
+                                   SearchMessagesFilter filter, Promise<Unit> &&promise);
 
   void on_load_active_live_location_full_message_ids_from_database(string value);
 

td/telegram/NotificationManager.cpp

@@ -164,7 +164,11 @@ void NotificationManager::on_flush_pending_updates_timeout_callback(void *notifi
 }
 
 bool NotificationManager::is_disabled() const {
-  return !td_->auth_manager_->is_authorized() || td_->auth_manager_->is_bot() || G()->close_flag();
+  if ( G()->shared_config().get_option_boolean("disable_notifications")) {
+    return true;
+  } else {
+    return !td_->auth_manager_->is_authorized() || td_->auth_manager_->is_bot() || G()->close_flag();
+  }
 }
 
 StringBuilder &operator<<(StringBuilder &string_builder, const NotificationManager::ActiveNotificationsUpdate &update) {

td/telegram/NotificationSettings.cpp

@@ -131,7 +131,7 @@ static int32 get_mute_until(int32 mute_for) {
 Result<DialogNotificationSettings> get_dialog_notification_settings(
     td_api::object_ptr<td_api::chatNotificationSettings> &&notification_settings, bool old_silent_send_message) {
   if (notification_settings == nullptr) {
-    return Status::Error(400, "New notification settings must not be empty");
+    return Status::Error(400, "New notification settings must be non-empty");
   }
   if (!clean_input_string(notification_settings->sound_)) {
     return Status::Error(400, "Notification settings sound must be encoded in UTF-8");
@@ -155,7 +155,7 @@ Result<DialogNotificationSettings> get_dialog_notification_settings(
 Result<ScopeNotificationSettings> get_scope_notification_settings(
     td_api::object_ptr<td_api::scopeNotificationSettings> &&notification_settings) {
   if (notification_settings == nullptr) {
-    return Status::Error(400, "New notification settings must not be empty");
+    return Status::Error(400, "New notification settings must be non-empty");
   }
   if (!clean_input_string(notification_settings->sound_)) {
     return Status::Error(400, "Notification settings sound must be encoded in UTF-8");

td/telegram/Payments.cpp

@@ -647,7 +647,7 @@ static Status check_postal_code(string &postal_code) {
 
 Result<Address> get_address(td_api::object_ptr<td_api::address> &&address) {
   if (address == nullptr) {
-    return Status::Error(400, "Address must not be empty");
+    return Status::Error(400, "Address must be non-empty");
   }
   TRY_STATUS(check_country_code(address->country_code_));
   TRY_STATUS(check_state(address->state_));
@@ -770,7 +770,7 @@ void answer_shipping_query(int64 shipping_query_id, vector<tl_object_ptr<td_api:
   vector<tl_object_ptr<telegram_api::shippingOption>> options;
   for (auto &option : shipping_options) {
     if (option == nullptr) {
-      return promise.set_error(Status::Error(400, "Shipping option must not be empty"));
+      return promise.set_error(Status::Error(400, "Shipping option must be non-empty"));
     }
     if (!clean_input_string(option->id_)) {
       return promise.set_error(Status::Error(400, "Shipping option id must be encoded in UTF-8"));
@@ -782,7 +782,7 @@ void answer_shipping_query(int64 shipping_query_id, vector<tl_object_ptr<td_api:
     vector<tl_object_ptr<telegram_api::labeledPrice>> prices;
     for (auto &price_part : option->price_parts_) {
       if (price_part == nullptr) {
-        return promise.set_error(Status::Error(400, "Shipping option price part must not be empty"));
+        return promise.set_error(Status::Error(400, "Shipping option price part must be non-empty"));
       }
       if (!clean_input_string(price_part->label_)) {
         return promise.set_error(Status::Error(400, "Shipping option price part label must be encoded in UTF-8"));

td/telegram/Photo.cpp

@@ -95,18 +95,32 @@ td_api::object_ptr<td_api::minithumbnail> get_minithumbnail_object(const string
   return nullptr;
 }
 
+static StringBuilder &operator<<(StringBuilder &string_builder, PhotoFormat format) {
+  switch (format) {
+    case PhotoFormat::Jpeg:
+      return string_builder << "jpg";
+    case PhotoFormat::Png:
+      return string_builder << "png";
+    case PhotoFormat::Webp:
+      return string_builder << "webp";
+    case PhotoFormat::Tgs:
+      return string_builder << "tgs";
+    default:
+      UNREACHABLE();
+      return string_builder;
+  }
+}
+
 static FileId register_photo(FileManager *file_manager, const PhotoSizeSource &source, int64 id, int64 access_hash,
                              std::string file_reference,
                              tl_object_ptr<telegram_api::fileLocationToBeDeprecated> &&location,
-                             DialogId owner_dialog_id, int32 file_size, DcId dc_id, bool is_webp = false,
-                             bool is_png = false) {
+                             DialogId owner_dialog_id, int32 file_size, DcId dc_id, PhotoFormat format) {
   int32 local_id = location->local_id_;
   int64 volume_id = location->volume_id_;
-  LOG(DEBUG) << "Receive " << (is_webp ? "webp" : (is_png ? "png" : "jpeg")) << " photo of type "
-             << source.get_file_type() << " in [" << dc_id << "," << volume_id << "," << local_id << "]. Id: (" << id
-             << ", " << access_hash << ")";
-  auto suggested_name = PSTRING() << static_cast<uint64>(volume_id) << "_" << static_cast<uint64>(local_id)
-                                  << (is_webp ? ".webp" : (is_png ? ".png" : ".jpg"));
+  LOG(DEBUG) << "Receive " << format << " photo of type " << source.get_file_type() << " in [" << dc_id << ","
+             << volume_id << "," << local_id << "]. Id: (" << id << ", " << access_hash << ")";
+  auto suggested_name = PSTRING() << static_cast<uint64>(volume_id) << "_" << static_cast<uint64>(local_id) << '.'
+                                  << format;
   auto file_location_source = owner_dialog_id.get_type() == DialogType::SecretChat ? FileLocationSource::FromUser
                                                                                    : FileLocationSource::FromServer;
   return file_manager->register_remote(
@@ -127,10 +141,12 @@ ProfilePhoto get_profile_photo(FileManager *file_manager, UserId user_id, int64
 
       auto dc_id = DcId::create(profile_photo->dc_id_);
       result.id = profile_photo->photo_id_;
-      result.small_file_id = register_photo(file_manager, {DialogId(user_id), user_access_hash, false}, result.id, 0,
-                                            "", std::move(profile_photo->photo_small_), DialogId(), 0, dc_id);
-      result.big_file_id = register_photo(file_manager, {DialogId(user_id), user_access_hash, true}, result.id, 0, "",
-                                          std::move(profile_photo->photo_big_), DialogId(), 0, dc_id);
+      result.small_file_id =
+          register_photo(file_manager, {DialogId(user_id), user_access_hash, false}, result.id, 0, "",
+                         std::move(profile_photo->photo_small_), DialogId(), 0, dc_id, PhotoFormat::Jpeg);
+      result.big_file_id =
+          register_photo(file_manager, {DialogId(user_id), user_access_hash, true}, result.id, 0, "",
+                         std::move(profile_photo->photo_big_), DialogId(), 0, dc_id, PhotoFormat::Jpeg);
       break;
     }
     default:
@@ -152,21 +168,7 @@ tl_object_ptr<td_api::profilePhoto> get_profile_photo_object(FileManager *file_m
 }
 
 bool operator==(const ProfilePhoto &lhs, const ProfilePhoto &rhs) {
-  bool location_differs = lhs.small_file_id != rhs.small_file_id || lhs.big_file_id != rhs.big_file_id;
-  bool id_differs;
-  if (lhs.id == -1 && rhs.id == -1) {
-    // group chat photo
-    id_differs = location_differs;
-  } else {
-    id_differs = lhs.id != rhs.id;
-  }
-
-  if (location_differs) {
-    LOG_IF(ERROR, !id_differs) << "Photo " << lhs.id << " location has changed. First profilePhoto: " << lhs
-                               << ", second profilePhoto: " << rhs;
-    return false;
-  }
-  return true;
+  return lhs.small_file_id == rhs.small_file_id || lhs.big_file_id == rhs.big_file_id;
 }
 
 bool operator!=(const ProfilePhoto &lhs, const ProfilePhoto &rhs) {
@@ -190,10 +192,11 @@ DialogPhoto get_dialog_photo(FileManager *file_manager, DialogId dialog_id, int6
       auto chat_photo = move_tl_object_as<telegram_api::chatPhoto>(chat_photo_ptr);
 
       auto dc_id = DcId::create(chat_photo->dc_id_);
-      result.small_file_id = register_photo(file_manager, {dialog_id, dialog_access_hash, false}, 0, 0, "",
-                                            std::move(chat_photo->photo_small_), DialogId(), 0, dc_id);
+      result.small_file_id =
+          register_photo(file_manager, {dialog_id, dialog_access_hash, false}, 0, 0, "",
+                         std::move(chat_photo->photo_small_), DialogId(), 0, dc_id, PhotoFormat::Jpeg);
       result.big_file_id = register_photo(file_manager, {dialog_id, dialog_access_hash, true}, 0, 0, "",
-                                          std::move(chat_photo->photo_big_), DialogId(), 0, dc_id);
+                                          std::move(chat_photo->photo_big_), DialogId(), 0, dc_id, PhotoFormat::Jpeg);
 
       break;
     }
@@ -283,7 +286,7 @@ PhotoSize get_secret_thumbnail_photo_size(FileManager *file_manager, BufferSlice
 Variant<PhotoSize, string> get_photo_size(FileManager *file_manager, PhotoSizeSource source, int64 id,
                                           int64 access_hash, std::string file_reference, DcId dc_id,
                                           DialogId owner_dialog_id, tl_object_ptr<telegram_api::PhotoSize> &&size_ptr,
-                                          bool is_webp, bool is_png) {
+                                          PhotoFormat format) {
   CHECK(size_ptr != nullptr);
 
   tl_object_ptr<telegram_api::fileLocationToBeDeprecated> location;
@@ -336,7 +339,7 @@ Variant<PhotoSize, string> get_photo_size(FileManager *file_manager, PhotoSizeSo
   }
 
   res.file_id = register_photo(file_manager, source, id, access_hash, file_reference, std::move(location),
-                               owner_dialog_id, res.size, dc_id, is_webp, is_png);
+                               owner_dialog_id, res.size, dc_id, format);
 
   if (!content.empty()) {
     file_manager->set_content(res.file_id, std::move(content));
@@ -535,7 +538,7 @@ Photo get_photo(FileManager *file_manager, tl_object_ptr<telegram_api::photo> &&
   for (auto &size_ptr : photo->sizes_) {
     auto photo_size = get_photo_size(file_manager, {FileType::Photo, 0}, photo->id_, photo->access_hash_,
                                      photo->file_reference_.as_slice().str(), DcId::create(photo->dc_id_),
-                                     owner_dialog_id, std::move(size_ptr), false, false);
+                                     owner_dialog_id, std::move(size_ptr), PhotoFormat::Jpeg);
     if (photo_size.get_offset() == 0) {
       PhotoSize &size = photo_size.get<0>();
       if (size.type == 0 || size.type == 't' || size.type == 'i') {

td/telegram/Photo.h

@@ -90,12 +90,14 @@ bool operator!=(const DialogPhoto &lhs, const DialogPhoto &rhs);
 
 StringBuilder &operator<<(StringBuilder &string_builder, const DialogPhoto &dialog_photo);
 
+enum class PhotoFormat : int32 { Jpeg, Png, Webp, Tgs };
+
 PhotoSize get_secret_thumbnail_photo_size(FileManager *file_manager, BufferSlice bytes, DialogId owner_dialog_id,
                                           int32 width, int32 height);
 Variant<PhotoSize, string> get_photo_size(FileManager *file_manager, PhotoSizeSource source, int64 id,
                                           int64 access_hash, string file_reference, DcId dc_id,
                                           DialogId owner_dialog_id, tl_object_ptr<telegram_api::PhotoSize> &&size_ptr,
-                                          bool is_webp, bool is_png);
+                                          PhotoFormat format);
 PhotoSize get_web_document_photo_size(FileManager *file_manager, FileType file_type, DialogId owner_dialog_id,
                                       tl_object_ptr<telegram_api::WebDocument> web_document_ptr);
 td_api::object_ptr<td_api::photoSize> get_photo_size_object(FileManager *file_manager, const PhotoSize *photo_size);

td/telegram/PhotoSizeSource.hpp

@@ -8,6 +8,7 @@
 
 #include "td/telegram/PhotoSizeSource.h"
 
+#include "td/utils/logging.h"
 #include "td/utils/tl_helpers.h"
 
 namespace td {
@@ -69,7 +70,7 @@ void parse(PhotoSizeSource::DialogPhoto &source, ParserT &parser) {
   switch (source.dialog_id.get_type()) {
     case DialogType::SecretChat:
     case DialogType::None:
-      return parser.set_error("Invalid chat identifier");
+      return parser.set_error(PSTRING() << "Invalid chat identifier " << source.dialog_id.get());
     default:
       break;
   }

td/telegram/PollManager.cpp

@@ -1297,7 +1297,8 @@ PollId PollManager::on_get_poll(PollId poll_id, tl_object_ptr<telegram_api::poll
     poll_id = PollId(poll_server->id_);
   }
   if (!poll_id.is_valid() || is_local_poll_id(poll_id)) {
-    LOG(ERROR) << "Receive " << poll_id << " from server";
+    LOG(ERROR) << "Receive " << poll_id << " from server: " << oneline(to_string(poll_server)) << " "
+               << oneline(to_string(poll_results));
     return PollId();
   }
   if (poll_server != nullptr && poll_server->id_ != poll_id.get()) {

td/telegram/PollManager.hpp

@@ -128,7 +128,9 @@ void PollManager::store_poll(PollId poll_id, StorerT &storer) const {
   td::store(poll_id.get(), storer);
   if (is_local_poll_id(poll_id)) {
     auto poll = get_poll(poll_id);
-    CHECK(poll != nullptr);
+    if (poll == nullptr) {
+        return;
+    }
     bool has_open_period = poll->open_period != 0;
     bool has_close_date = poll->close_date != 0;
     bool has_explanation = !poll->explanation.text.empty();

td/telegram/SecureValue.cpp

@@ -757,7 +757,7 @@ static Status check_gender(string &gender) {
 
 static Result<string> get_personal_details(td_api::object_ptr<td_api::personalDetails> &&personal_details) {
   if (personal_details == nullptr) {
-    return Status::Error(400, "Personal details must not be empty");
+    return Status::Error(400, "Personal details must be non-empty");
   }
   TRY_STATUS(check_name(personal_details->first_name_));
   TRY_STATUS(check_name(personal_details->middle_name_));
@@ -767,7 +767,7 @@ static Result<string> get_personal_details(td_api::object_ptr<td_api::personalDe
   TRY_STATUS(check_name(personal_details->native_last_name_));
   TRY_RESULT(birthdate, get_date(std::move(personal_details->birthdate_)));
   if (birthdate.empty()) {
-    return Status::Error(400, "Birthdate must not be empty");
+    return Status::Error(400, "Birthdate must be non-empty");
   }
   TRY_STATUS(check_gender(personal_details->gender_));
   TRY_STATUS(check_country_code(personal_details->country_code_));
@@ -808,7 +808,7 @@ static Result<td_api::object_ptr<td_api::personalDetails>> get_personal_details_
   TRY_RESULT(native_last_name, get_json_object_string_field(object, "last_name_native", true));
   TRY_RESULT(birthdate, get_json_object_string_field(object, "birth_date", true));
   if (birthdate.empty()) {
-    return Status::Error(400, "Birthdate must not be empty");
+    return Status::Error(400, "Birthdate must be non-empty");
   }
   TRY_RESULT(gender, get_json_object_string_field(object, "gender", true));
   TRY_RESULT(country_code, get_json_object_string_field(object, "country_code", true));
@@ -836,7 +836,7 @@ static Status check_document_number(string &number) {
     return Status::Error(400, "Document number must be encoded in UTF-8");
   }
   if (number.empty()) {
-    return Status::Error(400, "Document number must not be empty");
+    return Status::Error(400, "Document number must be non-empty");
   }
   if (utf8_length(number) > 24) {
     return Status::Error(400, "Document number is too long");
@@ -867,7 +867,7 @@ static Result<SecureValue> get_identity_document(SecureValueType type, FileManag
                                                  td_api::object_ptr<td_api::inputIdentityDocument> &&identity_document,
                                                  bool need_reverse_side) {
   if (identity_document == nullptr) {
-    return Status::Error(400, "Identity document must not be empty");
+    return Status::Error(400, "Identity document must be non-empty");
   }
   TRY_STATUS(check_document_number(identity_document->number_));
   TRY_RESULT(date, get_date(std::move(identity_document->expiry_date_)));
@@ -950,7 +950,7 @@ static Result<SecureValue> get_personal_document(
     SecureValueType type, FileManager *file_manager,
     td_api::object_ptr<td_api::inputPersonalDocument> &&personal_document) {
   if (personal_document == nullptr) {
-    return Status::Error(400, "Personal document must not be empty");
+    return Status::Error(400, "Personal document must be non-empty");
   }
 
   SecureValue res;
@@ -988,7 +988,7 @@ static Status check_email_address(string &email_address) {
 Result<SecureValue> get_secure_value(FileManager *file_manager,
                                      td_api::object_ptr<td_api::InputPassportElement> &&input_passport_element) {
   if (input_passport_element == nullptr) {
-    return Status::Error(400, "InputPassportElement must not be empty");
+    return Status::Error(400, "InputPassportElement must be non-empty");
   }
 
   SecureValue res;

td/telegram/SequenceDispatcher.cpp

@@ -255,13 +255,17 @@ void MultiSequenceDispatcher::send_with_callback(NetQueryPtr query, ActorShared<
 
 void MultiSequenceDispatcher::on_result() {
   auto it = dispatchers_.find(get_link_token());
-  CHECK(it != dispatchers_.end());
+  if (it == dispatchers_.end()) {
+      return;
+  }
   it->second.cnt_--;
 }
 
 void MultiSequenceDispatcher::ready_to_close() {
   auto it = dispatchers_.find(get_link_token());
-  CHECK(it != dispatchers_.end());
+  if (it == dispatchers_.end()) {
+      return;
+  }
   if (it->second.cnt_ == 0) {
     LOG(DEBUG) << "Close SequenceDispatcher " << get_link_token();
     dispatchers_.erase(it);

td/telegram/StickersManager.cpp

@@ -12,6 +12,7 @@
 
 #include "td/telegram/AccessRights.h"
 #include "td/telegram/AuthManager.h"
+#include "td/telegram/ConfigManager.h"
 #include "td/telegram/ConfigShared.h"
 #include "td/telegram/ContactsManager.h"
 #include "td/telegram/DialogId.h"
@@ -1328,9 +1329,17 @@ tl_object_ptr<td_api::sticker> StickersManager::get_sticker_object(FileId file_i
   }
 
   auto it = stickers_.find(file_id);
-  CHECK(it != stickers_.end());
+
+  if (it == stickers_.end() || it->second == nullptr) {
+    return nullptr;
+  }
+
   auto sticker = it->second.get();
-  CHECK(sticker != nullptr);
+
+  if (sticker == nullptr) {
+    return nullptr;
+  }
+
   sticker->is_changed = false;
 
   auto mask_position = sticker->point >= 0
@@ -1584,9 +1593,10 @@ std::pair<int64, FileId> StickersManager::on_get_sticker_document(
 
   PhotoSize thumbnail;
   for (auto &thumb : document->thumbs_) {
-    auto photo_size = get_photo_size(td_->file_manager_.get(), {FileType::Thumbnail, 0}, document_id,
-                                     document->access_hash_, document->file_reference_.as_slice().str(), dc_id,
-                                     DialogId(), std::move(thumb), has_webp_thumbnail(sticker), false);
+    auto photo_size =
+        get_photo_size(td_->file_manager_.get(), {FileType::Thumbnail, 0}, document_id, document->access_hash_,
+                       document->file_reference_.as_slice().str(), dc_id, DialogId(), std::move(thumb),
+                       has_webp_thumbnail(sticker) ? PhotoFormat::Webp : PhotoFormat::Jpeg);
     if (photo_size.get_offset() == 0) {
       thumbnail = std::move(photo_size.get<0>());
       break;
@@ -1601,8 +1611,8 @@ std::pair<int64, FileId> StickersManager::on_get_sticker_document(
 
 StickersManager::Sticker *StickersManager::get_sticker(FileId file_id) {
   auto sticker = stickers_.find(file_id);
-  if (sticker == stickers_.end()) {
-    return nullptr;
+  if (sticker == stickers_.end() || sticker->second == nullptr) {
+    return make_unique<Sticker>().get();
   }
 
   CHECK(sticker->second->file_id == file_id);
@@ -1611,18 +1621,22 @@ StickersManager::Sticker *StickersManager::get_sticker(FileId file_id) {
 
 const StickersManager::Sticker *StickersManager::get_sticker(FileId file_id) const {
   auto sticker = stickers_.find(file_id);
-  if (sticker == stickers_.end()) {
-    return nullptr;
+
+  if (sticker == stickers_.end() ||
+      sticker->second == nullptr ||
+      sticker->second->file_id != file_id) {
+    return make_unique<Sticker>().get();
   }
 
-  CHECK(sticker->second->file_id == file_id);
   return sticker->second.get();
 }
 
 StickersManager::StickerSet *StickersManager::get_sticker_set(StickerSetId sticker_set_id) {
   auto sticker_set = sticker_sets_.find(sticker_set_id);
-  if (sticker_set == sticker_sets_.end()) {
-    return nullptr;
+
+  if (sticker_set == sticker_sets_.end() ||
+      sticker_set->second == nullptr) {
+    return make_unique<StickerSet>().get();
   }
 
   return sticker_set->second.get();
@@ -1631,7 +1645,7 @@ StickersManager::StickerSet *StickersManager::get_sticker_set(StickerSetId stick
 const StickersManager::StickerSet *StickersManager::get_sticker_set(StickerSetId sticker_set_id) const {
   auto sticker_set = sticker_sets_.find(sticker_set_id);
   if (sticker_set == sticker_sets_.end()) {
-    return nullptr;
+    return make_unique<StickerSet>().get();
   }
 
   return sticker_set->second.get();
@@ -1649,9 +1663,11 @@ StickerSetId StickersManager::get_sticker_set_id(const tl_object_ptr<telegram_ap
       return search_sticker_set(static_cast<const telegram_api::inputStickerSetShortName *>(set_ptr.get())->short_name_,
                                 Auto());
     case telegram_api::inputStickerSetAnimatedEmoji::ID:
-    case telegram_api::inputStickerSetDice::ID:
       LOG(ERROR) << "Receive special sticker set " << to_string(set_ptr);
       return add_special_sticker_set(SpecialStickerSetType(set_ptr).type_).id_;
+    case telegram_api::inputStickerSetDice::ID:
+      LOG(ERROR) << "Receive special sticker set " << to_string(set_ptr);
+      return StickerSetId();
     default:
       UNREACHABLE();
       return StickerSetId();
@@ -1675,9 +1691,11 @@ StickerSetId StickersManager::add_sticker_set(tl_object_ptr<telegram_api::InputS
       return search_sticker_set(set->short_name_, Auto());
     }
     case telegram_api::inputStickerSetAnimatedEmoji::ID:
-    case telegram_api::inputStickerSetDice::ID:
       LOG(ERROR) << "Receive special sticker set " << to_string(set_ptr);
       return add_special_sticker_set(SpecialStickerSetType(set_ptr).type_).id_;
+    case telegram_api::inputStickerSetDice::ID:
+      LOG(ERROR) << "Receive special sticker set " << to_string(set_ptr);
+      return StickerSetId();
     default:
       UNREACHABLE();
       return StickerSetId();
@@ -1718,7 +1736,11 @@ void StickersManager::delete_sticker_thumbnail(FileId file_id) {
 vector<FileId> StickersManager::get_sticker_file_ids(FileId file_id) const {
   vector<FileId> result;
   auto sticker = get_sticker(file_id);
-  CHECK(sticker != nullptr);
+
+  if (sticker == nullptr) {
+    return result;
+  }
+
   result.push_back(file_id);
   if (sticker->s_thumbnail.file_id.is_valid()) {
     result.push_back(sticker->s_thumbnail.file_id);
@@ -1755,7 +1777,7 @@ bool StickersManager::merge_stickers(FileId new_id, FileId old_id, bool can_dele
   }
 
   auto new_it = stickers_.find(new_id);
-  if (new_it == stickers_.end()) {
+  if (new_it == stickers_.end() || new_it->second == nullptr) {
     auto &old = stickers_[old_id];
     old->is_changed = true;
     if (!can_delete_old) {
@@ -1768,9 +1790,9 @@ bool StickersManager::merge_stickers(FileId new_id, FileId old_id, bool can_dele
     Sticker *new_ = new_it->second.get();
     CHECK(new_ != nullptr);
 
-    if (old_->alt != new_->alt || old_->set_id != new_->set_id ||
-        (!old_->is_animated && !new_->is_animated && old_->dimensions.width != 0 && old_->dimensions.height != 0 &&
-         old_->dimensions != new_->dimensions)) {
+    if (old_->set_id == new_->set_id && (old_->alt != new_->alt || old_->set_id != new_->set_id ||
+                                         (!old_->is_animated && !new_->is_animated && old_->dimensions.width != 0 &&
+                                          old_->dimensions.height != 0 && old_->dimensions != new_->dimensions))) {
       LOG(ERROR) << "Sticker has changed: alt = (" << old_->alt << ", " << new_->alt << "), set_id = (" << old_->set_id
                  << ", " << new_->set_id << "), dimensions = (" << old_->dimensions << ", " << new_->dimensions << ")";
     }
@@ -1862,11 +1884,14 @@ StickerSetId StickersManager::on_get_input_sticker_set(FileId sticker_file_id,
               }
             }));
       }
-      return set_id;
+      // always return empty StickerSetId, because we can't trust the set_id provided by the peer in the secret chat
+      // the real sticker set id will be set in on_get_sticker if and only if the sticker is really from the set
+      return StickerSetId();
     }
     case telegram_api::inputStickerSetAnimatedEmoji::ID:
-    case telegram_api::inputStickerSetDice::ID:
       return add_special_sticker_set(SpecialStickerSetType(set_ptr).type_).id_;
+    case telegram_api::inputStickerSetDice::ID:
+      return StickerSetId();
     default:
       UNREACHABLE();
       return StickerSetId();
@@ -1930,13 +1955,19 @@ void StickersManager::create_sticker(FileId file_id, PhotoSize thumbnail, Dimens
       }
     }
   }
-  s->is_animated = is_animated;
+  if (s->set_id.is_valid()) {
+    s->is_animated = is_animated;
+  }
   on_get_sticker(std::move(s), sticker != nullptr);
 }
 
 bool StickersManager::has_input_media(FileId sticker_file_id, bool is_secret) const {
   const Sticker *sticker = get_sticker(sticker_file_id);
-  CHECK(sticker != nullptr);
+
+  if (sticker == nullptr) {
+    return false;
+  }
+
   auto file_view = td_->file_manager_->get_file_view(sticker_file_id);
   if (is_secret) {
     if (file_view.is_encrypted_secret()) {
@@ -1965,7 +1996,11 @@ SecretInputMedia StickersManager::get_secret_input_media(FileId sticker_file_id,
                                                          tl_object_ptr<telegram_api::InputEncryptedFile> input_file,
                                                          BufferSlice thumbnail) const {
   const Sticker *sticker = get_sticker(sticker_file_id);
-  CHECK(sticker != nullptr);
+
+  if (sticker == nullptr) {
+    return {};
+  }
+
   auto file_view = td_->file_manager_->get_file_view(sticker_file_id);
   if (file_view.is_encrypted_secret()) {
     if (file_view.has_remote_location()) {
@@ -2084,7 +2119,8 @@ StickerSetId StickersManager::on_get_sticker_set(tl_object_ptr<telegram_api::sti
   PhotoSize thumbnail;
   if (set->thumb_ != nullptr) {
     auto photo_size = get_photo_size(td_->file_manager_.get(), {set_id.get(), s->access_hash}, 0, 0, "",
-                                     DcId::create(set->thumb_dc_id_), DialogId(), std::move(set->thumb_), true, false);
+                                     DcId::create(set->thumb_dc_id_), DialogId(), std::move(set->thumb_),
+                                     is_animated ? PhotoFormat::Tgs : PhotoFormat::Webp);
     if (photo_size.get_offset() == 0) {
       thumbnail = std::move(photo_size.get<0>());
     } else {
@@ -3058,6 +3094,9 @@ void StickersManager::load_installed_sticker_sets(bool is_masks, Promise<Unit> &
 }
 
 void StickersManager::on_load_installed_sticker_sets_from_database(bool is_masks, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (value.empty()) {
     LOG(INFO) << "Installed " << (is_masks ? "mask " : "") << "sticker sets aren't found in database";
     reload_installed_sticker_sets(is_masks, true);
@@ -3169,7 +3208,7 @@ string StickersManager::get_sticker_set_database_value(const StickerSet *s, bool
 void StickersManager::update_sticker_set(StickerSet *sticker_set) {
   CHECK(sticker_set != nullptr);
   if (sticker_set->is_changed || sticker_set->need_save_to_database) {
-    if (G()->parameters().use_file_db) {
+    if (G()->parameters().use_file_db && !G()->close_flag()) {
       LOG(INFO) << "Save " << sticker_set->id << " to database";
       if (sticker_set->is_inited) {
         G()->td_db()->get_sqlite_pmc()->set(get_sticker_set_database_key(sticker_set->id),
@@ -3264,6 +3303,9 @@ void StickersManager::load_sticker_sets_without_stickers(vector<StickerSetId> &&
 }
 
 void StickersManager::on_load_sticker_set_from_database(StickerSetId sticker_set_id, bool with_stickers, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
   StickerSet *sticker_set = get_sticker_set(sticker_set_id);
   CHECK(sticker_set != nullptr);
   if (sticker_set->was_loaded) {
@@ -3387,12 +3429,22 @@ void StickersManager::on_update_dice_emojis() {
     return;
   }
 
-  auto dice_emojis_str = G()->shared_config().get_option_string("dice_emojis", "🎲\x01🎯");
+  auto dice_emojis_str = G()->shared_config().get_option_string("dice_emojis", "🎲\x01🎯\x01🏀");
   if (dice_emojis_str == dice_emojis_str_) {
     return;
   }
   dice_emojis_str_ = std::move(dice_emojis_str);
-  dice_emojis_ = full_split(dice_emojis_str_, '\x01');
+  auto new_dice_emojis = full_split(dice_emojis_str_, '\x01');
+  for (auto &emoji : new_dice_emojis) {
+    if (!td::contains(dice_emojis_, emoji)) {
+      auto &special_sticker_set = add_special_sticker_set(SpecialStickerSetType::animated_dice(emoji));
+      CHECK(!special_sticker_set.id_.is_valid());
+
+      LOG(INFO) << "Load new dice sticker set for emoji " << emoji;
+      load_special_sticker_set(special_sticker_set);
+    }
+  }
+  dice_emojis_ = std::move(new_dice_emojis);
 
   send_closure(G()->td(), &Td::send_update, get_update_dice_emojis_object());
 }
@@ -3406,7 +3458,7 @@ void StickersManager::on_update_dice_success_values() {
     return;
   }
 
-  auto dice_success_values_str = G()->shared_config().get_option_string("dice_success_values", "0,0");
+  auto dice_success_values_str = G()->shared_config().get_option_string("dice_success_values", "0,0,0");
   if (dice_success_values_str == dice_success_values_str_) {
     return;
   }
@@ -3440,7 +3492,12 @@ void StickersManager::register_dice(const string &emoji, int32 value, FullMessag
   bool is_inserted = dice_messages_[emoji].insert(full_message_id).second;
   LOG_CHECK(is_inserted) << source << " " << emoji << " " << value << " " << full_message_id;
 
-  if (!td::contains(dice_emojis_, emoji) && !full_message_id.get_message_id().is_any_server()) {
+  if (!td::contains(dice_emojis_, emoji)) {
+    if (full_message_id.get_message_id().is_any_server() &&
+        full_message_id.get_dialog_id().get_type() != DialogType::SecretChat) {
+      send_closure(G()->config_manager(), &ConfigManager::get_app_config,
+                   Promise<td_api::object_ptr<td_api::JsonValue>>());
+    }
     return;
   }
 
@@ -3475,8 +3532,8 @@ void StickersManager::unregister_dice(const string &emoji, int32 value, FullMess
   LOG(INFO) << "Unregister dice " << emoji << " with value " << value << " from " << full_message_id << " from "
             << source;
   auto &message_ids = dice_messages_[emoji];
-  auto is_deleted = message_ids.erase(full_message_id);
-  LOG_CHECK(is_deleted) << source << " " << emoji << " " << value << " " << full_message_id;
+  message_ids.erase(full_message_id);
+  LOG(INFO) << source << " " << emoji << " " << value << " " << full_message_id;
 
   if (message_ids.empty()) {
     dice_messages_.erase(emoji);
@@ -3673,6 +3730,10 @@ void StickersManager::on_old_featured_sticker_sets_invalidated() {
 }
 
 void StickersManager::invalidate_old_featured_sticker_sets() {
+  if (G()->close_flag()) {
+    return;
+  }
+
   LOG(INFO) << "Invalidate old featured sticker sets";
   if (G()->parameters().use_file_db) {
     G()->td_db()->get_binlog_pmc()->erase("invalidate_old_featured_sticker_sets");
@@ -3777,7 +3838,7 @@ void StickersManager::on_get_featured_sticker_sets(
 
   if (offset >= 0) {
     if (generation == old_featured_sticker_set_generation_) {
-      if (G()->parameters().use_file_db) {
+      if (G()->parameters().use_file_db && !G()->close_flag()) {
         LOG(INFO) << "Save old trending sticker sets to database with offset " << old_featured_sticker_set_ids_.size();
         CHECK(old_featured_sticker_set_ids_.size() % OLD_FEATURED_STICKER_SET_SLICE_SIZE == 0);
         StickerSetListLogEvent log_event(featured_sticker_set_ids);
@@ -3795,7 +3856,7 @@ void StickersManager::on_get_featured_sticker_sets(
 
   LOG_IF(ERROR, featured_sticker_sets_hash_ != featured_stickers->hash_) << "Trending sticker sets hash mismatch";
 
-  if (!G()->parameters().use_file_db) {
+  if (!G()->parameters().use_file_db || G()->close_flag()) {
     return;
   }
 
@@ -3850,6 +3911,9 @@ void StickersManager::load_featured_sticker_sets(Promise<Unit> &&promise) {
 }
 
 void StickersManager::on_load_featured_sticker_sets_from_database(string value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (value.empty()) {
     LOG(INFO) << "Trending sticker sets aren't found in database";
     reload_featured_sticker_sets(true);
@@ -3926,6 +3990,9 @@ void StickersManager::load_old_featured_sticker_sets(Promise<Unit> &&promise) {
 }
 
 void StickersManager::on_load_old_featured_sticker_sets_from_database(uint32 generation, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (generation != old_featured_sticker_set_generation_) {
     return;
   }
@@ -4131,7 +4198,7 @@ string &StickersManager::get_input_sticker_emojis(td_api::InputSticker *sticker)
 
 Result<std::tuple<FileId, bool, bool, bool>> StickersManager::prepare_input_sticker(td_api::InputSticker *sticker) {
   if (sticker == nullptr) {
-    return Status::Error(3, "Input sticker must not be empty");
+    return Status::Error(3, "Input sticker must be non-empty");
   }
 
   if (!clean_input_string(get_input_sticker_emojis(sticker))) {
@@ -4825,7 +4892,7 @@ void StickersManager::send_update_installed_sticker_sets(bool from_database) {
         installed_sticker_sets_hash_[is_masks] = get_sticker_sets_hash(installed_sticker_set_ids_[is_masks]);
         send_closure(G()->td(), &Td::send_update, get_update_installed_sticker_sets_object(is_masks));
 
-        if (G()->parameters().use_file_db && !from_database) {
+        if (G()->parameters().use_file_db && !from_database && !G()->close_flag()) {
           LOG(INFO) << "Save installed " << (is_masks ? "mask " : "") << "sticker sets to database";
           StickerSetListLogEvent log_event(installed_sticker_set_ids_[is_masks]);
           G()->td_db()->get_sqlite_pmc()->set(is_masks ? "sss1" : "sss0", log_event_store(log_event).as_slice().str(),
@@ -4912,6 +4979,9 @@ void StickersManager::load_recent_stickers(bool is_attached, Promise<Unit> &&pro
 }
 
 void StickersManager::on_load_recent_stickers_from_database(bool is_attached, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (value.empty()) {
     LOG(INFO) << "Recent " << (is_attached ? "attached " : "") << "stickers aren't found in database";
     reload_recent_stickers(is_attached, true);
@@ -5008,7 +5078,9 @@ int32 StickersManager::get_recent_stickers_hash(const vector<FileId> &sticker_id
   numbers.reserve(sticker_ids.size() * 2);
   for (auto sticker_id : sticker_ids) {
     auto sticker = get_sticker(sticker_id);
-    CHECK(sticker != nullptr);
+    if (sticker == nullptr) {
+      continue;
+    }
     auto file_view = td_->file_manager_->get_file_view(sticker_id);
     CHECK(file_view.has_remote_location());
     if (!file_view.remote_location().is_document()) {
@@ -5230,7 +5302,7 @@ void StickersManager::send_update_recent_stickers(bool from_database) {
 }
 
 void StickersManager::save_recent_stickers_to_database(bool is_attached) {
-  if (G()->parameters().use_file_db) {
+  if (G()->parameters().use_file_db && !G()->close_flag()) {
     LOG(INFO) << "Save recent " << (is_attached ? "attached " : "") << "stickers to database";
     StickerListLogEvent log_event(recent_sticker_ids_[is_attached]);
     G()->td_db()->get_sqlite_pmc()->set(is_attached ? "ssr1" : "ssr0", log_event_store(log_event).as_slice().str(),
@@ -5330,6 +5402,9 @@ void StickersManager::load_favorite_stickers(Promise<Unit> &&promise) {
 }
 
 void StickersManager::on_load_favorite_stickers_from_database(const string &value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (value.empty()) {
     LOG(INFO) << "Favorite stickers aren't found in database";
     reload_favorite_stickers(true);
@@ -5594,7 +5669,7 @@ void StickersManager::send_update_favorite_stickers(bool from_database) {
 }
 
 void StickersManager::save_favorite_stickers_to_database() {
-  if (G()->parameters().use_file_db) {
+  if (G()->parameters().use_file_db && !G()->close_flag()) {
     LOG(INFO) << "Save favorite stickers to database";
     StickerListLogEvent log_event(favorite_sticker_ids_);
     G()->td_db()->get_sqlite_pmc()->set("ssfav", log_event_store(log_event).as_slice().str(), Auto());
@@ -5764,7 +5839,10 @@ void StickersManager::on_get_language_codes(const string &key, Result<vector<str
   CHECK(it != emoji_language_codes_.end());
   if (it->second != language_codes) {
     LOG(INFO) << "Update emoji language codes for " << key << " to " << language_codes;
-    G()->td_db()->get_sqlite_pmc()->set(key, implode(language_codes, '$'), Auto());
+    if (!G()->close_flag()) {
+      CHECK(G()->parameters().use_file_db);
+      G()->td_db()->get_sqlite_pmc()->set(key, implode(language_codes, '$'), Auto());
+    }
     it->second = std::move(language_codes);
   }
 
@@ -5898,7 +5976,8 @@ void StickersManager::on_get_emoji_keywords(
             is_good = false;
           }
         }
-        if (is_good) {
+        if (is_good && !G()->close_flag()) {
+          CHECK(G()->parameters().use_file_db);
           G()->td_db()->get_sqlite_pmc()->set(get_language_emojis_database_key(language_code, text),
                                               implode(keyword->emoticons_, '$'), mpas.get_promise());
         }
@@ -5911,10 +5990,13 @@ void StickersManager::on_get_emoji_keywords(
         UNREACHABLE();
     }
   }
-  G()->td_db()->get_sqlite_pmc()->set(get_emoji_language_code_version_database_key(language_code), to_string(version),
-                                      mpas.get_promise());
-  G()->td_db()->get_sqlite_pmc()->set(get_emoji_language_code_last_difference_time_database_key(language_code),
-                                      to_string(G()->unix_time()), mpas.get_promise());
+  if (!G()->close_flag()) {
+    CHECK(G()->parameters().use_file_db);
+    G()->td_db()->get_sqlite_pmc()->set(get_emoji_language_code_version_database_key(language_code), to_string(version),
+                                        mpas.get_promise());
+    G()->td_db()->get_sqlite_pmc()->set(get_emoji_language_code_last_difference_time_database_key(language_code),
+                                        to_string(G()->unix_time()), mpas.get_promise());
+  }
   emoji_language_code_versions_[language_code] = version;
   emoji_language_code_last_difference_times_[language_code] = static_cast<int32>(Time::now_cached());
 
@@ -6181,4 +6263,33 @@ void StickersManager::get_current_state(vector<td_api::object_ptr<td_api::Update
   }
 }
 
+void StickersManager::memory_cleanup() {
+    stickers_.clear();
+    stickers_.rehash(0);
+    sticker_sets_.clear();
+    sticker_sets_.rehash(0);
+    short_name_to_sticker_set_id_.clear();
+    short_name_to_sticker_set_id_.rehash(0);
+    attached_sticker_sets_.clear();
+    attached_sticker_sets_.rehash(0);
+    found_stickers_.clear();
+    found_stickers_.rehash(0);
+    found_sticker_sets_.clear();
+    found_sticker_sets_.rehash(0);
+    special_sticker_sets_.clear();
+    special_sticker_sets_.rehash(0);
+    sticker_set_load_requests_.clear();
+    sticker_set_load_requests_.rehash(0);
+    emoji_language_codes_.clear();
+    emoji_language_codes_.rehash(0);
+    emoji_language_code_versions_.clear();
+    emoji_language_code_versions_.rehash(0);
+    emoji_language_code_last_difference_times_.clear();
+    emoji_language_code_last_difference_times_.rehash(0);
+    emoji_suggestions_urls_.clear();
+    emoji_suggestions_urls_.rehash(0);
+    dice_messages_.clear();
+    dice_messages_.rehash(0);
+}
+
 }  // namespace td

td/telegram/StickersManager.h

@@ -42,6 +42,8 @@ class StickersManager : public Actor {
  public:
   static constexpr int64 GREAT_MINDS_SET_ID = 1842540969984001;
 
+  void memory_cleanup();
+
   static vector<StickerSetId> convert_sticker_set_ids(const vector<int64> &sticker_set_ids);
   static vector<int64> convert_sticker_set_ids(const vector<StickerSetId> &sticker_set_ids);
 

td/telegram/StickersManager.hpp

@@ -22,7 +22,9 @@ namespace td {
 template <class StorerT>
 void StickersManager::store_sticker(FileId file_id, bool in_sticker_set, StorerT &storer) const {
   auto it = stickers_.find(file_id);
-  CHECK(it != stickers_.end());
+  if (it == stickers_.end() || it->second == nullptr) {
+    return;
+  }
   const Sticker *sticker = it->second.get();
   bool has_sticker_set_access_hash = sticker->set_id.is_valid() && !in_sticker_set;
   BEGIN_STORE_FLAGS();
@@ -35,8 +37,9 @@ void StickersManager::store_sticker(FileId file_id, bool in_sticker_set, StorerT
     store(sticker->set_id.get(), storer);
     if (has_sticker_set_access_hash) {
       auto sticker_set = get_sticker_set(sticker->set_id);
-      CHECK(sticker_set != nullptr);
-      store(sticker_set->access_hash, storer);
+      if (sticker_set != nullptr) {
+        store(sticker_set->access_hash, storer);
+      }
     }
   }
   store(sticker->alt, storer);
@@ -167,7 +170,9 @@ void StickersManager::store_sticker_set(const StickerSet *sticker_set, bool with
 
 template <class ParserT>
 void StickersManager::parse_sticker_set(StickerSet *sticker_set, ParserT &parser) {
-  CHECK(sticker_set != nullptr);
+  if (sticker_set == nullptr) {
+      return;
+  }
   CHECK(!sticker_set->was_loaded);
   bool was_inited = sticker_set->is_inited;
   bool is_installed;
@@ -268,7 +273,9 @@ void StickersManager::parse_sticker_set(StickerSet *sticker_set, ParserT &parser
       sticker_set->sticker_ids.push_back(sticker_id);
 
       Sticker *sticker = get_sticker(sticker_id);
-      CHECK(sticker != nullptr);
+      if (sticker == nullptr) {
+          return;
+      }
       if (sticker->set_id != sticker_set->id) {
         LOG_IF(ERROR, sticker->set_id.is_valid()) << "Sticker " << sticker_id << " set_id has changed";
         sticker->set_id = sticker_set->id;
@@ -297,7 +304,9 @@ template <class StorerT>
 void StickersManager::store_sticker_set_id(StickerSetId sticker_set_id, StorerT &storer) const {
   CHECK(sticker_set_id.is_valid());
   const StickerSet *sticker_set = get_sticker_set(sticker_set_id);
-  CHECK(sticker_set != nullptr);
+  if (sticker_set == nullptr) {
+      return;
+  }
   store(sticker_set_id.get(), storer);
   store(sticker_set->access_hash, storer);
 }

td/telegram/Td.cpp

@@ -99,11 +99,11 @@
 
 #include "td/db/binlog/BinlogEvent.h"
 
-#include "td/mtproto/crypto.h"
 #include "td/mtproto/DhHandshake.h"
 #include "td/mtproto/Handshake.h"
 #include "td/mtproto/HandshakeActor.h"
 #include "td/mtproto/RawConnection.h"
+#include "td/mtproto/RSA.h"
 #include "td/mtproto/TransportType.h"
 
 #include "td/utils/buffer.h"
@@ -587,10 +587,10 @@ class TestProxyRequest : public RequestOnceActor {
     }
 
     auto dc_options = ConnectionCreator::get_default_dc_options(false);
-    IPAddress mtproto_ip;
+    IPAddress mtproto_ip_address;
     for (auto &dc_option : dc_options.dc_options) {
       if (dc_option.get_dc_id().get_raw_id() == dc_id_) {
-        mtproto_ip = dc_option.get_ip_address();
+        mtproto_ip_address = dc_option.get_ip_address();
         break;
       }
     }
@@ -601,8 +601,8 @@ class TestProxyRequest : public RequestOnceActor {
         });
 
     child_ =
-        ConnectionCreator::prepare_connection(r_socket_fd.move_as_ok(), proxy_, mtproto_ip, get_transport(), "Test",
-                                              "TestPingDC2", nullptr, {}, false, std::move(connection_promise));
+        ConnectionCreator::prepare_connection(r_socket_fd.move_as_ok(), proxy_, mtproto_ip_address, get_transport(),
+                                              "Test", "TestPingDC2", nullptr, {}, false, std::move(connection_promise));
   }
 
   void on_connection_data(Result<ConnectionCreator::ConnectionData> r_data) {
@@ -4313,6 +4313,10 @@ void Td::init_file_manager() {
       send_closure(G()->storage_manager(), &StorageManager::on_new_file, size, real_size, cnt);
     }
 
+    void destroy_file_source(FileId file_id) final {
+      td_->file_reference_manager_->memory_cleanup(file_id);
+    }
+
     void on_file_updated(FileId file_id) final {
       send_closure(G()->td(), &Td::send_update,
                    make_tl_object<td_api::updateFile>(td_->file_manager_->get_file_object(file_id)));
@@ -5119,6 +5123,18 @@ void Td::on_request(uint64 id, td_api::getDatabaseStatistics &request) {
 }
 
 void Td::on_request(uint64 id, td_api::optimizeStorage &request) {
+  contacts_manager_->memory_cleanup();
+  web_pages_manager_->memory_cleanup();
+  stickers_manager_->memory_cleanup();
+  documents_manager_->memory_cleanup();
+  video_notes_manager_->memory_cleanup();
+  videos_manager_->memory_cleanup();
+  audios_manager_->memory_cleanup();
+  animations_manager_->memory_cleanup();
+  file_manager_->memory_cleanup();
+
+  malloc_trim(0);
+
   std::vector<FileType> file_types;
   for (auto &file_type : request.file_types_) {
     if (file_type == nullptr) {
@@ -5788,7 +5804,7 @@ void Td::on_request(uint64 id, td_api::createCall &request) {
   });
 
   if (!request.protocol_) {
-    return query_promise.set_error(Status::Error(5, "Call protocol must not be empty"));
+    return query_promise.set_error(Status::Error(5, "Call protocol must be non-empty"));
   }
 
   UserId user_id(request.user_id_);
@@ -5816,7 +5832,7 @@ void Td::on_request(uint64 id, td_api::acceptCall &request) {
   CHECK_IS_USER();
   CREATE_OK_REQUEST_PROMISE();
   if (!request.protocol_) {
-    return promise.set_error(Status::Error(5, "Call protocol must not be empty"));
+    return promise.set_error(Status::Error(5, "Call protocol must be non-empty"));
   }
   send_closure(G()->call_manager(), &CallManager::accept_call, CallId(request.call_id_),
                CallProtocol::from_td_api(*request.protocol_), std::move(promise));
@@ -6222,7 +6238,7 @@ void Td::on_request(uint64 id, const td_api::getBlockedUsers &request) {
 void Td::on_request(uint64 id, td_api::addContact &request) {
   CHECK_IS_USER();
   if (request.contact_ == nullptr) {
-    return send_error_raw(id, 5, "Contact must not be empty");
+    return send_error_raw(id, 5, "Contact must be non-empty");
   }
   CLEAN_INPUT_STRING(request.contact_->phone_number_);
   CLEAN_INPUT_STRING(request.contact_->first_name_);
@@ -6235,7 +6251,7 @@ void Td::on_request(uint64 id, td_api::importContacts &request) {
   CHECK_IS_USER();
   for (auto &contact : request.contacts_) {
     if (contact == nullptr) {
-      return send_error_raw(id, 5, "Contact must not be empty");
+      return send_error_raw(id, 5, "Contact must be non-empty");
     }
     CLEAN_INPUT_STRING(contact->phone_number_);
     CLEAN_INPUT_STRING(contact->first_name_);
@@ -6269,7 +6285,7 @@ void Td::on_request(uint64 id, td_api::changeImportedContacts &request) {
   CHECK_IS_USER();
   for (auto &contact : request.contacts_) {
     if (contact == nullptr) {
-      return send_error_raw(id, 5, "Contact must not be empty");
+      return send_error_raw(id, 5, "Contact must be non-empty");
     }
     CLEAN_INPUT_STRING(contact->phone_number_);
     CLEAN_INPUT_STRING(contact->first_name_);
@@ -6588,7 +6604,7 @@ void Td::on_request(uint64 id, const td_api::getChatNotificationSettingsExceptio
 void Td::on_request(uint64 id, const td_api::getScopeNotificationSettings &request) {
   CHECK_IS_USER();
   if (request.scope_ == nullptr) {
-    return send_error_raw(id, 400, "Scope must not be empty");
+    return send_error_raw(id, 400, "Scope must be non-empty");
   }
   CREATE_REQUEST(GetScopeNotificationSettingsRequest, get_notification_settings_scope(request.scope_));
 }
@@ -6636,7 +6652,7 @@ void Td::on_request(uint64 id, td_api::setChatNotificationSettings &request) {
 void Td::on_request(uint64 id, td_api::setScopeNotificationSettings &request) {
   CHECK_IS_USER();
   if (request.scope_ == nullptr) {
-    return send_error_raw(id, 400, "Scope must not be empty");
+    return send_error_raw(id, 400, "Scope must be non-empty");
   }
   answer_ok_query(id, messages_manager_->set_scope_notification_settings(
                           get_notification_settings_scope(request.scope_), std::move(request.notification_settings_)));
@@ -6901,8 +6917,19 @@ void Td::on_request(uint64 id, td_api::setOption &request) {
         return;
       }
       if (!is_bot && set_boolean_option("disable_top_chats")) {
-        return;
+	        return;
       }
+      // Start custom-patches
+      if (set_boolean_option("disable_document_filenames")) {
+	        return;
+      }
+      if (set_boolean_option("disable_minithumbnails")) {
+	        return;
+      }
+      if (set_boolean_option("disable_notifications")) {
+	        return;
+      }
+      // End custom-patches
       if (request.name_ == "drop_notification_ids") {
         G()->td_db()->get_binlog_pmc()->erase("notification_id_current");
         G()->td_db()->get_binlog_pmc()->erase("notification_group_id_current");
@@ -7180,7 +7207,7 @@ void Td::on_request(uint64 id, td_api::sendPaymentForm &request) {
   CLEAN_INPUT_STRING(request.order_info_id_);
   CLEAN_INPUT_STRING(request.shipping_option_id_);
   if (request.credentials_ == nullptr) {
-    return send_error_raw(id, 400, "Input payments credentials must not be empty");
+    return send_error_raw(id, 400, "Input payments credentials must be non-empty");
   }
   CREATE_REQUEST_PROMISE();
   messages_manager_->send_payment_form({DialogId(request.chat_id_), MessageId(request.message_id_)},
@@ -7229,7 +7256,7 @@ void Td::on_request(uint64 id, td_api::getPassportElement &request) {
   CHECK_IS_USER();
   CLEAN_INPUT_STRING(request.password_);
   if (request.type_ == nullptr) {
-    return send_error_raw(id, 400, "Type must not be empty");
+    return send_error_raw(id, 400, "Type must be non-empty");
   }
   CREATE_REQUEST_PROMISE();
   send_closure(secure_manager_, &SecureManager::get_secure_value, std::move(request.password_),
@@ -7259,7 +7286,7 @@ void Td::on_request(uint64 id, td_api::setPassportElement &request) {
 void Td::on_request(uint64 id, const td_api::deletePassportElement &request) {
   CHECK_IS_USER();
   if (request.type_ == nullptr) {
-    return send_error_raw(id, 400, "Type must not be empty");
+    return send_error_raw(id, 400, "Type must be non-empty");
   }
   CREATE_OK_REQUEST_PROMISE();
   send_closure(secure_manager_, &SecureManager::delete_secure_value, get_secure_value_type_td_api(request.type_),
@@ -7355,7 +7382,7 @@ void Td::on_request(uint64 id, td_api::sendPassportAuthorizationForm &request) {
   CHECK_IS_USER();
   for (auto &type : request.types_) {
     if (type == nullptr) {
-      return send_error_raw(id, 400, "Type must not be empty");
+      return send_error_raw(id, 400, "Type must be non-empty");
     }
   }
 

td/telegram/Td.h

@@ -30,6 +30,7 @@
 #include "td/utils/Slice.h"
 #include "td/utils/Status.h"
 
+#include <malloc.h>
 #include <memory>
 #include <unordered_map>
 #include <unordered_set>

td/telegram/TdDb.cpp

@@ -137,9 +137,13 @@ Status init_db(SqliteDb &db) {
   TRY_STATUS(db.exec("PRAGMA encoding=\"UTF-8\""));
   TRY_STATUS(db.exec("PRAGMA journal_mode=WAL"));
 
+  TRY_STATUS(db.exec("PRAGMA cache_size=4096"));
+  TRY_STATUS(db.exec("PRAGMA page_size=65536"));
   TRY_STATUS(db.exec("PRAGMA synchronous=NORMAL"));
-  TRY_STATUS(db.exec("PRAGMA temp_store=MEMORY"));
-  TRY_STATUS(db.exec("PRAGMA secure_delete=1"));
+  TRY_STATUS(db.exec("PRAGMA temp_store=FILE"));
+  TRY_STATUS(db.exec("PRAGMA wal_autocheckpoint=10000"));
+  TRY_STATUS(db.exec("PRAGMA shrink_memory"));
+  TRY_STATUS(db.exec("PRAGMA secure_delete=0"));
 
   return Status::OK();
 }
@@ -316,17 +320,16 @@ Status TdDb::init_sqlite(int32 scheduler_id, const TdParameters &parameters, DbK
 
   // init DialogDb
   bool dialog_db_was_created = false;
+
   if (use_dialog_db) {
     TRY_STATUS(init_dialog_db(db, user_version, dialog_db_was_created));
-  } else {
-    TRY_STATUS(drop_dialog_db(db, user_version));
   }
 
   // init MessagesDb
+  TRY_STATUS(drop_messages_db(db, user_version));
+
   if (use_message_db) {
     TRY_STATUS(init_messages_db(db, user_version));
-  } else {
-    TRY_STATUS(drop_messages_db(db, user_version));
   }
 
   // init filesDb
@@ -356,6 +359,7 @@ Status TdDb::init_sqlite(int32 scheduler_id, const TdParameters &parameters, DbK
   binlog_pmc.force_sync({});
 
   TRY_STATUS(db.exec("COMMIT TRANSACTION"));
+  TRY_STATUS(db.exec("VACUUM"));
 
   file_db_ = create_file_db(sql_connection_, scheduler_id);
 
@@ -512,8 +516,6 @@ Result<string> TdDb::get_stats() {
                               << mask << "'",
                      PSLICE() << table << ":" << mask);
   };
-  TRY_STATUS(run_query("SELECT 0, SUM(length(data)), COUNT(*) FROM messages WHERE 1", "messages"));
-  TRY_STATUS(run_query("SELECT 0, SUM(length(data)), COUNT(*) FROM dialogs WHERE 1", "dialogs"));
   TRY_STATUS(run_kv_query("%", "common"));
   TRY_STATUS(run_kv_query("%", "files"));
   TRY_STATUS(run_kv_query("wp%"));

td/telegram/VideoNotesManager.cpp

@@ -25,7 +25,9 @@ VideoNotesManager::VideoNotesManager(Td *td) : td_(td) {
 
 int32 VideoNotesManager::get_video_note_duration(FileId file_id) const {
   auto it = video_notes_.find(file_id);
-  CHECK(it != video_notes_.end());
+  if (it == video_notes_.end() || it->second == nullptr) {
+      return 0;
+  }
   return it->second->duration;
 }
 
@@ -35,7 +37,9 @@ tl_object_ptr<td_api::videoNote> VideoNotesManager::get_video_note_object(FileId
   }
 
   auto &video_note = video_notes_[file_id];
-  CHECK(video_note != nullptr);
+  if (video_note == nullptr) {
+      return nullptr;
+  }
   video_note->is_changed = false;
 
   return make_tl_object<td_api::videoNote>(video_note->duration, video_note->dimensions.width,
@@ -79,29 +83,37 @@ FileId VideoNotesManager::on_get_video_note(unique_ptr<VideoNote> new_video_note
 
 const VideoNotesManager::VideoNote *VideoNotesManager::get_video_note(FileId file_id) const {
   auto video_note = video_notes_.find(file_id);
-  if (video_note == video_notes_.end()) {
-    return nullptr;
+
+  if (video_note == video_notes_.end() ||
+      video_note->second == nullptr ||
+      video_note->second->file_id != file_id) {
+    return make_unique<VideoNote>().get();
   }
 
-  CHECK(video_note->second->file_id == file_id);
   return video_note->second.get();
 }
 
 FileId VideoNotesManager::get_video_note_thumbnail_file_id(FileId file_id) const {
   auto video_note = get_video_note(file_id);
-  CHECK(video_note != nullptr);
+  if (video_note == nullptr) {
+      return FileId();
+  }
   return video_note->thumbnail.file_id;
 }
 
 void VideoNotesManager::delete_video_note_thumbnail(FileId file_id) {
   auto &video_note = video_notes_[file_id];
-  CHECK(video_note != nullptr);
+  if (video_note == nullptr) {
+      return;
+  }
   video_note->thumbnail = PhotoSize();
 }
 
 FileId VideoNotesManager::dup_video_note(FileId new_id, FileId old_id) {
   const VideoNote *old_video_note = get_video_note(old_id);
-  CHECK(old_video_note != nullptr);
+  if (old_video_note == nullptr) {
+      return FileId();
+  }
   auto &new_video_note = video_notes_[new_id];
   CHECK(!new_video_note);
   new_video_note = make_unique<VideoNote>(*old_video_note);
@@ -124,7 +136,7 @@ bool VideoNotesManager::merge_video_notes(FileId new_id, FileId old_id, bool can
   }
 
   auto new_it = video_notes_.find(new_id);
-  if (new_it == video_notes_.end()) {
+  if (new_it == video_notes_.end() || new_it->second == nullptr) {
     auto &old = video_notes_[old_id];
     old->is_changed = true;
     if (!can_delete_old) {
@@ -211,7 +223,9 @@ tl_object_ptr<telegram_api::InputMedia> VideoNotesManager::get_input_media(
 
   if (input_file != nullptr) {
     const VideoNote *video_note = get_video_note(file_id);
-    CHECK(video_note != nullptr);
+    if (video_note == nullptr) {
+        return nullptr;
+    }
 
     vector<tl_object_ptr<telegram_api::DocumentAttribute>> attributes;
     attributes.push_back(make_tl_object<telegram_api::documentAttributeVideo>(
@@ -232,4 +246,9 @@ tl_object_ptr<telegram_api::InputMedia> VideoNotesManager::get_input_media(
   return nullptr;
 }
 
+void VideoNotesManager::memory_cleanup() {
+  video_notes_.clear();
+  video_notes_.rehash(0);
+}
+
 }  // namespace td

td/telegram/VideoNotesManager.h

@@ -26,6 +26,8 @@ class VideoNotesManager {
  public:
   explicit VideoNotesManager(Td *td);
 
+  void memory_cleanup();
+
   int32 get_video_note_duration(FileId file_id) const;
 
   tl_object_ptr<td_api::videoNote> get_video_note_object(FileId file_id);

td/telegram/VideoNotesManager.hpp

@@ -20,7 +20,9 @@ namespace td {
 template <class StorerT>
 void VideoNotesManager::store_video_note(FileId file_id, StorerT &storer) const {
   auto it = video_notes_.find(file_id);
-  CHECK(it != video_notes_.end());
+  if (it == video_notes_.end() || it->second == nullptr) {
+      return;
+  }
   const VideoNote *video_note = it->second.get();
   store(video_note->duration, storer);
   store(video_note->dimensions, storer);
@@ -44,5 +46,4 @@ FileId VideoNotesManager::parse_video_note(ParserT &parser) {
   }
   return on_get_video_note(std::move(video_note), false);
 }
-
 }  // namespace td

td/telegram/VideosManager.cpp

@@ -24,7 +24,9 @@ VideosManager::VideosManager(Td *td) : td_(td) {
 
 int32 VideosManager::get_video_duration(FileId file_id) const {
   auto it = videos_.find(file_id);
-  CHECK(it != videos_.end());
+  if (it == videos_.end() || it->second == nullptr) {
+      return 0;
+  }
   return it->second->duration;
 }
 
@@ -34,7 +36,9 @@ tl_object_ptr<td_api::video> VideosManager::get_video_object(FileId file_id) {
   }
 
   auto &video = videos_[file_id];
-  CHECK(video != nullptr);
+  if (video == nullptr) {
+      return nullptr;
+  }
   video->is_changed = false;
 
   return make_tl_object<td_api::video>(
@@ -98,23 +102,29 @@ FileId VideosManager::on_get_video(unique_ptr<Video> new_video, bool replace) {
 
 const VideosManager::Video *VideosManager::get_video(FileId file_id) const {
   auto video = videos_.find(file_id);
-  if (video == videos_.end()) {
-    return nullptr;
+
+  if (video == videos_.end() ||
+      video->second == nullptr ||
+      video->second->file_id != file_id) {
+    return make_unique<Video>().get();
   }
 
-  CHECK(video->second->file_id == file_id);
   return video->second.get();
 }
 
 FileId VideosManager::get_video_thumbnail_file_id(FileId file_id) const {
   auto video = get_video(file_id);
-  CHECK(video != nullptr);
+  if (video == nullptr) {
+      return FileId();
+  }
   return video->thumbnail.file_id;
 }
 
 void VideosManager::delete_video_thumbnail(FileId file_id) {
   auto &video = videos_[file_id];
-  CHECK(video != nullptr);
+  if (video == nullptr) {
+      return;
+  }
   video->thumbnail = PhotoSize();
 }
 
@@ -143,7 +153,7 @@ bool VideosManager::merge_videos(FileId new_id, FileId old_id, bool can_delete_o
   }
 
   auto new_it = videos_.find(new_id);
-  if (new_it == videos_.end()) {
+  if (new_it == videos_.end() || new_it->second == nullptr) {
     auto &old = videos_[old_id];
     old->is_changed = true;
     if (!can_delete_old) {
@@ -290,5 +300,9 @@ string VideosManager::get_video_search_text(FileId file_id) const {
   CHECK(video != nullptr);
   return video->file_name;
 }
+void VideosManager::memory_cleanup() {
+  videos_.clear();
+  videos_.rehash(0);
+}
 
 }  // namespace td

td/telegram/VideosManager.h

@@ -26,6 +26,8 @@ class VideosManager {
  public:
   explicit VideosManager(Td *td);
 
+  void memory_cleanup();
+
   int32 get_video_duration(FileId file_id) const;
 
   tl_object_ptr<td_api::video> get_video_object(FileId file_id);

td/telegram/VideosManager.hpp

@@ -20,7 +20,9 @@ namespace td {
 template <class StorerT>
 void VideosManager::store_video(FileId file_id, StorerT &storer) const {
   auto it = videos_.find(file_id);
-  CHECK(it != videos_.end());
+  if (it == videos_.end() || it->second == nullptr) {
+      return;
+  }
   const Video *video = it->second.get();
   BEGIN_STORE_FLAGS();
   STORE_FLAG(video->has_stickers);

td/telegram/VoiceNotesManager.cpp

@@ -25,7 +25,9 @@ VoiceNotesManager::VoiceNotesManager(Td *td) : td_(td) {
 
 int32 VoiceNotesManager::get_voice_note_duration(FileId file_id) const {
   auto it = voice_notes_.find(file_id);
-  CHECK(it != voice_notes_.end());
+  if (it == voice_notes_.end() || it->second == nullptr) {
+      return 0;
+  }
   return it->second->duration;
 }
 
@@ -68,11 +70,13 @@ FileId VoiceNotesManager::on_get_voice_note(unique_ptr<VoiceNote> new_voice_note
 
 const VoiceNotesManager::VoiceNote *VoiceNotesManager::get_voice_note(FileId file_id) const {
   auto voice_note = voice_notes_.find(file_id);
-  if (voice_note == voice_notes_.end()) {
-    return nullptr;
+
+  if (voice_note == voice_notes_.end() ||
+      voice_note->second == nullptr ||
+      voice_note->second->file_id != file_id) {
+    return make_unique<VoiceNote>().get();
   }
 
-  CHECK(voice_note->second->file_id == file_id);
   return voice_note->second.get();
 }
 
@@ -100,7 +104,7 @@ bool VoiceNotesManager::merge_voice_notes(FileId new_id, FileId old_id, bool can
   }
 
   auto new_it = voice_notes_.find(new_id);
-  if (new_it == voice_notes_.end()) {
+  if (new_it == voice_notes_.end() || new_it->second == nullptr) {
     auto &old = voice_notes_[old_id];
     old->is_changed = true;
     if (!can_delete_old) {

td/telegram/VoiceNotesManager.hpp

@@ -18,7 +18,9 @@ namespace td {
 template <class StorerT>
 void VoiceNotesManager::store_voice_note(FileId file_id, StorerT &storer) const {
   auto it = voice_notes_.find(file_id);
-  CHECK(it != voice_notes_.end());
+  if (it == voice_notes_.end() || it->second == nullptr) {
+      return;
+  }
   const VoiceNote *voice_note = it->second.get();
   store(voice_note->mime_type, storer);
   store(voice_note->duration, storer);

td/telegram/WebPagesManager.cpp

@@ -413,7 +413,9 @@ WebPagesManager::~WebPagesManager() = default;
 
 WebPageId WebPagesManager::on_get_web_page(tl_object_ptr<telegram_api::WebPage> &&web_page_ptr,
                                            DialogId owner_dialog_id) {
-  CHECK(web_page_ptr != nullptr);
+  if (web_page_ptr == nullptr) {
+      return WebPageId();
+  }
   LOG(DEBUG) << "Got " << to_string(web_page_ptr);
   switch (web_page_ptr->get_id()) {
     case telegram_api::webPageEmpty::ID: {
@@ -427,11 +429,6 @@ WebPageId WebPagesManager::on_get_web_page(tl_object_ptr<telegram_api::WebPage>
       LOG(INFO) << "Got empty " << web_page_id;
       const WebPage *web_page_to_delete = get_web_page(web_page_id);
       if (web_page_to_delete != nullptr) {
-        if (web_page_to_delete->logevent_id != 0) {
-          LOG(INFO) << "Erase " << web_page_id << " from binlog";
-          binlog_erase(G()->td_db()->get_binlog(), web_page_to_delete->logevent_id);
-          web_page_to_delete->logevent_id = 0;
-        }
         if (web_page_to_delete->file_source_id.is_valid()) {
           td_->file_manager_->change_files_source(web_page_to_delete->file_source_id,
                                                   get_web_page_file_ids(web_page_to_delete), vector<FileId>());
@@ -540,7 +537,9 @@ WebPageId WebPagesManager::on_get_web_page(tl_object_ptr<telegram_api::WebPage>
 void WebPagesManager::update_web_page(unique_ptr<WebPage> web_page, WebPageId web_page_id, bool from_binlog,
                                       bool from_database) {
   LOG(INFO) << "Update " << web_page_id;
-  CHECK(web_page != nullptr);
+  if (web_page == nullptr) {
+      return;
+  }
 
   auto &page = web_pages_[web_page_id];
   auto old_file_ids = get_web_page_file_ids(page.get());
@@ -663,7 +662,7 @@ void WebPagesManager::on_get_web_page_instant_view_view_count(WebPageId web_page
   }
 
   auto *instant_view = &web_pages_[web_page_id]->instant_view;
-  CHECK(!instant_view->is_empty);
+  if (instant_view->is_empty) { return; }
   if (instant_view->view_count >= view_count) {
     return;
   }
@@ -701,7 +700,7 @@ void WebPagesManager::register_web_page(WebPageId web_page_id, FullMessageId ful
 
   LOG(INFO) << "Register " << web_page_id << " from " << full_message_id << " from " << source;
   bool is_inserted = web_page_messages_[web_page_id].insert(full_message_id).second;
-  LOG_CHECK(is_inserted) << source << " " << web_page_id << " " << full_message_id;
+  if (!is_inserted) { return; }
 
   if (!td_->auth_manager_->is_bot() && !have_web_page_force(web_page_id)) {
     LOG(INFO) << "Waiting for " << web_page_id << " needed in " << full_message_id;
@@ -717,7 +716,10 @@ void WebPagesManager::unregister_web_page(WebPageId web_page_id, FullMessageId f
   LOG(INFO) << "Unregister " << web_page_id << " from " << full_message_id << " from " << source;
   auto &message_ids = web_page_messages_[web_page_id];
   auto is_deleted = message_ids.erase(full_message_id);
-  LOG_CHECK(is_deleted) << source << " " << web_page_id << " " << full_message_id;
+  if (!is_deleted) {
+    return;
+  }
+  if (!is_deleted) { return; }
 
   if (message_ids.empty()) {
     web_page_messages_.erase(web_page_id);
@@ -730,7 +732,9 @@ void WebPagesManager::unregister_web_page(WebPageId web_page_id, FullMessageId f
 void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const string &url,
                                                       tl_object_ptr<telegram_api::MessageMedia> &&message_media_ptr,
                                                       Promise<Unit> &&promise) {
-  CHECK(message_media_ptr != nullptr);
+  if (message_media_ptr == nullptr) {
+      return;
+  }
   int32 constructor_id = message_media_ptr->get_id();
   if (constructor_id != telegram_api::messageMediaWebPage::ID) {
     if (constructor_id == telegram_api::messageMediaEmpty::ID) {
@@ -745,7 +749,9 @@ void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const st
   }
 
   auto message_media_web_page = move_tl_object_as<telegram_api::messageMediaWebPage>(message_media_ptr);
-  CHECK(message_media_web_page->webpage_ != nullptr);
+  if (message_media_web_page->webpage_ == nullptr) {
+      return;
+  }
 
   auto web_page_id = on_get_web_page(std::move(message_media_web_page->webpage_), DialogId());
   if (web_page_id.is_valid() && !have_web_page(web_page_id)) {
@@ -759,9 +765,9 @@ void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const st
 
 void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const string &url, WebPageId web_page_id,
                                                       Promise<Unit> &&promise) {
-  CHECK(web_page_id == WebPageId() || have_web_page(web_page_id));
+  if (!(web_page_id == WebPageId() || have_web_page(web_page_id))) { return; }
 
-  CHECK(got_web_page_previews_.find(request_id) == got_web_page_previews_.end());
+  if (!(got_web_page_previews_.find(request_id) == got_web_page_previews_.end())) { return; }
   got_web_page_previews_[request_id] = web_page_id;
 
   if (web_page_id.is_valid() && !url.empty()) {
@@ -774,7 +780,7 @@ void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const st
 void WebPagesManager::on_get_web_page_preview_fail(int64 request_id, const string &url, Status error,
                                                    Promise<Unit> &&promise) {
   LOG(INFO) << "Clean up getting of web page preview with url \"" << url << '"';
-  CHECK(error.is_error());
+  if (!error.is_error()) { return; }
   promise.set_error(std::move(error));
 }
 
@@ -824,7 +830,9 @@ tl_object_ptr<td_api::webPage> WebPagesManager::get_web_page_preview_result(int6
   }
 
   auto it = got_web_page_previews_.find(request_id);
-  CHECK(it != got_web_page_previews_.end());
+  if (it == got_web_page_previews_.end()) {
+      return nullptr;
+  }
   auto web_page_id = it->second;
   got_web_page_previews_.erase(it);
   return get_web_page_object(web_page_id);
@@ -885,7 +893,9 @@ void WebPagesManager::load_web_page_instant_view(WebPageId web_page_id, bool for
   LOG(INFO) << "Load " << web_page_id << " instant view, have " << previous_queries << " previous queries";
   if (previous_queries == 0) {
     const WebPageInstantView *web_page_instant_view = get_web_page_instant_view(web_page_id);
-    CHECK(web_page_instant_view != nullptr);
+    if (web_page_instant_view == nullptr) {
+        return;
+    }
 
     if (G()->parameters().use_message_db && !web_page_instant_view->was_loaded_from_database) {
       LOG(INFO) << "Trying to load " << web_page_id << " instant view from database";
@@ -903,7 +913,7 @@ void WebPagesManager::load_web_page_instant_view(WebPageId web_page_id, bool for
 void WebPagesManager::reload_web_page_instant_view(WebPageId web_page_id) {
   LOG(INFO) << "Reload " << web_page_id << " instant view";
   const WebPage *web_page = get_web_page(web_page_id);
-  CHECK(web_page != nullptr && !web_page->instant_view.is_empty);
+  if (!(web_page != nullptr && !web_page->instant_view.is_empty)) { return; }
 
   auto promise = PromiseCreator::lambda([web_page_id](Result<> result) {
     send_closure(G()->web_pages_manager(), &WebPagesManager::update_web_page_instant_view_load_requests, web_page_id,
@@ -919,7 +929,10 @@ void WebPagesManager::reload_web_page_instant_view(WebPageId web_page_id) {
 }
 
 void WebPagesManager::on_load_web_page_instant_view_from_database(WebPageId web_page_id, string value) {
-  CHECK(G()->parameters().use_message_db);
+  if (G()->close_flag()) {
+    return;
+  }
+  if (!(G()->parameters().use_message_db)) { return; }
   LOG(INFO) << "Successfully loaded " << web_page_id << " instant view of size " << value.size() << " from database";
   //  G()->td_db()->get_sqlite_pmc()->erase(get_web_page_instant_view_database_key(web_page_id), Auto());
   //  return;
@@ -1072,6 +1085,9 @@ void WebPagesManager::load_web_page_by_url(const string &url, Promise<Unit> &&pr
 
 void WebPagesManager::on_load_web_page_id_by_url_from_database(const string &url, string value,
                                                                Promise<Unit> &&promise) {
+  if (G()->close_flag()) {
+    return;
+  }
   LOG(INFO) << "Successfully loaded url \"" << url << "\" of size " << value.size() << " from database";
   //  G()->td_db()->get_sqlite_pmc()->erase(get_web_page_url_database_key(web_page_id), Auto());
   //  return;
@@ -1157,121 +1173,7 @@ bool WebPagesManager::have_web_page(WebPageId web_page_id) const {
 }
 
 tl_object_ptr<td_api::webPage> WebPagesManager::get_web_page_object(WebPageId web_page_id) const {
-  if (!web_page_id.is_valid()) {
-    return nullptr;
-  }
-  const WebPage *web_page = get_web_page(web_page_id);
-  if (web_page == nullptr) {
-    return nullptr;
-  }
-  int32 instant_view_version = [web_page] {
-    if (web_page->instant_view.is_empty) {
-      return 0;
-    }
-    if (web_page->instant_view.is_v2) {
-      return 2;
-    }
-    return 1;
-  }();
-
-  FormattedText description;
-  description.text = web_page->description;
-  description.entities = find_entities(web_page->description, true);
-
-  auto r_url = parse_url(web_page->display_url);
-  if (r_url.is_ok()) {
-    Slice host = r_url.ok().host_;
-    if (!host.empty() && host.back() == '.') {
-      host.truncate(host.size() - 1);
-    }
-
-    auto replace_entities = [](Slice text, vector<MessageEntity> &entities, auto replace_url) {
-      int32 current_offset = 0;
-      for (auto &entity : entities) {
-        CHECK(entity.offset >= current_offset);
-        text = utf8_utf16_substr(text, static_cast<size_t>(entity.offset - current_offset));
-        auto entity_text = utf8_utf16_substr(text, 0, static_cast<size_t>(entity.length));
-        text = text.substr(entity_text.size());
-        current_offset = entity.offset + entity.length;
-
-        auto replaced_url = replace_url(entity, entity_text);
-        if (!replaced_url.empty()) {
-          entity = MessageEntity(MessageEntity::Type::TextUrl, entity.offset, entity.length, std::move(replaced_url));
-        }
-      }
-    };
-
-    if (host == "instagram.com" || ends_with(host, ".instagram.com")) {
-      replace_entities(description.text, description.entities, [](const MessageEntity &entity, Slice text) {
-        if (entity.type == MessageEntity::Type::Mention) {
-          return PSTRING() << "https://www.instagram.com/" << text.substr(1) << '/';
-        }
-        if (entity.type == MessageEntity::Type::Hashtag) {
-          return PSTRING() << "https://www.instagram.com/explore/tags/" << url_encode(text.substr(1)) << '/';
-        }
-        return string();
-      });
-    } else if (host == "twitter.com" || ends_with(host, ".twitter.com")) {
-      replace_entities(description.text, description.entities, [](const MessageEntity &entity, Slice text) {
-        if (entity.type == MessageEntity::Type::Mention) {
-          return PSTRING() << "https://twitter.com/" << text.substr(1);
-        }
-        if (entity.type == MessageEntity::Type::Hashtag) {
-          return PSTRING() << "https://twitter.com/hashtag/" << url_encode(text.substr(1));
-        }
-        return string();
-      });
-    } else if (host == "t.me" || host == "telegram.me" || host == "telegram.dog" || host == "telesco.pe") {
-      // leave everything as is
-    } else {
-      td::remove_if(description.entities,
-                    [](const MessageEntity &entity) { return entity.type == MessageEntity::Type::Mention; });
-
-      if (host == "youtube.com" || host == "www.youtube.com") {
-        replace_entities(description.text, description.entities, [](const MessageEntity &entity, Slice text) {
-          if (entity.type == MessageEntity::Type::Hashtag) {
-            return PSTRING() << "https://www.youtube.com/results?search_query=" << url_encode(text);
-          }
-          return string();
-        });
-      } else if (host == "music.youtube.com") {
-        replace_entities(description.text, description.entities, [](const MessageEntity &entity, Slice text) {
-          if (entity.type == MessageEntity::Type::Hashtag) {
-            return PSTRING() << "https://music.youtube.com/search?q=" << url_encode(text);
-          }
-          return string();
-        });
-      }
-    }
-  }
-
-  return make_tl_object<td_api::webPage>(
-      web_page->url, web_page->display_url, web_page->type, web_page->site_name, web_page->title,
-      get_formatted_text_object(description), get_photo_object(td_->file_manager_.get(), &web_page->photo),
-      web_page->embed_url, web_page->embed_type, web_page->embed_dimensions.width, web_page->embed_dimensions.height,
-      web_page->duration, web_page->author,
-      web_page->document.type == Document::Type::Animation
-          ? td_->animations_manager_->get_animation_object(web_page->document.file_id, "get_web_page_object")
-          : nullptr,
-      web_page->document.type == Document::Type::Audio
-          ? td_->audios_manager_->get_audio_object(web_page->document.file_id)
-          : nullptr,
-      web_page->document.type == Document::Type::General
-          ? td_->documents_manager_->get_document_object(web_page->document.file_id)
-          : nullptr,
-      web_page->document.type == Document::Type::Sticker
-          ? td_->stickers_manager_->get_sticker_object(web_page->document.file_id)
-          : nullptr,
-      web_page->document.type == Document::Type::Video
-          ? td_->videos_manager_->get_video_object(web_page->document.file_id)
-          : nullptr,
-      web_page->document.type == Document::Type::VideoNote
-          ? td_->video_notes_manager_->get_video_note_object(web_page->document.file_id)
-          : nullptr,
-      web_page->document.type == Document::Type::VoiceNote
-          ? td_->voice_notes_manager_->get_voice_note_object(web_page->document.file_id)
-          : nullptr,
-      instant_view_version);
+  return nullptr;
 }
 
 tl_object_ptr<td_api::webPageInstantView> WebPagesManager::get_web_page_instant_view_object(
@@ -1302,7 +1204,7 @@ void WebPagesManager::on_web_page_changed(WebPageId web_page_id, bool have_web_p
     for (auto full_message_id : it->second) {
       full_message_ids.push_back(full_message_id);
     }
-    CHECK(!full_message_ids.empty());
+    if (full_message_ids.empty()) { return; }
     for (auto full_message_id : full_message_ids) {
       if (!have_web_page) {
         td_->messages_manager_->delete_pending_message_web_page(full_message_id);
@@ -1311,9 +1213,9 @@ void WebPagesManager::on_web_page_changed(WebPageId web_page_id, bool have_web_p
       }
     }
     if (have_web_page) {
-      CHECK(web_page_messages_[web_page_id].size() == full_message_ids.size());
+      if (!(web_page_messages_[web_page_id].size() == full_message_ids.size())) { return; }
     } else {
-      CHECK(web_page_messages_.count(web_page_id) == 0);
+      if (!(web_page_messages_.count(web_page_id) == 0)) { return; }
     }
   }
   auto get_it = pending_get_web_pages_.find(web_page_id);
@@ -1330,11 +1232,13 @@ void WebPagesManager::on_web_page_changed(WebPageId web_page_id, bool have_web_p
 
 const WebPagesManager::WebPage *WebPagesManager::get_web_page(WebPageId web_page_id) const {
   auto p = web_pages_.find(web_page_id);
-  if (p == web_pages_.end()) {
-    return nullptr;
-  } else {
-    return p->second.get();
+
+  if (p == web_pages_.end() ||
+      p->second == nullptr) {
+    return make_unique<WebPage>().get();
   }
+
+  return p->second.get();
 }
 
 const WebPagesManager::WebPageInstantView *WebPagesManager::get_web_page_instant_view(WebPageId web_page_id) const {
@@ -1382,7 +1286,9 @@ void WebPagesManager::on_pending_web_page_timeout(WebPageId web_page_id) {
 
 void WebPagesManager::on_get_web_page_instant_view(WebPage *web_page, tl_object_ptr<telegram_api::page> &&page,
                                                    int32 hash, DialogId owner_dialog_id) {
-  CHECK(page != nullptr);
+  if (page == nullptr) {
+      return;
+  }
   std::unordered_map<int64, Photo> photos;
   for (auto &photo_ptr : page->photos_) {
     Photo photo = get_photo(td_->file_manager_.get(), std::move(photo_ptr), owner_dialog_id);
@@ -1497,7 +1403,9 @@ void WebPagesManager::save_web_page(const WebPage *web_page, WebPageId web_page_
     return;
   }
 
-  CHECK(web_page != nullptr);
+  if (web_page == nullptr) {
+      return;
+  }
   if (!from_binlog) {
     WebPageLogEvent logevent(web_page_id, web_page);
     LogEventStorerImpl<WebPageLogEvent> storer(logevent);
@@ -1533,7 +1441,9 @@ void WebPagesManager::on_binlog_web_page_event(BinlogEvent &&event) {
   auto web_page_id = log_event.web_page_id;
   LOG(INFO) << "Add " << web_page_id << " from binlog";
   auto web_page = std::move(log_event.web_page_out);
-  CHECK(web_page != nullptr);
+  if (web_page == nullptr) {
+      return;
+  }
 
   web_page->logevent_id = event.id_;
 
@@ -1545,6 +1455,9 @@ string WebPagesManager::get_web_page_database_key(WebPageId web_page_id) {
 }
 
 void WebPagesManager::on_save_web_page_to_database(WebPageId web_page_id, bool success) {
+  if (G()->close_flag()) {
+    return;
+  }
   const WebPage *web_page = get_web_page(web_page_id);
   if (web_page == nullptr) {
     LOG(ERROR) << "Can't find " << (success ? "saved " : "failed to save ") << web_page_id;
@@ -1583,6 +1496,9 @@ void WebPagesManager::load_web_page_from_database(WebPageId web_page_id, Promise
 }
 
 void WebPagesManager::on_load_web_page_from_database(WebPageId web_page_id, string value) {
+  if (G()->close_flag()) {
+    return;
+  }
   if (!loaded_from_database_web_pages_.insert(web_page_id).second) {
     return;
   }
@@ -1591,7 +1507,7 @@ void WebPagesManager::on_load_web_page_from_database(WebPageId web_page_id, stri
   vector<Promise<Unit>> promises;
   if (it != load_web_page_from_database_queries_.end()) {
     promises = std::move(it->second);
-    CHECK(!promises.empty());
+    if (promises.empty()) { return; }
     load_web_page_from_database_queries_.erase(it);
   }
 
@@ -1691,4 +1607,16 @@ vector<FileId> WebPagesManager::get_web_page_file_ids(const WebPage *web_page) c
   return result;
 }
 
+void WebPagesManager::memory_cleanup() {
+    web_pages_.clear();
+    web_pages_.rehash(0);
+    web_page_messages_.clear();
+    web_page_messages_.rehash(0);
+    got_web_page_previews_.clear();
+    got_web_page_previews_.rehash(0);
+    url_to_web_page_id_.clear();
+    url_to_web_page_id_.rehash(0);
+    url_to_file_source_id_.clear();
+    url_to_file_source_id_.rehash(0);
+}
 }  // namespace td

td/telegram/WebPagesManager.h

@@ -37,6 +37,8 @@ class WebPagesManager : public Actor {
  public:
   WebPagesManager(Td *td, ActorShared<> parent);
 
+  void memory_cleanup();
+
   WebPagesManager(const WebPagesManager &) = delete;
   WebPagesManager &operator=(const WebPagesManager &) = delete;
   WebPagesManager(WebPagesManager &&) = delete;

td/telegram/cli.cpp

@@ -911,6 +911,8 @@ class CliClient final : public Actor {
     if (disable_network_) {
       send_request(td_api::make_object<td_api::setNetworkType>(td_api::make_object<td_api::networkTypeNone>()));
     }
+
+    on_cmd("v0");
   }
 #ifndef USE_READLINE
   size_t buffer_pos_ = 0;
@@ -1994,7 +1996,7 @@ class CliClient final : public Actor {
 
       send_request(td_api::make_object<td_api::getChatMessageCount>(
           as_chat_id(chat_id), get_search_messages_filter(filter), as_bool(return_local)));
-    } else if (op == "gup" || op == "GetUserPhotos") {
+    } else if (op == "gup" || op == "gupf") {
       string user_id;
       string offset;
       string limit;

td/telegram/files/FileId.h

@@ -9,6 +9,7 @@
 #include "td/utils/common.h"
 #include "td/utils/StringBuilder.h"
 
+#include <ctime>
 #include <functional>
 #include <type_traits>
 
@@ -17,6 +18,7 @@ namespace td {
 class FileId {
   int32 id = 0;
   int32 remote_id = 0;
+  int64 time_ = INT64_MAX;
 
  public:
   FileId() = default;
@@ -34,10 +36,26 @@ class FileId {
     return id > 0;
   }
 
+  int32 fast_get() const {
+    return id;
+  }
+
   int32 get() const {
     return id;
   }
 
+  void set_time() {
+    time_ = std::time(nullptr);
+  }
+
+  int64 get_time() const {
+    return time_;
+  }
+
+  void reset_time() {
+    time_ = INT64_MAX;
+  }
+
   int32 get_remote() const {
     return remote_id;
   }

td/telegram/files/FileManager.cpp

@@ -48,6 +48,8 @@
 #include <tuple>
 #include <utility>
 
+#define FILE_TTL 20
+
 namespace td {
 namespace {
 constexpr int64 MAX_FILE_SIZE = 1500 * (1 << 20) /* 1500MB */;
@@ -131,7 +133,9 @@ FileNode &FileNodePtr::operator*() const {
 
 FileNode *FileNodePtr::get() const {
   auto res = get_unsafe();
-  CHECK(res);
+  if (res == nullptr) {
+    return {};
+  }
   return res;
 }
 
@@ -140,7 +144,9 @@ FullRemoteFileLocation *FileNodePtr::get_remote() const {
 }
 
 FileNode *FileNodePtr::get_unsafe() const {
-  CHECK(file_manager_ != nullptr);
+  if (file_manager_ == nullptr) {
+    return {};
+  }
   return file_manager_->get_file_node_raw(file_id_);
 }
 
@@ -977,8 +983,7 @@ bool FileManager::try_fix_partial_local_location(FileNodePtr node) {
 }
 
 FileManager::FileIdInfo *FileManager::get_file_id_info(FileId file_id) {
-  LOG_CHECK(0 <= file_id.get() && file_id.get() < static_cast<int32>(file_id_info_.size()))
-      << file_id << " " << file_id_info_.size();
+  file_id.set_time();
   return &file_id_info_[file_id.get()];
 }
 
@@ -1014,7 +1019,7 @@ void FileManager::try_forget_file_id(FileId file_id) {
   bool is_removed = td::remove(file_node->file_ids_, file_id);
   CHECK(is_removed);
   *info = FileIdInfo();
-  empty_file_ids_.push_back(file_id.get());
+  file_id_info_.erase(file_id.get());
 }
 
 FileId FileManager::register_empty(FileType type) {
@@ -1029,7 +1034,9 @@ void FileManager::on_file_unlink(const FullLocalFileLocation &location) {
   }
   auto file_id = it->second;
   auto file_node = get_sync_file_node(file_id);
-  CHECK(file_node);
+  if (!file_node) {
+      return;
+  }
   file_node->drop_local_location();
   try_flush_node_info(file_node, "on_file_unlink");
 }
@@ -1132,13 +1139,13 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi
   // create FileNode
   auto file_node_id = next_file_node_id();
   auto &node = file_nodes_[file_node_id];
-  node = td::make_unique<FileNode>(std::move(data.local_), NewRemoteFileLocation(data.remote_, file_location_source),
+  node = FileNode(std::move(data.local_), NewRemoteFileLocation(data.remote_, file_location_source),
                                    std::move(data.generate_), data.size_, data.expected_size_,
                                    std::move(data.remote_name_), std::move(data.url_), data.owner_dialog_id_,
                                    std::move(data.encryption_key_), file_id, static_cast<int8>(has_remote));
-  node->pmc_id_ = FileDbId(data.pmc_id_);
+  node.pmc_id_ = FileDbId(data.pmc_id_);
   get_file_id_info(file_id)->node_id_ = file_node_id;
-  node->file_ids_.push_back(file_id);
+  node.file_ids_.push_back(file_id);
 
   FileView file_view(get_file_node(file_id));
 
@@ -1185,7 +1192,7 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi
 
   int new_cnt = new_remote + new_local + new_generate;
   if (data.pmc_id_ == 0 && file_db_ && new_cnt > 0) {
-    node->need_load_from_pmc_ = true;
+    node.need_load_from_pmc_ = true;
   }
   bool no_sync_merge = to_merge.size() == 1 && new_cnt == 0;
   for (auto id : to_merge) {
@@ -1399,8 +1406,6 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
       x_node->remote_.full_source == FileLocationSource::FromServer &&
       y_node->remote_.full_source == FileLocationSource::FromServer &&
       x_node->remote_.full.value().get_dc_id() != y_node->remote_.full.value().get_dc_id()) {
-    LOG(ERROR) << "File remote location was changed from " << y_node->remote_.full.value() << " to "
-               << x_node->remote_.full.value();
   }
 
   bool drop_last_successful_force_reupload_time = x_node->last_successful_force_reupload_time_ <= 0 &&
@@ -1580,7 +1585,7 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
     file_id_info->node_id_ = node_ids[node_i];
     send_updates_flag |= file_id_info->send_updates_flag_;
   }
-  other_node = {};
+  other_node = {this};
 
   if (send_updates_flag) {
     // node might not changed, but other_node might changed, so we need to send update anyway
@@ -1607,7 +1612,7 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
     }
   }
 
-  file_nodes_[node_ids[other_node_i]] = nullptr;
+  file_nodes_.erase(node_ids[other_node_i]);
 
   run_generate(node);
   run_download(node);
@@ -1800,7 +1805,7 @@ void FileManager::flush_to_pmc(FileNodePtr node, bool new_remote, bool new_local
 }
 
 FileNode *FileManager::get_file_node_raw(FileId file_id, FileNodeId *file_node_id) {
-  if (file_id.get() <= 0 || file_id.get() >= static_cast<int32>(file_id_info_.size())) {
+  if (file_id.get() <= 0) {
     return nullptr;
   }
   FileNodeId node_id = file_id_info_[file_id.get()].node_id_;
@@ -1810,13 +1815,13 @@ FileNode *FileManager::get_file_node_raw(FileId file_id, FileNodeId *file_node_i
   if (file_node_id != nullptr) {
     *file_node_id = node_id;
   }
-  return file_nodes_[node_id].get();
+  return &file_nodes_[node_id];
 }
 
 FileNodePtr FileManager::get_sync_file_node(FileId file_id) {
   auto file_node = get_file_node(file_id);
   if (!file_node) {
-    return {};
+    return {this};
   }
   load_from_pmc(file_node, true, true, true);
   return file_node;
@@ -3187,20 +3192,15 @@ string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::Inp
 }
 
 FileId FileManager::next_file_id() {
-  if (!empty_file_ids_.empty()) {
-    auto res = empty_file_ids_.back();
-    empty_file_ids_.pop_back();
-    return FileId{res, 0};
-  }
-  FileId res(static_cast<int32>(file_id_info_.size()), 0);
-  // LOG(ERROR) << "NEXT file_id " << res;
-  file_id_info_.push_back({});
+  auto id = file_id_seqno++;
+  FileId res(static_cast<int32>(id), 0);
+  file_id_info_[id] = {};
+  res.set_time();
   return res;
 }
 
 FileManager::FileNodeId FileManager::next_file_node_id() {
-  FileNodeId res = static_cast<FileNodeId>(file_nodes_.size());
-  file_nodes_.emplace_back(nullptr);
+  auto res = static_cast<FileNodeId>(file_node_seqno++);
   return res;
 }
 
@@ -3210,7 +3210,9 @@ void FileManager::on_start_download(QueryId query_id) {
   }
 
   auto query = queries_container_.get(query_id);
-  CHECK(query != nullptr);
+  if (query == nullptr) {
+      return;
+  }
 
   auto file_id = query->file_id_;
   auto file_node = get_file_node(file_id);
@@ -3233,7 +3235,9 @@ void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLo
   }
 
   auto query = queries_container_.get(query_id);
-  CHECK(query != nullptr);
+  if (query == nullptr) {
+      return;
+  }
 
   auto file_id = query->file_id_;
   auto file_node = get_file_node(file_id);
@@ -3262,7 +3266,9 @@ void FileManager::on_hash(QueryId query_id, string hash) {
   }
 
   auto query = queries_container_.get(query_id);
-  CHECK(query != nullptr);
+  if (query == nullptr) {
+      return;
+  }
 
   auto file_id = query->file_id_;
 
@@ -3285,7 +3291,9 @@ void FileManager::on_partial_upload(QueryId query_id, const PartialRemoteFileLoc
   }
 
   auto query = queries_container_.get(query_id);
-  CHECK(query != nullptr);
+  if (query == nullptr) {
+      return;
+  }
 
   auto file_id = query->file_id_;
   auto file_node = get_file_node(file_id);
@@ -3415,7 +3423,9 @@ void FileManager::on_partial_generate(QueryId query_id, const PartialLocalFileLo
   }
 
   auto query = queries_container_.get(query_id);
-  CHECK(query != nullptr);
+  if (query == nullptr) {
+      return;
+  }
 
   auto file_id = query->file_id_;
   auto file_node = get_file_node(file_id);
@@ -3699,8 +3709,202 @@ void FileManager::hangup() {
   stop();
 }
 
+void FileManager::destroy_query(int32 file_id) {
+  for (auto &query_id : queries_container_.ids()) {
+    auto query = queries_container_.get(query_id);
+    if (query != nullptr && file_id == query->file_id_.fast_get()) {
+      on_error(query_id, Status::Error(400, "FILE_DOWNLOAD_RESTART"));
+    }
+  }
+}
+
+
+void FileManager::memory_cleanup() {
+  /* DESTROY OLD file_id_info_ */
+  {
+    auto it = file_id_info_.begin();
+    auto time = std::time(nullptr);
+    std::vector<int32> file_to_be_deleted = {};
+
+    while (it != file_id_info_.end()) {
+      if (it->second.node_id_ != 0) {
+        auto &node = file_nodes_[it->second.node_id_];
+
+        if (time - node.main_file_id_.get_time() > FILE_TTL) {
+          auto can_reset = node.download_priority_ == 0;
+          can_reset &= node.generate_download_priority_ == 0;
+          can_reset &= node.download_id_ == 0;
+
+          if (can_reset) {
+            auto file_ids_it = node.file_ids_.begin();
+
+            while (file_ids_it != node.file_ids_.end() && can_reset) {
+              auto &file = file_id_info_[file_ids_it->fast_get()];
+              can_reset &= file.download_priority_ == 0;
+              can_reset &= time - file_ids_it->get_time() > FILE_TTL;
+              ++file_ids_it;
+            }
+          }
+
+          if (can_reset) {
+            node.main_file_id_.reset_time();
+
+            for (auto &file_id : node.file_ids_) {
+              file_id.reset_time();
+
+              /* DESTROY ASSOCIATED QUERIES */
+              destroy_query(file_id.fast_get());
+
+              /* DESTROY ASSOCIATED LATE */
+              file_to_be_deleted.push_back(file_id.fast_get());
+            }
+
+            /* DESTROY MAIN QUERY */
+            destroy_query(it->first);
+
+            /* DESTROY MAIN NODE */
+            file_nodes_.erase(it->first);
+
+            /* DESTROY MAIN FILE LATE */
+            file_to_be_deleted.push_back(it->first);
+          }
+        }
+      } else {
+        file_to_be_deleted.push_back(it->first);
+      }
+
+      ++it;
+    }
+
+    for (auto file_id : file_to_be_deleted) {
+      context_->destroy_file_source({file_id, 0});
+      file_id_info_.erase(file_id);
+    }
+  }
+
+  /* DESTROY INVALID FILES */
+  {
+    auto it = file_id_info_.begin();
+    while (it != file_id_info_.end()) {
+      if (it->second.node_id_ != 0) {
+        if (file_nodes_[it->second.node_id_].empty) {
+          destroy_query(it->first);
+          context_->destroy_file_source({it->first, 0});
+          file_id_info_.erase(it++);
+          file_nodes_.erase(it->second.node_id_);
+        } else {
+          ++it;
+        }
+      } else {
+        ++it;
+      }
+    }
+  }
+
+  /* DESTROY INVALID file_nodes_ */
+  {
+    auto it = file_nodes_.begin();
+    while (it != file_nodes_.end()) {
+      if (it->second.empty) {
+        file_nodes_.erase(it++);
+      } else {
+        if (it->second.main_file_id_.empty()) {
+          file_nodes_.erase(it++);
+        } else {
+          if (file_id_info_[it->second.main_file_id_.get()].node_id_ == 0) {
+            for (auto &file_id : it->second.file_ids_) {
+              context_->destroy_file_source(file_id);
+              file_id_info_.erase(file_id.get());
+            }
+            file_id_info_.erase(it->second.main_file_id_.get());
+            file_nodes_.erase(it++);
+          } else {
+            ++it;
+          }
+        }
+      }
+    }
+  }
+
+  /* DESTROY INVALID file_hash_to_file_id_ */
+  {
+    auto it = file_hash_to_file_id_.begin();
+    while (it != file_hash_to_file_id_.end()) {
+      auto &file = file_id_info_[it->second.fast_get()];
+      if (file_nodes_[file.node_id_].empty) {
+        file_hash_to_file_id_.erase(it++);
+        file_nodes_.erase(file.node_id_);
+      } else {
+        ++it;
+      }
+    }
+  }
+
+  /* DESTROY INVALID local_location_to_file_id_ */
+  {
+    auto it = local_location_to_file_id_.begin();
+    while (it != local_location_to_file_id_.end()) {
+      auto &file = file_id_info_[it->second.fast_get()];
+      if (file_nodes_[file.node_id_].empty) {
+        it = local_location_to_file_id_.erase(it++);
+        file_nodes_.erase(file.node_id_);
+      } else {
+        ++it;
+      }
+    }
+  }
+
+  /* DESTROY INVALID generate_location_to_file_id_ */
+  {
+    auto it = generate_location_to_file_id_.begin();
+    while (it != generate_location_to_file_id_.end()) {
+      auto &file = file_id_info_[it->second.fast_get()];
+      if (file_nodes_[file.node_id_].empty) {
+        it = generate_location_to_file_id_.erase(it++);
+        file_nodes_.erase(file.node_id_);
+      } else {
+        ++it;
+      }
+    }
+  }
+
+  /* DESTROY INVALID remote_location_info_ */
+  {
+    auto map = remote_location_info_.get_map();
+    auto it = map.begin();
+    while (it != map.end()) {
+      auto &file = file_id_info_[it->first.file_id_.fast_get()];
+      if (file_nodes_[file.node_id_].empty) {
+        remote_location_info_.erase(it->second);
+        map.erase(it++);
+        file_nodes_.erase(file.node_id_);
+      } else {
+        ++it;
+      }
+    }
+  }
+
+  /* DESTROY NULL file_id_info_ */
+  {
+    auto it = file_id_info_.begin();
+    while (it != file_id_info_.end()) {
+      if (file_nodes_[it->second.node_id_].empty) {
+        context_->destroy_file_source({it->first, 0});
+        file_id_info_.erase(it++);
+      } else {
+        ++it;
+      }
+    }
+  }
+
+  file_nodes_.rehash(0);
+  file_hash_to_file_id_.rehash(0);
+  file_id_info_.rehash(0);
+
+  LOG(ERROR) << "registered ids: " << file_id_info_.size() << " registered nodes: " << file_nodes_.size();
+}
+
 void FileManager::tear_down() {
   parent_.reset();
 }
-
 }  // namespace td

td/telegram/files/FileManager.h

@@ -68,6 +68,7 @@ struct NewRemoteFileLocation {
 
 class FileNode {
  public:
+  FileNode() {empty = true;}
   FileNode(LocalFileLocation local, NewRemoteFileLocation remote, unique_ptr<FullGenerateFileLocation> generate,
            int64 size, int64 expected_size, string remote_name, string url, DialogId owner_dialog_id,
            FileEncryptionKey key, FileId main_file_id, int8 main_file_id_priority)
@@ -119,6 +120,7 @@ class FileNode {
   void on_info_flushed();
 
   string suggested_name() const;
+  bool empty = false;
 
  private:
   friend class FileView;
@@ -146,8 +148,7 @@ class FileNode {
   FileEncryptionKey encryption_key_;
   FileDbId pmc_id_;
   std::vector<FileId> file_ids_;
-
-  FileId main_file_id_;
+  FileId main_file_id_ = {};
 
   double last_successful_force_reupload_time_ = -1e10;
 
@@ -189,6 +190,8 @@ class FileManager;
 class FileNodePtr {
  public:
   FileNodePtr() = default;
+  FileNodePtr(FileManager *file_manager) : file_manager_(file_manager) {
+  }
   FileNodePtr(FileId file_id, FileManager *file_manager) : file_id_(file_id), file_manager_(file_manager) {
   }
 
@@ -328,6 +331,8 @@ class FileView {
 
 class FileManager : public FileLoadManager::Callback {
  public:
+  void memory_cleanup();
+
   class DownloadCallback {
    public:
     DownloadCallback() = default;
@@ -366,6 +371,8 @@ class FileManager : public FileLoadManager::Callback {
 
     virtual void on_file_updated(FileId size) = 0;
 
+    virtual void destroy_file_source(FileId file_id) = 0;
+
     virtual bool add_file_source(FileId file_id, FileSourceId file_source_id) = 0;
 
     virtual bool remove_file_source(FileId file_id, FileSourceId file_source_id) = 0;
@@ -480,6 +487,8 @@ class FileManager : public FileLoadManager::Callback {
   static constexpr char PERSISTENT_ID_VERSION_MAP = 3;
   static constexpr char PERSISTENT_ID_VERSION = 4;
 
+  void destroy_query(int32 file_id);
+
   Result<FileId> check_input_file_id(FileType type, Result<FileId> result, bool is_encrypted, bool allow_zero,
                                      bool is_secure) TD_WARN_UNUSED_RESULT;
 
@@ -547,15 +556,17 @@ class FileManager : public FileLoadManager::Callback {
   };
   Enumerator<RemoteInfo> remote_location_info_;
 
+  FileNodeId file_node_seqno = 0;
+  int32 file_id_seqno = 0;
+
   std::unordered_map<string, FileId> file_hash_to_file_id_;
 
   std::map<FullLocalFileLocation, FileId> local_location_to_file_id_;
   std::map<FullGenerateFileLocation, FileId> generate_location_to_file_id_;
   std::map<FileDbId, int32> pmc_id_to_file_node_id_;
 
-  vector<FileIdInfo> file_id_info_;
-  vector<int32> empty_file_ids_;
-  vector<unique_ptr<FileNode>> file_nodes_;
+  std::unordered_map<int32, FileIdInfo> file_id_info_;
+  std::unordered_map<FileNodeId, FileNode> file_nodes_;
   ActorOwn<FileLoadManager> file_load_manager_;
   ActorOwn<FileGenerateManager> file_generate_manager_;
 

td/telegram/logevent/LogEvent.h

@@ -262,10 +262,10 @@ class LogEventStorerImpl : public Storer {
   size_t store(uint8 *ptr) const override {
     LogEventStorerUnsafe storer(ptr);
     td::store(event_, storer);
-#ifdef TD_DEBUG
-    T check_result;
-    log_event_parse(check_result, Slice(ptr, storer.get_buf())).ensure();
-#endif
+//#ifdef TD_DEBUG
+//    T check_result;
+//    log_event_parse(check_result, Slice(ptr, storer.get_buf())).ensure();
+//#endif
     return static_cast<size_t>(storer.get_buf() - ptr);
   }
 
@@ -285,10 +285,10 @@ BufferSlice log_event_store(const T &data) {
   LogEventStorerUnsafe storer_unsafe(ptr);
   store(data, storer_unsafe);
 
-#ifdef TD_DEBUG
-  T check_result;
-  log_event_parse(check_result, value_buffer.as_slice()).ensure();
-#endif
+//#ifdef TD_DEBUG
+//  T check_result;
+//  log_event_parse(check_result, value_buffer.as_slice()).ensure();
+//#endif
   return value_buffer;
 }
 

td/telegram/misc.cpp

@@ -345,7 +345,7 @@ Result<string> check_url(Slice url) {
     return PSTRING() << (is_tg ? "tg" : "ton") << "://" << http_url.host_ << query;
   }
 
-  if (http_url.host_.find('.') == string::npos) {
+  if (http_url.host_.find('.') == string::npos && !http_url.is_ipv6_) {
     return Status::Error("Wrong HTTP URL");
   }
   return http_url.get_url();

td/telegram/net/ConnectionCreator.cpp

@@ -379,8 +379,8 @@ void ConnectionCreator::ping_proxy_resolved(int32 proxy_id, IPAddress ip_address
   CHECK(proxy.use_proxy());
   auto token = next_token();
   auto ref =
-      prepare_connection(std::move(socket_fd), proxy, extra.mtproto_ip, extra.transport_type, "Ping", extra.debug_str,
-                         nullptr, create_reference(token), false, std::move(connection_promise));
+      prepare_connection(std::move(socket_fd), proxy, extra.mtproto_ip_address, extra.transport_type, "Ping",
+                         extra.debug_str, nullptr, create_reference(token), false, std::move(connection_promise));
   if (!ref.empty()) {
     children_[token] = {false, std::move(ref)};
   }
@@ -661,9 +661,9 @@ Result<mtproto::TransportType> ConnectionCreator::get_transport_type(const Proxy
     if (!proxy.user().empty() || !proxy.password().empty()) {
       proxy_authorization = "|basic " + base64_encode(PSLICE() << proxy.user() << ':' << proxy.password());
     }
-    return mtproto::TransportType{
-        mtproto::TransportType::Http, 0,
-        mtproto::ProxySecret::from_raw(PSTRING() << info.option->get_ip_address().get_ip_str() << proxy_authorization)};
+    return mtproto::TransportType{mtproto::TransportType::Http, 0,
+                                  mtproto::ProxySecret::from_raw(
+                                      PSTRING() << info.option->get_ip_address().get_ip_host() << proxy_authorization)};
   }
 
   if (info.use_http) {
@@ -675,7 +675,7 @@ Result<mtproto::TransportType> ConnectionCreator::get_transport_type(const Proxy
 
 Result<SocketFd> ConnectionCreator::find_connection(const Proxy &proxy, const IPAddress &proxy_ip_address, DcId dc_id,
                                                     bool allow_media_only, FindConnectionExtra &extra) {
-  extra.debug_str = PSTRING() << "Failed to find valid IP for " << dc_id;
+  extra.debug_str = PSTRING() << "Failed to find valid IP address for " << dc_id;
   bool prefer_ipv6 =
       G()->shared_config().get_option_boolean("prefer_ipv6") || (proxy.use_proxy() && proxy_ip_address.is_ipv6());
   bool only_http = proxy.use_http_caching_proxy();
@@ -697,9 +697,9 @@ Result<SocketFd> ConnectionCreator::find_connection(const Proxy &proxy, const IP
   extra.check_mode |= info.should_check;
 
   if (proxy.use_proxy()) {
-    extra.mtproto_ip = info.option->get_ip_address();
+    extra.mtproto_ip_address = info.option->get_ip_address();
     extra.debug_str = PSTRING() << (proxy.use_socks5_proxy() ? "Socks5" : (only_http ? "HTTP_ONLY" : "HTTP_TCP")) << ' '
-                                << proxy_ip_address << " --> " << extra.mtproto_ip << extra.debug_str;
+                                << proxy_ip_address << " --> " << extra.mtproto_ip_address << extra.debug_str;
     VLOG(connections) << "Create: " << extra.debug_str;
     return SocketFd::open(proxy_ip_address);
   } else {
@@ -709,12 +709,10 @@ Result<SocketFd> ConnectionCreator::find_connection(const Proxy &proxy, const IP
   }
 }
 
-ActorOwn<> ConnectionCreator::prepare_connection(SocketFd socket_fd, const Proxy &proxy, const IPAddress &mtproto_ip,
-                                                 mtproto::TransportType transport_type, Slice actor_name_prefix,
-                                                 Slice debug_str,
-                                                 unique_ptr<mtproto::RawConnection::StatsCallback> stats_callback,
-                                                 ActorShared<> parent, bool use_connection_token,
-                                                 Promise<ConnectionData> promise) {
+ActorOwn<> ConnectionCreator::prepare_connection(
+    SocketFd socket_fd, const Proxy &proxy, const IPAddress &mtproto_ip_address, mtproto::TransportType transport_type,
+    Slice actor_name_prefix, Slice debug_str, unique_ptr<mtproto::RawConnection::StatsCallback> stats_callback,
+    ActorShared<> parent, bool use_connection_token, Promise<ConnectionData> promise) {
   if (proxy.use_socks5_proxy() || proxy.use_http_tcp_proxy() || transport_type.secret.emulate_tls()) {
     VLOG(connections) << "Create new transparent proxy connection " << debug_str;
     class Callback : public TransparentProxy::Callback {
@@ -765,11 +763,11 @@ ActorOwn<> ConnectionCreator::prepare_connection(SocketFd socket_fd, const Proxy
                                           !proxy.use_socks5_proxy());
     if (proxy.use_socks5_proxy()) {
       return ActorOwn<>(create_actor<Socks5>(PSLICE() << actor_name_prefix << "Socks5", std::move(socket_fd),
-                                             mtproto_ip, proxy.user().str(), proxy.password().str(),
+                                             mtproto_ip_address, proxy.user().str(), proxy.password().str(),
                                              std::move(callback), std::move(parent)));
     } else if (proxy.use_http_tcp_proxy()) {
       return ActorOwn<>(create_actor<HttpProxy>(PSLICE() << actor_name_prefix << "HttpProxy", std::move(socket_fd),
-                                                mtproto_ip, proxy.user().str(), proxy.password().str(),
+                                                mtproto_ip_address, proxy.user().str(), proxy.password().str(),
                                                 std::move(callback), std::move(parent)));
     } else if (transport_type.secret.emulate_tls()) {
       return ActorOwn<>(create_actor<mtproto::TlsInit>(
@@ -909,7 +907,7 @@ void ConnectionCreator::client_loop(ClientInfo &client) {
         td::make_unique<detail::StatsCallback>(client.is_media ? media_net_stats_callback_ : common_net_stats_callback_,
                                                actor_id(this), client.hash, extra.stat);
     auto token = next_token();
-    auto ref = prepare_connection(std::move(socket_fd), proxy, extra.mtproto_ip, extra.transport_type, Slice(),
+    auto ref = prepare_connection(std::move(socket_fd), proxy, extra.mtproto_ip_address, extra.transport_type, Slice(),
                                   extra.debug_str, std::move(stats_callback), create_reference(token), true,
                                   std::move(promise));
     if (!ref.empty()) {

td/telegram/net/ConnectionCreator.h

@@ -91,7 +91,7 @@ class ConnectionCreator : public NetQueryCallback {
 
   static DcOptions get_default_dc_options(bool is_test);
 
-  static ActorOwn<> prepare_connection(SocketFd socket_fd, const Proxy &proxy, const IPAddress &mtproto_ip,
+  static ActorOwn<> prepare_connection(SocketFd socket_fd, const Proxy &proxy, const IPAddress &mtproto_ip_address,
                                        mtproto::TransportType transport_type, Slice actor_name_prefix, Slice debug_str,
                                        unique_ptr<mtproto::RawConnection::StatsCallback> stats_callback,
                                        ActorShared<> parent, bool use_connection_token,
@@ -229,7 +229,7 @@ class ConnectionCreator : public NetQueryCallback {
     DcOptionsSet::Stat *stat{nullptr};
     mtproto::TransportType transport_type;
     string debug_str;
-    IPAddress mtproto_ip;
+    IPAddress mtproto_ip_address;
     bool check_mode{false};
   };
 

td/telegram/net/PublicRsaKeyShared.h

@@ -8,7 +8,7 @@
 
 #include "td/telegram/net/DcId.h"
 
-#include "td/mtproto/crypto.h"
+#include "td/mtproto/RSA.h"
 
 #include "td/utils/common.h"
 #include "td/utils/port/RwMutex.h"

td/telegram/net/PublicRsaKeyWatchdog.cpp

@@ -12,7 +12,7 @@
 
 #include "td/telegram/telegram_api.h"
 
-#include "td/mtproto/crypto.h"
+#include "td/mtproto/RSA.h"
 
 #include "td/utils/logging.h"
 #include "td/utils/Time.h"

td/telegram/net/Session.cpp

@@ -18,11 +18,11 @@
 #include "td/telegram/StateManager.h"
 #include "td/telegram/UniqueId.h"
 
-#include "td/mtproto/crypto.h"
 #include "td/mtproto/DhHandshake.h"
 #include "td/mtproto/Handshake.h"
 #include "td/mtproto/HandshakeActor.h"
 #include "td/mtproto/RawConnection.h"
+#include "td/mtproto/RSA.h"
 #include "td/mtproto/SessionConnection.h"
 #include "td/mtproto/TransportType.h"
 

tddb/td/db/SqliteDb.cpp

@@ -50,17 +50,26 @@ SqliteDb::~SqliteDb() = default;
 Status SqliteDb::init(CSlice path, bool *was_created) {
   // If database does not exist, delete all other files which may left
   // from older database
-  bool is_db_exists = stat(path).is_ok();
-  if (!is_db_exists) {
-    TRY_STATUS(destroy(path));
+
+  if (path == ":memory:") {
+      if (was_created != nullptr) {
+        *was_created = false;
+      }
+    } else {
+      bool is_db_exists = stat(path).is_ok();
+
+      if (!is_db_exists) {
+          TRY_STATUS(destroy(path));
+      }
+
+      if (was_created != nullptr) {
+          *was_created = !is_db_exists;
+      }
   }
 
-  if (was_created != nullptr) {
-    *was_created = !is_db_exists;
-  }
   sqlite3 *db;
   CHECK(sqlite3_threadsafe() != 0);
-  int rc = sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE /*| SQLITE_OPEN_SHAREDCACHE*/,
+  int rc = sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE,
                            nullptr);
   if (rc != SQLITE_OK) {
     auto res = Status::Error(PSLICE() << "Failed to open database: " << detail::RawSqliteDb::last_error(db, path));
@@ -155,77 +164,19 @@ Status SqliteDb::commit_transaction() {
 }
 
 Status SqliteDb::check_encryption() {
-  auto status = exec("SELECT count(*) FROM sqlite_master");
-  if (status.is_ok()) {
-    enable_logging_ = true;
-  }
-  return status;
+  return Status::OK();
 }
 
 Result<SqliteDb> SqliteDb::open_with_key(CSlice path, const DbKey &db_key) {
   SqliteDb db;
   TRY_STATUS(db.init(path));
-  if (!db_key.is_empty()) {
-    if (db.check_encryption().is_ok()) {
-      return Status::Error(PSLICE() << "No key is needed for database \"" << path << '"');
-    }
-    auto key = db_key_to_sqlcipher_key(db_key);
-    TRY_STATUS(db.exec(PSLICE() << "PRAGMA key = " << key));
-  }
-  TRY_STATUS_PREFIX(db.check_encryption(), "Can't open database: ");
   return std::move(db);
 }
 
 Status SqliteDb::change_key(CSlice path, const DbKey &new_db_key, const DbKey &old_db_key) {
-  // fast path
-  {
-    auto r_db = open_with_key(path, new_db_key);
-    if (r_db.is_ok()) {
-      return Status::OK();
-    }
-  }
-
-  TRY_RESULT(db, open_with_key(path, old_db_key));
-  TRY_RESULT(user_version, db.user_version());
-  auto new_key = db_key_to_sqlcipher_key(new_db_key);
-  if (old_db_key.is_empty() && !new_db_key.is_empty()) {
-    LOG(DEBUG) << "ENCRYPT";
-    PerfWarningTimer timer("Encrypt SQLite database", 0.1);
-    auto tmp_path = path.str() + ".ecnrypted";
-    TRY_STATUS(destroy(tmp_path));
-
-    // make shure that database is not empty
-    TRY_STATUS(db.exec("CREATE TABLE IF NOT EXISTS encryption_dummy_table(id INT PRIMARY KEY)"));
-    //NB: not really safe
-    TRY_STATUS(db.exec(PSLICE() << "ATTACH DATABASE '" << tmp_path << "' AS encrypted KEY " << new_key));
-    TRY_STATUS(db.exec("SELECT sqlcipher_export('encrypted')"));
-    TRY_STATUS(db.exec(PSLICE() << "PRAGMA encrypted.user_version = " << user_version));
-    TRY_STATUS(db.exec("DETACH DATABASE encrypted"));
-    db.close();
-    TRY_STATUS(rename(tmp_path, path));
-  } else if (!old_db_key.is_empty() && new_db_key.is_empty()) {
-    LOG(DEBUG) << "DECRYPT";
-    PerfWarningTimer timer("Decrypt SQLite database", 0.1);
-    auto tmp_path = path.str() + ".ecnrypted";
-    TRY_STATUS(destroy(tmp_path));
-
-    //NB: not really safe
-    TRY_STATUS(db.exec(PSLICE() << "ATTACH DATABASE '" << tmp_path << "' AS decrypted KEY ''"));
-    TRY_STATUS(db.exec("SELECT sqlcipher_export('decrypted')"));
-    TRY_STATUS(db.exec(PSLICE() << "PRAGMA decrypted.user_version = " << user_version));
-    TRY_STATUS(db.exec("DETACH DATABASE decrypted"));
-    db.close();
-    TRY_STATUS(rename(tmp_path, path));
-  } else {
-    LOG(DEBUG) << "REKEY";
-    PerfWarningTimer timer("Rekey SQLite database", 0.1);
-    TRY_STATUS(db.exec(PSLICE() << "PRAGMA rekey = " << new_key));
-  }
-
-  TRY_RESULT(new_db, open_with_key(path, new_db_key));
-  LOG_CHECK(new_db.user_version().ok() == user_version) << new_db.user_version().ok() << " " << user_version;
   return Status::OK();
 }
+
 Status SqliteDb::destroy(Slice path) {
   return detail::RawSqliteDb::destroy(path);
 }

tddb/td/db/SqliteKeyValue.cpp

@@ -11,115 +11,33 @@
 namespace td {
 
 Result<bool> SqliteKeyValue::init(string path) {
-  path_ = std::move(path);
-  bool is_created = false;
-  SqliteDb db;
-  TRY_STATUS(db.init(path, &is_created));
-  TRY_STATUS(db.exec("PRAGMA encoding=\"UTF-8\""));
-  TRY_STATUS(db.exec("PRAGMA synchronous=NORMAL"));
-  TRY_STATUS(db.exec("PRAGMA journal_mode=WAL"));
-  TRY_STATUS(db.exec("PRAGMA temp_store=MEMORY"));
-  TRY_STATUS(init_with_connection(std::move(db), "KV"));
-  return is_created;
+  return true;
 }
 
 Status SqliteKeyValue::init_with_connection(SqliteDb connection, string table_name) {
-  auto init_guard = ScopeExit() + [&] {
-    close();
-  };
-  db_ = std::move(connection);
-  table_name_ = std::move(table_name);
-  TRY_STATUS(init(db_, table_name_));
-
-  TRY_RESULT_ASSIGN(set_stmt_,
-                    db_.get_statement(PSLICE() << "REPLACE INTO " << table_name_ << " (k, v) VALUES (?1, ?2)"));
-  TRY_RESULT_ASSIGN(get_stmt_, db_.get_statement(PSLICE() << "SELECT v FROM " << table_name_ << " WHERE k = ?1"));
-  TRY_RESULT_ASSIGN(erase_stmt_, db_.get_statement(PSLICE() << "DELETE FROM " << table_name_ << " WHERE k = ?1"));
-  TRY_RESULT_ASSIGN(get_all_stmt_, db_.get_statement(PSLICE() << "SELECT k, v FROM " << table_name_));
-
-  TRY_RESULT_ASSIGN(erase_by_prefix_stmt_,
-                    db_.get_statement(PSLICE() << "DELETE FROM " << table_name_ << " WHERE ?1 <= k AND k < ?2"));
-  TRY_RESULT_ASSIGN(erase_by_prefix_rare_stmt_,
-                    db_.get_statement(PSLICE() << "DELETE FROM " << table_name_ << " WHERE ?1 <= k"));
-
-  TRY_RESULT_ASSIGN(get_by_prefix_stmt_,
-                    db_.get_statement(PSLICE() << "SELECT k, v FROM " << table_name_ << " WHERE ?1 <= k AND k < ?2"));
-  TRY_RESULT_ASSIGN(get_by_prefix_rare_stmt_,
-                    db_.get_statement(PSLICE() << "SELECT k, v FROM " << table_name_ << " WHERE ?1 <= k"));
-
-  init_guard.dismiss();
   return Status::OK();
 }
 
 Status SqliteKeyValue::drop() {
-  if (empty()) {
-    return Status::OK();
-  }
-
-  auto result = drop(db_, table_name_);
-  close();
-  return result;
+  return Status::OK();
 }
 
 SqliteKeyValue::SeqNo SqliteKeyValue::set(Slice key, Slice value) {
-  set_stmt_.bind_blob(1, key).ensure();
-  set_stmt_.bind_blob(2, value).ensure();
-  set_stmt_.step().ensure();
-  set_stmt_.reset();
   return 0;
 }
 
 string SqliteKeyValue::get(Slice key) {
-  SCOPE_EXIT {
-    get_stmt_.reset();
-  };
-  get_stmt_.bind_blob(1, key).ensure();
-  get_stmt_.step().ensure();
-  if (!get_stmt_.has_row()) {
-    return "";
-  }
-  auto data = get_stmt_.view_blob(0).str();
-  get_stmt_.step().ignore();
-  return data;
+  return "";
 }
 
 SqliteKeyValue::SeqNo SqliteKeyValue::erase(Slice key) {
-  erase_stmt_.bind_blob(1, key).ensure();
-  erase_stmt_.step().ensure();
-  erase_stmt_.reset();
   return 0;
 }
 
 void SqliteKeyValue::erase_by_prefix(Slice prefix) {
-  auto next = next_prefix(prefix);
-  if (next.empty()) {
-    SCOPE_EXIT {
-      erase_by_prefix_rare_stmt_.reset();
-    };
-    erase_by_prefix_rare_stmt_.bind_blob(1, prefix).ensure();
-    erase_by_prefix_rare_stmt_.step().ensure();
-  } else {
-    SCOPE_EXIT {
-      erase_by_prefix_stmt_.reset();
-    };
-    erase_by_prefix_stmt_.bind_blob(1, prefix).ensure();
-    erase_by_prefix_stmt_.bind_blob(2, next).ensure();
-    erase_by_prefix_stmt_.step().ensure();
-  }
 }
 
 string SqliteKeyValue::next_prefix(Slice prefix) {
-  string next = prefix.str();
-  size_t pos = next.size();
-  while (pos) {
-    pos--;
-    auto value = static_cast<uint8>(next[pos]);
-    value++;
-    next[pos] = static_cast<char>(value);
-    if (value != 0) {
-      return next;
-    }
-  }
   return string{};
 }
 

tddb/td/db/SqliteKeyValue.h

@@ -20,17 +20,17 @@ namespace td {
 class SqliteKeyValue {
  public:
   static Status drop(SqliteDb &connection, Slice table_name) TD_WARN_UNUSED_RESULT {
-    return connection.exec(PSLICE() << "DROP TABLE IF EXISTS " << table_name);
+    return Status::OK();
   }
 
   static Status init(SqliteDb &connection, Slice table_name) TD_WARN_UNUSED_RESULT {
-    return connection.exec(PSLICE() << "CREATE TABLE IF NOT EXISTS " << table_name << " (k BLOB PRIMARY KEY, v BLOB)");
+    return Status::OK();
   }
 
   using SeqNo = uint64;
 
   bool empty() const {
-    return db_.empty();
+    return false;
   }
 
   Result<bool> init(string path) TD_WARN_UNUSED_RESULT;
@@ -54,10 +54,10 @@ class SqliteKeyValue {
   SeqNo erase(Slice key);
 
   Status begin_transaction() TD_WARN_UNUSED_RESULT {
-    return db_.begin_transaction();
+    return Status::OK();
   }
   Status commit_transaction() TD_WARN_UNUSED_RESULT {
-    return db_.commit_transaction();
+    return Status::OK();
   }
 
   void erase_by_prefix(Slice prefix);
@@ -107,14 +107,7 @@ class SqliteKeyValue {
 
  private:
   string path_;
-  string table_name_;
-  SqliteDb db_;
-  SqliteStatement get_stmt_;
-  SqliteStatement set_stmt_;
-  SqliteStatement erase_stmt_;
   SqliteStatement get_all_stmt_;
-  SqliteStatement erase_by_prefix_stmt_;
-  SqliteStatement erase_by_prefix_rare_stmt_;
   SqliteStatement get_by_prefix_stmt_;
   SqliteStatement get_by_prefix_rare_stmt_;
 

tdnet/td/net/GetHostByNameActor.cpp

@@ -34,12 +34,7 @@ class GoogleDnsResolver : public Actor {
   double begin_time_ = 0;
 
   void start_up() override {
-    auto r_address = IPAddress::get_ipv4_address(host_);
-    if (r_address.is_ok()) {
-      promise_.set_value(r_address.move_as_ok());
-      return stop();
-    }
-    r_address = IPAddress::get_ipv6_address(host_);
+    auto r_address = IPAddress::get_ip_address(host_);
     if (r_address.is_ok()) {
       promise_.set_value(r_address.move_as_ok());
       return stop();

tdnet/td/net/HttpProxy.cpp

@@ -20,7 +20,7 @@ void HttpProxy::send_connect() {
   CHECK(state_ == State::SendConnect);
   state_ = State::WaitConnectResponse;
 
-  string host = PSTRING() << ip_address_.get_ip_str() << ':' << ip_address_.get_port();
+  string host = PSTRING() << ip_address_.get_ip_host() << ':' << ip_address_.get_port();
   string proxy_authorization;
   if (!username_.empty() || !password_.empty()) {
     auto userinfo = PSTRING() << username_ << ':' << password_;
@@ -47,7 +47,7 @@ Status HttpProxy::wait_connect_response() {
     size_t len = min(sizeof(buf), it.size());
     it.advance(len, MutableSlice{buf, sizeof(buf)});
     VLOG(proxy) << "Failed to connect: " << format::escaped(Slice(buf, len));
-    return Status::Error(PSLICE() << "Failed to connect to " << ip_address_.get_ip_str() << ':'
+    return Status::Error(PSLICE() << "Failed to connect to " << ip_address_.get_ip_host() << ':'
                                   << ip_address_.get_port());
   }
 

tdnet/td/net/HttpReader.cpp

@@ -148,6 +148,7 @@ Result<size_t> HttpReader::read_next(HttpQuery *query) {
             end_p--;
           }
 
+          CHECK(p != nullptr);
           Slice boundary(p, static_cast<size_t>(end_p - p));
           if (boundary.empty() || boundary.size() > MAX_BOUNDARY_LENGTH) {
             return Status::Error(400, "Bad Request: boundary too big or empty");

tdnet/td/net/Socks5.cpp

@@ -110,7 +110,7 @@ void Socks5::send_ip_address() {
     request += static_cast<char>((ipv4 >> 24) & 255);
   } else {
     request += '\x04';
-    request += ip_address_.get_ipv6().str();
+    request += ip_address_.get_ipv6();
   }
   auto port = ip_address_.get_port();
   request += static_cast<char>((port >> 8) & 255);

tdnet/td/net/SslStream.cpp

@@ -10,6 +10,7 @@
 #include "td/utils/common.h"
 #include "td/utils/logging.h"
 #include "td/utils/misc.h"
+#include "td/utils/port/IPAddress.h"
 #include "td/utils/port/wstring_convert.h"
 #include "td/utils/StackAllocator.h"
 #include "td/utils/Status.h"
@@ -235,6 +236,9 @@ class SslStreamImpl {
     options |= SSL_OP_NO_SSLv3;
 #endif
     SSL_CTX_set_options(ssl_ctx, options);
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+    SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_VERSION);
+#endif
     SSL_CTX_set_mode(ssl_ctx, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_ENABLE_PARTIAL_WRITE);
 
     if (cert_file.empty()) {
@@ -318,12 +322,18 @@ class SslStreamImpl {
       SSL_free(ssl_handle);
     };
 
+    auto r_ip_address = IPAddress::get_ip_address(host);
+
 #if OPENSSL_VERSION_NUMBER >= 0x10002000L
     X509_VERIFY_PARAM *param = SSL_get0_param(ssl_handle);
-    /* Enable automatic hostname checks */
-    // TODO: X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS
     X509_VERIFY_PARAM_set_hostflags(param, 0);
-    X509_VERIFY_PARAM_set1_host(param, host.c_str(), 0);
+    if (r_ip_address.is_ok()) {
+      LOG(DEBUG) << "Set verification IP address to " << r_ip_address.ok().get_ip_str();
+      X509_VERIFY_PARAM_set1_ip_asc(param, r_ip_address.ok().get_ip_str().c_str());
+    } else {
+      LOG(DEBUG) << "Set verification host to " << host;
+      X509_VERIFY_PARAM_set1_host(param, host.c_str(), 0);
+    }
 #else
 #warning DANGEROUS! HTTPS HOST WILL NOT BE CHECKED. INSTALL OPENSSL >= 1.0.2 OR IMPLEMENT HTTPS HOST CHECK MANUALLY
 #endif
@@ -333,8 +343,11 @@ class SslStreamImpl {
     SSL_set_bio(ssl_handle, bio, bio);
 
 #if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT)
-    auto host_str = host.str();
-    SSL_set_tlsext_host_name(ssl_handle, MutableCSlice(host_str).begin());
+    if (r_ip_address.is_error()) {  // IP address must not be send as SNI
+      LOG(DEBUG) << "Set SNI host name to " << host;
+      auto host_str = host.str();
+      SSL_set_tlsext_host_name(ssl_handle, MutableCSlice(host_str).begin());
+    }
 #endif
     SSL_set_connect_state(ssl_handle);
 
@@ -466,25 +479,26 @@ class SslStreamImpl {
         LOG(ERROR) << "SSL_get_error returned no error";
         return 0;
       case SSL_ERROR_ZERO_RETURN:
-        LOG(DEBUG) << "SSL_ERROR_ZERO_RETURN";
+        LOG(DEBUG) << "SSL_ZERO_RETURN";
         return 0;
       case SSL_ERROR_WANT_READ:
-        LOG(DEBUG) << "SSL_ERROR_WANT_READ";
+        LOG(DEBUG) << "SSL_WANT_READ";
         return 0;
       case SSL_ERROR_WANT_WRITE:
-        LOG(DEBUG) << "SSL_ERROR_WANT_WRITE";
+        LOG(DEBUG) << "SSL_WANT_WRITE";
         return 0;
       case SSL_ERROR_WANT_CONNECT:
       case SSL_ERROR_WANT_ACCEPT:
       case SSL_ERROR_WANT_X509_LOOKUP:
-        LOG(DEBUG) << "SSL_ERROR: CONNECT ACCEPT LOOKUP";
+        LOG(DEBUG) << "SSL: CONNECT ACCEPT LOOKUP";
         return 0;
       case SSL_ERROR_SYSCALL:
-        LOG(DEBUG) << "SSL_ERROR_SYSCALL";
         if (ERR_peek_error() == 0) {
           if (os_error.code() != 0) {
+            LOG(DEBUG) << "SSL_ERROR_SYSCALL";
             return std::move(os_error);
           } else {
+            LOG(DEBUG) << "SSL_SYSCALL";
             return 0;
           }
         }
@@ -501,6 +515,7 @@ int strm_read(BIO *b, char *buf, int len) {
   auto *stream = static_cast<SslStreamImpl *>(BIO_get_data(b));
   CHECK(stream != nullptr);
   BIO_clear_retry_flags(b);
+  CHECK(buf != nullptr);
   int res = narrow_cast<int>(stream->flow_read(MutableSlice(buf, len)));
   if (res == 0) {
     BIO_set_retry_read(b);
@@ -512,6 +527,7 @@ int strm_write(BIO *b, const char *buf, int len) {
   auto *stream = static_cast<SslStreamImpl *>(BIO_get_data(b));
   CHECK(stream != nullptr);
   BIO_clear_retry_flags(b);
+  CHECK(buf != nullptr);
   return narrow_cast<int>(stream->flow_write(Slice(buf, len)));
 }
 }  // namespace

tdutils/td/utils/Enumerator.h

@@ -19,25 +19,46 @@ class Enumerator {
  public:
   using Key = int32;
 
+  std::map<ValueT, int32> get_map() const {
+    return map_;
+  }
+
+  std::pair<Key, bool> next() {
+    if (!empty_id_.empty()) {
+      auto res = empty_id_.back();
+      empty_id_.pop_back();
+      return std::make_pair(res, true);
+    }
+
+    return std::make_pair((Key) (arr_.size() + 1), false);
+  }
+
+  void erase(Key key_y) {
+    empty_id_.push_back(key_y);
+  }
+
   Key add(ValueT v) {
-    CHECK(arr_.size() < static_cast<size_t>(std::numeric_limits<int32>::max() - 1));
-    int32 next_id = static_cast<int32>(arr_.size() + 1);
+    auto next_id = next();
     bool was_inserted;
     decltype(map_.begin()) it;
-    std::tie(it, was_inserted) = map_.emplace(std::move(v), next_id);
-    if (was_inserted) {
+    std::tie(it, was_inserted) = map_.emplace(std::move(v), next_id.first);
+    if (was_inserted && next_id.second) {
+      arr_[next_id.first - 1] = &it->first;
+    } else if (was_inserted) {
       arr_.push_back(&it->first);
+    } else if (next_id.second) {
+      empty_id_.push_back(next_id.first);
     }
     return it->second;
   }
 
   const ValueT &get(Key key) const {
-    auto pos = static_cast<size_t>(key - 1);
-    CHECK(pos < arr_.size());
+    auto pos = static_cast<Key>(key - 1);
     return *arr_[pos];
   }
 
  private:
+  std::vector<Key> empty_id_;
   std::map<ValueT, int32> map_;
   std::vector<const ValueT *> arr_;
 };

tdutils/td/utils/HttpUrl.cpp

@@ -10,6 +10,7 @@
 #include "td/utils/logging.h"
 #include "td/utils/misc.h"
 #include "td/utils/Parser.h"
+#include "td/utils/port/IPAddress.h"
 
 namespace td {
 
@@ -29,13 +30,7 @@ string HttpUrl::get_url() const {
     result += userinfo_;
     result += '@';
   }
-  if (is_ipv6_) {
-    result += '[';
-  }
   result += host_;
-  if (is_ipv6_) {
-    result += ']';
-  }
   if (specified_port_ > 0) {
     result += ':';
     result += to_string(specified_port_);
@@ -88,8 +83,11 @@ Result<HttpUrl> parse_url(Slice url, HttpUrl::Protocol default_protocol) {
 
   bool is_ipv6 = false;
   if (!host.empty() && host[0] == '[' && host.back() == ']') {
-    host.remove_prefix(1);
-    host.remove_suffix(1);
+    IPAddress ip_address;
+    if (ip_address.init_ipv6_port(host.str(), 1).is_error()) {
+      return Status::Error("Wrong IPv6 address specified in the URL");
+    }
+    CHECK(ip_address.is_ipv6());
     is_ipv6 = true;
   }
   if (host.empty()) {

tdutils/td/utils/Parser.h

@@ -45,7 +45,7 @@ class ParserImpl {
     return ptr_ == end_;
   }
   void clear() {
-    ptr_ = nullptr;
+    ptr_ = SliceT().begin();
     end_ = ptr_;
     status_ = Status::OK();
   }

tdutils/td/utils/common.h

@@ -48,6 +48,7 @@
 
 #include <string>
 #include <vector>
+#include <unordered_map>
 
 #define TD_DEBUG
 

tdutils/td/utils/port/IPAddress.cpp

@@ -267,11 +267,11 @@ uint32 IPAddress::get_ipv4() const {
   return htonl(ipv4_addr_.sin_addr.s_addr);
 }
 
-Slice IPAddress::get_ipv6() const {
+string IPAddress::get_ipv6() const {
   static_assert(sizeof(ipv6_addr_.sin6_addr) == 16, "ipv6 size == 16");
   CHECK(is_valid());
   CHECK(!is_ipv4());
-  return Slice(ipv6_addr_.sin6_addr.s6_addr, 16);
+  return Slice(ipv6_addr_.sin6_addr.s6_addr, 16).str();
 }
 
 IPAddress IPAddress::get_any_addr() const {
@@ -308,7 +308,12 @@ void IPAddress::init_ipv6_any() {
 Status IPAddress::init_ipv6_port(CSlice ipv6, int port) {
   is_valid_ = false;
   if (port <= 0 || port >= (1 << 16)) {
-    return Status::Error(PSLICE() << "Invalid [port=" << port << "]");
+    return Status::Error(PSLICE() << "Invalid [IPv6 address port=" << port << "]");
+  }
+  string ipv6_plain;
+  if (ipv6.size() > 2 && ipv6[0] == '[' && ipv6.back() == ']') {
+    ipv6_plain.assign(ipv6.begin() + 1, ipv6.size() - 2);
+    ipv6 = ipv6_plain;
   }
   std::memset(&ipv6_addr_, 0, sizeof(ipv6_addr_));
   ipv6_addr_.sin6_family = AF_INET6;
@@ -330,7 +335,7 @@ Status IPAddress::init_ipv6_as_ipv4_port(CSlice ipv4, int port) {
 Status IPAddress::init_ipv4_port(CSlice ipv4, int port) {
   is_valid_ = false;
   if (port <= 0 || port >= (1 << 16)) {
-    return Status::Error(PSLICE() << "Invalid [port=" << port << "]");
+    return Status::Error(PSLICE() << "Invalid [IPv4 address port=" << port << "]");
   }
   std::memset(&ipv4_addr_, 0, sizeof(ipv4_addr_));
   ipv4_addr_.sin_family = AF_INET;
@@ -345,6 +350,18 @@ Status IPAddress::init_ipv4_port(CSlice ipv4, int port) {
   return Status::OK();
 }
 
+Result<IPAddress> IPAddress::get_ip_address(CSlice host) {
+  auto r_address = get_ipv4_address(host);
+  if (r_address.is_ok()) {
+    return r_address.move_as_ok();
+  }
+  r_address = get_ipv6_address(host);
+  if (r_address.is_ok()) {
+    return r_address.move_as_ok();
+  }
+  return Status::Error("Not a valid IP address");
+}
+
 Result<IPAddress> IPAddress::get_ipv4_address(CSlice host) {
   // sometimes inet_addr allows much more valid IPv4 hosts than inet_pton,
   // like 0x12.0x34.0x56.0x78, or 0x12345678, or 0x7f.001
@@ -372,6 +389,10 @@ Result<IPAddress> IPAddress::get_ipv6_address(CSlice host) {
 }
 
 Status IPAddress::init_host_port(CSlice host, int port, bool prefer_ipv6) {
+  if (host.size() > 2 && host[0] == '[' && host.back() == ']') {
+    return init_ipv6_port(host, port == 0 ? 1 : port);
+  }
+
   return init_host_port(host, PSLICE() << port, prefer_ipv6);
 }
 
@@ -388,6 +409,11 @@ Status IPAddress::init_host_port(CSlice host, CSlice port, bool prefer_ipv6) {
   TRY_RESULT(ascii_host, idn_to_ascii(host));
   host = ascii_host;  // assign string to CSlice
 
+  if (host[0] == '[' && host.back() == ']') {
+    auto port_int = to_integer<int>(port);
+    return init_ipv6_port(host, port_int == 0 ? 1 : port_int);
+  }
+
   // some getaddrinfo implementations use inet_pton instead of inet_aton and support only decimal-dotted IPv4 form,
   // and so doesn't recognize 0x12.0x34.0x56.0x78, or 0x12345678, or 0x7f.001 as valid IPv4 addresses
   auto ipv4_numeric_addr = inet_addr(host.c_str());
@@ -494,19 +520,19 @@ Status IPAddress::init_peer_address(const SocketFd &socket_fd) {
   return Status::OK();
 }
 
-CSlice IPAddress::ipv4_to_str(uint32 ipv4) {
+string IPAddress::ipv4_to_str(uint32 ipv4) {
   ipv4 = ntohl(ipv4);
-  return ::td::get_ip_str(AF_INET, &ipv4);
+  return ::td::get_ip_str(AF_INET, &ipv4).str();
 }
 
-CSlice IPAddress::ipv6_to_str(Slice ipv6) {
+string IPAddress::ipv6_to_str(Slice ipv6) {
   CHECK(ipv6.size() == 16);
-  return ::td::get_ip_str(AF_INET6, ipv6.ubegin());
+  return ::td::get_ip_str(AF_INET6, ipv6.ubegin()).str();
 }
 
-Slice IPAddress::get_ip_str() const {
+CSlice IPAddress::get_ip_str() const {
   if (!is_valid()) {
-    return Slice("0.0.0.0");
+    return CSlice("0.0.0.0");
   }
 
   switch (get_address_family()) {
@@ -516,7 +542,23 @@ Slice IPAddress::get_ip_str() const {
       return ::td::get_ip_str(AF_INET, &ipv4_addr_.sin_addr);
     default:
       UNREACHABLE();
-      return Slice();
+      return CSlice();
+  }
+}
+
+string IPAddress::get_ip_host() const {
+  if (!is_valid()) {
+    return "0.0.0.0";
+  }
+
+  switch (get_address_family()) {
+    case AF_INET6:
+      return PSTRING() << '[' << ::td::get_ip_str(AF_INET6, &ipv6_addr_.sin6_addr) << ']';
+    case AF_INET:
+      return ::td::get_ip_str(AF_INET, &ipv4_addr_.sin_addr).str();
+    default:
+      UNREACHABLE();
+      return string();
   }
 }
 
@@ -599,12 +641,7 @@ StringBuilder &operator<<(StringBuilder &builder, const IPAddress &address) {
   if (!address.is_valid()) {
     return builder << "[invalid]";
   }
-  if (address.get_address_family() == AF_INET) {
-    return builder << "[" << address.get_ip_str() << ":" << address.get_port() << "]";
-  } else {
-    CHECK(address.get_address_family() == AF_INET6);
-    return builder << "[[" << address.get_ip_str() << "]:" << address.get_port() << "]";
-  }
+  return builder << "[" << address.get_ip_host() << ":" << address.get_port() << "]";
 }
 
 }  // namespace td

tdutils/td/utils/port/IPAddress.h

@@ -39,11 +39,20 @@ class IPAddress {
   void set_port(int port);
 
   uint32 get_ipv4() const;
-  Slice get_ipv6() const;
-  Slice get_ip_str() const;
+  string get_ipv6() const;
+
+  // returns result in a static thread-local buffer, which may be overwritten by any subsequent method call
+  CSlice get_ip_str() const;
+
+  // returns IP address as a host, i.e. IPv4 or [IPv6]
+  string get_ip_host() const;
+
+  static string ipv4_to_str(uint32 ipv4);
+  static string ipv6_to_str(Slice ipv6);
 
   IPAddress get_any_addr() const;
 
+  static Result<IPAddress> get_ip_address(CSlice host);  // host must be any IPv4 or IPv6
   static Result<IPAddress> get_ipv4_address(CSlice host);
   static Result<IPAddress> get_ipv6_address(CSlice host);
 
@@ -63,8 +72,6 @@ class IPAddress {
   const sockaddr *get_sockaddr() const;
   size_t get_sockaddr_len() const;
   int get_address_family() const;
-  static CSlice ipv4_to_str(uint32 ipv4);
-  static CSlice ipv6_to_str(Slice ipv6);
   Status init_sockaddr(sockaddr *addr);
   Status init_sockaddr(sockaddr *addr, socklen_t len) TD_WARN_UNUSED_RESULT;
 

tdutils/td/utils/port/MemoryMapping.cpp

@@ -84,7 +84,7 @@ Result<MemoryMapping> MemoryMapping::create_from_file(const FileFd &file_fd, con
   auto data_offset = begin - fixed_begin;
   auto data_size = narrow_cast<size_t>(end - fixed_begin);
 
-  void *data = mmap(nullptr, data_size, PROT_READ, MAP_PRIVATE, fd, narrow_cast<off_t>(fixed_begin));
+  void *data = mmap(nullptr, data_size, PROT_READ, MADV_MERGEABLE, fd, narrow_cast<off_t>(fixed_begin));
   if (data == MAP_FAILED) {
     return OS_ERROR("mmap call failed");
   }

tdutils/td/utils/port/signals.cpp

@@ -45,7 +45,7 @@ Status setup_signals_alt_stack() {
   auto page_size = getpagesize();
   auto stack_size = (MINSIGSTKSZ + 16 * page_size - 1) / page_size * page_size;
 
-  void *stack = mmap(nullptr, stack_size + 2 * page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
+  void *stack = mmap(nullptr, stack_size + 2 * page_size, PROT_READ | PROT_WRITE, MAP_ANON | MADV_MERGEABLE, -1, 0);
   if (stack == MAP_FAILED) {
     return OS_ERROR("Mmap failed");
   }

tdutils/test/heap.cpp

@@ -10,31 +10,27 @@
 #include "td/utils/Heap.h"
 #include "td/utils/Random.h"
 
-#include <algorithm>
 #include <cstdio>
-#include <cstdlib>
 #include <set>
 #include <utility>
 
 REGISTER_TESTS(heap)
 
-using namespace td;
-
 TEST(Heap, sort_random_perm) {
   int n = 1000000;
 
-  std::vector<int> v(n);
+  td::vector<int> v(n);
   for (int i = 0; i < n; i++) {
     v[i] = i;
   }
 
   // random shuffle
   for (int i = 1; i < n; i++) {
-    std::swap(v[Random::fast(0, i)], v[i]);
+    std::swap(v[td::Random::fast(0, i)], v[i]);
   }
 
-  std::vector<HeapNode> nodes(n);
-  KHeap<int> kheap;
+  td::vector<td::HeapNode> nodes(n);
+  td::KHeap<int> kheap;
   for (int i = 0; i < n; i++) {
     kheap.insert(v[i], &nodes[i]);
   }
@@ -55,7 +51,7 @@ class CheckedHeap {
       nodes[i].value = i;
     }
   }
-  static void xx(int key, const HeapNode *heap_node) {
+  static void xx(int key, const td::HeapNode *heap_node) {
     const Node *node = static_cast<const Node *>(heap_node);
     std::fprintf(stderr, "(%d;%d)", node->key, node->value);
   }
@@ -70,9 +66,9 @@ class CheckedHeap {
   }
   int random_id() const {
     CHECK(!empty());
-    return ids[Random::fast(0, static_cast<int>(ids.size() - 1))];
+    return ids[td::Random::fast(0, static_cast<int>(ids.size() - 1))];
   }
-  size_t size() const {
+  std::size_t size() const {
     return ids.size();
   }
   bool empty() const {
@@ -144,19 +140,19 @@ class CheckedHeap {
   }
 
  private:
-  struct Node : public HeapNode {
+  struct Node : public td::HeapNode {
     Node() = default;
     Node(int key, int value) : key(key), value(value) {
     }
     int key = 0;
     int value = 0;
   };
-  vector<int> ids;
-  vector<int> rev_ids;
-  vector<int> free_ids;
-  vector<Node> nodes;
+  td::vector<int> ids;
+  td::vector<int> rev_ids;
+  td::vector<int> free_ids;
+  td::vector<Node> nodes;
   std::set<std::pair<int, int>> set_heap;
-  KHeap<int> kheap;
+  td::KHeap<int> kheap;
 };
 
 TEST(Heap, random_events) {
@@ -167,11 +163,11 @@ TEST(Heap, random_events) {
       heap.top_key();
     }
 
-    int x = Random::fast(0, 4);
+    int x = td::Random::fast(0, 4);
     if (heap.empty() || (x < 2 && heap.size() < 1000)) {
-      heap.insert(Random::fast(0, 99));
+      heap.insert(td::Random::fast(0, 99));
     } else if (x < 3) {
-      heap.fix_key(Random::fast(0, 99), heap.random_id());
+      heap.fix_key(td::Random::fast(0, 99), heap.random_id());
     } else if (x < 4) {
       heap.erase(heap.random_id());
     } else if (x < 5) {

test/http.cpp

@@ -36,7 +36,6 @@
 #include "data.h"
 
 #include <algorithm>
-#include <cstdlib>
 #include <limits>
 
 REGISTER_TESTS(http)