tdlight-team/tdlight
tdlight-team/tdlight
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Patches

Browse Source
This commit is contained in:
Andrea Cavalli 2020-05-23 21:27:24 +02:00
parent e2fd1c13e9
commit e3bf0f63f0
57 changed files with 51258 additions and 25172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
sqlite/CMakeLists.txt
View File

@ -11,7 +11,6 @@ endif()
set(SQLITE_SOURCE
sqlite/sqlite3.c
sqlite/sqlite3.h
sqlite/sqlite3ext.h
sqlite/sqlite3session.h
@ -21,15 +20,14 @@ add_library(tdsqlite STATIC ${SQLITE_SOURCE})
target_include_directories(tdsqlite PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
target_include_directories(tdsqlite SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR})
target_link_libraries(tdsqlite PRIVATE ${OPENSSL_CRYPTO_LIBRARY} ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES})
target_compile_definitions(tdsqlite PRIVATE
-DSQLITE_DEFAULT_MEMSTATUS=0
-DSQLITE_OMIT_LOAD_EXTENSION
-DSQLITE_OMIT_DECLTYPE
-DSQLITE_OMIT_PROGRESS_CALLBACK
#-DSQLITE_OMIT_DEPRECATED # SQLCipher uses deprecated sqlite3_profile
#-DSQLITE_OMIT_SHARED_CACHE
-DSQLITE_MAX_MMAP_SIZE=50331648
-DSQLITE_MAX_MEMORY=50331648
-DSQLITE_ENABLE_SORTER_REFERENCES
-DSQLITE_DIRECT_OVERFLOW_READ
-DSQLITE_ENABLE_MEMORY_MANAGEMENT
)
target_compile_definitions(tdsqlite PRIVATE -DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=2 -DSQLITE_ENABLE_FTS5 -DSQLITE_DISABLE_LFS)
if (NOT WIN32)
target_compile_definitions(tdsqlite PRIVATE -DHAVE_USLEEP -DNDEBUG=1)

71573
sqlite/sqlite/sqlite3.c vendored
View File

File diff suppressed because it is too large Load Diff

2409
sqlite/sqlite/sqlite3.h vendored
View File

File diff suppressed because it is too large Load Diff

90
sqlite/sqlite/sqlite3ext.h vendored
View File

@ -134,7 +134,7 @@ struct sqlite3_api_routines {
int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*,
const char*,const char*),void*);
void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*));
char * (*snprintf)(int,char*,const char*,...);
char * (*xsnprintf)(int,char*,const char*,...);
int (*step)(sqlite3_stmt*);
int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*,
char const**,char const**,int*,int*,int*);
@ -246,7 +246,7 @@ struct sqlite3_api_routines {
int (*uri_boolean)(const char*,const char*,int);
sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64);
const char *(*uri_parameter)(const char*,const char*);
char *(*vsnprintf)(int,char*,const char*,va_list);
char *(*xvsnprintf)(int,char*,const char*,va_list);
int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*);
/* Version 3.8.7 and later */
int (*auto_extension)(void(*)(void));
@ -282,6 +282,49 @@ struct sqlite3_api_routines {
/* Version 3.14.0 and later */
int (*trace_v2)(sqlite3*,unsigned,int(*)(unsigned,void*,void*,void*),void*);
char *(*expanded_sql)(sqlite3_stmt*);
/* Version 3.18.0 and later */
void (*set_last_insert_rowid)(sqlite3*,sqlite3_int64);
/* Version 3.20.0 and later */
int (*prepare_v3)(sqlite3*,const char*,int,unsigned int,
sqlite3_stmt**,const char**);
int (*prepare16_v3)(sqlite3*,const void*,int,unsigned int,
sqlite3_stmt**,const void**);
int (*bind_pointer)(sqlite3_stmt*,int,void*,const char*,void(*)(void*));
void (*result_pointer)(sqlite3_context*,void*,const char*,void(*)(void*));
void *(*value_pointer)(sqlite3_value*,const char*);
int (*vtab_nochange)(sqlite3_context*);
int (*value_nochange)(sqlite3_value*);
const char *(*vtab_collation)(sqlite3_index_info*,int);
/* Version 3.24.0 and later */
int (*keyword_count)(void);
int (*keyword_name)(int,const char**,int*);
int (*keyword_check)(const char*,int);
sqlite3_str *(*str_new)(sqlite3*);
char *(*str_finish)(sqlite3_str*);
void (*str_appendf)(sqlite3_str*, const char *zFormat, ...);
void (*str_vappendf)(sqlite3_str*, const char *zFormat, va_list);
void (*str_append)(sqlite3_str*, const char *zIn, int N);
void (*str_appendall)(sqlite3_str*, const char *zIn);
void (*str_appendchar)(sqlite3_str*, int N, char C);
void (*str_reset)(sqlite3_str*);
int (*str_errcode)(sqlite3_str*);
int (*str_length)(sqlite3_str*);
char *(*str_value)(sqlite3_str*);
/* Version 3.25.0 and later */
int (*create_window_function)(sqlite3*,const char*,int,int,void*,
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
void (*xFinal)(sqlite3_context*),
void (*xValue)(sqlite3_context*),
void (*xInv)(sqlite3_context*,int,sqlite3_value**),
void(*xDestroy)(void*));
/* Version 3.26.0 and later */
const char *(*normalized_sql)(sqlite3_stmt*);
/* Version 3.28.0 and later */
int (*stmt_isexplain)(sqlite3_stmt*);
int (*value_frombind)(sqlite3_value*);
/* Version 3.30.0 and later */
int (*drop_modules)(sqlite3*,const char**);
sqlite3_int64 (*hard_heap_limit64)(sqlite3_int64);
};
/*
@ -408,7 +451,7 @@ typedef int (*sqlite3_loadext_entry)(
#define sqlite3_rollback_hook sqlite3_api->rollback_hook
#define sqlite3_set_authorizer sqlite3_api->set_authorizer
#define sqlite3_set_auxdata sqlite3_api->set_auxdata
#define sqlite3_snprintf sqlite3_api->snprintf
#define sqlite3_snprintf sqlite3_api->xsnprintf
#define sqlite3_step sqlite3_api->step
#define sqlite3_table_column_metadata sqlite3_api->table_column_metadata
#define sqlite3_thread_cleanup sqlite3_api->thread_cleanup
@ -432,7 +475,7 @@ typedef int (*sqlite3_loadext_entry)(
#define sqlite3_value_text16le sqlite3_api->value_text16le
#define sqlite3_value_type sqlite3_api->value_type
#define sqlite3_vmprintf sqlite3_api->vmprintf
#define sqlite3_vsnprintf sqlite3_api->vsnprintf
#define sqlite3_vsnprintf sqlite3_api->xvsnprintf
#define sqlite3_overload_function sqlite3_api->overload_function
#define sqlite3_prepare_v2 sqlite3_api->prepare_v2
#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2
@ -508,7 +551,7 @@ typedef int (*sqlite3_loadext_entry)(
#define sqlite3_uri_boolean sqlite3_api->uri_boolean
#define sqlite3_uri_int64 sqlite3_api->uri_int64
#define sqlite3_uri_parameter sqlite3_api->uri_parameter
#define sqlite3_uri_vsnprintf sqlite3_api->vsnprintf
#define sqlite3_uri_vsnprintf sqlite3_api->xvsnprintf
#define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2
/* Version 3.8.7 and later */
#define sqlite3_auto_extension sqlite3_api->auto_extension
@ -540,6 +583,43 @@ typedef int (*sqlite3_loadext_entry)(
/* Version 3.14.0 and later */
#define sqlite3_trace_v2 sqlite3_api->trace_v2
#define sqlite3_expanded_sql sqlite3_api->expanded_sql
/* Version 3.18.0 and later */
#define sqlite3_set_last_insert_rowid sqlite3_api->set_last_insert_rowid
/* Version 3.20.0 and later */
#define sqlite3_prepare_v3 sqlite3_api->prepare_v3
#define sqlite3_prepare16_v3 sqlite3_api->prepare16_v3
#define sqlite3_bind_pointer sqlite3_api->bind_pointer
#define sqlite3_result_pointer sqlite3_api->result_pointer
#define sqlite3_value_pointer sqlite3_api->value_pointer
/* Version 3.22.0 and later */
#define sqlite3_vtab_nochange sqlite3_api->vtab_nochange
#define sqlite3_value_nochange sqlite3_api->value_nochange
#define sqlite3_vtab_collation sqlite3_api->vtab_collation
/* Version 3.24.0 and later */
#define sqlite3_keyword_count sqlite3_api->keyword_count
#define sqlite3_keyword_name sqlite3_api->keyword_name
#define sqlite3_keyword_check sqlite3_api->keyword_check
#define sqlite3_str_new sqlite3_api->str_new
#define sqlite3_str_finish sqlite3_api->str_finish
#define sqlite3_str_appendf sqlite3_api->str_appendf
#define sqlite3_str_vappendf sqlite3_api->str_vappendf
#define sqlite3_str_append sqlite3_api->str_append
#define sqlite3_str_appendall sqlite3_api->str_appendall
#define sqlite3_str_appendchar sqlite3_api->str_appendchar
#define sqlite3_str_reset sqlite3_api->str_reset
#define sqlite3_str_errcode sqlite3_api->str_errcode
#define sqlite3_str_length sqlite3_api->str_length
#define sqlite3_str_value sqlite3_api->str_value
/* Version 3.25.0 and later */
#define sqlite3_create_window_function sqlite3_api->create_window_function
/* Version 3.26.0 and later */
#define sqlite3_normalized_sql sqlite3_api->normalized_sql
/* Version 3.28.0 and later */
#define sqlite3_stmt_isexplain sqlite3_api->isexplain
#define sqlite3_value_frombind sqlite3_api->frombind
/* Version 3.30.0 and later */
#define sqlite3_drop_modules sqlite3_api->drop_modules
#define sqlite3_hard_heap_limit64 sqlite3_api->hard_heap_limit64
#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)

442
sqlite/sqlite/sqlite3session.h vendored
View File

@ -1,3 +1,4 @@
#if !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION)
#define __SQLITESESSION_H_ 1
@ -12,16 +13,23 @@ extern "C" {
/*
** CAPI3REF: Session Object Handle
**
** An instance of this object is a [session] that can be used to
** record changes to a database.
*/
typedef struct sqlite3_session sqlite3_session;
/*
** CAPI3REF: Changeset Iterator Handle
**
** An instance of this object acts as a cursor for iterating
** over the elements of a [changeset] or [patchset].
*/
typedef struct sqlite3_changeset_iter sqlite3_changeset_iter;
/*
** CAPI3REF: Create A New Session Object
** CONSTRUCTOR: sqlite3_session
**
** Create a new session object attached to database handle db. If successful,
** a pointer to the new object is written to *ppSession and SQLITE_OK is
@ -58,6 +66,7 @@ int sqlite3session_create(
/*
** CAPI3REF: Delete A Session Object
** DESTRUCTOR: sqlite3_session
**
** Delete a session object previously allocated using
** [sqlite3session_create()]. Once a session object has been deleted, the
@ -73,6 +82,7 @@ void sqlite3session_delete(sqlite3_session *pSession);
/*
** CAPI3REF: Enable Or Disable A Session Object
** METHOD: sqlite3_session
**
** Enable or disable the recording of changes by a session object. When
** enabled, a session object records changes made to the database. When
@ -92,6 +102,7 @@ int sqlite3session_enable(sqlite3_session *pSession, int bEnable);
/*
** CAPI3REF: Set Or Clear the Indirect Change Flag
** METHOD: sqlite3_session
**
** Each change recorded by a session object is marked as either direct or
** indirect. A change is marked as indirect if either:
@ -121,6 +132,7 @@ int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect);
/*
** CAPI3REF: Attach A Table To A Session Object
** METHOD: sqlite3_session
**
** If argument zTab is not NULL, then it is the name of a table to attach
** to the session object passed as the first argument. All subsequent changes
@ -146,6 +158,35 @@ int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect);
**
** SQLITE_OK is returned if the call completes without error. Or, if an error
** occurs, an SQLite error code (e.g. SQLITE_NOMEM) is returned.
**
** <h3>Special sqlite_stat1 Handling</h3>
**
** As of SQLite version 3.22.0, the "sqlite_stat1" table is an exception to
** some of the rules above. In SQLite, the schema of sqlite_stat1 is:
** <pre>
** &nbsp; CREATE TABLE sqlite_stat1(tbl,idx,stat)
** </pre>
**
** Even though sqlite_stat1 does not have a PRIMARY KEY, changes are
** recorded for it as if the PRIMARY KEY is (tbl,idx). Additionally, changes
** are recorded for rows for which (idx IS NULL) is true. However, for such
** rows a zero-length blob (SQL value X'') is stored in the changeset or
** patchset instead of a NULL value. This allows such changesets to be
** manipulated by legacy implementations of sqlite3changeset_invert(),
** concat() and similar.
**
** The sqlite3changeset_apply() function automatically converts the
** zero-length blob back to a NULL value when updating the sqlite_stat1
** table. However, if the application calls sqlite3changeset_new(),
** sqlite3changeset_old() or sqlite3changeset_conflict on a changeset
** iterator directly (including on a changeset iterator passed to a
** conflict-handler callback) then the X'' value is returned. The application
** must translate X'' to NULL itself if required.
**
** Legacy (older than 3.22.0) versions of the sessions module cannot capture
** changes made to the sqlite_stat1 table. Legacy versions of the
** sqlite3changeset_apply() function silently ignore any modifications to the
** sqlite_stat1 table that are part of a changeset or patchset.
*/
int sqlite3session_attach(
sqlite3_session *pSession, /* Session object */
@ -154,6 +195,7 @@ int sqlite3session_attach(
/*
** CAPI3REF: Set a table filter on a Session Object.
** METHOD: sqlite3_session
**
** The second argument (xFilter) is the "filter callback". For changes to rows
** in tables that are not attached to the Session object, the filter is called
@ -172,6 +214,7 @@ void sqlite3session_table_filter(
/*
** CAPI3REF: Generate A Changeset From A Session Object
** METHOD: sqlite3_session
**
** Obtain a changeset containing changes to the tables attached to the
** session object passed as the first argument. If successful,
@ -281,7 +324,8 @@ int sqlite3session_changeset(
);
/*
** CAPI3REF: Load The Difference Between Tables Into A Session
** CAPI3REF: Load The Difference Between Tables Into A Session
** METHOD: sqlite3_session
**
** If it is not already attached to the session object passed as the first
** argument, this function attaches table zTbl in the same manner as the
@ -318,7 +362,8 @@ int sqlite3session_changeset(
** the from-table, a DELETE record is added to the session object.
**
** <li> For each row (primary key) that exists in both tables, but features
** different in each, an UPDATE record is added to the session.
** different non-PK values in each, an UPDATE record is added to the
** session.
** </ul>
**
** To clarify, if this function is called and then a changeset constructed
@ -345,6 +390,7 @@ int sqlite3session_diff(
/*
** CAPI3REF: Generate A Patchset From A Session Object
** METHOD: sqlite3_session
**
** The differences between a patchset and a changeset are that:
**
@ -373,8 +419,8 @@ int sqlite3session_diff(
*/
int sqlite3session_patchset(
sqlite3_session *pSession, /* Session object */
int *pnPatchset, /* OUT: Size of buffer at *ppChangeset */
void **ppPatchset /* OUT: Buffer containing changeset */
int *pnPatchset, /* OUT: Size of buffer at *ppPatchset */
void **ppPatchset /* OUT: Buffer containing patchset */
);
/*
@ -396,6 +442,7 @@ int sqlite3session_isempty(sqlite3_session *pSession);
/*
** CAPI3REF: Create An Iterator To Traverse A Changeset
** CONSTRUCTOR: sqlite3_changeset_iter
**
** Create an iterator used to iterate through the contents of a changeset.
** If successful, *pp is set to point to the iterator handle and SQLITE_OK
@ -426,16 +473,43 @@ int sqlite3session_isempty(sqlite3_session *pSession);
** consecutively. There is no chance that the iterator will visit a change
** the applies to table X, then one for table Y, and then later on visit
** another change for table X.
**
** The behavior of sqlite3changeset_start_v2() and its streaming equivalent
** may be modified by passing a combination of
** [SQLITE_CHANGESETSTART_INVERT | supported flags] as the 4th parameter.
**
** Note that the sqlite3changeset_start_v2() API is still <b>experimental</b>
** and therefore subject to change.
*/
int sqlite3changeset_start(
sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */
int nChangeset, /* Size of changeset blob in bytes */
void *pChangeset /* Pointer to blob containing changeset */
);
int sqlite3changeset_start_v2(
sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */
int nChangeset, /* Size of changeset blob in bytes */
void *pChangeset, /* Pointer to blob containing changeset */
int flags /* SESSION_CHANGESETSTART_* flags */
);
/*
** CAPI3REF: Flags for sqlite3changeset_start_v2
**
** The following flags may passed via the 4th parameter to
** [sqlite3changeset_start_v2] and [sqlite3changeset_start_v2_strm]:
**
** <dt>SQLITE_CHANGESETAPPLY_INVERT <dd>
** Invert the changeset while iterating through it. This is equivalent to
** inverting a changeset using sqlite3changeset_invert() before applying it.
** It is an error to specify this flag with a patchset.
*/
#define SQLITE_CHANGESETSTART_INVERT 0x0002
/*
** CAPI3REF: Advance A Changeset Iterator
** METHOD: sqlite3_changeset_iter
**
** This function may only be used with iterators created by function
** [sqlite3changeset_start()]. If it is called on an iterator passed to
@ -460,6 +534,7 @@ int sqlite3changeset_next(sqlite3_changeset_iter *pIter);
/*
** CAPI3REF: Obtain The Current Operation From A Changeset Iterator
** METHOD: sqlite3_changeset_iter
**
** The pIter argument passed to this function may either be an iterator
** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator
@ -473,7 +548,7 @@ int sqlite3changeset_next(sqlite3_changeset_iter *pIter);
** sqlite3changeset_next() is called on the iterator or until the
** conflict-handler function returns. If pnCol is not NULL, then *pnCol is
** set to the number of columns in the table affected by the change. If
** pbIncorrect is not NULL, then *pbIndirect is set to true (1) if the change
** pbIndirect is not NULL, then *pbIndirect is set to true (1) if the change
** is an indirect change, or false (0) otherwise. See the documentation for
** [sqlite3session_indirect()] for a description of direct and indirect
** changes. Finally, if pOp is not NULL, then *pOp is set to one of
@ -494,6 +569,7 @@ int sqlite3changeset_op(
/*
** CAPI3REF: Obtain The Primary Key Definition Of A Table
** METHOD: sqlite3_changeset_iter
**
** For each modified table, a changeset includes the following:
**
@ -525,6 +601,7 @@ int sqlite3changeset_pk(
/*
** CAPI3REF: Obtain old.* Values From A Changeset Iterator
** METHOD: sqlite3_changeset_iter
**
** The pIter argument passed to this function may either be an iterator
** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator
@ -555,6 +632,7 @@ int sqlite3changeset_old(
/*
** CAPI3REF: Obtain new.* Values From A Changeset Iterator
** METHOD: sqlite3_changeset_iter
**
** The pIter argument passed to this function may either be an iterator
** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator
@ -588,6 +666,7 @@ int sqlite3changeset_new(
/*
** CAPI3REF: Obtain Conflicting Row Values From A Changeset Iterator
** METHOD: sqlite3_changeset_iter
**
** This function should only be used with iterator objects passed to a
** conflict-handler callback by [sqlite3changeset_apply()] with either
@ -615,6 +694,7 @@ int sqlite3changeset_conflict(
/*
** CAPI3REF: Determine The Number Of Foreign Key Constraint Violations
** METHOD: sqlite3_changeset_iter
**
** This function may only be called with an iterator passed to an
** SQLITE_CHANGESET_FOREIGN_KEY conflict handler callback. In this case
@ -631,6 +711,7 @@ int sqlite3changeset_fk_conflicts(
/*
** CAPI3REF: Finalize A Changeset Iterator
** METHOD: sqlite3_changeset_iter
**
** This function is used to finalize an iterator allocated with
** [sqlite3changeset_start()].
@ -647,6 +728,7 @@ int sqlite3changeset_fk_conflicts(
** to that error is returned by this function. Otherwise, SQLITE_OK is
** returned. This is to allow the following pattern (pseudo-code):
**
** <pre>
** sqlite3changeset_start();
** while( SQLITE_ROW==sqlite3changeset_next() ){
** // Do something with change.
@ -655,6 +737,7 @@ int sqlite3changeset_fk_conflicts(
** if( rc!=SQLITE_OK ){
** // An error has occurred
** }
** </pre>
*/
int sqlite3changeset_finalize(sqlite3_changeset_iter *pIter);
@ -702,6 +785,7 @@ int sqlite3changeset_invert(
** sqlite3_changegroup object. Calling it produces similar results as the
** following code fragment:
**
** <pre>
** sqlite3_changegroup *pGrp;
** rc = sqlite3_changegroup_new(&pGrp);
** if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nA, pA);
@ -712,6 +796,7 @@ int sqlite3changeset_invert(
** *ppOut = 0;
** *pnOut = 0;
** }
** </pre>
**
** Refer to the sqlite3_changegroup documentation below for details.
*/
@ -727,11 +812,15 @@ int sqlite3changeset_concat(
/*
** CAPI3REF: Changegroup Handle
**
** A changegroup is an object used to combine two or more
** [changesets] or [patchsets]
*/
typedef struct sqlite3_changegroup sqlite3_changegroup;
/*
** CAPI3REF: Create A New Changegroup Object
** CONSTRUCTOR: sqlite3_changegroup
**
** An sqlite3_changegroup object is used to combine two or more changesets
** (or patchsets) into a single changeset (or patchset). A single changegroup
@ -769,6 +858,7 @@ int sqlite3changegroup_new(sqlite3_changegroup **pp);
/*
** CAPI3REF: Add A Changeset To A Changegroup
** METHOD: sqlite3_changegroup
**
** Add all changes within the changeset (or patchset) in buffer pData (size
** nData bytes) to the changegroup.
@ -846,6 +936,7 @@ int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData);
/*
** CAPI3REF: Obtain A Composite Changeset From A Changegroup
** METHOD: sqlite3_changegroup
**
** Obtain a buffer containing a changeset (or patchset) representing the
** current contents of the changegroup. If the inputs to the changegroup
@ -876,25 +967,25 @@ int sqlite3changegroup_output(
/*
** CAPI3REF: Delete A Changegroup Object
** DESTRUCTOR: sqlite3_changegroup
*/
void sqlite3changegroup_delete(sqlite3_changegroup*);
/*
** CAPI3REF: Apply A Changeset To A Database
**
** Apply a changeset to a database. This function attempts to update the
** "main" database attached to handle db with the changes found in the
** changeset passed via the second and third arguments.
** Apply a changeset or patchset to a database. These functions attempt to
** update the "main" database attached to handle db with the changes found in
** the changeset passed via the second and third arguments.
**
** The fourth argument (xFilter) passed to this function is the "filter
** The fourth argument (xFilter) passed to these functions is the "filter
** callback". If it is not NULL, then for each table affected by at least one
** change in the changeset, the filter callback is invoked with
** the table name as the second argument, and a copy of the context pointer
** passed as the sixth argument to this function as the first. If the "filter
** callback" returns zero, then no attempt is made to apply any changes to
** the table. Otherwise, if the return value is non-zero or the xFilter
** argument to this function is NULL, all changes related to the table are
** attempted.
** passed as the sixth argument as the first. If the "filter callback"
** returns zero, then no attempt is made to apply any changes to the table.
** Otherwise, if the return value is non-zero or the xFilter argument to
** is NULL, all changes related to the table are attempted.
**
** For each table that is not excluded by the filter callback, this function
** tests that the target database contains a compatible table. A table is
@ -903,7 +994,7 @@ void sqlite3changegroup_delete(sqlite3_changegroup*);
** <ul>
** <li> The table has the same name as the name recorded in the
** changeset, and
** <li> The table has the same number of columns as recorded in the
** <li> The table has at least as many columns as recorded in the
** changeset, and
** <li> The table has primary key columns in the same position as
** recorded in the changeset.
@ -939,7 +1030,7 @@ void sqlite3changegroup_delete(sqlite3_changegroup*);
**
** <dl>
** <dt>DELETE Changes<dd>
** For each DELETE change, this function checks if the target database
** For each DELETE change, the function checks if the target database
** contains a row with the same primary key value (or values) as the
** original row values stored in the changeset. If it does, and the values
** stored in all non-primary key columns also match the values stored in
@ -948,7 +1039,11 @@ void sqlite3changegroup_delete(sqlite3_changegroup*);
** If a row with matching primary key values is found, but one or more of
** the non-primary key fields contains a value different from the original
** row value stored in the changeset, the conflict-handler function is
** invoked with [SQLITE_CHANGESET_DATA] as the second argument.
** invoked with [SQLITE_CHANGESET_DATA] as the second argument. If the
** database table has more columns than are recorded in the changeset,
** only the values of those non-primary key fields are compared against
** the current database contents - any trailing database table columns
** are ignored.
**
** If no row with matching primary key values is found in the database,
** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND]
@ -963,7 +1058,9 @@ void sqlite3changegroup_delete(sqlite3_changegroup*);
**
** <dt>INSERT Changes<dd>
** For each INSERT change, an attempt is made to insert the new row into
** the database.
** the database. If the changeset row contains fewer fields than the
** database table, the trailing fields are populated with their default
** values.
**
** If the attempt to insert the row fails because the database already
** contains a row with the same primary key values, the conflict handler
@ -978,16 +1075,16 @@ void sqlite3changegroup_delete(sqlite3_changegroup*);
** [SQLITE_CHANGESET_REPLACE].
**
** <dt>UPDATE Changes<dd>
** For each UPDATE change, this function checks if the target database
** For each UPDATE change, the function checks if the target database
** contains a row with the same primary key value (or values) as the
** original row values stored in the changeset. If it does, and the values
** stored in all non-primary key columns also match the values stored in
** the changeset the row is updated within the target database.
** stored in all modified non-primary key columns also match the values
** stored in the changeset the row is updated within the target database.
**
** If a row with matching primary key values is found, but one or more of
** the non-primary key fields contains a value different from an original
** row value stored in the changeset, the conflict-handler function is
** invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since
** the modified non-primary key fields contains a value different from an
** original row value stored in the changeset, the conflict-handler function
** is invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since
** UPDATE changes only contain values for non-primary key fields that are
** to be modified, only those fields need to match the original values to
** avoid the SQLITE_CHANGESET_DATA conflict-handler callback.
@ -1009,11 +1106,28 @@ void sqlite3changegroup_delete(sqlite3_changegroup*);
** This can be used to further customize the applications conflict
** resolution strategy.
**
** All changes made by this function are enclosed in a savepoint transaction.
** All changes made by these functions are enclosed in a savepoint transaction.
** If any other error (aside from a constraint failure when attempting to
** write to the target database) occurs, then the savepoint transaction is
** rolled back, restoring the target database to its original state, and an
** SQLite error code returned.
**
** If the output parameters (ppRebase) and (pnRebase) are non-NULL and
** the input is a changeset (not a patchset), then sqlite3changeset_apply_v2()
** may set (*ppRebase) to point to a "rebase" that may be used with the
** sqlite3_rebaser APIs buffer before returning. In this case (*pnRebase)
** is set to the size of the buffer in bytes. It is the responsibility of the
** caller to eventually free any such buffer using sqlite3_free(). The buffer
** is only allocated and populated if one or more conflicts were encountered
** while applying the patchset. See comments surrounding the sqlite3_rebaser
** APIs for further details.
**
** The behavior of sqlite3changeset_apply_v2() and its streaming equivalent
** may be modified by passing a combination of
** [SQLITE_CHANGESETAPPLY_NOSAVEPOINT | supported flags] as the 9th parameter.
**
** Note that the sqlite3changeset_apply_v2() API is still <b>experimental</b>
** and therefore subject to change.
*/
int sqlite3changeset_apply(
sqlite3 *db, /* Apply change to "main" db of this handle */
@ -1030,6 +1144,47 @@ int sqlite3changeset_apply(
),
void *pCtx /* First argument passed to xConflict */
);
int sqlite3changeset_apply_v2(
sqlite3 *db, /* Apply change to "main" db of this handle */
int nChangeset, /* Size of changeset in bytes */
void *pChangeset, /* Changeset blob */
int(*xFilter)(
void *pCtx, /* Copy of sixth arg to _apply() */
const char *zTab /* Table name */
),
int(*xConflict)(
void *pCtx, /* Copy of sixth arg to _apply() */
int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */
sqlite3_changeset_iter *p /* Handle describing change and conflict */
),
void *pCtx, /* First argument passed to xConflict */
void **ppRebase, int *pnRebase, /* OUT: Rebase data */
int flags /* SESSION_CHANGESETAPPLY_* flags */
);
/*
** CAPI3REF: Flags for sqlite3changeset_apply_v2
**
** The following flags may passed via the 9th parameter to
** [sqlite3changeset_apply_v2] and [sqlite3changeset_apply_v2_strm]:
**
** <dl>
** <dt>SQLITE_CHANGESETAPPLY_NOSAVEPOINT <dd>
** Usually, the sessions module encloses all operations performed by
** a single call to apply_v2() or apply_v2_strm() in a [SAVEPOINT]. The
** SAVEPOINT is committed if the changeset or patchset is successfully
** applied, or rolled back if an error occurs. Specifying this flag
** causes the sessions module to omit this savepoint. In this case, if the
** caller has an open transaction or savepoint when apply_v2() is called,
** it may revert the partially applied changeset by rolling it back.
**
** <dt>SQLITE_CHANGESETAPPLY_INVERT <dd>
** Invert the changeset before applying it. This is equivalent to inverting
** a changeset using sqlite3changeset_invert() before applying it. It is
** an error to specify this flag with a patchset.
*/
#define SQLITE_CHANGESETAPPLY_NOSAVEPOINT 0x0001
#define SQLITE_CHANGESETAPPLY_INVERT 0x0002
/*
** CAPI3REF: Constants Passed To The Conflict Handler
@ -1127,6 +1282,161 @@ int sqlite3changeset_apply(
#define SQLITE_CHANGESET_REPLACE 1
#define SQLITE_CHANGESET_ABORT 2
/*
** CAPI3REF: Rebasing changesets
** EXPERIMENTAL
**
** Suppose there is a site hosting a database in state S0. And that
** modifications are made that move that database to state S1 and a
** changeset recorded (the "local" changeset). Then, a changeset based
** on S0 is received from another site (the "remote" changeset) and
** applied to the database. The database is then in state
** (S1+"remote"), where the exact state depends on any conflict
** resolution decisions (OMIT or REPLACE) made while applying "remote".
** Rebasing a changeset is to update it to take those conflict
** resolution decisions into account, so that the same conflicts
** do not have to be resolved elsewhere in the network.
**
** For example, if both the local and remote changesets contain an
** INSERT of the same key on "CREATE TABLE t1(a PRIMARY KEY, b)":
**
** local: INSERT INTO t1 VALUES(1, 'v1');
** remote: INSERT INTO t1 VALUES(1, 'v2');
**
** and the conflict resolution is REPLACE, then the INSERT change is
** removed from the local changeset (it was overridden). Or, if the
** conflict resolution was "OMIT", then the local changeset is modified
** to instead contain:
**
** UPDATE t1 SET b = 'v2' WHERE a=1;
**
** Changes within the local changeset are rebased as follows:
**
** <dl>
** <dt>Local INSERT<dd>
** This may only conflict with a remote INSERT. If the conflict
** resolution was OMIT, then add an UPDATE change to the rebased
** changeset. Or, if the conflict resolution was REPLACE, add
** nothing to the rebased changeset.
**
** <dt>Local DELETE<dd>
** This may conflict with a remote UPDATE or DELETE. In both cases the
** only possible resolution is OMIT. If the remote operation was a
** DELETE, then add no change to the rebased changeset. If the remote
** operation was an UPDATE, then the old.* fields of change are updated
** to reflect the new.* values in the UPDATE.
**
** <dt>Local UPDATE<dd>
** This may conflict with a remote UPDATE or DELETE. If it conflicts
** with a DELETE, and the conflict resolution was OMIT, then the update
** is changed into an INSERT. Any undefined values in the new.* record
** from the update change are filled in using the old.* values from
** the conflicting DELETE. Or, if the conflict resolution was REPLACE,
** the UPDATE change is simply omitted from the rebased changeset.
**
** If conflict is with a remote UPDATE and the resolution is OMIT, then
** the old.* values are rebased using the new.* values in the remote
** change. Or, if the resolution is REPLACE, then the change is copied
** into the rebased changeset with updates to columns also updated by
** the conflicting remote UPDATE removed. If this means no columns would
** be updated, the change is omitted.
** </dl>
**
** A local change may be rebased against multiple remote changes
** simultaneously. If a single key is modified by multiple remote
** changesets, they are combined as follows before the local changeset
** is rebased:
**
** <ul>
** <li> If there has been one or more REPLACE resolutions on a
** key, it is rebased according to a REPLACE.
**
** <li> If there have been no REPLACE resolutions on a key, then
** the local changeset is rebased according to the most recent
** of the OMIT resolutions.
** </ul>
**
** Note that conflict resolutions from multiple remote changesets are
** combined on a per-field basis, not per-row. This means that in the
** case of multiple remote UPDATE operations, some fields of a single
** local change may be rebased for REPLACE while others are rebased for
** OMIT.
**
** In order to rebase a local changeset, the remote changeset must first
** be applied to the local database using sqlite3changeset_apply_v2() and
** the buffer of rebase information captured. Then:
**
** <ol>
** <li> An sqlite3_rebaser object is created by calling
** sqlite3rebaser_create().
** <li> The new object is configured with the rebase buffer obtained from
** sqlite3changeset_apply_v2() by calling sqlite3rebaser_configure().
** If the local changeset is to be rebased against multiple remote
** changesets, then sqlite3rebaser_configure() should be called
** multiple times, in the same order that the multiple
** sqlite3changeset_apply_v2() calls were made.
** <li> Each local changeset is rebased by calling sqlite3rebaser_rebase().
** <li> The sqlite3_rebaser object is deleted by calling
** sqlite3rebaser_delete().
** </ol>
*/
typedef struct sqlite3_rebaser sqlite3_rebaser;
/*
** CAPI3REF: Create a changeset rebaser object.
** EXPERIMENTAL
**
** Allocate a new changeset rebaser object. If successful, set (*ppNew) to
** point to the new object and return SQLITE_OK. Otherwise, if an error
** occurs, return an SQLite error code (e.g. SQLITE_NOMEM) and set (*ppNew)
** to NULL.
*/
int sqlite3rebaser_create(sqlite3_rebaser **ppNew);
/*
** CAPI3REF: Configure a changeset rebaser object.
** EXPERIMENTAL
**
** Configure the changeset rebaser object to rebase changesets according
** to the conflict resolutions described by buffer pRebase (size nRebase
** bytes), which must have been obtained from a previous call to
** sqlite3changeset_apply_v2().
*/
int sqlite3rebaser_configure(
sqlite3_rebaser*,
int nRebase, const void *pRebase
);
/*
** CAPI3REF: Rebase a changeset
** EXPERIMENTAL
**
** Argument pIn must point to a buffer containing a changeset nIn bytes
** in size. This function allocates and populates a buffer with a copy
** of the changeset rebased rebased according to the configuration of the
** rebaser object passed as the first argument. If successful, (*ppOut)
** is set to point to the new buffer containing the rebased changeset and
** (*pnOut) to its size in bytes and SQLITE_OK returned. It is the
** responsibility of the caller to eventually free the new buffer using
** sqlite3_free(). Otherwise, if an error occurs, (*ppOut) and (*pnOut)
** are set to zero and an SQLite error code returned.
*/
int sqlite3rebaser_rebase(
sqlite3_rebaser*,
int nIn, const void *pIn,
int *pnOut, void **ppOut
);
/*
** CAPI3REF: Delete a changeset rebaser object.
** EXPERIMENTAL
**
** Delete the changeset rebaser object and all associated resources. There
** should be one call to this function for each successful invocation
** of sqlite3rebaser_create().
*/
void sqlite3rebaser_delete(sqlite3_rebaser *p);
/*
** CAPI3REF: Streaming Versions of API functions.
**
@ -1135,12 +1445,13 @@ int sqlite3changeset_apply(
**
** <table border=1 style="margin-left:8ex;margin-right:8ex">
** <tr><th>Streaming function<th>Non-streaming equivalent</th>
** <tr><td>sqlite3changeset_apply_str<td>[sqlite3changeset_apply]
** <tr><td>sqlite3changeset_concat_str<td>[sqlite3changeset_concat]
** <tr><td>sqlite3changeset_invert_str<td>[sqlite3changeset_invert]
** <tr><td>sqlite3changeset_start_str<td>[sqlite3changeset_start]
** <tr><td>sqlite3session_changeset_str<td>[sqlite3session_changeset]
** <tr><td>sqlite3session_patchset_str<td>[sqlite3session_patchset]
** <tr><td>sqlite3changeset_apply_strm<td>[sqlite3changeset_apply]
** <tr><td>sqlite3changeset_apply_strm_v2<td>[sqlite3changeset_apply_v2]
** <tr><td>sqlite3changeset_concat_strm<td>[sqlite3changeset_concat]
** <tr><td>sqlite3changeset_invert_strm<td>[sqlite3changeset_invert]
** <tr><td>sqlite3changeset_start_strm<td>[sqlite3changeset_start]
** <tr><td>sqlite3session_changeset_strm<td>[sqlite3session_changeset]
** <tr><td>sqlite3session_patchset_strm<td>[sqlite3session_patchset]
** </table>
**
** Non-streaming functions that accept changesets (or patchsets) as input
@ -1231,6 +1542,23 @@ int sqlite3changeset_apply_strm(
),
void *pCtx /* First argument passed to xConflict */
);
int sqlite3changeset_apply_v2_strm(
sqlite3 *db, /* Apply change to "main" db of this handle */
int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */
void *pIn, /* First arg for xInput */
int(*xFilter)(
void *pCtx, /* Copy of sixth arg to _apply() */
const char *zTab /* Table name */
),
int(*xConflict)(
void *pCtx, /* Copy of sixth arg to _apply() */
int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */
sqlite3_changeset_iter *p /* Handle describing change and conflict */
),
void *pCtx, /* First argument passed to xConflict */
void **ppRebase, int *pnRebase,
int flags
);
int sqlite3changeset_concat_strm(
int (*xInputA)(void *pIn, void *pData, int *pnData),
void *pInA,
@ -1250,6 +1578,12 @@ int sqlite3changeset_start_strm(
int (*xInput)(void *pIn, void *pData, int *pnData),
void *pIn
);
int sqlite3changeset_start_v2_strm(
sqlite3_changeset_iter **pp,
int (*xInput)(void *pIn, void *pData, int *pnData),
void *pIn,
int flags
);
int sqlite3session_changeset_strm(
sqlite3_session *pSession,
int (*xOutput)(void *pOut, const void *pData, int nData),
@ -1268,7 +1602,53 @@ int sqlite3changegroup_output_strm(sqlite3_changegroup*,
int (*xOutput)(void *pOut, const void *pData, int nData),
void *pOut
);
int sqlite3rebaser_rebase_strm(
sqlite3_rebaser *pRebaser,
int (*xInput)(void *pIn, void *pData, int *pnData),
void *pIn,
int (*xOutput)(void *pOut, const void *pData, int nData),
void *pOut
);
/*
** CAPI3REF: Configure global parameters
**
** The sqlite3session_config() interface is used to make global configuration
** changes to the sessions module in order to tune it to the specific needs
** of the application.
**
** The sqlite3session_config() interface is not threadsafe. If it is invoked
** while any other thread is inside any other sessions method then the
** results are undefined. Furthermore, if it is invoked after any sessions
** related objects have been created, the results are also undefined.
**
** The first argument to the sqlite3session_config() function must be one
** of the SQLITE_SESSION_CONFIG_XXX constants defined below. The
** interpretation of the (void*) value passed as the second parameter and
** the effect of calling this function depends on the value of the first
** parameter.
**
** <dl>
** <dt>SQLITE_SESSION_CONFIG_STRMSIZE<dd>
** By default, the sessions module streaming interfaces attempt to input
** and output data in approximately 1 KiB chunks. This operand may be used
** to set and query the value of this configuration setting. The pointer
** passed as the second argument must point to a value of type (int).
** If this value is greater than 0, it is used as the new streaming data
** chunk size for both input and output. Before returning, the (int) value
** pointed to by pArg is set to the final value of the streaming interface
** chunk size.
** </dl>
**
** This function returns SQLITE_OK if successful, or an SQLite error code
** otherwise.
*/
int sqlite3session_config(int op, void *pArg);
/*
** CAPI3REF: Values for sqlite3session_config().
*/
#define SQLITE_SESSION_CONFIG_STRMSIZE 1
/*
** Make sure we can call this stuff from C++.

45
td/telegram/AnimationsManager.cpp
View File

@ -78,7 +78,9 @@ class SaveGifQuery : public Td::ResultHandler {
}
void send(FileId file_id, tl_object_ptr<telegram_api::inputDocument> &&input_document, bool unsave) {
CHECK(input_document != nullptr);
if (input_document == nullptr) {
return;
}
CHECK(file_id.is_valid());
file_id_ = file_id;
file_reference_ = input_document->file_reference_.as_slice().str();
@ -145,7 +147,9 @@ void AnimationsManager::tear_down() {
int32 AnimationsManager::get_animation_duration(FileId file_id) const {
auto it = animations_.find(file_id);
CHECK(it != animations_.end());
if (it == animations_.end() || it->second == nullptr) {
return 0;
}
return it->second->duration;
}
@ -155,6 +159,9 @@ tl_object_ptr<td_api::animation> AnimationsManager::get_animation_object(FileId
}
auto &animation = animations_[file_id];
if (animation == nullptr) {
return nullptr;
}
LOG_CHECK(animation != nullptr) << source << " " << file_id << " "
<< static_cast<int32>(td_->file_manager_->get_file_view(file_id).get_type());
// TODO can we make that function const?
@ -217,8 +224,8 @@ FileId AnimationsManager::on_get_animation(unique_ptr<Animation> new_animation,
const AnimationsManager::Animation *AnimationsManager::get_animation(FileId file_id) const {
auto animation = animations_.find(file_id);
if (animation == animations_.end()) {
return nullptr;
if (animation == animations_.end() || animation->second == nullptr) {
return make_unique<Animation>().get();
}
CHECK(animation->second->file_id == file_id);
@ -227,13 +234,17 @@ const AnimationsManager::Animation *AnimationsManager::get_animation(FileId file
FileId AnimationsManager::get_animation_thumbnail_file_id(FileId file_id) const {
auto animation = get_animation(file_id);
CHECK(animation != nullptr);
if (animation == nullptr) {
return FileId();
}
return animation->thumbnail.file_id;
}
void AnimationsManager::delete_animation_thumbnail(FileId file_id) {
auto &animation = animations_[file_id];
CHECK(animation != nullptr);
if (animation == nullptr) {
return;
}
animation->thumbnail = PhotoSize();
}
@ -263,7 +274,7 @@ bool AnimationsManager::merge_animations(FileId new_id, FileId old_id, bool can_
}
auto new_it = animations_.find(new_id);
if (new_it == animations_.end()) {
if (new_it == animations_.end() || new_it->second == nullptr) {
auto &old = animations_[old_id];
old->is_changed = true;
if (!can_delete_old) {
@ -317,7 +328,9 @@ tl_object_ptr<telegram_api::InputMedia> AnimationsManager::get_input_media(
if (input_file != nullptr) {
const Animation *animation = get_animation(file_id);
CHECK(animation != nullptr);
if (animation == nullptr) {
return nullptr;
}
vector<tl_object_ptr<telegram_api::DocumentAttribute>> attributes;
if (!animation->file_name.empty()) {
@ -354,7 +367,9 @@ SecretInputMedia AnimationsManager::get_secret_input_media(FileId animation_file
const string &caption, BufferSlice thumbnail,
int32 layer) const {
auto *animation = get_animation(animation_file_id);
CHECK(animation != nullptr);
if (animation == nullptr) {
return SecretInputMedia{};
}
auto file_view = td_->file_manager_->get_file_view(animation_file_id);
auto &encryption_key = file_view.encryption_key();
if (!file_view.is_encrypted_secret() || encryption_key.empty()) {
@ -605,7 +620,9 @@ int32 AnimationsManager::get_saved_animations_hash(const char *source) const {
numbers.reserve(saved_animation_ids_.size() * 2);
for (auto animation_id : saved_animation_ids_) {
auto animation = get_animation(animation_id);
CHECK(animation != nullptr);
if (animation == nullptr) {
continue;
}
auto file_view = td_->file_manager_->get_file_view(animation_id);
CHECK(file_view.has_remote_location());
if (!file_view.remote_location().is_document()) {
@ -808,7 +825,9 @@ FileSourceId AnimationsManager::get_saved_animations_file_source_id() {
string AnimationsManager::get_animation_search_text(FileId file_id) const {
auto animation = get_animation(file_id);
CHECK(animation != nullptr);
if (animation == nullptr) {
return "";
}
return animation->file_name;
}
@ -827,5 +846,9 @@ void AnimationsManager::get_current_state(vector<td_api::object_ptr<td_api::Upda
updates.push_back(get_update_saved_animations_object());
}
}
void AnimationsManager::memory_cleanup() {
animations_.clear();
animations_.rehash(0);
}
} // namespace td

2
td/telegram/AnimationsManager.h
View File

@ -31,6 +31,8 @@ class AnimationsManager : public Actor {
public:
AnimationsManager(Td *td, ActorShared<> parent);
void memory_cleanup();
int32 get_animation_duration(FileId file_id) const;
tl_object_ptr<td_api::animation> get_animation_object(FileId file_id, const char *source);

4
td/telegram/AnimationsManager.hpp
View File

@ -20,7 +20,9 @@ namespace td {
template <class StorerT>
void AnimationsManager::store_animation(FileId file_id, StorerT &storer) const {
auto it = animations_.find(file_id);
CHECK(it != animations_.end());
if (it == animations_.end() || it->second == nullptr) {
return;
}
const Animation *animation = it->second.get();
store(animation->duration, storer);
store(animation->dimensions, storer);

38
td/telegram/AudiosManager.cpp
View File

@ -24,7 +24,9 @@ AudiosManager::AudiosManager(Td *td) : td_(td) {
int32 AudiosManager::get_audio_duration(FileId file_id) const {
auto it = audios_.find(file_id);
CHECK(it != audios_.end());
if (it == audios_.end() || it->second == nullptr) {
return 0;
}
return it->second->duration;
}
@ -34,7 +36,9 @@ tl_object_ptr<td_api::audio> AudiosManager::get_audio_object(FileId file_id) {
}
auto &audio = audios_[file_id];
CHECK(audio != nullptr);
if (audio == nullptr) {
return nullptr;
}
audio->is_changed = false;
return make_tl_object<td_api::audio>(audio->duration, audio->title, audio->performer, audio->file_name,
audio->mime_type, get_minithumbnail_object(audio->minithumbnail),
@ -89,8 +93,8 @@ FileId AudiosManager::on_get_audio(unique_ptr<Audio> new_audio, bool replace) {
const AudiosManager::Audio *AudiosManager::get_audio(FileId file_id) const {
auto audio = audios_.find(file_id);
if (audio == audios_.end()) {
return nullptr;
if (audio == audios_.end() || audio->second == nullptr) {
return make_unique<Audio>().get();
}
CHECK(audio->second->file_id == file_id);
@ -122,7 +126,7 @@ bool AudiosManager::merge_audios(FileId new_id, FileId old_id, bool can_delete_o
}
auto new_it = audios_.find(new_id);
if (new_it == audios_.end()) {
if (new_it == audios_.end() || new_it->second == nullptr) {
auto &old = audios_[old_id];
old->is_changed = true;
if (!can_delete_old) {
@ -153,19 +157,25 @@ bool AudiosManager::merge_audios(FileId new_id, FileId old_id, bool can_delete_o
string AudiosManager::get_audio_search_text(FileId file_id) const {
auto audio = get_audio(file_id);
CHECK(audio != nullptr);
if (audio == nullptr) {
return "";
}
return PSTRING() << audio->file_name << " " << audio->title << " " << audio->performer;
}
FileId AudiosManager::get_audio_thumbnail_file_id(FileId file_id) const {
auto audio = get_audio(file_id);
CHECK(audio != nullptr);
if (audio == nullptr) {
return FileId();
}
return audio->thumbnail.file_id;
}
void AudiosManager::delete_audio_thumbnail(FileId file_id) {
auto &audio = audios_[file_id];
CHECK(audio != nullptr);
if (audio == nullptr) {
return;
}
audio->thumbnail = PhotoSize();
}
@ -187,7 +197,9 @@ SecretInputMedia AudiosManager::get_secret_input_media(FileId audio_file_id,
tl_object_ptr<telegram_api::InputEncryptedFile> input_file,
const string &caption, BufferSlice thumbnail) const {
auto *audio = get_audio(audio_file_id);
CHECK(audio != nullptr);
if (audio == nullptr) {
return SecretInputMedia{};
}
auto file_view = td_->file_manager_->get_file_view(audio_file_id);
auto &encryption_key = file_view.encryption_key();
if (!file_view.is_encrypted_secret() || encryption_key.empty()) {
@ -234,7 +246,9 @@ tl_object_ptr<telegram_api::InputMedia> AudiosManager::get_input_media(
if (input_file != nullptr) {
const Audio *audio = get_audio(file_id);
CHECK(audio != nullptr);
if (audio == nullptr) {
return nullptr;
}
vector<tl_object_ptr<telegram_api::DocumentAttribute>> attributes;
attributes.push_back(make_tl_object<telegram_api::documentAttributeAudio>(
@ -260,5 +274,9 @@ tl_object_ptr<telegram_api::InputMedia> AudiosManager::get_input_media(
return nullptr;
}
void AudiosManager::memory_cleanup() {
audios_.clear();
audios_.rehash(0);
}
} // namespace td

2
td/telegram/AudiosManager.h
View File

@ -26,6 +26,8 @@ class AudiosManager {
public:
explicit AudiosManager(Td *td);
void memory_cleanup();
int32 get_audio_duration(FileId file_id) const;
tl_object_ptr<td_api::audio> get_audio_object(FileId file_id);

4
td/telegram/AudiosManager.hpp
View File

@ -20,7 +20,9 @@ namespace td {
template <class StorerT>
void AudiosManager::store_audio(FileId file_id, StorerT &storer) const {
auto it = audios_.find(file_id);
CHECK(it != audios_.end());
if (it == audios_.end() || it->second == nullptr) {
return;
}
const Audio *audio = it->second.get();
store(audio->file_name, storer);
store(audio->mime_type, storer);

12
td/telegram/AutoDownloadSettings.cpp
View File

@ -116,13 +116,13 @@ class SaveAutoDownloadSettingsQuery : public Td::ResultHandler {
AutoDownloadSettings get_auto_download_settings(const td_api::object_ptr<td_api::autoDownloadSettings> &settings) {
CHECK(settings != nullptr);
AutoDownloadSettings result;
result.max_photo_file_size = settings->max_photo_file_size_;
result.max_video_file_size = settings->max_video_file_size_;
result.max_other_file_size = settings->max_other_file_size_;
result.max_photo_file_size = -1;
result.max_video_file_size = -1;
result.max_other_file_size = -1;
result.video_upload_bitrate = settings->video_upload_bitrate_;
result.is_enabled = settings->is_auto_download_enabled_;
result.preload_large_videos = settings->preload_large_videos_;
result.preload_next_audio = settings->preload_next_audio_;
result.is_enabled = false;
result.preload_large_videos = false;
result.preload_next_audio = false;
result.use_less_data_for_calls = settings->use_less_data_for_calls_;
return result;
}

12
td/telegram/BackgroundManager.cpp
View File

@ -137,7 +137,9 @@ class UploadBackgroundQuery : public Td::ResultHandler {
void send(FileId file_id, tl_object_ptr<telegram_api::InputFile> &&input_file, const BackgroundType &type,
bool for_dark_theme) {
CHECK(input_file != nullptr);
if (input_file == nullptr) {
return;
}
file_id_ = file_id;
type_ = type;
for_dark_theme_ = for_dark_theme;
@ -694,7 +696,9 @@ void BackgroundManager::on_upload_background_file(FileId file_id, tl_object_ptr<
LOG(INFO) << "Background file " << file_id << " has been uploaded";
auto it = being_uploaded_files_.find(file_id);
CHECK(it != being_uploaded_files_.end());
if (it == being_uploaded_files_.end()) {
return;
}
auto type = it->second.type;
auto for_dark_theme = it->second.for_dark_theme;
@ -715,7 +719,9 @@ void BackgroundManager::on_upload_background_file_error(FileId file_id, Status s
CHECK(status.is_error());
auto it = being_uploaded_files_.find(file_id);
CHECK(it != being_uploaded_files_.end());
if (it == being_uploaded_files_.end()) {
return;
}
auto promise = std::move(it->second.promise);

4
td/telegram/CallbackQueriesManager.cpp
View File

@ -44,7 +44,9 @@ class GetBotCallbackAnswerQuery : public Td::ResultHandler {
message_id_ = message_id;
auto input_peer = td->messages_manager_->get_input_peer(dialog_id, AccessRights::Read);
CHECK(input_peer != nullptr);
if (input_peer == nullptr) {
return;
}
int32 flags = 0;
BufferSlice data;

71
td/telegram/ContactsManager.cpp
View File

@ -4463,7 +4463,9 @@ void ContactsManager::on_get_blocked_users_result(int32 offset, int32 limit, int
vector<tl_object_ptr<telegram_api::contactBlocked>> &&blocked_users) {
LOG(INFO) << "Receive " << blocked_users.size() << " blocked users out of " << total_count;
auto it = found_blocked_users_.find(random_id);
CHECK(it != found_blocked_users_.end());
if (it == found_blocked_users_.end()) {
return;
}
auto &result = it->second.second;
CHECK(result.empty());
@ -4481,13 +4483,17 @@ void ContactsManager::on_get_blocked_users_result(int32 offset, int32 limit, int
void ContactsManager::on_failed_get_blocked_users(int64 random_id) {
auto it = found_blocked_users_.find(random_id);
CHECK(it != found_blocked_users_.end());
if (it == found_blocked_users_.end()) {
return;
}
found_blocked_users_.erase(it);
}
tl_object_ptr<td_api::users> ContactsManager::get_blocked_users_object(int64 random_id) {
auto it = found_blocked_users_.find(random_id);
CHECK(it != found_blocked_users_.end());
if (it == found_blocked_users_.end()) {
return nullptr;
}
auto result = get_users_object(it->second.first, it->second.second);
found_blocked_users_.erase(it);
return result;
@ -9394,6 +9400,8 @@ void ContactsManager::on_update_user_photo(User *u, UserId user_id,
void ContactsManager::do_update_user_photo(User *u, UserId user_id,
tl_object_ptr<telegram_api::UserProfilePhoto> &&photo, const char *source) {
u->is_photo_inited = true;
LOG_IF(INFO, u->access_hash == -1) << "Update profile photo of " << user_id << " without access hash from "
<< source;
ProfilePhoto new_photo = get_profile_photo(td_->file_manager_.get(), user_id, u->access_hash, std::move(photo));
if (new_photo != u->photo) {
@ -9743,8 +9751,9 @@ void ContactsManager::update_user_online_member_count(User *u) {
case DialogType::Chat: {
auto chat_id = dialog_id.get_chat_id();
auto chat_full = get_chat_full(chat_id);
CHECK(chat_full != nullptr);
update_chat_online_member_count(chat_full, chat_id, false);
if (chat_full != nullptr) {
update_chat_online_member_count(chat_full, chat_id, false);
}
break;
}
case DialogType::Channel: {
@ -13435,7 +13444,9 @@ tl_object_ptr<td_api::supergroupFullInfo> ContactsManager::get_supergroup_full_i
tl_object_ptr<td_api::supergroupFullInfo> ContactsManager::get_supergroup_full_info_object(
const ChannelFull *channel_full) const {
CHECK(channel_full != nullptr);
if (channel_full == nullptr) {
return nullptr;
}
double slow_mode_delay_expires_in = 0;
if (channel_full->slow_mode_next_send_date != 0) {
slow_mode_delay_expires_in = max(channel_full->slow_mode_next_send_date - G()->server_time(), 1e-3);
@ -13637,4 +13648,52 @@ void ContactsManager::get_current_state(vector<td_api::object_ptr<td_api::Update
}
}
void ContactsManager::memory_cleanup() {
users_full_.clear();
users_full_.rehash(0);
bot_infos_.clear();
bot_infos_.rehash(0);
user_photos_.clear();
user_photos_.rehash(0);
user_profile_photo_file_source_ids_.clear();
user_profile_photo_file_source_ids_.rehash(0);
chat_photo_file_source_ids_.clear();
chat_photo_file_source_ids_.rehash(0);
channels_full_.clear();
channels_full_.rehash(0);
channel_photo_file_source_ids_.clear();
channel_photo_file_source_ids_.rehash(0);
secret_chats_.clear();
secret_chats_.rehash(0);
secret_chats_with_user_.clear();
secret_chats_with_user_.rehash(0);
chat_invite_links_.clear();
chat_invite_links_.rehash(0);
channel_invite_links_.clear();
channel_invite_links_.rehash(0);
invite_link_infos_.clear();
invite_link_infos_.rehash(0);
load_user_from_database_queries_.clear();
load_user_from_database_queries_.rehash(0);
load_chat_from_database_queries_.clear();
load_chat_from_database_queries_.rehash(0);
load_channel_from_database_queries_.clear();
load_channel_from_database_queries_.rehash(0);
load_secret_chat_from_database_queries_.clear();
load_secret_chat_from_database_queries_.rehash(0);
dialog_administrators_.clear();
dialog_administrators_.rehash(0);
uploaded_profile_photos_.clear();
uploaded_profile_photos_.rehash(0);
imported_contacts_.clear();
imported_contacts_.rehash(0);
received_channel_participant_.clear();
received_channel_participant_.rehash(0);
received_channel_participants_.clear();
received_channel_participants_.rehash(0);
cached_channel_participants_.clear();
cached_channel_participants_.rehash(0);
linked_channel_ids_.clear();
linked_channel_ids_.rehash(0);
}
} // namespace td

2
td/telegram/ContactsManager.h
View File

@ -77,6 +77,8 @@ class ContactsManager : public Actor {
static UserId load_my_id();
void memory_cleanup();
static UserId get_user_id(const tl_object_ptr<telegram_api::User> &user);
static ChatId get_chat_id(const tl_object_ptr<telegram_api::Chat> &chat);
static ChannelId get_channel_id(const tl_object_ptr<telegram_api::Chat> &chat);

248
td/telegram/DialogDb.cpp
View File

@ -5,289 +5,71 @@
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/telegram/DialogDb.h"
#include "td/telegram/Version.h"
#include "td/actor/actor.h"
#include "td/actor/SchedulerLocalStorage.h"
#include "td/db/SqliteConnectionSafe.h"
#include "td/db/SqliteDb.h"
#include "td/db/SqliteKeyValue.h"
#include "td/db/SqliteStatement.h"
#include "td/utils/format.h"
#include "td/utils/logging.h"
#include "td/utils/ScopeGuard.h"
#include "td/utils/Time.h"
namespace td {
// NB: must happen inside a transaction
Status init_dialog_db(SqliteDb &db, int32 version, bool &was_created) {
LOG(INFO) << "Init dialog database " << tag("version", version);
was_created = false;
// Check if database exists
TRY_RESULT(has_table, db.has_table("dialogs"));
if (!has_table) {
version = 0;
}
if (version < static_cast<int32>(DbVersion::DialogDbCreated) || version > current_db_version()) {
TRY_STATUS(drop_dialog_db(db, version));
version = 0;
}
auto create_notification_group_table = [&db] {
return db.exec(
"CREATE TABLE IF NOT EXISTS notification_groups (notification_group_id INT4 PRIMARY KEY, dialog_id "
"INT8, last_notification_date INT4)");
};
auto create_last_notification_date_index = [&db] {
return db.exec(
"CREATE INDEX IF NOT EXISTS notification_group_by_last_notification_date ON notification_groups "
"(last_notification_date, dialog_id, notification_group_id) WHERE last_notification_date IS NOT NULL");
};
auto add_dialogs_in_folder_index = [&db] {
return db.exec(
"CREATE INDEX IF NOT EXISTS dialog_in_folder_by_dialog_order ON dialogs (folder_id, dialog_order, dialog_id) "
"WHERE folder_id IS NOT NULL");
};
if (version == 0) {
LOG(INFO) << "Create new dialog database";
was_created = true;
TRY_STATUS(
db.exec("CREATE TABLE IF NOT EXISTS dialogs (dialog_id INT8 PRIMARY KEY, dialog_order INT8, data BLOB, "
"folder_id INT4)"));
TRY_STATUS(create_notification_group_table());
TRY_STATUS(create_last_notification_date_index());
TRY_STATUS(add_dialogs_in_folder_index());
version = current_db_version();
}
if (version < static_cast<int32>(DbVersion::AddNotificationsSupport)) {
TRY_STATUS(create_notification_group_table());
TRY_STATUS(create_last_notification_date_index());
}
if (version < static_cast<int32>(DbVersion::AddFolders)) {
TRY_STATUS(db.exec("DROP INDEX IF EXISTS dialog_by_dialog_order"));
TRY_STATUS(db.exec("ALTER TABLE dialogs ADD COLUMN folder_id INT4"));
TRY_STATUS(add_dialogs_in_folder_index());
TRY_STATUS(db.exec("UPDATE dialogs SET folder_id = 0 WHERE dialog_id < -1500000000000 AND dialog_order > 0"));
}
return Status::OK();
}
// NB: must happen inside a transaction
Status drop_dialog_db(SqliteDb &db, int version) {
if (version < static_cast<int32>(DbVersion::DialogDbCreated)) {
LOG(WARNING) << "Drop old pmc dialog_db";
SqliteKeyValue kv;
kv.init_with_connection(db.clone(), "common").ensure();
kv.erase_by_prefix("di");
}
LOG(WARNING) << "Drop dialog_db " << tag("version", version) << tag("current_db_version", current_db_version());
auto status = db.exec("DROP TABLE IF EXISTS dialogs");
TRY_STATUS(db.exec("DROP TABLE IF EXISTS notification_groups"));
return status;
return Status::OK();
}
class DialogDbImpl : public DialogDbSyncInterface {
public:
explicit DialogDbImpl(SqliteDb db) : db_(std::move(db)) {
explicit DialogDbImpl(SqliteDb db) {
init().ensure();
}
Status init() {
TRY_RESULT_ASSIGN(add_dialog_stmt_, db_.get_statement("INSERT OR REPLACE INTO dialogs VALUES(?1, ?2, ?3, ?4)"));
TRY_RESULT_ASSIGN(add_notification_group_stmt_,
db_.get_statement("INSERT OR REPLACE INTO notification_groups VALUES(?1, ?2, ?3)"));
TRY_RESULT_ASSIGN(delete_notification_group_stmt_,
db_.get_statement("DELETE FROM notification_groups WHERE notification_group_id = ?1"));
TRY_RESULT_ASSIGN(get_dialog_stmt_, db_.get_statement("SELECT data FROM dialogs WHERE dialog_id = ?1"));
TRY_RESULT_ASSIGN(
get_dialogs_stmt_,
db_.get_statement("SELECT data, dialog_id, dialog_order FROM dialogs WHERE "
"folder_id == ?1 AND (dialog_order < ?2 OR (dialog_order = ?2 AND dialog_id < ?3)) ORDER "
"BY dialog_order DESC, dialog_id DESC LIMIT ?4"));
TRY_RESULT_ASSIGN(
get_notification_groups_by_last_notification_date_stmt_,
db_.get_statement("SELECT notification_group_id, dialog_id, last_notification_date FROM notification_groups "
"WHERE last_notification_date < ?1 OR (last_notification_date = ?1 "
"AND (dialog_id < ?2 OR (dialog_id = ?2 AND notification_group_id < ?3))) ORDER BY "
"last_notification_date DESC, dialog_id DESC LIMIT ?4"));
// "WHERE (last_notification_date, dialog_id, notification_group_id) < (?1, ?2, ?3) ORDER BY "
// "last_notification_date DESC, dialog_id DESC, notification_group_id DESC LIMIT ?4"));
TRY_RESULT_ASSIGN(
get_notification_group_stmt_,
db_.get_statement(
"SELECT dialog_id, last_notification_date FROM notification_groups WHERE notification_group_id = ?1"));
TRY_RESULT_ASSIGN(
get_secret_chat_count_stmt_,
db_.get_statement(
"SELECT COUNT(*) FROM dialogs WHERE folder_id = ?1 AND dialog_order > 0 AND dialog_id < -1500000000000"));
// LOG(ERROR) << get_dialog_stmt_.explain().ok();
// LOG(ERROR) << get_dialogs_stmt_.explain().ok();
// LOG(ERROR) << get_notification_groups_by_last_notification_date_stmt_.explain().ok();
// LOG(ERROR) << get_notification_group_stmt_.explain().ok();
// LOG(FATAL) << "EXPLAINED";
return Status::OK();
}
Status add_dialog(DialogId dialog_id, FolderId folder_id, int64 order, BufferSlice data,
vector<NotificationGroupKey> notification_groups) override {
SCOPE_EXIT {
add_dialog_stmt_.reset();
};
add_dialog_stmt_.bind_int64(1, dialog_id.get()).ensure();
add_dialog_stmt_.bind_int64(2, order).ensure();
add_dialog_stmt_.bind_blob(3, data.as_slice()).ensure();
if (order > 0) {
add_dialog_stmt_.bind_int32(4, folder_id.get()).ensure();
} else {
add_dialog_stmt_.bind_null(4).ensure();
}
TRY_STATUS(add_dialog_stmt_.step());
for (auto &to_add : notification_groups) {
if (to_add.dialog_id.is_valid()) {
SCOPE_EXIT {
add_notification_group_stmt_.reset();
};
add_notification_group_stmt_.bind_int32(1, to_add.group_id.get()).ensure();
add_notification_group_stmt_.bind_int64(2, to_add.dialog_id.get()).ensure();
if (to_add.last_notification_date != 0) {
add_notification_group_stmt_.bind_int32(3, to_add.last_notification_date).ensure();
} else {
add_notification_group_stmt_.bind_null(3).ensure();
}
TRY_STATUS(add_notification_group_stmt_.step());
} else {
SCOPE_EXIT {
delete_notification_group_stmt_.reset();
};
delete_notification_group_stmt_.bind_int32(1, to_add.group_id.get()).ensure();
TRY_STATUS(delete_notification_group_stmt_.step());
}
}
return Status::OK();
}
Result<BufferSlice> get_dialog(DialogId dialog_id) override {
SCOPE_EXIT {
get_dialog_stmt_.reset();
};
get_dialog_stmt_.bind_int64(1, dialog_id.get()).ensure();
TRY_STATUS(get_dialog_stmt_.step());
if (!get_dialog_stmt_.has_row()) {
return Status::Error("Not found");
}
return BufferSlice(get_dialog_stmt_.view_blob(0));
return Status::Error("Not found");
}
Result<NotificationGroupKey> get_notification_group(NotificationGroupId notification_group_id) override {
SCOPE_EXIT {
get_notification_group_stmt_.reset();
};
get_notification_group_stmt_.bind_int32(1, notification_group_id.get()).ensure();
TRY_STATUS(get_notification_group_stmt_.step());
if (!get_notification_group_stmt_.has_row()) {
return Status::Error("Not found");
}
return NotificationGroupKey(notification_group_id, DialogId(get_notification_group_stmt_.view_int64(0)),
get_last_notification_date(get_notification_group_stmt_, 1));
return Status::Error("Not found");
}
Result<int32> get_secret_chat_count(FolderId folder_id) override {
SCOPE_EXIT {
get_secret_chat_count_stmt_.reset();
};
get_secret_chat_count_stmt_.bind_int32(1, folder_id.get()).ensure();
TRY_STATUS(get_secret_chat_count_stmt_.step());
CHECK(get_secret_chat_count_stmt_.has_row());
return get_secret_chat_count_stmt_.view_int32(0);
return 0;
}
Result<DialogDbGetDialogsResult> get_dialogs(FolderId folder_id, int64 order, DialogId dialog_id,
int32 limit) override {
SCOPE_EXIT {
get_dialogs_stmt_.reset();
};
get_dialogs_stmt_.bind_int32(1, folder_id.get()).ensure();
get_dialogs_stmt_.bind_int64(2, order).ensure();
get_dialogs_stmt_.bind_int64(3, dialog_id.get()).ensure();
get_dialogs_stmt_.bind_int32(4, limit).ensure();
DialogDbGetDialogsResult result;
TRY_STATUS(get_dialogs_stmt_.step());
while (get_dialogs_stmt_.has_row()) {
BufferSlice data(get_dialogs_stmt_.view_blob(0));
result.next_dialog_id = DialogId(get_dialogs_stmt_.view_int64(1));
result.next_order = get_dialogs_stmt_.view_int64(2);
LOG(INFO) << "Load " << result.next_dialog_id << " with order " << result.next_order;
result.dialogs.emplace_back(std::move(data));
TRY_STATUS(get_dialogs_stmt_.step());
}
return std::move(result);
}
Result<vector<NotificationGroupKey>> get_notification_groups_by_last_notification_date(
NotificationGroupKey notification_group_key, int32 limit) override {
auto &stmt = get_notification_groups_by_last_notification_date_stmt_;
SCOPE_EXIT {
stmt.reset();
};
stmt.bind_int32(1, notification_group_key.last_notification_date).ensure();
stmt.bind_int64(2, notification_group_key.dialog_id.get()).ensure();
stmt.bind_int32(3, notification_group_key.group_id.get()).ensure();
stmt.bind_int32(4, limit).ensure();
vector<NotificationGroupKey> notification_groups;
TRY_STATUS(stmt.step());
while (stmt.has_row()) {
notification_groups.emplace_back(NotificationGroupId(stmt.view_int32(0)), DialogId(stmt.view_int64(1)),
get_last_notification_date(stmt, 2));
TRY_STATUS(stmt.step());
}
return std::move(notification_groups);
}
Status begin_transaction() override {
return db_.begin_transaction();
return Status::OK();
}
Status commit_transaction() override {
return db_.commit_transaction();
}
private:
SqliteDb db_;
SqliteStatement add_dialog_stmt_;
SqliteStatement add_notification_group_stmt_;
SqliteStatement delete_notification_group_stmt_;
SqliteStatement get_dialog_stmt_;
SqliteStatement get_dialogs_stmt_;
SqliteStatement get_notification_groups_by_last_notification_date_stmt_;
SqliteStatement get_notification_group_stmt_;
SqliteStatement get_secret_chat_count_stmt_;
static int32 get_last_notification_date(SqliteStatement &stmt, int id) {
if (stmt.view_datatype(id) == SqliteStatement::Datatype::Null) {
return 0;
}
return stmt.view_int32(id);
return Status::OK();
}
};
@ -431,28 +213,12 @@ class DialogDbAsync : public DialogDbAsyncInterface {
}
void add_read_query() {
do_flush();
}
void do_flush() {
if (pending_writes_.empty()) {
return;
}
sync_db_->begin_transaction().ensure();
for (auto &query : pending_writes_) {
query.set_value(Unit());
}
sync_db_->commit_transaction().ensure();
pending_writes_.clear();
for (auto &p : pending_write_results_) {
p.first.set_result(std::move(p.second));
}
pending_write_results_.clear();
cancel_timeout();
}
void timeout_expired() override {
do_flush();
}
void start_up() override {

8
td/telegram/Document.hpp
View File

@ -33,7 +33,9 @@ namespace td {
template <class StorerT>
void store(const Document &document, StorerT &storer) {
Td *td = storer.context()->td().get_actor_unsafe();
CHECK(td != nullptr);
if (td == nullptr) {
return;
}
store(document.type, storer);
switch (document.type) {
@ -67,7 +69,9 @@ void store(const Document &document, StorerT &storer) {
template <class ParserT>
void parse(Document &document, ParserT &parser) {
Td *td = parser.context()->td().get_actor_unsafe();
CHECK(td != nullptr);
if (td == nullptr) {
return;
}
parse(document.type, parser);
switch (document.type) {

34
td/telegram/DocumentsManager.cpp
View File

@ -55,6 +55,9 @@ tl_object_ptr<td_api::document> DocumentsManager::get_document_object(FileId fil
LOG(INFO) << "Return document " << file_id << " object";
auto &document = documents_[file_id];
if (document == nullptr) {
return nullptr;
}
LOG_CHECK(document != nullptr) << tag("file_id", file_id);
document->is_changed = false;
return make_tl_object<td_api::document>(document->file_name, document->mime_type,
@ -493,11 +496,13 @@ void DocumentsManager::create_document(FileId file_id, string minithumbnail, Pho
const DocumentsManager::GeneralDocument *DocumentsManager::get_document(FileId file_id) const {
auto document = documents_.find(file_id);
if (document == documents_.end()) {
return nullptr;
if (document == documents_.end() ||
document->second == nullptr ||
document->second->file_id != file_id) {
return make_unique<GeneralDocument>().get();
}
CHECK(document->second->file_id == file_id);
return document->second.get();
}
@ -527,7 +532,9 @@ SecretInputMedia DocumentsManager::get_secret_input_media(FileId document_file_i
tl_object_ptr<telegram_api::InputEncryptedFile> input_file,
const string &caption, BufferSlice thumbnail) const {
const GeneralDocument *document = get_document(document_file_id);
CHECK(document != nullptr);
if (document == nullptr) {
return SecretInputMedia{};
}
auto file_view = td_->file_manager_->get_file_view(document_file_id);
auto &encryption_key = file_view.encryption_key();
if (!file_view.is_encrypted_secret() || encryption_key.empty()) {
@ -570,7 +577,9 @@ tl_object_ptr<telegram_api::InputMedia> DocumentsManager::get_input_media(
if (input_file != nullptr) {
const GeneralDocument *document = get_document(file_id);
CHECK(document != nullptr);
if (document == nullptr) {
return nullptr;
}
vector<tl_object_ptr<telegram_api::DocumentAttribute>> attributes;
if (document->file_name.size()) {
@ -592,13 +601,17 @@ tl_object_ptr<telegram_api::InputMedia> DocumentsManager::get_input_media(
FileId DocumentsManager::get_document_thumbnail_file_id(FileId file_id) const {
auto document = get_document(file_id);
CHECK(document != nullptr);
if (document == nullptr) {
return FileId();
}
return document->thumbnail.file_id;
}
void DocumentsManager::delete_document_thumbnail(FileId file_id) {
auto &document = documents_[file_id];
CHECK(document != nullptr);
if (document == nullptr) {
return;
}
document->thumbnail = PhotoSize();
}
@ -627,7 +640,7 @@ bool DocumentsManager::merge_documents(FileId new_id, FileId old_id, bool can_de
}
auto new_it = documents_.find(new_id);
if (new_it == documents_.end()) {
if (new_it == documents_.end() || new_it->second == nullptr) {
auto &old = documents_[old_id];
old->is_changed = true;
if (!can_delete_old) {
@ -653,6 +666,11 @@ bool DocumentsManager::merge_documents(FileId new_id, FileId old_id, bool can_de
return true;
}
void DocumentsManager::memory_cleanup() {
documents_.clear();
documents_.rehash(0);
}
string DocumentsManager::get_document_search_text(FileId file_id) const {
auto document = get_document(file_id);
CHECK(document);

2
td/telegram/DocumentsManager.h
View File

@ -76,6 +76,8 @@ class DocumentsManager {
tl_object_ptr<td_api::document> get_document_object(FileId file_id);
void memory_cleanup();
Document on_get_document(RemoteDocument remote_document, DialogId owner_dialog_id,
MultiPromiseActor *load_data_multipromise_ptr = nullptr,
Document::Type default_document_type = Document::Type::General, bool is_background = false,

36
td/telegram/DocumentsManager.hpp
View File

@ -15,14 +15,21 @@
#include "td/utils/logging.h"
#include "td/utils/tl_helpers.h"
#include "td/telegram/ConfigShared.h"
namespace td {
template <class StorerT>
void DocumentsManager::store_document(FileId file_id, StorerT &storer) const {
LOG(DEBUG) << "Store document " << file_id;
//LOG(DEBUG) << "Store document " << file_id;
auto it = documents_.find(file_id);
CHECK(it != documents_.end());
if (it == documents_.end() || it->second == nullptr) {
return;
}
const GeneralDocument *document = it->second.get();
if (document == nullptr) {
return;
}
store(document->file_name, storer);
store(document->mime_type, storer);
store(document->minithumbnail, storer);
@ -33,13 +40,32 @@ void DocumentsManager::store_document(FileId file_id, StorerT &storer) const {
template <class ParserT>
FileId DocumentsManager::parse_document(ParserT &parser) {
auto document = make_unique<GeneralDocument>();
parse(document->file_name, parser);
string tmp_filename;
parse(tmp_filename, parser);
parse(document->mime_type, parser);
if (parser.version() >= static_cast<int32>(Version::SupportMinithumbnails)) {
parse(document->minithumbnail, parser);
if ( G()->shared_config().get_option_boolean("disable_document_filenames") && (
document->mime_type.rfind("image/") == 0 ||
document->mime_type.rfind("video/") == 0 ||
document->mime_type.rfind("audio/") == 0)) {
document->file_name = "0";
} else {
document->file_name = tmp_filename;
}
if (parser.version() >= static_cast<int32>(Version::SupportMinithumbnails)) {
string tmp_minithumbnail;
parse(tmp_minithumbnail, parser);
if (!G()->shared_config().get_option_boolean("disable_minithumbnails")) {
document->minithumbnail = tmp_minithumbnail;
}
}
parse(document->thumbnail, parser);
parse(document->file_id, parser);
LOG(DEBUG) << "Parsed document " << document->file_id;
if (parser.get_error() != nullptr || !document->file_id.is_valid()) {
return FileId();

7
td/telegram/FileReferenceManager.cpp
View File

@ -364,4 +364,11 @@ void FileReferenceManager::reload_photo(PhotoSizeSource source, Promise<Unit> pr
}
}
void FileReferenceManager::memory_cleanup(FileId file_id) {
auto &node = nodes_[file_id];
node.query.reset();
node.file_source_ids.reset_position();
nodes_.erase(file_id);
}
} // namespace td

2
td/telegram/FileReferenceManager.h
View File

@ -38,6 +38,8 @@ class FileReferenceManager : public Actor {
static bool is_file_reference_error(const Status &error);
static size_t get_file_reference_error_pos(const Status &error);
void memory_cleanup(FileId file_id);
FileSourceId create_message_file_source(FullMessageId full_message_id);
FileSourceId create_user_photo_file_source(UserId user_id, int64 photo_id);
FileSourceId create_chat_photo_file_source(ChatId chat_id);

226
td/telegram/MessagesDb.cpp
View File

@ -53,44 +53,44 @@ Status init_messages_db(SqliteDb &db, int32 version) {
}
auto add_media_indices = [&db](int begin, int end) {
for (int i = begin; i < end; i++) {
TRY_STATUS(db.exec(PSLICE() << "CREATE INDEX IF NOT EXISTS message_index_" << i
<< " ON messages (dialog_id, message_id) WHERE (index_mask & " << (1 << i)
<< ") != 0"));
}
// for (int i = begin; i < end; i++) {
// TRY_STATUS(db.exec(PSLICE() << "CREATE INDEX IF NOT EXISTS message_index_" << i
// << " ON messages (dialog_id, message_id) WHERE (index_mask & " << (1 << i)
// << ") != 0"));
// }
return Status::OK();
};
auto add_fts = [&db] {
TRY_STATUS(
db.exec("CREATE INDEX IF NOT EXISTS message_by_search_id ON messages "
"(search_id) WHERE search_id IS NOT NULL"));
// TRY_STATUS(
// db.exec("CREATE INDEX IF NOT EXISTS message_by_search_id ON messages "
// "(search_id) WHERE search_id IS NOT NULL"));
TRY_STATUS(
db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(text, content='messages', "
"content_rowid='search_id', tokenize = \"unicode61 remove_diacritics 0 tokenchars '\a'\")"));
TRY_STATUS(db.exec(
"CREATE TRIGGER IF NOT EXISTS trigger_fts_delete BEFORE DELETE ON messages WHEN OLD.search_id IS NOT NULL"
" BEGIN INSERT INTO messages_fts(messages_fts, rowid, text) VALUES(\'delete\', OLD.search_id, OLD.text); END"));
TRY_STATUS(db.exec(
"CREATE TRIGGER IF NOT EXISTS trigger_fts_insert AFTER INSERT ON messages WHEN NEW.search_id IS NOT NULL"
" BEGIN INSERT INTO messages_fts(rowid, text) VALUES(NEW.search_id, NEW.text); END"));
//TRY_STATUS(db.exec(
//"CREATE TRIGGER IF NOT EXISTS trigger_fts_update AFTER UPDATE ON messages WHEN NEW.search_id IS NOT NULL OR "
//"OLD.search_id IS NOT NULL"
//" BEGIN "
//"INSERT INTO messages_fts(messages_fts, rowid, text) VALUES(\'delete\', OLD.search_id, OLD.text); "
//"INSERT INTO messages_fts(rowid, text) VALUES(NEW.search_id, NEW.text); "
//" END"));
// TRY_STATUS(
// db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(text, content='messages', "
// "content_rowid='search_id', tokenize = \"unicode61 remove_diacritics 0 tokenchars '\a'\")"));
// TRY_STATUS(db.exec(
// "CREATE TRIGGER IF NOT EXISTS trigger_fts_delete BEFORE DELETE ON messages WHEN OLD.search_id IS NOT NULL"
// " BEGIN INSERT INTO messages_fts(messages_fts, rowid, text) VALUES(\'delete\', OLD.search_id, OLD.text); END"));
// TRY_STATUS(db.exec(
// "CREATE TRIGGER IF NOT EXISTS trigger_fts_insert AFTER INSERT ON messages WHEN NEW.search_id IS NOT NULL"
// " BEGIN INSERT INTO messages_fts(rowid, text) VALUES(NEW.search_id, NEW.text); END"));
// //TRY_STATUS(db.exec(
// //"CREATE TRIGGER IF NOT EXISTS trigger_fts_update AFTER UPDATE ON messages WHEN NEW.search_id IS NOT NULL OR "
// //"OLD.search_id IS NOT NULL"
// //" BEGIN "
// //"INSERT INTO messages_fts(messages_fts, rowid, text) VALUES(\'delete\', OLD.search_id, OLD.text); "
// //"INSERT INTO messages_fts(rowid, text) VALUES(NEW.search_id, NEW.text); "
// //" END"));
return Status::OK();
};
auto add_call_index = [&db] {
for (int i = static_cast<int>(SearchMessagesFilter::Call) - 1;
i < static_cast<int>(SearchMessagesFilter::MissedCall); i++) {
TRY_STATUS(db.exec(PSLICE() << "CREATE INDEX IF NOT EXISTS full_message_index_" << i
<< " ON messages (unique_message_id) WHERE (index_mask & " << (1 << i) << ") != 0"));
}
auto add_call_index = [&db]() {
// for (int i = static_cast<int>(SearchMessagesFilter::Call) - 1;
// i < static_cast<int>(SearchMessagesFilter::MissedCall); i++) {
// TRY_STATUS(db.exec(PSLICE() << "CREATE INDEX IF NOT EXISTS full_message_index_" << i
// << " ON messages (unique_message_id) WHERE (index_mask & " << (1 << i) << ") != 0"));
// }
return Status::OK();
};
auto add_notification_id_index = [&db] {
@ -114,7 +114,7 @@ Status init_messages_db(SqliteDb &db, int32 version) {
TRY_STATUS(
db.exec("CREATE TABLE IF NOT EXISTS messages (dialog_id INT8, message_id INT8, "
"unique_message_id INT4, sender_user_id INT4, random_id INT8, data BLOB, "
"ttl_expires_at INT4, index_mask INT4, search_id INT8, text STRING, notification_id INT4, PRIMARY KEY "
"ttl_expires_at INT4, index_mask INT4, search_id INT8, text STRING, notification_id INT4, seqno INT32, PRIMARY KEY "
"(dialog_id, message_id))"));
TRY_STATUS(
@ -180,56 +180,60 @@ class MessagesDbImpl : public MessagesDbSyncInterface {
}
Status init() {
seqno_ = 0;
db_memory_ = SqliteDb::open_with_key(":memory:", DbKey::empty()).move_as_ok();
TRY_STATUS(init_messages_db(db_memory_, db_.user_version().move_as_ok()));
TRY_RESULT_ASSIGN(
add_message_stmt_,
db_.get_statement("INSERT OR REPLACE INTO messages VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"));
db_memory_.get_statement("INSERT OR REPLACE INTO messages VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)"));
TRY_RESULT_ASSIGN(delete_message_stmt_,
db_.get_statement("DELETE FROM messages WHERE dialog_id = ?1 AND message_id = ?2"));
db_memory_.get_statement("DELETE FROM messages WHERE dialog_id = ?1 AND message_id = ?2"));
TRY_RESULT_ASSIGN(delete_all_dialog_messages_stmt_,
db_.get_statement("DELETE FROM messages WHERE dialog_id = ?1 AND message_id <= ?2"));
db_memory_.get_statement("DELETE FROM messages WHERE dialog_id = ?1 AND message_id <= ?2"));
TRY_RESULT_ASSIGN(delete_dialog_messages_from_user_stmt_,
db_.get_statement("DELETE FROM messages WHERE dialog_id = ?1 AND sender_user_id == ?2"));
db_memory_.get_statement("DELETE FROM messages WHERE dialog_id = ?1 AND sender_user_id == ?2"));
TRY_RESULT_ASSIGN(get_message_stmt_,
db_.get_statement("SELECT data FROM messages WHERE dialog_id = ?1 AND message_id = ?2"));
db_memory_.get_statement("SELECT data FROM messages WHERE dialog_id = ?1 AND message_id = ?2"));
TRY_RESULT_ASSIGN(get_message_by_random_id_stmt_,
db_.get_statement("SELECT data FROM messages WHERE dialog_id = ?1 AND random_id = ?2"));
db_memory_.get_statement("SELECT data FROM messages WHERE dialog_id = ?1 AND random_id = ?2"));
TRY_RESULT_ASSIGN(get_message_by_unique_message_id_stmt_,
db_.get_statement("SELECT dialog_id, data FROM messages WHERE unique_message_id = ?1"));
db_memory_.get_statement("SELECT dialog_id, data FROM messages WHERE unique_message_id = ?1"));
TRY_RESULT_ASSIGN(
get_expiring_messages_stmt_,
db_.get_statement("SELECT dialog_id, data FROM messages WHERE ?1 < ttl_expires_at AND ttl_expires_at <= ?2"));
db_memory_.get_statement("SELECT dialog_id, data FROM messages WHERE ?1 < ttl_expires_at AND ttl_expires_at <= ?2"));
TRY_RESULT_ASSIGN(get_expiring_messages_helper_stmt_,
db_.get_statement("SELECT MAX(ttl_expires_at), COUNT(*) FROM (SELECT ttl_expires_at FROM "
db_memory_.get_statement("SELECT MAX(ttl_expires_at), COUNT(*) FROM (SELECT ttl_expires_at FROM "
"messages WHERE ?1 < ttl_expires_at LIMIT ?2) AS T"));
TRY_RESULT_ASSIGN(get_messages_stmt_.asc_stmt_,
db_.get_statement("SELECT data, message_id FROM messages WHERE dialog_id = ?1 AND message_id > "
db_memory_.get_statement("SELECT data, message_id FROM messages WHERE dialog_id = ?1 AND message_id > "
"?2 ORDER BY message_id ASC LIMIT ?3"));
TRY_RESULT_ASSIGN(get_messages_stmt_.desc_stmt_,
db_.get_statement("SELECT data, message_id FROM messages WHERE dialog_id = ?1 AND message_id < "
db_memory_.get_statement("SELECT data, message_id FROM messages WHERE dialog_id = ?1 AND message_id < "
"?2 ORDER BY message_id DESC LIMIT ?3"));
TRY_RESULT_ASSIGN(get_scheduled_messages_stmt_,
db_.get_statement("SELECT data, message_id FROM scheduled_messages WHERE dialog_id = ?1 AND "
db_memory_.get_statement("SELECT data, message_id FROM scheduled_messages WHERE dialog_id = ?1 AND "
"message_id < ?2 ORDER BY message_id DESC LIMIT ?3"));
TRY_RESULT_ASSIGN(get_messages_from_notification_id_stmt_,
db_.get_statement("SELECT data, message_id FROM messages WHERE dialog_id = ?1 AND "
db_memory_.get_statement("SELECT data, message_id FROM messages WHERE dialog_id = ?1 AND "
"notification_id < ?2 ORDER BY notification_id DESC LIMIT ?3"));
TRY_RESULT_ASSIGN(
get_messages_fts_stmt_,
db_.get_statement(
"SELECT dialog_id, data, search_id FROM messages WHERE search_id IN (SELECT rowid FROM messages_fts WHERE "
"messages_fts MATCH ?1 AND rowid < ?2 ORDER BY rowid DESC LIMIT ?3) ORDER BY search_id DESC"));
// TRY_RESULT_ASSIGN(
// get_messages_fts_stmt_,
// db_memory_.get_statement(
// "SELECT dialog_id, data, search_id FROM messages WHERE search_id IN (SELECT rowid FROM messages_fts WHERE "
// "messages_fts MATCH ?1 AND rowid < ?2 ORDER BY rowid DESC LIMIT ?3) ORDER BY search_id DESC"));
for (int32 i = 0; i < MESSAGES_DB_INDEX_COUNT; i++) {
TRY_RESULT_ASSIGN(get_messages_from_index_stmts_[i].desc_stmt_,
db_.get_statement(PSLICE() << "SELECT data, message_id FROM messages WHERE dialog_id = ?1 "
db_memory_.get_statement(PSLICE() << "SELECT data, message_id FROM messages WHERE dialog_id = ?1 "
"AND message_id < ?2 AND (index_mask & "
<< (1 << i) << ") != 0 ORDER BY message_id DESC LIMIT ?3"));
TRY_RESULT_ASSIGN(get_messages_from_index_stmts_[i].asc_stmt_,
db_.get_statement(PSLICE() << "SELECT data, message_id FROM messages WHERE dialog_id = ?1 "
db_memory_.get_statement(PSLICE() << "SELECT data, message_id FROM messages WHERE dialog_id = ?1 "
"AND message_id > ?2 AND (index_mask & "
<< (1 << i) << ") != 0 ORDER BY message_id ASC LIMIT ?3"));
@ -241,24 +245,24 @@ class MessagesDbImpl : public MessagesDbSyncInterface {
i < static_cast<int>(SearchMessagesFilter::MissedCall); i++, pos++) {
TRY_RESULT_ASSIGN(
get_calls_stmts_[pos],
db_.get_statement(
db_memory_.get_statement(
PSLICE() << "SELECT dialog_id, data FROM messages WHERE unique_message_id < ?1 AND (index_mask & "
<< (1 << i) << ") != 0 ORDER BY unique_message_id DESC LIMIT ?2"));
}
TRY_RESULT_ASSIGN(add_scheduled_message_stmt_,
db_.get_statement("INSERT OR REPLACE INTO scheduled_messages VALUES(?1, ?2, ?3, ?4)"));
db_memory_.get_statement("INSERT OR REPLACE INTO scheduled_messages VALUES(?1, ?2, ?3, ?4)"));
TRY_RESULT_ASSIGN(
get_scheduled_message_stmt_,
db_.get_statement("SELECT data FROM scheduled_messages WHERE dialog_id = ?1 AND message_id = ?2"));
db_memory_.get_statement("SELECT data FROM scheduled_messages WHERE dialog_id = ?1 AND message_id = ?2"));
TRY_RESULT_ASSIGN(
get_scheduled_server_message_stmt_,
db_.get_statement("SELECT data FROM scheduled_messages WHERE dialog_id = ?1 AND server_message_id = ?2"));
db_memory_.get_statement("SELECT data FROM scheduled_messages WHERE dialog_id = ?1 AND server_message_id = ?2"));
TRY_RESULT_ASSIGN(delete_scheduled_message_stmt_,
db_.get_statement("DELETE FROM scheduled_messages WHERE dialog_id = ?1 AND message_id = ?2"));
db_memory_.get_statement("DELETE FROM scheduled_messages WHERE dialog_id = ?1 AND message_id = ?2"));
TRY_RESULT_ASSIGN(
delete_scheduled_server_message_stmt_,
db_.get_statement("DELETE FROM scheduled_messages WHERE dialog_id = ?1 AND server_message_id = ?2"));
db_memory_.get_statement("DELETE FROM scheduled_messages WHERE dialog_id = ?1 AND server_message_id = ?2"));
// LOG(ERROR) << get_message_stmt_.explain().ok();
// LOG(ERROR) << get_messages_from_notification_id_stmt.explain().ok();
@ -284,6 +288,16 @@ class MessagesDbImpl : public MessagesDbSyncInterface {
SCOPE_EXIT {
add_message_stmt_.reset();
};
seqno_++;
if (seqno_ % 8128 == 0) {
TRY_STATUS(db_memory_.exec("DELETE FROM messages WHERE seqno < " + to_string(seqno_ - 7168)));
TRY_STATUS(db_.exec("PRAGMA shrink_memory"));
}
add_message_stmt_.bind_int32(12, seqno_).ensure();
add_message_stmt_.bind_int64(1, dialog_id.get()).ensure();
add_message_stmt_.bind_int64(2, message_id.get()).ensure();
@ -664,56 +678,57 @@ class MessagesDbImpl : public MessagesDbSyncInterface {
}
Result<MessagesDbFtsResult> get_messages_fts(MessagesDbFtsQuery query) override {
SCOPE_EXIT {
get_messages_fts_stmt_.reset();
};
// SCOPE_EXIT {
// get_messages_fts_stmt_.reset();
// };
LOG(INFO) << tag("query", query.query) << query.dialog_id << tag("index_mask", query.index_mask)
<< tag("from_search_id", query.from_search_id) << tag("limit", query.limit);
string words = prepare_query(query.query);
LOG(INFO) << tag("from", query.query) << tag("to", words);
// LOG(INFO) << tag("query", query.query) << query.dialog_id << tag("index_mask", query.index_mask)
// << tag("from_search_id", query.from_search_id) << tag("limit", query.limit);
// string words = prepare_query(query.query);
// LOG(INFO) << tag("from", query.query) << tag("to", words);
// dialog_id kludge
if (query.dialog_id.is_valid()) {
words += PSTRING() << " \"\a" << query.dialog_id.get() << "\"";
}
// // dialog_id kludge
// if (query.dialog_id.is_valid()) {
// words += PSTRING() << " \"\a" << query.dialog_id.get() << "\"";
// }
// index_mask kludge
if (query.index_mask != 0) {
int index_i = -1;
for (int i = 0; i < MESSAGES_DB_INDEX_COUNT; i++) {
if (query.index_mask == (1 << i)) {
index_i = i;
break;
}
}
if (index_i == -1) {
return Status::Error("Union of index types is not supported");
}
words += PSTRING() << " \"\a\a" << index_i << "\"";
}
// // index_mask kludge
// if (query.index_mask != 0) {
// int index_i = -1;
// for (int i = 0; i < MESSAGES_DB_INDEX_COUNT; i++) {
// if (query.index_mask == (1 << i)) {
// index_i = i;
// break;
// }
// }
// if (index_i == -1) {
// return Status::Error("Union of index types is not supported");
// }
// words += PSTRING() << " \"\a\a" << index_i << "\"";
// }
auto &stmt = get_messages_fts_stmt_;
stmt.bind_string(1, words).ensure();
if (query.from_search_id == 0) {
query.from_search_id = std::numeric_limits<int64>::max();
}
stmt.bind_int64(2, query.from_search_id).ensure();
stmt.bind_int32(3, query.limit).ensure();
MessagesDbFtsResult result;
auto status = stmt.step();
if (status.is_error()) {
LOG(ERROR) << status;
return std::move(result);
}
while (stmt.has_row()) {
auto dialog_id = stmt.view_int64(0);
auto data_slice = stmt.view_blob(1);
auto search_id = stmt.view_int64(2);
result.next_search_id = search_id;
result.messages.push_back(MessagesDbMessage{DialogId(dialog_id), BufferSlice(data_slice)});
stmt.step().ensure();
}
// auto &stmt = get_messages_fts_stmt_;
// stmt.bind_string(1, words).ensure();
// if (query.from_search_id == 0) {
// query.from_search_id = std::numeric_limits<int64>::max();
// }
// stmt.bind_int64(2, query.from_search_id).ensure();
// stmt.bind_int32(3, query.limit).ensure();
// MessagesDbFtsResult result;
// auto status = stmt.step();
// if (status.is_error()) {
// LOG(ERROR) << status;
// return std::move(result);
// }
// while (stmt.has_row()) {
// auto dialog_id = stmt.view_int64(0);
// auto data_slice = stmt.view_blob(1);
// auto search_id = stmt.view_int64(2);
// result.next_search_id = search_id;
// result.messages.push_back(MessagesDbMessage{DialogId(dialog_id), BufferSlice(data_slice)});
// stmt.step().ensure();
// }
MessagesDbFtsResult result;
return std::move(result);
}
@ -786,6 +801,9 @@ class MessagesDbImpl : public MessagesDbSyncInterface {
private:
SqliteDb db_;
SqliteDb db_memory_;
int32_t seqno_;
SqliteStatement add_message_stmt_;
@ -810,7 +828,7 @@ class MessagesDbImpl : public MessagesDbSyncInterface {
std::array<GetMessagesStmt, MESSAGES_DB_INDEX_COUNT> get_messages_from_index_stmts_;
std::array<SqliteStatement, 2> get_calls_stmts_;
SqliteStatement get_messages_fts_stmt_;
// SqliteStatement get_messages_fts_stmt_;
SqliteStatement add_scheduled_message_stmt_;
SqliteStatement get_scheduled_message_stmt_;

85
td/telegram/MessagesManager.cpp
View File

@ -293,7 +293,9 @@ class GetChannelMessagesQuery : public Td::ResultHandler {
void send(ChannelId channel_id, tl_object_ptr<telegram_api::InputChannel> &&input_channel,
vector<tl_object_ptr<telegram_api::InputMessage>> &&message_ids) {
channel_id_ = channel_id;
CHECK(input_channel != nullptr);
if (input_channel == nullptr) {
return;
}
send_query(G()->net_query_creator().create(
telegram_api::channels_getMessages(std::move(input_channel), std::move(message_ids))));
}
@ -433,7 +435,9 @@ class ExportChannelMessageLinkQuery : public Td::ResultHandler {
for_group_ = for_group;
ignore_result_ = ignore_result;
auto input_channel = td->contacts_manager_->get_input_channel(channel_id);
CHECK(input_channel != nullptr);
if (input_channel == nullptr) {
return;
}
send_query(G()->net_query_creator().create(telegram_api::channels_exportMessageLink(
std::move(input_channel), message_id.get_server_message_id().get(), for_group)));
}
@ -572,7 +576,9 @@ class GetCommonDialogsQuery : public Td::ResultHandler {
LOG(INFO) << "Get common dialogs with " << user_id << " from " << offset_chat_id << " with limit " << limit;
auto input_user = td->contacts_manager_->get_input_user(user_id);
CHECK(input_user != nullptr);
if (input_user == nullptr) {
return;
}
send_query(G()->net_query_creator().create(
telegram_api::messages_getCommonChats(std::move(input_user), offset_chat_id, limit)));
@ -696,7 +702,9 @@ class EditDialogPhotoQuery : public Td::ResultHandler {
}
void send(DialogId dialog_id, FileId file_id, tl_object_ptr<telegram_api::InputChatPhoto> &&input_chat_photo) {
CHECK(input_chat_photo != nullptr);
if (input_chat_photo == nullptr) {
return;
}
file_id_ = file_id;
was_uploaded_ = FileManager::extract_was_uploaded(input_chat_photo);
file_reference_ = FileManager::extract_file_reference(input_chat_photo);
@ -710,7 +718,9 @@ class EditDialogPhotoQuery : public Td::ResultHandler {
case DialogType::Channel: {
auto channel_id = dialog_id.get_channel_id();
auto input_channel = td->contacts_manager_->get_input_channel(channel_id);
CHECK(input_channel != nullptr);
if (input_channel == nullptr) {
return;
}
send_query(G()->net_query_creator().create(
telegram_api::channels_editPhoto(std::move(input_channel), std::move(input_chat_photo))));
break;
@ -2714,7 +2724,9 @@ class GetGameHighScoresQuery : public Td::ResultHandler {
random_id_ = random_id;
auto input_peer = td->messages_manager_->get_input_peer(dialog_id, AccessRights::Read);
CHECK(input_peer != nullptr);
if (input_peer == nullptr) {
return;
}
CHECK(input_user != nullptr);
send_query(G()->net_query_creator().create(telegram_api::messages_getGameHighScores(
@ -2898,7 +2910,9 @@ class SendScreenshotNotificationQuery : public Td::ResultHandler {
dialog_id_ = dialog_id;
auto input_peer = td->messages_manager_->get_input_peer(dialog_id, AccessRights::Write);
CHECK(input_peer != nullptr);
if (input_peer == nullptr) {
return;
}
auto query = G()->net_query_creator().create(
telegram_api::messages_sendScreenshotNotification(std::move(input_peer), 0, random_id));
@ -3056,7 +3070,9 @@ class DeleteChannelMessagesQuery : public Td::ResultHandler {
query_count_++;
auto input_channel = td->contacts_manager_->get_input_channel(channel_id);
CHECK(input_channel != nullptr);
if (input_channel == nullptr) {
return;
}
send_query(G()->net_query_creator().create(
telegram_api::channels_deleteMessages(std::move(input_channel), std::move(slice))));
}
@ -7955,8 +7971,9 @@ void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_
CHECK(message_id > last_server_message_id);
if (message_id.is_server()) {
auto message = delete_message(d, message_id, true, &need_update_dialog_pos, "on_get_gistory 1");
CHECK(message != nullptr);
deleted_message_ids.push_back(message->message_id.get());
if (message != nullptr) {
deleted_message_ids.push_back(message->message_id.get());
}
}
}
if (need_update_dialog_pos) {
@ -7973,7 +7990,7 @@ void MessagesManager::on_get_history(DialogId dialog_id, MessageId from_message_
// connect all messages with ID > last_server_message_id
for (size_t i = 0; i + 1 < message_ids.size(); i++) {
auto m = get_message(d, message_ids[i]);
CHECK(m != nullptr);
if (m == nullptr) { continue; }
if (!m->have_next) {
m->have_next = true;
attach_message_to_next(d, message_ids[i], "on_get_history 3");
@ -17868,8 +17885,8 @@ void MessagesManager::load_dialog_scheduled_messages(DialogId dialog_id, bool fr
void MessagesManager::on_get_scheduled_messages_from_database(DialogId dialog_id, vector<BufferSlice> &&messages) {
if (G()->close_flag()) {
auto it = load_scheduled_messages_from_database_queries_.find(dialog_id);
CHECK(it != load_scheduled_messages_from_database_queries_.end());
CHECK(!it->second.empty());
if (it == load_scheduled_messages_from_database_queries_.end()) { return; }
if (it->second.empty()) { return; }
auto promises = std::move(it->second);
load_scheduled_messages_from_database_queries_.erase(it);
@ -17879,7 +17896,7 @@ void MessagesManager::on_get_scheduled_messages_from_database(DialogId dialog_id
return;
}
auto d = get_dialog(dialog_id);
CHECK(d != nullptr);
if (d == nullptr) { return; }
d->has_loaded_scheduled_messages_from_database = true;
LOG(INFO) << "Receive " << messages.size() << " scheduled messages from database in " << dialog_id;
@ -27248,47 +27265,11 @@ void MessagesManager::delete_message_files(DialogId dialog_id, const Message *m)
}
bool MessagesManager::need_delete_file(FullMessageId full_message_id, FileId file_id) const {
if (being_readded_message_id_ == full_message_id) {
return false;
}
auto main_file_id = td_->file_manager_->get_file_view(file_id).file_id();
auto full_message_ids = td_->file_reference_manager_->get_some_message_file_sources(main_file_id);
LOG(INFO) << "Receive " << full_message_ids << " as sources for file " << main_file_id << "/" << file_id << " from "
<< full_message_id;
for (auto other_full_messsage_id : full_message_ids) {
if (other_full_messsage_id != full_message_id) {
return false;
}
}
return true;
return false;
}
bool MessagesManager::need_delete_message_files(DialogId dialog_id, const Message *m) const {
if (m == nullptr) {
return false;
}
auto dialog_type = dialog_id.get_type();
if (!m->message_id.is_scheduled() && !m->message_id.is_server() && dialog_type != DialogType::SecretChat) {
return false;
}
if (being_readded_message_id_ == FullMessageId{dialog_id, m->message_id}) {
return false;
}
if (m->forward_info != nullptr && m->forward_info->from_dialog_id.is_valid() &&
m->forward_info->from_message_id.is_valid()) {
// this function must not try to load the message, because it can be called from
// do_delete_message or add_scheduled_message_to_dialog
const Message *old_m = get_message({m->forward_info->from_dialog_id, m->forward_info->from_message_id});
if (old_m != nullptr && get_message_file_ids(old_m) == get_message_file_ids(m)) {
return false;
}
}
return true;
return false;
}
void MessagesManager::delete_message_from_database(Dialog *d, MessageId message_id, const Message *m,

6
td/telegram/NotificationManager.cpp
View File

@ -164,7 +164,11 @@ void NotificationManager::on_flush_pending_updates_timeout_callback(void *notifi
}
bool NotificationManager::is_disabled() const {
return !td_->auth_manager_->is_authorized() || td_->auth_manager_->is_bot() || G()->close_flag();
if ( G()->shared_config().get_option_boolean("disable_notifications")) {
return true;
} else {
return !td_->auth_manager_->is_authorized() || td_->auth_manager_->is_bot() || G()->close_flag();
}
}
StringBuilder &operator<<(StringBuilder &string_builder, const NotificationManager::ActiveNotificationsUpdate &update) {

16
td/telegram/Photo.cpp
View File

@ -168,21 +168,7 @@ tl_object_ptr<td_api::profilePhoto> get_profile_photo_object(FileManager *file_m
}
bool operator==(const ProfilePhoto &lhs, const ProfilePhoto &rhs) {
bool location_differs = lhs.small_file_id != rhs.small_file_id || lhs.big_file_id != rhs.big_file_id;
bool id_differs;
if (lhs.id == -1 && rhs.id == -1) {
// group chat photo
id_differs = location_differs;
} else {
id_differs = lhs.id != rhs.id;
}
if (location_differs) {
LOG_IF(ERROR, !id_differs) << "Photo " << lhs.id << " location has changed. First profilePhoto: " << lhs
<< ", second profilePhoto: " << rhs;
return false;
}
return true;
return lhs.small_file_id == rhs.small_file_id || lhs.big_file_id == rhs.big_file_id;
}
bool operator!=(const ProfilePhoto &lhs, const ProfilePhoto &rhs) {

4
td/telegram/PollManager.hpp
View File

@ -128,7 +128,9 @@ void PollManager::store_poll(PollId poll_id, StorerT &storer) const {
td::store(poll_id.get(), storer);
if (is_local_poll_id(poll_id)) {
auto poll = get_poll(poll_id);
CHECK(poll != nullptr);
if (poll == nullptr) {
return;
}
bool has_open_period = poll->open_period != 0;
bool has_close_date = poll->close_date != 0;
bool has_explanation = !poll->explanation.text.empty();

8
td/telegram/SequenceDispatcher.cpp
View File

@ -255,13 +255,17 @@ void MultiSequenceDispatcher::send_with_callback(NetQueryPtr query, ActorShared<
void MultiSequenceDispatcher::on_result() {
auto it = dispatchers_.find(get_link_token());
CHECK(it != dispatchers_.end());
if (it == dispatchers_.end()) {
return;
}
it->second.cnt_--;
}
void MultiSequenceDispatcher::ready_to_close() {
auto it = dispatchers_.find(get_link_token());
CHECK(it != dispatchers_.end());
if (it == dispatchers_.end()) {
return;
}
if (it->second.cnt_ == 0) {
LOG(DEBUG) << "Close SequenceDispatcher " << get_link_token();
dispatchers_.erase(it);

89
td/telegram/StickersManager.cpp
View File

@ -1329,9 +1329,17 @@ tl_object_ptr<td_api::sticker> StickersManager::get_sticker_object(FileId file_i
}
auto it = stickers_.find(file_id);
CHECK(it != stickers_.end());
if (it == stickers_.end() || it->second == nullptr) {
return nullptr;
}
auto sticker = it->second.get();
CHECK(sticker != nullptr);
if (sticker == nullptr) {
return nullptr;
}
sticker->is_changed = false;
auto mask_position = sticker->point >= 0
@ -1603,8 +1611,8 @@ std::pair<int64, FileId> StickersManager::on_get_sticker_document(
StickersManager::Sticker *StickersManager::get_sticker(FileId file_id) {
auto sticker = stickers_.find(file_id);
if (sticker == stickers_.end()) {
return nullptr;
if (sticker == stickers_.end() || sticker->second == nullptr) {
return make_unique<Sticker>().get();
}
CHECK(sticker->second->file_id == file_id);
@ -1613,18 +1621,22 @@ StickersManager::Sticker *StickersManager::get_sticker(FileId file_id) {
const StickersManager::Sticker *StickersManager::get_sticker(FileId file_id) const {
auto sticker = stickers_.find(file_id);
if (sticker == stickers_.end()) {
return nullptr;
if (sticker == stickers_.end() ||
sticker->second == nullptr ||
sticker->second->file_id != file_id) {
return make_unique<Sticker>().get();
}
CHECK(sticker->second->file_id == file_id);
return sticker->second.get();
}
StickersManager::StickerSet *StickersManager::get_sticker_set(StickerSetId sticker_set_id) {
auto sticker_set = sticker_sets_.find(sticker_set_id);
if (sticker_set == sticker_sets_.end()) {
return nullptr;
if (sticker_set == sticker_sets_.end() ||
sticker_set->second == nullptr) {
return make_unique<StickerSet>().get();
}
return sticker_set->second.get();
@ -1633,7 +1645,7 @@ StickersManager::StickerSet *StickersManager::get_sticker_set(StickerSetId stick
const StickersManager::StickerSet *StickersManager::get_sticker_set(StickerSetId sticker_set_id) const {
auto sticker_set = sticker_sets_.find(sticker_set_id);
if (sticker_set == sticker_sets_.end()) {
return nullptr;
return make_unique<StickerSet>().get();
}
return sticker_set->second.get();
@ -1724,7 +1736,11 @@ void StickersManager::delete_sticker_thumbnail(FileId file_id) {
vector<FileId> StickersManager::get_sticker_file_ids(FileId file_id) const {
vector<FileId> result;
auto sticker = get_sticker(file_id);
CHECK(sticker != nullptr);
if (sticker == nullptr) {
return result;
}
result.push_back(file_id);
if (sticker->s_thumbnail.file_id.is_valid()) {
result.push_back(sticker->s_thumbnail.file_id);
@ -1761,7 +1777,7 @@ bool StickersManager::merge_stickers(FileId new_id, FileId old_id, bool can_dele
}
auto new_it = stickers_.find(new_id);
if (new_it == stickers_.end()) {
if (new_it == stickers_.end() || new_it->second == nullptr) {
auto &old = stickers_[old_id];
old->is_changed = true;
if (!can_delete_old) {
@ -1947,7 +1963,11 @@ void StickersManager::create_sticker(FileId file_id, PhotoSize thumbnail, Dimens
bool StickersManager::has_input_media(FileId sticker_file_id, bool is_secret) const {
const Sticker *sticker = get_sticker(sticker_file_id);
CHECK(sticker != nullptr);
if (sticker == nullptr) {
return false;
}
auto file_view = td_->file_manager_->get_file_view(sticker_file_id);
if (is_secret) {
if (file_view.is_encrypted_secret()) {
@ -1976,7 +1996,11 @@ SecretInputMedia StickersManager::get_secret_input_media(FileId sticker_file_id,
tl_object_ptr<telegram_api::InputEncryptedFile> input_file,
BufferSlice thumbnail) const {
const Sticker *sticker = get_sticker(sticker_file_id);
CHECK(sticker != nullptr);
if (sticker == nullptr) {
return {};
}
auto file_view = td_->file_manager_->get_file_view(sticker_file_id);
if (file_view.is_encrypted_secret()) {
if (file_view.has_remote_location()) {
@ -3508,8 +3532,8 @@ void StickersManager::unregister_dice(const string &emoji, int32 value, FullMess
LOG(INFO) << "Unregister dice " << emoji << " with value " << value << " from " << full_message_id << " from "
<< source;
auto &message_ids = dice_messages_[emoji];
auto is_deleted = message_ids.erase(full_message_id);
LOG_CHECK(is_deleted) << source << " " << emoji << " " << value << " " << full_message_id;
message_ids.erase(full_message_id);
LOG(INFO) << source << " " << emoji << " " << value << " " << full_message_id;
if (message_ids.empty()) {
dice_messages_.erase(emoji);
@ -5054,7 +5078,9 @@ int32 StickersManager::get_recent_stickers_hash(const vector<FileId> &sticker_id
numbers.reserve(sticker_ids.size() * 2);
for (auto sticker_id : sticker_ids) {
auto sticker = get_sticker(sticker_id);
CHECK(sticker != nullptr);
if (sticker == nullptr) {
continue;
}
auto file_view = td_->file_manager_->get_file_view(sticker_id);
CHECK(file_view.has_remote_location());
if (!file_view.remote_location().is_document()) {
@ -6237,4 +6263,33 @@ void StickersManager::get_current_state(vector<td_api::object_ptr<td_api::Update
}
}
void StickersManager::memory_cleanup() {
stickers_.clear();
stickers_.rehash(0);
sticker_sets_.clear();
sticker_sets_.rehash(0);
short_name_to_sticker_set_id_.clear();
short_name_to_sticker_set_id_.rehash(0);
attached_sticker_sets_.clear();
attached_sticker_sets_.rehash(0);
found_stickers_.clear();
found_stickers_.rehash(0);
found_sticker_sets_.clear();
found_sticker_sets_.rehash(0);
special_sticker_sets_.clear();
special_sticker_sets_.rehash(0);
sticker_set_load_requests_.clear();
sticker_set_load_requests_.rehash(0);
emoji_language_codes_.clear();
emoji_language_codes_.rehash(0);
emoji_language_code_versions_.clear();
emoji_language_code_versions_.rehash(0);
emoji_language_code_last_difference_times_.clear();
emoji_language_code_last_difference_times_.rehash(0);
emoji_suggestions_urls_.clear();
emoji_suggestions_urls_.rehash(0);
dice_messages_.clear();
dice_messages_.rehash(0);
}
} // namespace td

2
td/telegram/StickersManager.h
View File

@ -42,6 +42,8 @@ class StickersManager : public Actor {
public:
static constexpr int64 GREAT_MINDS_SET_ID = 1842540969984001;
void memory_cleanup();
static vector<StickerSetId> convert_sticker_set_ids(const vector<int64> &sticker_set_ids);
static vector<int64> convert_sticker_set_ids(const vector<StickerSetId> &sticker_set_ids);

21
td/telegram/StickersManager.hpp
View File

@ -22,7 +22,9 @@ namespace td {
template <class StorerT>
void StickersManager::store_sticker(FileId file_id, bool in_sticker_set, StorerT &storer) const {
auto it = stickers_.find(file_id);
CHECK(it != stickers_.end());
if (it == stickers_.end() || it->second == nullptr) {
return;
}
const Sticker *sticker = it->second.get();
bool has_sticker_set_access_hash = sticker->set_id.is_valid() && !in_sticker_set;
BEGIN_STORE_FLAGS();
@ -35,8 +37,9 @@ void StickersManager::store_sticker(FileId file_id, bool in_sticker_set, StorerT
store(sticker->set_id.get(), storer);
if (has_sticker_set_access_hash) {
auto sticker_set = get_sticker_set(sticker->set_id);
CHECK(sticker_set != nullptr);
store(sticker_set->access_hash, storer);
if (sticker_set != nullptr) {
store(sticker_set->access_hash, storer);
}
}
}
store(sticker->alt, storer);
@ -167,7 +170,9 @@ void StickersManager::store_sticker_set(const StickerSet *sticker_set, bool with
template <class ParserT>
void StickersManager::parse_sticker_set(StickerSet *sticker_set, ParserT &parser) {
CHECK(sticker_set != nullptr);
if (sticker_set == nullptr) {
return;
}
CHECK(!sticker_set->was_loaded);
bool was_inited = sticker_set->is_inited;
bool is_installed;
@ -268,7 +273,9 @@ void StickersManager::parse_sticker_set(StickerSet *sticker_set, ParserT &parser
sticker_set->sticker_ids.push_back(sticker_id);
Sticker *sticker = get_sticker(sticker_id);
CHECK(sticker != nullptr);
if (sticker == nullptr) {
return;
}
if (sticker->set_id != sticker_set->id) {
LOG_IF(ERROR, sticker->set_id.is_valid()) << "Sticker " << sticker_id << " set_id has changed";
sticker->set_id = sticker_set->id;
@ -297,7 +304,9 @@ template <class StorerT>
void StickersManager::store_sticker_set_id(StickerSetId sticker_set_id, StorerT &storer) const {
CHECK(sticker_set_id.is_valid());
const StickerSet *sticker_set = get_sticker_set(sticker_set_id);
CHECK(sticker_set != nullptr);
if (sticker_set == nullptr) {
return;
}
store(sticker_set_id.get(), storer);
store(sticker_set->access_hash, storer);
}

29
td/telegram/Td.cpp
View File

@ -4313,6 +4313,10 @@ void Td::init_file_manager() {
send_closure(G()->storage_manager(), &StorageManager::on_new_file, size, real_size, cnt);
}
void destroy_file_source(FileId file_id) final {
td_->file_reference_manager_->memory_cleanup(file_id);
}
void on_file_updated(FileId file_id) final {
send_closure(G()->td(), &Td::send_update,
make_tl_object<td_api::updateFile>(td_->file_manager_->get_file_object(file_id)));
@ -5119,6 +5123,18 @@ void Td::on_request(uint64 id, td_api::getDatabaseStatistics &request) {
}
void Td::on_request(uint64 id, td_api::optimizeStorage &request) {
contacts_manager_->memory_cleanup();
web_pages_manager_->memory_cleanup();
stickers_manager_->memory_cleanup();
documents_manager_->memory_cleanup();
video_notes_manager_->memory_cleanup();
videos_manager_->memory_cleanup();
audios_manager_->memory_cleanup();
animations_manager_->memory_cleanup();
file_manager_->memory_cleanup();
malloc_trim(0);
std::vector<FileType> file_types;
for (auto &file_type : request.file_types_) {
if (file_type == nullptr) {
@ -6901,8 +6917,19 @@ void Td::on_request(uint64 id, td_api::setOption &request) {
return;
}
if (!is_bot && set_boolean_option("disable_top_chats")) {
return;
return;
}
// Start custom-patches
if (set_boolean_option("disable_document_filenames")) {
return;
}
if (set_boolean_option("disable_minithumbnails")) {
return;
}
if (set_boolean_option("disable_notifications")) {
return;
}
// End custom-patches
if (request.name_ == "drop_notification_ids") {
G()->td_db()->get_binlog_pmc()->erase("notification_id_current");
G()->td_db()->get_binlog_pmc()->erase("notification_group_id_current");

1
td/telegram/Td.h
View File

@ -30,6 +30,7 @@
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include <malloc.h>
#include <memory>
#include <unordered_map>
#include <unordered_set>

18
td/telegram/TdDb.cpp
View File

@ -137,9 +137,13 @@ Status init_db(SqliteDb &db) {
TRY_STATUS(db.exec("PRAGMA encoding=\"UTF-8\""));
TRY_STATUS(db.exec("PRAGMA journal_mode=WAL"));
TRY_STATUS(db.exec("PRAGMA cache_size=4096"));
TRY_STATUS(db.exec("PRAGMA page_size=65536"));
TRY_STATUS(db.exec("PRAGMA synchronous=NORMAL"));
TRY_STATUS(db.exec("PRAGMA temp_store=MEMORY"));
TRY_STATUS(db.exec("PRAGMA secure_delete=1"));
TRY_STATUS(db.exec("PRAGMA temp_store=FILE"));
TRY_STATUS(db.exec("PRAGMA wal_autocheckpoint=10000"));
TRY_STATUS(db.exec("PRAGMA shrink_memory"));
TRY_STATUS(db.exec("PRAGMA secure_delete=0"));
return Status::OK();
}
@ -316,17 +320,16 @@ Status TdDb::init_sqlite(int32 scheduler_id, const TdParameters &parameters, DbK
// init DialogDb
bool dialog_db_was_created = false;
if (use_dialog_db) {
TRY_STATUS(init_dialog_db(db, user_version, dialog_db_was_created));
} else {
TRY_STATUS(drop_dialog_db(db, user_version));
}
// init MessagesDb
TRY_STATUS(drop_messages_db(db, user_version));
if (use_message_db) {
TRY_STATUS(init_messages_db(db, user_version));
} else {
TRY_STATUS(drop_messages_db(db, user_version));
}
// init filesDb
@ -356,6 +359,7 @@ Status TdDb::init_sqlite(int32 scheduler_id, const TdParameters &parameters, DbK
binlog_pmc.force_sync({});
TRY_STATUS(db.exec("COMMIT TRANSACTION"));
TRY_STATUS(db.exec("VACUUM"));
file_db_ = create_file_db(sql_connection_, scheduler_id);
@ -512,8 +516,6 @@ Result<string> TdDb::get_stats() {
<< mask << "'",
PSLICE() << table << ":" << mask);
};
TRY_STATUS(run_query("SELECT 0, SUM(length(data)), COUNT(*) FROM messages WHERE 1", "messages"));
TRY_STATUS(run_query("SELECT 0, SUM(length(data)), COUNT(*) FROM dialogs WHERE 1", "dialogs"));
TRY_STATUS(run_kv_query("%", "common"));
TRY_STATUS(run_kv_query("%", "files"));
TRY_STATUS(run_kv_query("wp%"));

39
td/telegram/VideoNotesManager.cpp
View File

@ -25,7 +25,9 @@ VideoNotesManager::VideoNotesManager(Td *td) : td_(td) {
int32 VideoNotesManager::get_video_note_duration(FileId file_id) const {
auto it = video_notes_.find(file_id);
CHECK(it != video_notes_.end());
if (it == video_notes_.end() || it->second == nullptr) {
return 0;
}
return it->second->duration;
}
@ -35,7 +37,9 @@ tl_object_ptr<td_api::videoNote> VideoNotesManager::get_video_note_object(FileId
}
auto &video_note = video_notes_[file_id];
CHECK(video_note != nullptr);
if (video_note == nullptr) {
return nullptr;
}
video_note->is_changed = false;
return make_tl_object<td_api::videoNote>(video_note->duration, video_note->dimensions.width,
@ -79,29 +83,37 @@ FileId VideoNotesManager::on_get_video_note(unique_ptr<VideoNote> new_video_note
const VideoNotesManager::VideoNote *VideoNotesManager::get_video_note(FileId file_id) const {
auto video_note = video_notes_.find(file_id);
if (video_note == video_notes_.end()) {
return nullptr;
if (video_note == video_notes_.end() ||
video_note->second == nullptr ||
video_note->second->file_id != file_id) {
return make_unique<VideoNote>().get();
}
CHECK(video_note->second->file_id == file_id);
return video_note->second.get();
}
FileId VideoNotesManager::get_video_note_thumbnail_file_id(FileId file_id) const {
auto video_note = get_video_note(file_id);
CHECK(video_note != nullptr);
if (video_note == nullptr) {
return FileId();
}
return video_note->thumbnail.file_id;
}
void VideoNotesManager::delete_video_note_thumbnail(FileId file_id) {
auto &video_note = video_notes_[file_id];
CHECK(video_note != nullptr);
if (video_note == nullptr) {
return;
}
video_note->thumbnail = PhotoSize();
}
FileId VideoNotesManager::dup_video_note(FileId new_id, FileId old_id) {
const VideoNote *old_video_note = get_video_note(old_id);
CHECK(old_video_note != nullptr);
if (old_video_note == nullptr) {
return FileId();
}
auto &new_video_note = video_notes_[new_id];
CHECK(!new_video_note);
new_video_note = make_unique<VideoNote>(*old_video_note);
@ -124,7 +136,7 @@ bool VideoNotesManager::merge_video_notes(FileId new_id, FileId old_id, bool can
}
auto new_it = video_notes_.find(new_id);
if (new_it == video_notes_.end()) {
if (new_it == video_notes_.end() || new_it->second == nullptr) {
auto &old = video_notes_[old_id];
old->is_changed = true;
if (!can_delete_old) {
@ -211,7 +223,9 @@ tl_object_ptr<telegram_api::InputMedia> VideoNotesManager::get_input_media(
if (input_file != nullptr) {
const VideoNote *video_note = get_video_note(file_id);
CHECK(video_note != nullptr);
if (video_note == nullptr) {
return nullptr;
}
vector<tl_object_ptr<telegram_api::DocumentAttribute>> attributes;
attributes.push_back(make_tl_object<telegram_api::documentAttributeVideo>(
@ -232,4 +246,9 @@ tl_object_ptr<telegram_api::InputMedia> VideoNotesManager::get_input_media(
return nullptr;
}
void VideoNotesManager::memory_cleanup() {
video_notes_.clear();
video_notes_.rehash(0);
}
} // namespace td

2
td/telegram/VideoNotesManager.h
View File

@ -26,6 +26,8 @@ class VideoNotesManager {
public:
explicit VideoNotesManager(Td *td);
void memory_cleanup();
int32 get_video_note_duration(FileId file_id) const;
tl_object_ptr<td_api::videoNote> get_video_note_object(FileId file_id);

5
td/telegram/VideoNotesManager.hpp
View File

@ -20,7 +20,9 @@ namespace td {
template <class StorerT>
void VideoNotesManager::store_video_note(FileId file_id, StorerT &storer) const {
auto it = video_notes_.find(file_id);
CHECK(it != video_notes_.end());
if (it == video_notes_.end() || it->second == nullptr) {
return;
}
const VideoNote *video_note = it->second.get();
store(video_note->duration, storer);
store(video_note->dimensions, storer);
@ -44,5 +46,4 @@ FileId VideoNotesManager::parse_video_note(ParserT &parser) {
}
return on_get_video_note(std::move(video_note), false);
}
} // namespace td

30
td/telegram/VideosManager.cpp
View File

@ -24,7 +24,9 @@ VideosManager::VideosManager(Td *td) : td_(td) {
int32 VideosManager::get_video_duration(FileId file_id) const {
auto it = videos_.find(file_id);
CHECK(it != videos_.end());
if (it == videos_.end() || it->second == nullptr) {
return 0;
}
return it->second->duration;
}
@ -34,7 +36,9 @@ tl_object_ptr<td_api::video> VideosManager::get_video_object(FileId file_id) {
}
auto &video = videos_[file_id];
CHECK(video != nullptr);
if (video == nullptr) {
return nullptr;
}
video->is_changed = false;
return make_tl_object<td_api::video>(
@ -98,23 +102,29 @@ FileId VideosManager::on_get_video(unique_ptr<Video> new_video, bool replace) {
const VideosManager::Video *VideosManager::get_video(FileId file_id) const {
auto video = videos_.find(file_id);
if (video == videos_.end()) {
return nullptr;
if (video == videos_.end() ||
video->second == nullptr ||
video->second->file_id != file_id) {
return make_unique<Video>().get();
}
CHECK(video->second->file_id == file_id);
return video->second.get();
}
FileId VideosManager::get_video_thumbnail_file_id(FileId file_id) const {
auto video = get_video(file_id);
CHECK(video != nullptr);
if (video == nullptr) {
return FileId();
}
return video->thumbnail.file_id;
}
void VideosManager::delete_video_thumbnail(FileId file_id) {
auto &video = videos_[file_id];
CHECK(video != nullptr);
if (video == nullptr) {
return;
}
video->thumbnail = PhotoSize();
}
@ -143,7 +153,7 @@ bool VideosManager::merge_videos(FileId new_id, FileId old_id, bool can_delete_o
}
auto new_it = videos_.find(new_id);
if (new_it == videos_.end()) {
if (new_it == videos_.end() || new_it->second == nullptr) {
auto &old = videos_[old_id];
old->is_changed = true;
if (!can_delete_old) {
@ -290,5 +300,9 @@ string VideosManager::get_video_search_text(FileId file_id) const {
CHECK(video != nullptr);
return video->file_name;
}
void VideosManager::memory_cleanup() {
videos_.clear();
videos_.rehash(0);
}
} // namespace td

2
td/telegram/VideosManager.h
View File

@ -26,6 +26,8 @@ class VideosManager {
public:
explicit VideosManager(Td *td);
void memory_cleanup();
int32 get_video_duration(FileId file_id) const;
tl_object_ptr<td_api::video> get_video_object(FileId file_id);

4
td/telegram/VideosManager.hpp
View File

@ -20,7 +20,9 @@ namespace td {
template <class StorerT>
void VideosManager::store_video(FileId file_id, StorerT &storer) const {
auto it = videos_.find(file_id);
CHECK(it != videos_.end());
if (it == videos_.end() || it->second == nullptr) {
return;
}
const Video *video = it->second.get();
BEGIN_STORE_FLAGS();
STORE_FLAG(video->has_stickers);

14
td/telegram/VoiceNotesManager.cpp
View File

@ -25,7 +25,9 @@ VoiceNotesManager::VoiceNotesManager(Td *td) : td_(td) {
int32 VoiceNotesManager::get_voice_note_duration(FileId file_id) const {
auto it = voice_notes_.find(file_id);
CHECK(it != voice_notes_.end());
if (it == voice_notes_.end() || it->second == nullptr) {
return 0;
}
return it->second->duration;
}
@ -68,11 +70,13 @@ FileId VoiceNotesManager::on_get_voice_note(unique_ptr<VoiceNote> new_voice_note
const VoiceNotesManager::VoiceNote *VoiceNotesManager::get_voice_note(FileId file_id) const {
auto voice_note = voice_notes_.find(file_id);
if (voice_note == voice_notes_.end()) {
return nullptr;
if (voice_note == voice_notes_.end() ||
voice_note->second == nullptr ||
voice_note->second->file_id != file_id) {
return make_unique<VoiceNote>().get();
}
CHECK(voice_note->second->file_id == file_id);
return voice_note->second.get();
}
@ -100,7 +104,7 @@ bool VoiceNotesManager::merge_voice_notes(FileId new_id, FileId old_id, bool can
}
auto new_it = voice_notes_.find(new_id);
if (new_it == voice_notes_.end()) {
if (new_it == voice_notes_.end() || new_it->second == nullptr) {
auto &old = voice_notes_[old_id];
old->is_changed = true;
if (!can_delete_old) {

4
td/telegram/VoiceNotesManager.hpp
View File

@ -18,7 +18,9 @@ namespace td {
template <class StorerT>
void VoiceNotesManager::store_voice_note(FileId file_id, StorerT &storer) const {
auto it = voice_notes_.find(file_id);
CHECK(it != voice_notes_.end());
if (it == voice_notes_.end() || it->second == nullptr) {
return;
}
const VoiceNote *voice_note = it->second.get();
store(voice_note->mime_type, storer);
store(voice_note->duration, storer);

206
td/telegram/WebPagesManager.cpp
View File

@ -413,7 +413,9 @@ WebPagesManager::~WebPagesManager() = default;
WebPageId WebPagesManager::on_get_web_page(tl_object_ptr<telegram_api::WebPage> &&web_page_ptr,
DialogId owner_dialog_id) {
CHECK(web_page_ptr != nullptr);
if (web_page_ptr == nullptr) {
return WebPageId();
}
LOG(DEBUG) << "Got " << to_string(web_page_ptr);
switch (web_page_ptr->get_id()) {
case telegram_api::webPageEmpty::ID: {
@ -427,11 +429,6 @@ WebPageId WebPagesManager::on_get_web_page(tl_object_ptr<telegram_api::WebPage>
LOG(INFO) << "Got empty " << web_page_id;
const WebPage *web_page_to_delete = get_web_page(web_page_id);
if (web_page_to_delete != nullptr) {
if (web_page_to_delete->logevent_id != 0) {
LOG(INFO) << "Erase " << web_page_id << " from binlog";
binlog_erase(G()->td_db()->get_binlog(), web_page_to_delete->logevent_id);
web_page_to_delete->logevent_id = 0;
}
if (web_page_to_delete->file_source_id.is_valid()) {
td_->file_manager_->change_files_source(web_page_to_delete->file_source_id,
get_web_page_file_ids(web_page_to_delete), vector<FileId>());
@ -540,7 +537,9 @@ WebPageId WebPagesManager::on_get_web_page(tl_object_ptr<telegram_api::WebPage>
void WebPagesManager::update_web_page(unique_ptr<WebPage> web_page, WebPageId web_page_id, bool from_binlog,
bool from_database) {
LOG(INFO) << "Update " << web_page_id;
CHECK(web_page != nullptr);
if (web_page == nullptr) {
return;
}
auto &page = web_pages_[web_page_id];
auto old_file_ids = get_web_page_file_ids(page.get());
@ -663,7 +662,7 @@ void WebPagesManager::on_get_web_page_instant_view_view_count(WebPageId web_page
}
auto *instant_view = &web_pages_[web_page_id]->instant_view;
CHECK(!instant_view->is_empty);
if (instant_view->is_empty) { return; }
if (instant_view->view_count >= view_count) {
return;
}
@ -701,7 +700,7 @@ void WebPagesManager::register_web_page(WebPageId web_page_id, FullMessageId ful
LOG(INFO) << "Register " << web_page_id << " from " << full_message_id << " from " << source;
bool is_inserted = web_page_messages_[web_page_id].insert(full_message_id).second;
LOG_CHECK(is_inserted) << source << " " << web_page_id << " " << full_message_id;
if (!is_inserted) { return; }
if (!td_->auth_manager_->is_bot() && !have_web_page_force(web_page_id)) {
LOG(INFO) << "Waiting for " << web_page_id << " needed in " << full_message_id;
@ -717,7 +716,10 @@ void WebPagesManager::unregister_web_page(WebPageId web_page_id, FullMessageId f
LOG(INFO) << "Unregister " << web_page_id << " from " << full_message_id << " from " << source;
auto &message_ids = web_page_messages_[web_page_id];
auto is_deleted = message_ids.erase(full_message_id);
LOG_CHECK(is_deleted) << source << " " << web_page_id << " " << full_message_id;
if (!is_deleted) {
return;
}
if (!is_deleted) { return; }
if (message_ids.empty()) {
web_page_messages_.erase(web_page_id);
@ -730,7 +732,9 @@ void WebPagesManager::unregister_web_page(WebPageId web_page_id, FullMessageId f
void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const string &url,
tl_object_ptr<telegram_api::MessageMedia> &&message_media_ptr,
Promise<Unit> &&promise) {
CHECK(message_media_ptr != nullptr);
if (message_media_ptr == nullptr) {
return;
}
int32 constructor_id = message_media_ptr->get_id();
if (constructor_id != telegram_api::messageMediaWebPage::ID) {
if (constructor_id == telegram_api::messageMediaEmpty::ID) {
@ -745,7 +749,9 @@ void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const st
}
auto message_media_web_page = move_tl_object_as<telegram_api::messageMediaWebPage>(message_media_ptr);
CHECK(message_media_web_page->webpage_ != nullptr);
if (message_media_web_page->webpage_ == nullptr) {
return;
}
auto web_page_id = on_get_web_page(std::move(message_media_web_page->webpage_), DialogId());
if (web_page_id.is_valid() && !have_web_page(web_page_id)) {
@ -759,9 +765,9 @@ void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const st
void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const string &url, WebPageId web_page_id,
Promise<Unit> &&promise) {
CHECK(web_page_id == WebPageId() || have_web_page(web_page_id));
if (!(web_page_id == WebPageId() || have_web_page(web_page_id))) { return; }
CHECK(got_web_page_previews_.find(request_id) == got_web_page_previews_.end());
if (!(got_web_page_previews_.find(request_id) == got_web_page_previews_.end())) { return; }
got_web_page_previews_[request_id] = web_page_id;
if (web_page_id.is_valid() && !url.empty()) {
@ -774,7 +780,7 @@ void WebPagesManager::on_get_web_page_preview_success(int64 request_id, const st
void WebPagesManager::on_get_web_page_preview_fail(int64 request_id, const string &url, Status error,
Promise<Unit> &&promise) {
LOG(INFO) << "Clean up getting of web page preview with url \"" << url << '"';
CHECK(error.is_error());
if (!error.is_error()) { return; }
promise.set_error(std::move(error));
}
@ -824,7 +830,9 @@ tl_object_ptr<td_api::webPage> WebPagesManager::get_web_page_preview_result(int6
}
auto it = got_web_page_previews_.find(request_id);
CHECK(it != got_web_page_previews_.end());
if (it == got_web_page_previews_.end()) {
return nullptr;
}
auto web_page_id = it->second;
got_web_page_previews_.erase(it);
return get_web_page_object(web_page_id);
@ -885,7 +893,9 @@ void WebPagesManager::load_web_page_instant_view(WebPageId web_page_id, bool for
LOG(INFO) << "Load " << web_page_id << " instant view, have " << previous_queries << " previous queries";
if (previous_queries == 0) {
const WebPageInstantView *web_page_instant_view = get_web_page_instant_view(web_page_id);
CHECK(web_page_instant_view != nullptr);
if (web_page_instant_view == nullptr) {
return;
}
if (G()->parameters().use_message_db && !web_page_instant_view->was_loaded_from_database) {
LOG(INFO) << "Trying to load " << web_page_id << " instant view from database";
@ -903,7 +913,7 @@ void WebPagesManager::load_web_page_instant_view(WebPageId web_page_id, bool for
void WebPagesManager::reload_web_page_instant_view(WebPageId web_page_id) {
LOG(INFO) << "Reload " << web_page_id << " instant view";
const WebPage *web_page = get_web_page(web_page_id);
CHECK(web_page != nullptr && !web_page->instant_view.is_empty);
if (!(web_page != nullptr && !web_page->instant_view.is_empty)) { return; }
auto promise = PromiseCreator::lambda([web_page_id](Result<> result) {
send_closure(G()->web_pages_manager(), &WebPagesManager::update_web_page_instant_view_load_requests, web_page_id,
@ -922,7 +932,7 @@ void WebPagesManager::on_load_web_page_instant_view_from_database(WebPageId web_
if (G()->close_flag()) {
return;
}
CHECK(G()->parameters().use_message_db);
if (!(G()->parameters().use_message_db)) { return; }
LOG(INFO) << "Successfully loaded " << web_page_id << " instant view of size " << value.size() << " from database";
// G()->td_db()->get_sqlite_pmc()->erase(get_web_page_instant_view_database_key(web_page_id), Auto());
// return;
@ -1163,121 +1173,7 @@ bool WebPagesManager::have_web_page(WebPageId web_page_id) const {
}
tl_object_ptr<td_api::webPage> WebPagesManager::get_web_page_object(WebPageId web_page_id) const {
if (!web_page_id.is_valid()) {
return nullptr;
}
const WebPage *web_page = get_web_page(web_page_id);
if (web_page == nullptr) {
return nullptr;
}
int32 instant_view_version = [web_page] {
if (web_page->instant_view.is_empty) {
return 0;
}
if (web_page->instant_view.is_v2) {
return 2;
}
return 1;
}();
FormattedText description;
description.text = web_page->description;
description.entities = find_entities(web_page->description, true);
auto r_url = parse_url(web_page->display_url);
if (r_url.is_ok()) {
Slice host = r_url.ok().host_;
if (!host.empty() && host.back() == '.') {
host.truncate(host.size() - 1);
}
auto replace_entities = [](Slice text, vector<MessageEntity> &entities, auto replace_url) {
int32 current_offset = 0;
for (auto &entity : entities) {
CHECK(entity.offset >= current_offset);
text = utf8_utf16_substr(text, static_cast<size_t>(entity.offset - current_offset));
auto entity_text = utf8_utf16_substr(text, 0, static_cast<size_t>(entity.length));
text = text.substr(entity_text.size());
current_offset = entity.offset + entity.length;
auto replaced_url = replace_url(entity, entity_text);
if (!replaced_url.empty()) {
entity = MessageEntity(MessageEntity::Type::TextUrl, entity.offset, entity.length, std::move(replaced_url));
}
}
};
if (host == "instagram.com" || ends_with(host, ".instagram.com")) {
replace_entities(description.text, description.entities, [](const MessageEntity &entity, Slice text) {
if (entity.type == MessageEntity::Type::Mention) {
return PSTRING() << "https://www.instagram.com/" << text.substr(1) << '/';
}
if (entity.type == MessageEntity::Type::Hashtag) {
return PSTRING() << "https://www.instagram.com/explore/tags/" << url_encode(text.substr(1)) << '/';
}
return string();
});
} else if (host == "twitter.com" || ends_with(host, ".twitter.com")) {
replace_entities(description.text, description.entities, [](const MessageEntity &entity, Slice text) {
if (entity.type == MessageEntity::Type::Mention) {
return PSTRING() << "https://twitter.com/" << text.substr(1);
}
if (entity.type == MessageEntity::Type::Hashtag) {
return PSTRING() << "https://twitter.com/hashtag/" << url_encode(text.substr(1));
}
return string();
});
} else if (host == "t.me" || host == "telegram.me" || host == "telegram.dog" || host == "telesco.pe") {
// leave everything as is
} else {
td::remove_if(description.entities,
[](const MessageEntity &entity) { return entity.type == MessageEntity::Type::Mention; });
if (host == "youtube.com" || host == "www.youtube.com") {
replace_entities(description.text, description.entities, [](const MessageEntity &entity, Slice text) {
if (entity.type == MessageEntity::Type::Hashtag) {
return PSTRING() << "https://www.youtube.com/results?search_query=" << url_encode(text);
}
return string();
});
} else if (host == "music.youtube.com") {
replace_entities(description.text, description.entities, [](const MessageEntity &entity, Slice text) {
if (entity.type == MessageEntity::Type::Hashtag) {
return PSTRING() << "https://music.youtube.com/search?q=" << url_encode(text);
}
return string();
});
}
}
}
return make_tl_object<td_api::webPage>(
web_page->url, web_page->display_url, web_page->type, web_page->site_name, web_page->title,
get_formatted_text_object(description), get_photo_object(td_->file_manager_.get(), &web_page->photo),
web_page->embed_url, web_page->embed_type, web_page->embed_dimensions.width, web_page->embed_dimensions.height,
web_page->duration, web_page->author,
web_page->document.type == Document::Type::Animation
? td_->animations_manager_->get_animation_object(web_page->document.file_id, "get_web_page_object")
: nullptr,
web_page->document.type == Document::Type::Audio
? td_->audios_manager_->get_audio_object(web_page->document.file_id)
: nullptr,
web_page->document.type == Document::Type::General
? td_->documents_manager_->get_document_object(web_page->document.file_id)
: nullptr,
web_page->document.type == Document::Type::Sticker
? td_->stickers_manager_->get_sticker_object(web_page->document.file_id)
: nullptr,
web_page->document.type == Document::Type::Video
? td_->videos_manager_->get_video_object(web_page->document.file_id)
: nullptr,
web_page->document.type == Document::Type::VideoNote
? td_->video_notes_manager_->get_video_note_object(web_page->document.file_id)
: nullptr,
web_page->document.type == Document::Type::VoiceNote
? td_->voice_notes_manager_->get_voice_note_object(web_page->document.file_id)
: nullptr,
instant_view_version);
return nullptr;
}
tl_object_ptr<td_api::webPageInstantView> WebPagesManager::get_web_page_instant_view_object(
@ -1308,7 +1204,7 @@ void WebPagesManager::on_web_page_changed(WebPageId web_page_id, bool have_web_p
for (auto full_message_id : it->second) {
full_message_ids.push_back(full_message_id);
}
CHECK(!full_message_ids.empty());
if (full_message_ids.empty()) { return; }
for (auto full_message_id : full_message_ids) {
if (!have_web_page) {
td_->messages_manager_->delete_pending_message_web_page(full_message_id);
@ -1317,9 +1213,9 @@ void WebPagesManager::on_web_page_changed(WebPageId web_page_id, bool have_web_p
}
}
if (have_web_page) {
CHECK(web_page_messages_[web_page_id].size() == full_message_ids.size());
if (!(web_page_messages_[web_page_id].size() == full_message_ids.size())) { return; }
} else {
CHECK(web_page_messages_.count(web_page_id) == 0);
if (!(web_page_messages_.count(web_page_id) == 0)) { return; }
}
}
auto get_it = pending_get_web_pages_.find(web_page_id);
@ -1336,11 +1232,13 @@ void WebPagesManager::on_web_page_changed(WebPageId web_page_id, bool have_web_p
const WebPagesManager::WebPage *WebPagesManager::get_web_page(WebPageId web_page_id) const {
auto p = web_pages_.find(web_page_id);
if (p == web_pages_.end()) {
return nullptr;
} else {
return p->second.get();
if (p == web_pages_.end() ||
p->second == nullptr) {
return make_unique<WebPage>().get();
}
return p->second.get();
}
const WebPagesManager::WebPageInstantView *WebPagesManager::get_web_page_instant_view(WebPageId web_page_id) const {
@ -1388,7 +1286,9 @@ void WebPagesManager::on_pending_web_page_timeout(WebPageId web_page_id) {
void WebPagesManager::on_get_web_page_instant_view(WebPage *web_page, tl_object_ptr<telegram_api::page> &&page,
int32 hash, DialogId owner_dialog_id) {
CHECK(page != nullptr);
if (page == nullptr) {
return;
}
std::unordered_map<int64, Photo> photos;
for (auto &photo_ptr : page->photos_) {
Photo photo = get_photo(td_->file_manager_.get(), std::move(photo_ptr), owner_dialog_id);
@ -1503,7 +1403,9 @@ void WebPagesManager::save_web_page(const WebPage *web_page, WebPageId web_page_
return;
}
CHECK(web_page != nullptr);
if (web_page == nullptr) {
return;
}
if (!from_binlog) {
WebPageLogEvent logevent(web_page_id, web_page);
LogEventStorerImpl<WebPageLogEvent> storer(logevent);
@ -1539,7 +1441,9 @@ void WebPagesManager::on_binlog_web_page_event(BinlogEvent &&event) {
auto web_page_id = log_event.web_page_id;
LOG(INFO) << "Add " << web_page_id << " from binlog";
auto web_page = std::move(log_event.web_page_out);
CHECK(web_page != nullptr);
if (web_page == nullptr) {
return;
}
web_page->logevent_id = event.id_;
@ -1603,7 +1507,7 @@ void WebPagesManager::on_load_web_page_from_database(WebPageId web_page_id, stri
vector<Promise<Unit>> promises;
if (it != load_web_page_from_database_queries_.end()) {
promises = std::move(it->second);
CHECK(!promises.empty());
if (promises.empty()) { return; }
load_web_page_from_database_queries_.erase(it);
}
@ -1703,4 +1607,16 @@ vector<FileId> WebPagesManager::get_web_page_file_ids(const WebPage *web_page) c
return result;
}
void WebPagesManager::memory_cleanup() {
web_pages_.clear();
web_pages_.rehash(0);
web_page_messages_.clear();
web_page_messages_.rehash(0);
got_web_page_previews_.clear();
got_web_page_previews_.rehash(0);
url_to_web_page_id_.clear();
url_to_web_page_id_.rehash(0);
url_to_file_source_id_.clear();
url_to_file_source_id_.rehash(0);
}
} // namespace td

2
td/telegram/WebPagesManager.h
View File

@ -37,6 +37,8 @@ class WebPagesManager : public Actor {
public:
WebPagesManager(Td *td, ActorShared<> parent);
void memory_cleanup();
WebPagesManager(const WebPagesManager &) = delete;
WebPagesManager &operator=(const WebPagesManager &) = delete;
WebPagesManager(WebPagesManager &&) = delete;

2
td/telegram/cli.cpp
View File

@ -4084,6 +4084,8 @@ class CliClient final : public Actor {
}
#endif
on_cmd("v0");
while (!cmd_queue_.empty() && !close_flag_) {
auto cmd = std::move(cmd_queue_.front());
cmd_queue_.pop();

18
td/telegram/files/FileId.h
View File

@ -9,6 +9,7 @@
#include "td/utils/common.h"
#include "td/utils/StringBuilder.h"
#include <ctime>
#include <functional>
#include <type_traits>
@ -17,6 +18,7 @@ namespace td {
class FileId {
int32 id = 0;
int32 remote_id = 0;
int64 time_ = INT64_MAX;
public:
FileId() = default;
@ -34,10 +36,26 @@ class FileId {
return id > 0;
}
int32 fast_get() const {
return id;
}
int32 get() const {
return id;
}
void set_time() {
time_ = std::time(nullptr);
}
int64 get_time() const {
return time_;
}
void reset_time() {
time_ = INT64_MAX;
}
int32 get_remote() const {
return remote_id;
}

267
td/telegram/files/FileManager.cpp
View File

@ -48,6 +48,8 @@
#include <tuple>
#include <utility>
#define FILE_TTL 20
namespace td {
namespace {
constexpr int64 MAX_FILE_SIZE = 1500 * (1 << 20) /* 1500MB */;
@ -131,7 +133,9 @@ FileNode &FileNodePtr::operator*() const {
FileNode *FileNodePtr::get() const {
auto res = get_unsafe();
CHECK(res);
if (res == nullptr) {
return {};
}
return res;
}
@ -140,7 +144,9 @@ FullRemoteFileLocation *FileNodePtr::get_remote() const {
}
FileNode *FileNodePtr::get_unsafe() const {
CHECK(file_manager_ != nullptr);
if (file_manager_ == nullptr) {
return {};
}
return file_manager_->get_file_node_raw(file_id_);
}
@ -977,8 +983,7 @@ bool FileManager::try_fix_partial_local_location(FileNodePtr node) {
}
FileManager::FileIdInfo *FileManager::get_file_id_info(FileId file_id) {
LOG_CHECK(0 <= file_id.get() && file_id.get() < static_cast<int32>(file_id_info_.size()))
<< file_id << " " << file_id_info_.size();
file_id.set_time();
return &file_id_info_[file_id.get()];
}
@ -1014,7 +1019,7 @@ void FileManager::try_forget_file_id(FileId file_id) {
bool is_removed = td::remove(file_node->file_ids_, file_id);
CHECK(is_removed);
*info = FileIdInfo();
empty_file_ids_.push_back(file_id.get());
file_id_info_.erase(file_id.get());
}
FileId FileManager::register_empty(FileType type) {
@ -1029,7 +1034,9 @@ void FileManager::on_file_unlink(const FullLocalFileLocation &location) {
}
auto file_id = it->second;
auto file_node = get_sync_file_node(file_id);
CHECK(file_node);
if (!file_node) {
return;
}
file_node->drop_local_location();
try_flush_node_info(file_node, "on_file_unlink");
}
@ -1132,13 +1139,13 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi
// create FileNode
auto file_node_id = next_file_node_id();
auto &node = file_nodes_[file_node_id];
node = td::make_unique<FileNode>(std::move(data.local_), NewRemoteFileLocation(data.remote_, file_location_source),
node = FileNode(std::move(data.local_), NewRemoteFileLocation(data.remote_, file_location_source),
std::move(data.generate_), data.size_, data.expected_size_,
std::move(data.remote_name_), std::move(data.url_), data.owner_dialog_id_,
std::move(data.encryption_key_), file_id, static_cast<int8>(has_remote));
node->pmc_id_ = FileDbId(data.pmc_id_);
node.pmc_id_ = FileDbId(data.pmc_id_);
get_file_id_info(file_id)->node_id_ = file_node_id;
node->file_ids_.push_back(file_id);
node.file_ids_.push_back(file_id);
FileView file_view(get_file_node(file_id));
@ -1185,7 +1192,7 @@ Result<FileId> FileManager::register_file(FileData &&data, FileLocationSource fi
int new_cnt = new_remote + new_local + new_generate;
if (data.pmc_id_ == 0 && file_db_ && new_cnt > 0) {
node->need_load_from_pmc_ = true;
node.need_load_from_pmc_ = true;
}
bool no_sync_merge = to_merge.size() == 1 && new_cnt == 0;
for (auto id : to_merge) {
@ -1399,8 +1406,6 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
x_node->remote_.full_source == FileLocationSource::FromServer &&
y_node->remote_.full_source == FileLocationSource::FromServer &&
x_node->remote_.full.value().get_dc_id() != y_node->remote_.full.value().get_dc_id()) {
LOG(ERROR) << "File remote location was changed from " << y_node->remote_.full.value() << " to "
<< x_node->remote_.full.value();
}
bool drop_last_successful_force_reupload_time = x_node->last_successful_force_reupload_time_ <= 0 &&
@ -1580,7 +1585,7 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
file_id_info->node_id_ = node_ids[node_i];
send_updates_flag |= file_id_info->send_updates_flag_;
}
other_node = {};
other_node = {this};
if (send_updates_flag) {
// node might not changed, but other_node might changed, so we need to send update anyway
@ -1607,7 +1612,7 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
}
}
file_nodes_[node_ids[other_node_i]] = nullptr;
file_nodes_.erase(node_ids[other_node_i]);
run_generate(node);
run_download(node);
@ -1800,7 +1805,7 @@ void FileManager::flush_to_pmc(FileNodePtr node, bool new_remote, bool new_local
}
FileNode *FileManager::get_file_node_raw(FileId file_id, FileNodeId *file_node_id) {
if (file_id.get() <= 0 || file_id.get() >= static_cast<int32>(file_id_info_.size())) {
if (file_id.get() <= 0) {
return nullptr;
}
FileNodeId node_id = file_id_info_[file_id.get()].node_id_;
@ -1810,13 +1815,13 @@ FileNode *FileManager::get_file_node_raw(FileId file_id, FileNodeId *file_node_i
if (file_node_id != nullptr) {
*file_node_id = node_id;
}
return file_nodes_[node_id].get();
return &file_nodes_[node_id];
}
FileNodePtr FileManager::get_sync_file_node(FileId file_id) {
auto file_node = get_file_node(file_id);
if (!file_node) {
return {};
return {this};
}
load_from_pmc(file_node, true, true, true);
return file_node;
@ -3187,20 +3192,15 @@ string FileManager::extract_file_reference(const tl_object_ptr<telegram_api::Inp
}
FileId FileManager::next_file_id() {
if (!empty_file_ids_.empty()) {
auto res = empty_file_ids_.back();
empty_file_ids_.pop_back();
return FileId{res, 0};
}
FileId res(static_cast<int32>(file_id_info_.size()), 0);
// LOG(ERROR) << "NEXT file_id " << res;
file_id_info_.push_back({});
auto id = file_id_seqno++;
FileId res(static_cast<int32>(id), 0);
file_id_info_[id] = {};
res.set_time();
return res;
}
FileManager::FileNodeId FileManager::next_file_node_id() {
FileNodeId res = static_cast<FileNodeId>(file_nodes_.size());
file_nodes_.emplace_back(nullptr);
auto res = static_cast<FileNodeId>(file_node_seqno++);
return res;
}
@ -3210,7 +3210,9 @@ void FileManager::on_start_download(QueryId query_id) {
}
auto query = queries_container_.get(query_id);
CHECK(query != nullptr);
if (query == nullptr) {
return;
}
auto file_id = query->file_id_;
auto file_node = get_file_node(file_id);
@ -3233,7 +3235,9 @@ void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLo
}
auto query = queries_container_.get(query_id);
CHECK(query != nullptr);
if (query == nullptr) {
return;
}
auto file_id = query->file_id_;
auto file_node = get_file_node(file_id);
@ -3262,7 +3266,9 @@ void FileManager::on_hash(QueryId query_id, string hash) {
}
auto query = queries_container_.get(query_id);
CHECK(query != nullptr);
if (query == nullptr) {
return;
}
auto file_id = query->file_id_;
@ -3285,7 +3291,9 @@ void FileManager::on_partial_upload(QueryId query_id, const PartialRemoteFileLoc
}
auto query = queries_container_.get(query_id);
CHECK(query != nullptr);
if (query == nullptr) {
return;
}
auto file_id = query->file_id_;
auto file_node = get_file_node(file_id);
@ -3415,7 +3423,9 @@ void FileManager::on_partial_generate(QueryId query_id, const PartialLocalFileLo
}
auto query = queries_container_.get(query_id);
CHECK(query != nullptr);
if (query == nullptr) {
return;
}
auto file_id = query->file_id_;
auto file_node = get_file_node(file_id);
@ -3699,8 +3709,199 @@ void FileManager::hangup() {
stop();
}
void FileManager::destroy_query(int32 file_id) {
for (auto &query_id : queries_container_.ids()) {
auto query = queries_container_.get(query_id);
if (query != nullptr && file_id == query->file_id_.fast_get()) {
on_error(query_id, Status::Error(400, "FILE_DOWNLOAD_RESTART"));
}
}
}
void FileManager::memory_cleanup() {
/* DESTROY OLD file_id_info_ */
{
auto it = file_id_info_.begin();
auto time = std::time(nullptr);
std::vector<int32> file_to_be_deleted = {};
while (it != file_id_info_.end()) {
if (it->second.node_id_ != 0) {
auto &node = file_nodes_[it->second.node_id_];
if (time - node.main_file_id_.get_time() > FILE_TTL) {
auto can_reset = node.download_priority_ == 0;
can_reset &= node.generate_download_priority_ == 0;
can_reset &= node.download_id_ == 0;
if (can_reset) {
auto file_ids_it = node.file_ids_.begin();
while (file_ids_it != node.file_ids_.end() && can_reset) {
auto &file = file_id_info_[file_ids_it->fast_get()];
can_reset &= file.download_priority_ == 0;
can_reset &= time - file_ids_it->get_time() > FILE_TTL;
++file_ids_it;
}
}
if (can_reset) {
node.main_file_id_.reset_time();
for (auto &file_id : node.file_ids_) {
file_id.reset_time();
/* DESTROY ASSOCIATED QUERIES */
destroy_query(file_id.fast_get());
/* DESTROY ASSOCIATED LATE */
file_to_be_deleted.push_back(file_id.fast_get());
}
/* DESTROY MAIN QUERY */
destroy_query(it->first);
/* DESTROY FILE REFERENCE */
context_->destroy_file_source(node.main_file_id_);
/* DESTROY MAIN NODE */
file_nodes_.erase(it->first);
/* DESTROY MAIN FILE LATE */
file_to_be_deleted.push_back(it->first);
}
}
} else {
file_to_be_deleted.push_back(it->first);
}
++it;
}
for (auto file_id : file_to_be_deleted) {
file_id_info_.erase(file_id);
}
}
/* DESTROY INVALID FILES */
{
auto it = file_id_info_.begin();
while (it != file_id_info_.end()) {
if (it->second.node_id_ != 0) {
if (file_nodes_[it->second.node_id_].empty) {
destroy_query(it->first);
context_->destroy_file_source({it->first, 0});
file_id_info_.erase(it++);
file_nodes_.erase(it->second.node_id_);
} else {
++it;
}
} else {
++it;
}
}
}
/* DESTROY INVALID file_nodes_ */
{
auto it = file_nodes_.begin();
while (it != file_nodes_.end()) {
if (it->second.empty) {
file_nodes_.erase(it++);
} else {
if (it->second.main_file_id_.empty()) {
file_nodes_.erase(it++);
} else {
if (file_id_info_[it->second.main_file_id_.get()].node_id_ == 0) {
file_id_info_.erase(it->second.main_file_id_.get());
file_nodes_.erase(it++);
} else {
++it;
}
}
}
}
}
/* DESTROY INVALID file_hash_to_file_id_ */
{
auto it = file_hash_to_file_id_.begin();
while (it != file_hash_to_file_id_.end()) {
auto &file = file_id_info_[it->second.fast_get()];
if (file_nodes_[file.node_id_].empty) {
file_hash_to_file_id_.erase(it++);
file_nodes_.erase(file.node_id_);
} else {
++it;
}
}
}
/* DESTROY INVALID local_location_to_file_id_ */
{
auto it = local_location_to_file_id_.begin();
while (it != local_location_to_file_id_.end()) {
auto &file = file_id_info_[it->second.fast_get()];
if (file_nodes_[file.node_id_].empty) {
it = local_location_to_file_id_.erase(it++);
file_nodes_.erase(file.node_id_);
} else {
++it;
}
}
}
/* DESTROY INVALID generate_location_to_file_id_ */
{
auto it = generate_location_to_file_id_.begin();
while (it != generate_location_to_file_id_.end()) {
auto &file = file_id_info_[it->second.fast_get()];
if (file_nodes_[file.node_id_].empty) {
it = generate_location_to_file_id_.erase(it++);
file_nodes_.erase(file.node_id_);
} else {
++it;
}
}
}
/* DESTROY INVALID remote_location_info_ */
{
auto map = remote_location_info_.get_map();
auto it = map.begin();
while (it != map.end()) {
auto &file = file_id_info_[it->first.file_id_.fast_get()];
if (file_nodes_[file.node_id_].empty) {
remote_location_info_.erase(it->second);
map.erase(it++);
file_nodes_.erase(file.node_id_);
} else {
++it;
}
}
}
/* DESTROY NULL file_id_info_ */
{
auto it = file_id_info_.begin();
while (it != file_id_info_.end()) {
if (&it->second == nullptr || file_nodes_[it->second.node_id_].empty) {
file_id_info_.erase(it++);
} else {
++it;
}
}
}
file_nodes_.rehash(0);
file_hash_to_file_id_.rehash(0);
file_id_info_.rehash(0);
LOG(ERROR) << "registered ids: " << file_id_info_.size() << " registered nodes: " << file_nodes_.size();
}
void FileManager::tear_down() {
parent_.reset();
}
} // namespace td

21
td/telegram/files/FileManager.h
View File

@ -68,6 +68,7 @@ struct NewRemoteFileLocation {
class FileNode {
public:
FileNode() {empty = true;}
FileNode(LocalFileLocation local, NewRemoteFileLocation remote, unique_ptr<FullGenerateFileLocation> generate,
int64 size, int64 expected_size, string remote_name, string url, DialogId owner_dialog_id,
FileEncryptionKey key, FileId main_file_id, int8 main_file_id_priority)
@ -119,6 +120,7 @@ class FileNode {
void on_info_flushed();
string suggested_name() const;
bool empty = false;
private:
friend class FileView;
@ -146,8 +148,7 @@ class FileNode {
FileEncryptionKey encryption_key_;
FileDbId pmc_id_;
std::vector<FileId> file_ids_;
FileId main_file_id_;
FileId main_file_id_ = {};
double last_successful_force_reupload_time_ = -1e10;
@ -189,6 +190,8 @@ class FileManager;
class FileNodePtr {
public:
FileNodePtr() = default;
FileNodePtr(FileManager *file_manager) : file_manager_(file_manager) {
}
FileNodePtr(FileId file_id, FileManager *file_manager) : file_id_(file_id), file_manager_(file_manager) {
}
@ -328,6 +331,8 @@ class FileView {
class FileManager : public FileLoadManager::Callback {
public:
void memory_cleanup();
class DownloadCallback {
public:
DownloadCallback() = default;
@ -366,6 +371,8 @@ class FileManager : public FileLoadManager::Callback {
virtual void on_file_updated(FileId size) = 0;
virtual void destroy_file_source(FileId file_id) = 0;
virtual bool add_file_source(FileId file_id, FileSourceId file_source_id) = 0;
virtual bool remove_file_source(FileId file_id, FileSourceId file_source_id) = 0;
@ -480,6 +487,8 @@ class FileManager : public FileLoadManager::Callback {
static constexpr char PERSISTENT_ID_VERSION_MAP = 3;
static constexpr char PERSISTENT_ID_VERSION = 4;
void destroy_query(int32 file_id);
Result<FileId> check_input_file_id(FileType type, Result<FileId> result, bool is_encrypted, bool allow_zero,
bool is_secure) TD_WARN_UNUSED_RESULT;
@ -547,15 +556,17 @@ class FileManager : public FileLoadManager::Callback {
};
Enumerator<RemoteInfo> remote_location_info_;
FileNodeId file_node_seqno = 0;
int32 file_id_seqno = 0;
std::unordered_map<string, FileId> file_hash_to_file_id_;
std::map<FullLocalFileLocation, FileId> local_location_to_file_id_;
std::map<FullGenerateFileLocation, FileId> generate_location_to_file_id_;
std::map<FileDbId, int32> pmc_id_to_file_node_id_;
vector<FileIdInfo> file_id_info_;
vector<int32> empty_file_ids_;
vector<unique_ptr<FileNode>> file_nodes_;
std::unordered_map<int32, FileIdInfo> file_id_info_;
std::unordered_map<FileNodeId, FileNode> file_nodes_;
ActorOwn<FileLoadManager> file_load_manager_;
ActorOwn<FileGenerateManager> file_generate_manager_;

16
td/telegram/logevent/LogEvent.h
View File

@ -262,10 +262,10 @@ class LogEventStorerImpl : public Storer {
size_t store(uint8 *ptr) const override {
LogEventStorerUnsafe storer(ptr);
td::store(event_, storer);
#ifdef TD_DEBUG
T check_result;
log_event_parse(check_result, Slice(ptr, storer.get_buf())).ensure();
#endif
//#ifdef TD_DEBUG
// T check_result;
// log_event_parse(check_result, Slice(ptr, storer.get_buf())).ensure();
//#endif
return static_cast<size_t>(storer.get_buf() - ptr);
}
@ -285,10 +285,10 @@ BufferSlice log_event_store(const T &data) {
LogEventStorerUnsafe storer_unsafe(ptr);
store(data, storer_unsafe);
#ifdef TD_DEBUG
T check_result;
log_event_parse(check_result, value_buffer.as_slice()).ensure();
#endif
//#ifdef TD_DEBUG
// T check_result;
// log_event_parse(check_result, value_buffer.as_slice()).ensure();
//#endif
return value_buffer;
}

85
tddb/td/db/SqliteDb.cpp
View File

@ -50,17 +50,26 @@ SqliteDb::~SqliteDb() = default;
Status SqliteDb::init(CSlice path, bool *was_created) {
// If database does not exist, delete all other files which may left
// from older database
bool is_db_exists = stat(path).is_ok();
if (!is_db_exists) {
TRY_STATUS(destroy(path));
if (path == ":memory:") {
if (was_created != nullptr) {
*was_created = false;
}
} else {
bool is_db_exists = stat(path).is_ok();
if (!is_db_exists) {
TRY_STATUS(destroy(path));
}
if (was_created != nullptr) {
*was_created = !is_db_exists;
}
}
if (was_created != nullptr) {
*was_created = !is_db_exists;
}
sqlite3 *db;
CHECK(sqlite3_threadsafe() != 0);
int rc = sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE /*| SQLITE_OPEN_SHAREDCACHE*/,
int rc = sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE,
nullptr);
if (rc != SQLITE_OK) {
auto res = Status::Error(PSLICE() << "Failed to open database: " << detail::RawSqliteDb::last_error(db, path));
@ -155,77 +164,19 @@ Status SqliteDb::commit_transaction() {
}
Status SqliteDb::check_encryption() {
auto status = exec("SELECT count(*) FROM sqlite_master");
if (status.is_ok()) {
enable_logging_ = true;
}
return status;
return Status::OK();
}
Result<SqliteDb> SqliteDb::open_with_key(CSlice path, const DbKey &db_key) {
SqliteDb db;
TRY_STATUS(db.init(path));
if (!db_key.is_empty()) {
if (db.check_encryption().is_ok()) {
return Status::Error(PSLICE() << "No key is needed for database \"" << path << '"');
}
auto key = db_key_to_sqlcipher_key(db_key);
TRY_STATUS(db.exec(PSLICE() << "PRAGMA key = " << key));
}
TRY_STATUS_PREFIX(db.check_encryption(), "Can't open database: ");
return std::move(db);
}
Status SqliteDb::change_key(CSlice path, const DbKey &new_db_key, const DbKey &old_db_key) {
// fast path
{
auto r_db = open_with_key(path, new_db_key);
if (r_db.is_ok()) {
return Status::OK();
}
}
TRY_RESULT(db, open_with_key(path, old_db_key));
TRY_RESULT(user_version, db.user_version());
auto new_key = db_key_to_sqlcipher_key(new_db_key);
if (old_db_key.is_empty() && !new_db_key.is_empty()) {
LOG(DEBUG) << "ENCRYPT";
PerfWarningTimer timer("Encrypt SQLite database", 0.1);
auto tmp_path = path.str() + ".ecnrypted";
TRY_STATUS(destroy(tmp_path));
// make shure that database is not empty
TRY_STATUS(db.exec("CREATE TABLE IF NOT EXISTS encryption_dummy_table(id INT PRIMARY KEY)"));
//NB: not really safe
TRY_STATUS(db.exec(PSLICE() << "ATTACH DATABASE '" << tmp_path << "' AS encrypted KEY " << new_key));
TRY_STATUS(db.exec("SELECT sqlcipher_export('encrypted')"));
TRY_STATUS(db.exec(PSLICE() << "PRAGMA encrypted.user_version = " << user_version));
TRY_STATUS(db.exec("DETACH DATABASE encrypted"));
db.close();
TRY_STATUS(rename(tmp_path, path));
} else if (!old_db_key.is_empty() && new_db_key.is_empty()) {
LOG(DEBUG) << "DECRYPT";
PerfWarningTimer timer("Decrypt SQLite database", 0.1);
auto tmp_path = path.str() + ".ecnrypted";
TRY_STATUS(destroy(tmp_path));
//NB: not really safe
TRY_STATUS(db.exec(PSLICE() << "ATTACH DATABASE '" << tmp_path << "' AS decrypted KEY ''"));
TRY_STATUS(db.exec("SELECT sqlcipher_export('decrypted')"));
TRY_STATUS(db.exec(PSLICE() << "PRAGMA decrypted.user_version = " << user_version));
TRY_STATUS(db.exec("DETACH DATABASE decrypted"));
db.close();
TRY_STATUS(rename(tmp_path, path));
} else {
LOG(DEBUG) << "REKEY";
PerfWarningTimer timer("Rekey SQLite database", 0.1);
TRY_STATUS(db.exec(PSLICE() << "PRAGMA rekey = " << new_key));
}
TRY_RESULT(new_db, open_with_key(path, new_db_key));
LOG_CHECK(new_db.user_version().ok() == user_version) << new_db.user_version().ok() << " " << user_version;
return Status::OK();
}
Status SqliteDb::destroy(Slice path) {
return detail::RawSqliteDb::destroy(path);
}

92
tddb/td/db/SqliteKeyValue.cpp
View File

@ -11,119 +11,33 @@
namespace td {
Result<bool> SqliteKeyValue::init(string path) {
path_ = std::move(path);
bool is_created = false;
SqliteDb db;
TRY_STATUS(db.init(path, &is_created));
TRY_STATUS(db.exec("PRAGMA encoding=\"UTF-8\""));
TRY_STATUS(db.exec("PRAGMA synchronous=NORMAL"));
TRY_STATUS(db.exec("PRAGMA journal_mode=WAL"));
TRY_STATUS(db.exec("PRAGMA temp_store=MEMORY"));
TRY_STATUS(init_with_connection(std::move(db), "KV"));
return is_created;
return true;
}
Status SqliteKeyValue::init_with_connection(SqliteDb connection, string table_name) {
auto init_guard = ScopeExit() + [&] {
close();
};
db_ = std::move(connection);
table_name_ = std::move(table_name);
TRY_STATUS(init(db_, table_name_));
TRY_RESULT_ASSIGN(set_stmt_,
db_.get_statement(PSLICE() << "REPLACE INTO " << table_name_ << " (k, v) VALUES (?1, ?2)"));
TRY_RESULT_ASSIGN(get_stmt_, db_.get_statement(PSLICE() << "SELECT v FROM " << table_name_ << " WHERE k = ?1"));
TRY_RESULT_ASSIGN(erase_stmt_, db_.get_statement(PSLICE() << "DELETE FROM " << table_name_ << " WHERE k = ?1"));
TRY_RESULT_ASSIGN(get_all_stmt_, db_.get_statement(PSLICE() << "SELECT k, v FROM " << table_name_));
TRY_RESULT_ASSIGN(erase_by_prefix_stmt_,
db_.get_statement(PSLICE() << "DELETE FROM " << table_name_ << " WHERE ?1 <= k AND k < ?2"));
TRY_RESULT_ASSIGN(erase_by_prefix_rare_stmt_,
db_.get_statement(PSLICE() << "DELETE FROM " << table_name_ << " WHERE ?1 <= k"));
TRY_RESULT_ASSIGN(get_by_prefix_stmt_,
db_.get_statement(PSLICE() << "SELECT k, v FROM " << table_name_ << " WHERE ?1 <= k AND k < ?2"));
TRY_RESULT_ASSIGN(get_by_prefix_rare_stmt_,
db_.get_statement(PSLICE() << "SELECT k, v FROM " << table_name_ << " WHERE ?1 <= k"));
init_guard.dismiss();
return Status::OK();
}
Status SqliteKeyValue::drop() {
if (empty()) {
return Status::OK();
}
auto result = drop(db_, table_name_);
close();
return result;
return Status::OK();
}
SqliteKeyValue::SeqNo SqliteKeyValue::set(Slice key, Slice value) {
set_stmt_.bind_blob(1, key).ensure();
set_stmt_.bind_blob(2, value).ensure();
auto status = set_stmt_.step();
if (status.is_error()) {
LOG(FATAL) << "Failed to set \"" << key << '"';
}
// set_stmt_.step().ensure();
set_stmt_.reset();
return 0;
}
string SqliteKeyValue::get(Slice key) {
SCOPE_EXIT {
get_stmt_.reset();
};
get_stmt_.bind_blob(1, key).ensure();
get_stmt_.step().ensure();
if (!get_stmt_.has_row()) {
return "";
}
auto data = get_stmt_.view_blob(0).str();
get_stmt_.step().ignore();
return data;
return "";
}
SqliteKeyValue::SeqNo SqliteKeyValue::erase(Slice key) {
erase_stmt_.bind_blob(1, key).ensure();
erase_stmt_.step().ensure();
erase_stmt_.reset();
return 0;
}
void SqliteKeyValue::erase_by_prefix(Slice prefix) {
auto next = next_prefix(prefix);
if (next.empty()) {
SCOPE_EXIT {
erase_by_prefix_rare_stmt_.reset();
};
erase_by_prefix_rare_stmt_.bind_blob(1, prefix).ensure();
erase_by_prefix_rare_stmt_.step().ensure();
} else {
SCOPE_EXIT {
erase_by_prefix_stmt_.reset();
};
erase_by_prefix_stmt_.bind_blob(1, prefix).ensure();
erase_by_prefix_stmt_.bind_blob(2, next).ensure();
erase_by_prefix_stmt_.step().ensure();
}
}
string SqliteKeyValue::next_prefix(Slice prefix) {
string next = prefix.str();
size_t pos = next.size();
while (pos) {
pos--;
auto value = static_cast<uint8>(next[pos]);
value++;
next[pos] = static_cast<char>(value);
if (value != 0) {
return next;
}
}
return string{};
}

17
tddb/td/db/SqliteKeyValue.h
View File

@ -20,17 +20,17 @@ namespace td {
class SqliteKeyValue {
public:
static Status drop(SqliteDb &connection, Slice table_name) TD_WARN_UNUSED_RESULT {
return connection.exec(PSLICE() << "DROP TABLE IF EXISTS " << table_name);
return Status::OK();
}
static Status init(SqliteDb &connection, Slice table_name) TD_WARN_UNUSED_RESULT {
return connection.exec(PSLICE() << "CREATE TABLE IF NOT EXISTS " << table_name << " (k BLOB PRIMARY KEY, v BLOB)");
return Status::OK();
}
using SeqNo = uint64;
bool empty() const {
return db_.empty();
return false;
}
Result<bool> init(string path) TD_WARN_UNUSED_RESULT;
@ -54,10 +54,10 @@ class SqliteKeyValue {
SeqNo erase(Slice key);
Status begin_transaction() TD_WARN_UNUSED_RESULT {
return db_.begin_transaction();
return Status::OK();
}
Status commit_transaction() TD_WARN_UNUSED_RESULT {
return db_.commit_transaction();
return Status::OK();
}
void erase_by_prefix(Slice prefix);
@ -107,14 +107,7 @@ class SqliteKeyValue {
private:
string path_;
string table_name_;
SqliteDb db_;
SqliteStatement get_stmt_;
SqliteStatement set_stmt_;
SqliteStatement erase_stmt_;
SqliteStatement get_all_stmt_;
SqliteStatement erase_by_prefix_stmt_;
SqliteStatement erase_by_prefix_rare_stmt_;
SqliteStatement get_by_prefix_stmt_;
SqliteStatement get_by_prefix_rare_stmt_;

33
tdutils/td/utils/Enumerator.h
View File

@ -19,25 +19,46 @@ class Enumerator {
public:
using Key = int32;
std::map<ValueT, int32> get_map() const {
return map_;
}
std::pair<Key, bool> next() {
if (!empty_id_.empty()) {
auto res = empty_id_.back();
empty_id_.pop_back();
return std::make_pair(res, true);
}
return std::make_pair((Key) (arr_.size() + 1), false);
}
void erase(Key key_y) {
empty_id_.push_back(key_y);
}
Key add(ValueT v) {
CHECK(arr_.size() < static_cast<size_t>(std::numeric_limits<int32>::max() - 1));
int32 next_id = static_cast<int32>(arr_.size() + 1);
auto next_id = next();
bool was_inserted;
decltype(map_.begin()) it;
std::tie(it, was_inserted) = map_.emplace(std::move(v), next_id);
if (was_inserted) {
std::tie(it, was_inserted) = map_.emplace(std::move(v), next_id.first);
if (was_inserted && next_id.second) {
arr_[next_id.first - 1] = &it->first;
} else if (was_inserted) {
arr_.push_back(&it->first);
} else if (next_id.second) {
empty_id_.push_back(next_id.first);
}
return it->second;
}
const ValueT &get(Key key) const {
auto pos = static_cast<size_t>(key - 1);
CHECK(pos < arr_.size());
auto pos = static_cast<Key>(key - 1);
return *arr_[pos];
}
private:
std::vector<Key> empty_id_;
std::map<ValueT, int32> map_;
std::vector<const ValueT *> arr_;
};

1
tdutils/td/utils/common.h
View File

@ -48,6 +48,7 @@
#include <string>
#include <vector>
#include <unordered_map>
#define TD_DEBUG

2
tdutils/td/utils/port/MemoryMapping.cpp
View File

@ -84,7 +84,7 @@ Result<MemoryMapping> MemoryMapping::create_from_file(const FileFd &file_fd, con
auto data_offset = begin - fixed_begin;
auto data_size = narrow_cast<size_t>(end - fixed_begin);
void *data = mmap(nullptr, data_size, PROT_READ, MAP_PRIVATE, fd, narrow_cast<off_t>(fixed_begin));
void *data = mmap(nullptr, data_size, PROT_READ, MADV_MERGEABLE, fd, narrow_cast<off_t>(fixed_begin));
if (data == MAP_FAILED) {
return OS_ERROR("mmap call failed");
}

2
tdutils/td/utils/port/signals.cpp
View File

@ -45,7 +45,7 @@ Status setup_signals_alt_stack() {
auto page_size = getpagesize();
auto stack_size = (MINSIGSTKSZ + 16 * page_size - 1) / page_size * page_size;
void *stack = mmap(nullptr, stack_size + 2 * page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
void *stack = mmap(nullptr, stack_size + 2 * page_size, PROT_READ | PROT_WRITE, MAP_ANON | MADV_MERGEABLE, -1, 0);
if (stack == MAP_FAILED) {
return OS_ERROR("Mmap failed");
}