From b6c5bd07bbed6b58ec9426904affee05a4ca78e0 Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Sun, 8 Oct 2023 16:59:20 +0200 Subject: [PATCH] Apply tdlight patch - Memory manager - TD_SKIP_TEST/TD_SKIP_BENCHMARK/TD_SKIP_TG_CLI - Rebranding - github workflow - .gitignore - Add crypt32 to windows - Fatal crash log handler - optionally unencrypted database - malloc_trim support - getMemoryStatistics Options: - disable_document_filenames - disable_minithumbnails - disable_notifications - disable_group_calls - disable_auto_download - ignore_server_deletes_and_reads - ignore_update_chat_last_message - ignore_update_chat_read_inbox - ignore_update_user_chat_action - receive_access_hashes - message_unload_delay modification --- .github/workflows/build.yaml | 50 +++ .gitignore | 2 + CHANGELOG.md | 6 +- CMakeLists.txt | 13 +- README.md | 54 ++- UpdateMemoryManager.javash | 381 +++++++++++++++++++ build.html | 40 +- example/README.md | 24 +- example/ios/README.md | 4 +- example/java/README.md | 2 +- example/uwp/extension.vsixmanifest | 2 +- example/web/README.md | 2 +- example/web/tdweb/README.md | 4 +- example/web/tdweb/src/index.js | 2 +- td/generate/scheme/.gitignore | 1 + td/generate/scheme/td_api.tl | 26 ++ td/generate/tl-parser/tl-parser.c | 112 +++--- td/generate/tl-parser/tlc.c | 2 +- td/telegram/AccountManager.cpp | 3 + td/telegram/AccountManager.h | 2 + td/telegram/AnimationsManager.cpp | 25 +- td/telegram/AnimationsManager.h | 1 + td/telegram/AnimationsManager.hpp | 21 +- td/telegram/AttachMenuManager.cpp | 12 + td/telegram/AttachMenuManager.h | 2 + td/telegram/AudiosManager.cpp | 4 + td/telegram/AudiosManager.h | 1 + td/telegram/AudiosManager.hpp | 14 +- td/telegram/AuthManager.cpp | 6 + td/telegram/AuthManager.h | 2 + td/telegram/AutoDownloadSettings.cpp | 27 +- td/telegram/AutosaveManager.cpp | 4 + td/telegram/AutosaveManager.h | 2 + td/telegram/BackgroundManager.cpp | 22 ++ td/telegram/BackgroundManager.h | 1 + td/telegram/BoostManager.cpp | 3 + td/telegram/BoostManager.h | 2 + td/telegram/BotInfoManager.cpp | 12 + td/telegram/BotInfoManager.h | 2 + td/telegram/BusinessConnectionManager.cpp | 10 + td/telegram/BusinessConnectionManager.h | 2 + td/telegram/BusinessManager.cpp | 3 + td/telegram/BusinessManager.h | 2 + td/telegram/CallbackQueriesManager.cpp | 3 + td/telegram/CallbackQueriesManager.h | 2 + td/telegram/ChannelRecommendationManager.cpp | 8 + td/telegram/ChannelRecommendationManager.h | 2 + td/telegram/ChatManager.cpp | 44 +++ td/telegram/ChatManager.h | 2 + td/telegram/CommonDialogManager.cpp | 4 + td/telegram/CommonDialogManager.h | 2 + td/telegram/ConnectionStateManager.cpp | 3 + td/telegram/ConnectionStateManager.h | 2 + td/telegram/CountryInfoManager.cpp | 4 + td/telegram/CountryInfoManager.h | 2 + td/telegram/DialogActionManager.cpp | 6 + td/telegram/DialogActionManager.h | 2 + td/telegram/DialogFilterManager.cpp | 10 + td/telegram/DialogFilterManager.h | 2 + td/telegram/DialogInviteLinkManager.cpp | 6 + td/telegram/DialogInviteLinkManager.h | 2 + td/telegram/DialogManager.cpp | 16 + td/telegram/DialogManager.h | 2 + td/telegram/DialogParticipantManager.cpp | 14 + td/telegram/DialogParticipantManager.h | 2 + td/telegram/Document.cpp | 37 +- td/telegram/DocumentsManager.cpp | 4 + td/telegram/DocumentsManager.h | 1 + td/telegram/DocumentsManager.hpp | 18 +- td/telegram/DownloadManager.cpp | 3 + td/telegram/DownloadManager.h | 2 + td/telegram/FileReferenceManager.cpp | 4 + td/telegram/FileReferenceManager.h | 1 + td/telegram/ForumTopicManager.cpp | 4 + td/telegram/ForumTopicManager.h | 2 + td/telegram/GameManager.cpp | 3 + td/telegram/GameManager.h | 2 + td/telegram/Global.cpp | 4 + td/telegram/Global.h | 57 +++ td/telegram/GroupCallManager.cpp | 32 +- td/telegram/GroupCallManager.h | 3 + td/telegram/InlineMessageManager.cpp | 3 + td/telegram/InlineMessageManager.h | 2 + td/telegram/InlineQueriesManager.cpp | 11 + td/telegram/InlineQueriesManager.h | 1 + td/telegram/LinkManager.cpp | 8 + td/telegram/LinkManager.h | 2 + td/telegram/MemoryManager.cpp | 250 ++++++++++++ td/telegram/MemoryManager.h | 65 ++++ td/telegram/MessageContent.cpp | 2 + td/telegram/MessageImportManager.cpp | 8 + td/telegram/MessageImportManager.h | 2 + td/telegram/MessagesManager.cpp | 182 ++++++++- td/telegram/MessagesManager.h | 1 + td/telegram/NotificationManager.cpp | 32 ++ td/telegram/NotificationManager.h | 2 + td/telegram/NotificationSettingsManager.cpp | 14 + td/telegram/NotificationSettingsManager.h | 4 +- td/telegram/OnlineManager.cpp | 3 + td/telegram/OnlineManager.h | 2 + td/telegram/OptionManager.cpp | 43 +++ td/telegram/OptionManager.h | 2 + td/telegram/PeopleNearbyManager.cpp | 8 + td/telegram/PeopleNearbyManager.h | 2 + td/telegram/Photo.cpp | 10 +- td/telegram/Photo.hpp | 20 +- td/telegram/PhotoSize.cpp | 23 +- td/telegram/PollManager.cpp | 20 + td/telegram/PollManager.h | 1 + td/telegram/PrivacyManager.cpp | 3 + td/telegram/PrivacyManager.h | 2 + td/telegram/PromoDataManager.cpp | 3 + td/telegram/PromoDataManager.h | 2 + td/telegram/QuickReplyManager.cpp | 18 + td/telegram/QuickReplyManager.h | 2 + td/telegram/ReactionManager.cpp | 12 + td/telegram/ReactionManager.h | 2 + td/telegram/Requests.cpp | 16 +- td/telegram/Requests.h | 2 + td/telegram/SavedMessagesManager.cpp | 4 + td/telegram/SavedMessagesManager.h | 2 + td/telegram/SponsoredMessageManager.cpp | 4 + td/telegram/SponsoredMessageManager.h | 2 + td/telegram/StarManager.cpp | 3 + td/telegram/StarManager.h | 2 + td/telegram/StatisticsManager.cpp | 3 + td/telegram/StatisticsManager.h | 2 + td/telegram/StickersManager.cpp | 86 +++++ td/telegram/StickersManager.h | 1 + td/telegram/StoryManager.cpp | 68 ++++ td/telegram/StoryManager.h | 2 + td/telegram/Td.cpp | 25 +- td/telegram/Td.h | 9 + td/telegram/TdDb.cpp | 21 +- td/telegram/TdDb.h | 7 +- td/telegram/TermsOfServiceManager.cpp | 3 + td/telegram/TermsOfServiceManager.h | 2 + td/telegram/ThemeManager.cpp | 3 + td/telegram/ThemeManager.h | 2 + td/telegram/TimeZoneManager.cpp | 4 + td/telegram/TimeZoneManager.h | 2 + td/telegram/TopDialogManager.cpp | 4 + td/telegram/TopDialogManager.h | 2 + td/telegram/TranscriptionManager.cpp | 8 + td/telegram/TranscriptionManager.h | 2 + td/telegram/TranslationManager.cpp | 3 + td/telegram/TranslationManager.h | 2 + td/telegram/UpdatesManager.cpp | 4 + td/telegram/UpdatesManager.h | 2 + td/telegram/UserManager.cpp | 64 ++++ td/telegram/UserManager.h | 2 + td/telegram/VideoNotesManager.cpp | 12 +- td/telegram/VideoNotesManager.h | 1 + td/telegram/VideosManager.cpp | 12 +- td/telegram/VideosManager.h | 1 + td/telegram/VideosManager.hpp | 21 +- td/telegram/VoiceNotesManager.cpp | 4 + td/telegram/VoiceNotesManager.h | 2 + td/telegram/WebPagesManager.cpp | 20 + td/telegram/WebPagesManager.h | 1 + td/telegram/files/FileManager.cpp | 9 +- td/telegram/files/FileManager.h | 1 + tdutils/CMakeLists.txt | 2 +- tdutils/td/utils/logging.cpp | 12 + tdutils/td/utils/logging.h | 4 + 165 files changed, 2277 insertions(+), 189 deletions(-) create mode 100644 .github/workflows/build.yaml create mode 100755 UpdateMemoryManager.javash create mode 100644 td/generate/scheme/.gitignore create mode 100644 td/telegram/MemoryManager.cpp create mode 100644 td/telegram/MemoryManager.h diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 000000000..41bbce82b --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,50 @@ +name: Build TDLib + +on: + push: + pull_request: + schedule: + - cron: '0 0 * * 0' # weekly + +jobs: + build: + runs-on: ubuntu-20.04 + strategy: + matrix: + arch: [linux/386, linux/amd64, linux/arm/v6, linux/arm/v7, linux/arm64, linux/ppc64le] + + steps: + - name: Setup variables + run: | + ARCH=${{ matrix.arch }} + SAFE_ARCH=$(echo $ARCH | sed 's/\//\-/g') + echo "SAFE_ARCH=$SAFE_ARCH" >> $GITHUB_ENV + - name: Install sudo package + run: | + (apt-get update || true) 2>/dev/null + (apt-get install -y sudo || true) 2>/dev/null + sudo apt update + - uses: actions/checkout@v2 + with: + submodules: "recursive" + - name: Cache ccache + id: cache-ccache + uses: actions/cache@v2 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ env.SAFE_ARCH }}-ccache-all + restore-keys: | + ${{ runner.os }}-${{ env.SAFE_ARCH }}-ccache- + - name: Install build tools + run: sudo apt-get install -y make git zlib1g-dev libssl-dev gperf php-cli cmake clang-6.0 libc++-dev libc++abi-dev ccache + - name: Build + run: | + mkdir build + cd build + CXXFLAGS="-stdlib=libc++" CC=/usr/bin/clang-10 CXX=/usr/bin/clang++-10 cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX:PATH=../tdlib .. + cmake --build . --target install -- -j4 + - uses: actions/upload-artifact@v2 + with: + name: tdlight-${{ env.SAFE_ARCH }} + path: tdlib/lib + diff --git a/.gitignore b/.gitignore index 2691379ef..6c1a20a8f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,9 @@ **/auto/ docs/ /tdlib/ +.idea/ vcpkg/ +*.tlo td.binlog diff --git a/CHANGELOG.md b/CHANGELOG.md index f590b3ef6..7bc1e6bf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1024,7 +1024,7 @@ Changes in 1.5.0 (9 Sep 2019): * Added the class `chatEventPollStopped` representing the closing of a poll in a message in the chat event log. * Added ability to specify the exact types of problems with a call in the method `sendCallRating` and the new class `CallProblem`. -* Changes in [tdweb](https://github.com/tdlib/td/blob/master/example/web/): +* Changes in [tdweb](https://github.com/tdlight-team/tdlight/blob/master/example/web/): - Supported non-zero `offset` and `limit` in `readFilePart`. ----------------------------------------------------------------------------------------------------------------------- @@ -1034,7 +1034,7 @@ Changes in 1.4.0 (1 May 2019): * Added a [TDLib build instructions generator](https://tdlib.github.io/td/build.html), covering in details TDLib building on the most popular operating systems. * Added an example of TDLib building and usage from a browser. - See https://github.com/tdlib/td/blob/master/example/web/ for more details. + See https://github.com/tdlight-team/tdlight/blob/master/example/web/ for more details. * Allowed to pass NULL pointer to `td_json_client_execute` instead of a previously created JSON client. Now you can use synchronous TDLib methods through a JSON interface before creating a TDLib JSON client. * Added support for media streaming by allowing to download any part of a file: @@ -1285,7 +1285,7 @@ Changes in 1.4.0 (1 May 2019): Changes in 1.3.0 (5 Sep 2018): -* Added a review of existing TDLib based [frameworks](https://github.com/tdlib/td/blob/master/example/README.md) +* Added a review of existing TDLib based [frameworks](https://github.com/tdlight-team/tdlight/blob/master/example/README.md) in different programming languages. * Added a [Getting started](https://core.telegram.org/tdlib/getting-started) guide describing the main TDLib concepts and basic principles required for library usage. diff --git a/CMakeLists.txt b/CMakeLists.txt index f5c92dd58..5176c566f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -207,9 +207,13 @@ add_subdirectory(sqlite) add_subdirectory(tddb) -add_subdirectory(test) +option(TD_SKIP_TEST "Use \"ON\" to skip building/running the test harness.") +if (NOT TD_SKIP_TEST) + add_subdirectory(test) +endif() -if (NOT CMAKE_CROSSCOMPILING) +option(TD_SKIP_BENCHMARK "Use \"ON\" to skip building/running the benchmarks.") +if (NOT CMAKE_CROSSCOMPILING AND NOT TD_SKIP_BENCHMARK) add_subdirectory(benchmark) endif() @@ -569,6 +573,7 @@ set(TDLIB_SOURCE_PART2 td/telegram/StickersManager.cpp td/telegram/StickerType.cpp td/telegram/StorageManager.cpp + td/telegram/MemoryManager.cpp td/telegram/StoryContent.cpp td/telegram/StoryContentType.cpp td/telegram/StoryDb.cpp @@ -918,6 +923,7 @@ set(TDLIB_SOURCE_PART2 td/telegram/StickersManager.h td/telegram/StickerType.h td/telegram/StorageManager.h + td/telegram/MemoryManager.h td/telegram/StoryContent.h td/telegram/StoryContentType.h td/telegram/StoryDb.h @@ -1241,6 +1247,8 @@ if (EMSCRIPTEN) endif() if (NOT CMAKE_CROSSCOMPILING) +option(TD_SKIP_TG_CLI "Use \"ON\" to skip building tg_cli.") +if (NOT CMAKE_CROSSCOMPILING AND NOT TD_SKIP_TG_CLI) add_executable(tg_cli td/telegram/cli.cpp ${TL_TD_JSON_SOURCE}) if (NOT READLINE_FOUND) @@ -1269,6 +1277,7 @@ if (NOT CMAKE_CROSSCOMPILING) target_link_libraries(tg_cli PRIVATE memprof tdclient tdcore) add_dependencies(tg_cli tl_generate_json) endif() +endif() # Exported libraries add_library(TdStatic INTERFACE) diff --git a/README.md b/README.md index ae6e1922b..05305b59d 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,12 @@ -# TDLib +# TDLight -TDLib (Telegram Database library) is a cross-platform library for building [Telegram](https://telegram.org) clients. It can be easily used from almost any programming language. +TDLight is a fork of TDLib, a cross-platform library for building [Telegram](https://telegram.org) clients. It can be easily used from almost any programming language. ## Table of Contents - [Features](#features) +- [TDLight extra features](#tdlight-extra-features) +- [TDLight extra API functions](#tdlight-extra-api-functions) +- [TDLight recommended options](#tdlight-recommended-options) - [Examples and documentation](#usage) - [Dependencies](#dependencies) - [Building](#building) @@ -28,17 +31,42 @@ TDLib (Telegram Database library) is a cross-platform library for building [Tele * **Secure**: all local data is encrypted using a user-provided encryption key. * **Fully-asynchronous**: requests to `TDLib` don't block each other or anything else, responses are sent when they are available. + +### TDLight extra features +#### TDLight extra options +* **disable_minithumbnails** (true/**false**) This setting removes minithumbnails everywhere. It reduces memory usage because tdlib keeps them in RAM +* **disable_document_filenames** (true/**false**) If you don't care about having the original filenames of every file stored in RAM, you can disable them using this option. It reduces memory usage +* **disable_notifications** (true/**false**) In TDLib pending notification updates are stored in ram until you "read" them. This option disables completely notifications and keeps the pending notifications queue empty, reducing memory usage +* **ignore_update_chat_last_message** (true/**false**) If you don't care about have updateChatLastMessage updates enable this +* **ignore_update_chat_read_inbox** (true/**false**) If you don't care about have updateChatReadInbox updates enable this +* **ignore_update_user_chat_action** (true/**false**) If you don't care about have updateUserChatAction updates enable this +* **ignore_server_deletes_and_reads** (true/**false**) If you don't care about receiving read receipts and remote deletes from other users, enable this, it will reduce memory usage +* **receive_access_hashes** (true/**false**) Receive chats and users access hash as updates +* **disable_auto_download** (true/**false**) Forcefully ignore auto download settings of all sessions + +### TDLight extra API functions +#### TdApi.GetMemoryStatistics +This method is used to read the size of all the internal TDLib data structures. +The output contains a string that can be parsed as a JSON. + +## TDLight recommended options +* Options: + * ignore_inline_thumbnails: true + * disable_top_chats: true + * ignore_platform_restrictions: true + * ignore_sensitive_content_restrictions: true +* Disable all the databases (messages_db, users_db, files_db) ## Examples and documentation See our [Getting Started](https://core.telegram.org/tdlib/getting-started) tutorial for a description of basic TDLib concepts. -Take a look at our [examples](https://github.com/tdlib/td/blob/master/example/README.md#tdlib-usage-and-build-examples). +Take a look at our [examples](https://github.com/tdlight-team/tdlight/blob/master/example/README.md#tdlib-usage-and-build-examples). -See a [TDLib build instructions generator](https://tdlib.github.io/td/build.html) for detailed instructions on how to build TDLib. +See a [TDLight build instructions generator](https://tdlight-team.github.io/tdlight/build.html) for detailed instructions on how to build TDLib. See description of our [JSON](#using-json), [C++](#using-cxx), [Java](#using-java) and [.NET](#using-dotnet) interfaces. -See the [td_api.tl](https://github.com/tdlib/td/blob/master/td/generate/scheme/td_api.tl) scheme or the automatically generated [HTML documentation](https://core.telegram.org/tdlib/docs/td__api_8h.html) +See the [td_api.tl](https://github.com/tdlight-team/tdlight/blob/master/td/generate/scheme/td_api.tl) scheme or the automatically generated [HTML documentation](https://core.telegram.org/tdlib/docs/td__api_8h.html) for a list of all available `TDLib` [methods](https://core.telegram.org/tdlib/docs/classtd_1_1td__api_1_1_function.html) and [classes](https://core.telegram.org/tdlib/docs/classtd_1_1td__api_1_1_object.html). @@ -55,7 +83,7 @@ for a list of all available `TDLib` [methods](https://core.telegram.org/tdlib/do ## Building -The simplest way to build `TDLib` is to use our [TDLib build instructions generator](https://tdlib.github.io/td/build.html). +The simplest way to build `TDLight` is to use our [TDLight build instructions generator](https://tdlight-team.github.io/tdlight/build.html). You need only to choose your programming language and target operating system to receive complete build instructions. In general, you need to install all `TDLib` [dependencies](#dependencies), enter directory containing `TDLib` sources and compile them using CMake: @@ -67,7 +95,7 @@ cmake -DCMAKE_BUILD_TYPE=Release .. cmake --build . ``` -To build `TDLib` on low memory devices you can run [SplitSource.php](https://github.com/tdlib/td/blob/master/SplitSource.php) script +To build `TDLib` on low memory devices you can run [SplitSource.php](https://github.com/tdlight-team/tdlight/blob/master/SplitSource.php) script before compiling main `TDLib` source code and compile only needed targets: ``` mkdir build @@ -106,21 +134,21 @@ Or you could install `TDLib` and then reference it in your CMakeLists.txt like t find_package(Td 1.8.35 REQUIRED) target_link_libraries(YourTarget PRIVATE Td::TdStatic) ``` -See [example/cpp/CMakeLists.txt](https://github.com/tdlib/td/blob/master/example/cpp/CMakeLists.txt). +See [example/cpp/CMakeLists.txt](https://github.com/tdlight-team/tdlight/blob/master/example/cpp/CMakeLists.txt). ## Using in Java projects `TDLib` provides native Java interface through JNI. To enable it, specify option `-DTD_ENABLE_JNI=ON` to CMake. -See [example/java](https://github.com/tdlib/td/tree/master/example/java) for example of using `TDLib` from Java and detailed build and usage instructions. +See [example/java](https://github.com/tdlight-team/tdlight/tree/master/example/java) for example of using `TDLib` from Java and detailed build and usage instructions. ## Using in .NET projects `TDLib` provides native .NET interface through `C++/CLI` and `C++/CX`. To enable it, specify option `-DTD_ENABLE_DOTNET=ON` to CMake. .NET Core supports `C++/CLI` only since version 3.1 and only on Windows, so if older .NET Core is used or portability is needed, then `TDLib` JSON interface should be used through P/Invoke instead. -See [example/csharp](https://github.com/tdlib/td/tree/master/example/csharp) for example of using `TDLib` from C# and detailed build and usage instructions. -See [example/uwp](https://github.com/tdlib/td/tree/master/example/uwp) for example of using `TDLib` from C# UWP application and detailed build and usage instructions for Visual Studio Extension "TDLib for Universal Windows Platform". +See [example/csharp](https://github.com/tdlight-team/tdlight/tree/master/example/csharp) for example of using `TDLib` from C# and detailed build and usage instructions. +See [example/uwp](https://github.com/tdlight-team/tdlight/tree/master/example/uwp) for example of using `TDLib` from C# UWP application and detailed build and usage instructions for Visual Studio Extension "TDLib for Universal Windows Platform". When `TDLib` is built with `TD_ENABLE_DOTNET` option enabled, `C++` documentation is removed from some files. You need to checkout these files to return `C++` documentation back: ``` @@ -132,13 +160,13 @@ git checkout td/telegram/Client.h td/telegram/Log.h td/tl/TlObject.h `TDLib` provides efficient native C++, Java, and .NET interfaces. But for most use cases we suggest to use the JSON interface, which can be easily used with any programming language that is able to execute C functions. See [td_json_client](https://core.telegram.org/tdlib/docs/td__json__client_8h.html) documentation for detailed JSON interface description, -the [td_api.tl](https://github.com/tdlib/td/blob/master/td/generate/scheme/td_api.tl) scheme or the automatically generated [HTML documentation](https://core.telegram.org/tdlib/docs/td__api_8h.html) for a list of +the [td_api.tl](https://github.com/tdlight-team/tdlight/blob/master/td/generate/scheme/td_api.tl) scheme or the automatically generated [HTML documentation](https://core.telegram.org/tdlib/docs/td__api_8h.html) for a list of all available `TDLib` [methods](https://core.telegram.org/tdlib/docs/classtd_1_1td__api_1_1_function.html) and [classes](https://core.telegram.org/tdlib/docs/classtd_1_1td__api_1_1_object.html). `TDLib` JSON interface adheres to semantic versioning and versions with the same major version number are binary and backward compatible, but the underlying `TDLib` API can be different for different minor and even patch versions. If you need to support different `TDLib` versions, then you can use a value of the `version` option to find exact `TDLib` version to use appropriate API methods. -See [example/python/tdjson_example.py](https://github.com/tdlib/td/blob/master/example/python/tdjson_example.py) for an example of such usage. +See [example/python/tdjson_example.py](https://github.com/tdlight-team/tdlight/tree/master/example/python/tdjson_example.py) for an example of such usage. ## License diff --git a/UpdateMemoryManager.javash b/UpdateMemoryManager.javash new file mode 100755 index 000000000..5926fdbaa --- /dev/null +++ b/UpdateMemoryManager.javash @@ -0,0 +1,381 @@ +#!/usr/bin/java --source 21 +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; +import java.util.SequencedSet; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class UpdateMemoryManager { + + static final Set EXCLUDED_MANAGERS = Set.of("Memory", + "Call", + "DeviceToken", + "LanguagePack", + "Pts", + "Password", + "SecretChats", + "Secure", + "Config", + "Storage", + "FileLoad", + "Parts", + "FileGenerate", + "Resource", + "NetStats", + "DcAuth", + "State", + "PhoneNumber", + "FileDownload", + "FileUpload", + "Alarm" + ); + + record Manager(Path directory, String name) { + String includePath() { + return directory.toString().substring(2) + "/" + name + "Manager.h"; + } + } + + public static void main(String[] args) throws Exception { + if (args.length == 0) { + System.err.println("Arguments: PATH"); + System.exit(1); + } + + var path = Path.of(args[0]); + var telegramPath = path.resolve("td/telegram"); + + var memoryManager = new Manager(telegramPath, "Memory"); + SequencedSet managers; + try (var stream = Files.walk(telegramPath, 16)) { + var endNamePattern = "Manager.h"; + managers = stream + .filter(Files::isRegularFile) + .filter(p -> p.getFileName().toString().endsWith(endNamePattern)) + .map(p -> new Manager(p.getParent(), p.getFileName().toString().substring(0, p.getFileName().toString().length() - endNamePattern.length()))) + .filter(name -> !EXCLUDED_MANAGERS.contains(name.name)) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + System.out.printf("Found %d managers%n", managers.size()); + + int fieldsFound = 0; + int updatedManagers = 0; + int totalManagers = 0; + List invalidManagers = new ArrayList<>(); + for (Manager manager : managers) { + totalManagers++; + var result = updateManagerJson(manager); + if (result != null) { + fieldsFound += result.fieldsFound(); + if (result.changed) { + updatedManagers++; + } + } else { + invalidManagers.add(manager); + } + } + + updateMemoryManagerJson(memoryManager, managers); + + System.out.printf("%n%nDone.%n"); + + if (!invalidManagers.isEmpty()) { + System.out.printf("%d invalid managers found:%n%s", + invalidManagers.size(), + invalidManagers.stream() + .map(x -> "\t\"" + x.directory + "\": " + x.name + "\n") + .collect(Collectors.joining(", "))); + } + System.out.printf("%d/%d managers updated, %d total fields%n", updatedManagers, totalManagers, fieldsFound); + } + + enum FieldType { + WaitFreeHashMap("WaitFreeHashMap", "calc_size"), + WaitFreeHashSet("WaitFreeHashSet", "calc_size"), + Vector("vector", "size"), + FlatHashMap("FlatHashMap", "size"), + FlatHashSet("FlatHashSet", "size") + ; + + private final String fieldName; + private final String sizeMethodName; + public final Pattern pattern; + + FieldType(String fieldName, String sizeMethodName) { + this.fieldName = fieldName; + this.sizeMethodName = sizeMethodName; + this.pattern = Pattern.compile("^ {2}(mutable )?" + fieldName + "(<([^ ]|, )+>)? +(?[a-zA-Z_]+);?[ \t/]*$"); + } + + Pattern getPattern() { + return pattern; + } + } + + record FoundField(FieldType type, String name) {} + record UpdateResult(boolean changed, int fieldsFound) {} + + private static UpdateResult updateManagerJson(Manager manager) throws IOException { + Path hFile = manager.directory.resolve(manager.name + "Manager.h"); + Path cppFile = manager.directory.resolve(manager.name + "Manager.cpp"); + + System.out.printf("Updating manager \"%s\" files: [\"%s\", \"%s\"]%n", manager.name, hFile, cppFile); + + if (Files.notExists(hFile)) { + System.out.printf("File not found, ignoring manager \"%s\": \"%s\"%n", manager.name, hFile); + return null; + } + if (Files.notExists(cppFile)) { + System.out.printf("File not found, ignoring manager \"%s\": \"%s\"%n", manager.name, cppFile); + return null; + } + + List fields = new ArrayList<>(); + + var hLines = normalizeSourceFile(readSourceFile(hFile)); + + boolean currentClass = false; + for (String hLine : hLines) { + FoundField field = null; + if (hLine.startsWith("class ")) { + currentClass = hLine.contains(" " + manager.name + "Manager"); + } + + if (currentClass) { + for (FieldType possibleFieldType : FieldType.values()) { + var m = possibleFieldType.getPattern().matcher(hLine); + if (m.matches()) { + var fieldName = m.group("field"); + field = new FoundField(possibleFieldType, fieldName); + break; + } + } + } + + if (field != null) { + System.out.println("\tFound field: (%s) %s".formatted(field.type, field.name)); + fields.add(field); + } + + } + + StringBuilder memoryStatsMethod = new StringBuilder(); + + memoryStatsMethod.append("void %sManager::memory_stats(vector &output) {\n".formatted(manager.name)); + memoryStatsMethod.append(fields.stream() + .map(field -> " output.push_back(\"\\\"%s\\\":\"); output.push_back(std::to_string(this->%s.%s()));\n".formatted(field.name, field.name, field.type.sizeMethodName)) + .collect(Collectors.joining(" output.push_back(\",\");\n"))); + memoryStatsMethod.append("}\n"); + + List memoryStatsMethodLines = Arrays.asList(memoryStatsMethod.toString().split("\n")); + + var cppLines = readSourceFile(cppFile); + var inputCppLines = new ArrayList<>(cppLines); + + // Remove the old memory_stats method + var indexOfMemoryStatsStart = -1; + var indexOfMemoryStatsEnd = -1; + for (int i = 0; i < cppLines.size(); i++) { + if (cppLines.get(i).contains("::memory_stats(")) { + indexOfMemoryStatsStart = i; + for (int j = i - 1; j >= 0; j--) { + if (cppLines.get(j).isBlank()) { + indexOfMemoryStatsStart = j; + } else { + break; + } + } + break; + } + } + if (indexOfMemoryStatsStart != -1) { + for (int i = indexOfMemoryStatsStart + 1; i < cppLines.size(); i++) { + if (cppLines.get(i).trim().equals("}")) { + indexOfMemoryStatsEnd = i; + break; + } + } + if (indexOfMemoryStatsEnd == -1) { + throw new IllegalStateException("memory_stats method end not found"); + } + cppLines.subList(indexOfMemoryStatsStart, indexOfMemoryStatsEnd + 1).clear(); + } + + var last = cppLines.removeLast(); + cppLines.addAll(memoryStatsMethodLines); + cppLines.add(""); + cppLines.addLast(last); + + boolean changed = !Objects.equals(inputCppLines, cppLines); + + if (changed) { + System.out.printf("\tDone: %s.cpp file has been updated!%n", manager.name); + Files.write(cppFile, cppLines, StandardCharsets.UTF_8); + } else { + System.out.printf("\tDone: %s.cpp file did not change.%n", manager.name); + } + + return new UpdateResult(changed, fields.size()); + } + + private static void updateMemoryManagerJson(Manager manager, SequencedSet managers) throws IOException { + Path hFile = manager.directory.resolve(manager.name + "Manager.h"); + Path cppFile = manager.directory.resolve(manager.name + "Manager.cpp"); + + System.out.printf("Updating memory manager \"%s\" files: [\"%s\", \"%s\"]%n", manager.name, hFile, cppFile); + + if (Files.notExists(hFile)) { + System.out.printf("File not found for manager \"%s\": \"%s\"%n", manager.name, hFile); + System.exit(1); + return; + } + if (Files.notExists(cppFile)) { + System.out.printf("File not found for manager \"%s\": \"%s\"%n", manager.name, cppFile); + System.exit(1); + return; + } + + StringBuilder memoryStatsMethod = new StringBuilder(); + + memoryStatsMethod.append("void %sManager::print_managers_memory_stats(vector &output) const {\n".formatted(manager.name)); + memoryStatsMethod.append(managers.stream() + .map(m -> """ + output.push_back("\\"%s_manager_\\":{"); td_->%s_manager_->memory_stats(output); output.push_back("}"); + """.formatted(toSnakeCase(m.name), toSnakeCase(m.name))) + .collect(Collectors.joining(" output.push_back(\",\");\n"))); + memoryStatsMethod.append("}\n"); + + List memoryStatsMethodLines = Arrays.asList(memoryStatsMethod.toString().split("\n")); + + var cppLines = readSourceFile(cppFile); + var inputCppLines = new ArrayList<>(cppLines); + + // Remove the old memory_stats method + var indexOfMemoryStatsStart = -1; + var indexOfMemoryStatsEnd = -1; + for (int i = 0; i < cppLines.size(); i++) { + if (cppLines.get(i).contains("::print_managers_memory_stats(")) { + indexOfMemoryStatsStart = i; + for (int j = i - 1; j >= 0; j--) { + if (cppLines.get(j).isBlank()) { + indexOfMemoryStatsStart = j; + } else { + break; + } + } + break; + } + } + if (indexOfMemoryStatsStart != -1) { + for (int i = indexOfMemoryStatsStart + 1; i < cppLines.size(); i++) { + if (cppLines.get(i).trim().equals("}")) { + indexOfMemoryStatsEnd = i; + break; + } + } + if (indexOfMemoryStatsEnd == -1) { + throw new IllegalStateException("print_managers_memory_stats method end not found"); + } + cppLines.subList(indexOfMemoryStatsStart, indexOfMemoryStatsEnd + 1).clear(); + } + + var last = cppLines.removeLast(); + cppLines.addAll(memoryStatsMethodLines); + cppLines.add(""); + cppLines.addLast(last); + + for (Manager m : managers) { + var mInclude = "#include \"%s\"".formatted(m.includePath()); + if (!cppLines.contains(mInclude)) { + System.out.printf("\tMissing include, adding \"" + m.includePath() + "\"%n"); + int includeInsertIndex = -1; + for (int i = 0; i < cppLines.size(); i++) { + if (cppLines.get(i).startsWith("#include")) { + includeInsertIndex = i; + } + } + if (includeInsertIndex == -1) { + throw new IllegalStateException("Cannot find a place to put the include"); + } + cppLines.add(includeInsertIndex + 1, mInclude); + } + } + + boolean changed = !Objects.equals(inputCppLines, cppLines); + + if (changed) { + System.out.printf("\tDone: %s.cpp file has been updated!%n", manager.name); + Files.write(cppFile, cppLines, StandardCharsets.UTF_8); + } else { + System.out.printf("\tDone: %s.cpp file did not change.%n", manager.name); + } + } + + private static String toSnakeCase(String name) { + var initialChar = name.codePoints() + .limit(1) + .map(Character::toLowerCase); + var restOfString = name.codePoints() + .skip(1) + .flatMap(codePoint -> { + if (Character.isUpperCase(codePoint)) { + return IntStream.of('_', Character.toLowerCase(codePoint)); + } else { + return IntStream.of(codePoint); + } + }); + var resultStream = IntStream.concat(initialChar, restOfString); + return resultStream.collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append).toString(); + } + + private static List readSourceFile(Path path) throws IOException { + return Files.readAllLines(path, StandardCharsets.UTF_8); + } + + private static List normalizeSourceFile(List lines) throws IOException { + List srcLines = new ArrayList<>(lines); + + // Remove empty lines + srcLines.removeIf(String::isBlank); + + // Remove unexpected newlines + List srcLinesWithoutNewlines = new ArrayList<>(); + StringBuilder buf = new StringBuilder(); + for (String srcLine : srcLines) { + if (!buf.isEmpty()) { + buf.append(" "); + } + buf.append(srcLine); + var trimmedLine = srcLine.trim(); + if (!trimmedLine.endsWith(">")) { + srcLinesWithoutNewlines.add(buf.toString()); + buf.setLength(0); + } + } + if (!buf.isEmpty()) { + srcLinesWithoutNewlines.add(buf.toString()); + } + + srcLinesWithoutNewlines.replaceAll(p -> { + var commentStart = p.indexOf("//"); + if (commentStart >= 0) { + return p.substring(0, commentStart); + } else { + return p; + } + }); + + return srcLinesWithoutNewlines; + } +} diff --git a/build.html b/build.html index c854181a5..3b62df2e5 100644 --- a/build.html +++ b/build.html @@ -2,7 +2,7 @@ - TDLib build instructions + TDLight build instructions