From 505316d9f9a556577672978c55142c4f5108cbb6 Mon Sep 17 00:00:00 2001
From: levlam <levlam@telegram.org>
Date: Tue, 2 Oct 2018 15:28:42 +0300
Subject: [PATCH] Add SplitSource.php and instruciton for building TDLib on low
 memory devices.

GitOrigin-RevId: 566c90ad274f95a39e2d141cd2541bd8816b9515
---
 README.md       |  17 ++++
 SplitSource.php | 219 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 236 insertions(+)
 create mode 100644 SplitSource.php

diff --git a/README.md b/README.md
index 657418298..ee123d292 100644
--- a/README.md
+++ b/README.md
@@ -62,6 +62,23 @@ cmake -DCMAKE_BUILD_TYPE=Release ..
 cmake --build .
 ```
 
+To build `TDLib` on low memory devices you can use [SplitSource.php](https://github.com/tdlib/td/blob/master/SplitSource.php)
+before compiling main `TDLib` source code and compile only needed targets:
+```
+mkdir build
+cd build
+cmake -DCMAKE_BUILD_TYPE=Release ..
+cmake --build . --target prepare_cross_compiling
+cd ..
+php SplitSource.php
+cd build
+cmake --build . --target tdjson
+cmake --build . --target tdjson_static
+cd ..
+php SplitSource.php --undo
+```
+In our tests clang 6.0 with libc++ required less than 500MB RAM per file and GCC 6.3 used less than 1GB RAM per file.
+
 <a name="installing-dependencies"></a>
 ### Installing dependencies
 
diff --git a/SplitSource.php b/SplitSource.php
new file mode 100644
index 000000000..826826449
--- /dev/null
+++ b/SplitSource.php
@@ -0,0 +1,219 @@
+<?php
+
+function disjoint_set_find(&$parents, $x) {
+    if ($parents[$x] !== $x) {
+        return $parents[$x] = disjoint_set_find($parents, $parents[$x]);
+    }
+    return $x;
+}
+
+function disjoint_set_union(&$parents, $x, $y) {
+    $x = disjoint_set_find($parents, $x);
+    $y = disjoint_set_find($parents, $y);
+
+    if ($x !== $y) {
+        if (rand(0, 1) == 0) {
+            $parents[$x] = $y;
+        } else {
+            $parents[$y] = $x;
+        } 
+    }
+}
+
+function split_file($file, $chunks, $undo) {
+    $cpp_name = "$file.cpp";
+
+    echo "Processing file $cpp_name".PHP_EOL;
+
+    $new_files = array();
+    foreach (range(0, $chunks - 1) as $n) {
+        $new_files[] = "$file$n.cpp";
+    }
+
+    $is_generated = (strpos($file, 'td/generate/') === 0);
+
+    $cmake_file = $is_generated ? 'td/generate/CMakeLists.txt' : 'CMakeLists.txt';
+    $cmake = file_get_contents($cmake_file);
+
+    $cmake_cpp_name = $cpp_name;
+    $cmake_new_files = $new_files;
+    if ($is_generated) {
+        foreach ($cmake_new_files as &$file_ref) {
+            $file_ref = str_replace('td/generate', '${CMAKE_CURRENT_SOURCE_DIR}', $file_ref);
+        }
+        $cmake_cpp_name = str_replace('td/generate', '${CMAKE_CURRENT_SOURCE_DIR}', $cmake_cpp_name);
+    }
+
+    if ($undo) {
+        foreach ($new_files as $file) {
+            if (file_exists($file)) {
+                echo "Unlinking ".$file.PHP_EOL;
+                unlink($file);
+            }
+        }
+
+        if (strpos($cmake, $cmake_cpp_name) === false) {
+            $cmake = str_replace(implode(PHP_EOL.'  ', $cmake_new_files), $cmake_cpp_name, $cmake);
+            file_put_contents($cmake_file, $cmake);
+        }
+
+        return;
+    }
+
+    if (strpos($cmake, $cmake_cpp_name) !== false) {
+        $cmake = str_replace($cmake_cpp_name, implode(PHP_EOL.'  ', $cmake_new_files), $cmake);
+        file_put_contents($cmake_file, $cmake);
+    }
+
+    $lines = file($cpp_name);
+    $depth = 0;
+    $target_depth = 1 + $is_generated;
+    $is_static = false;
+    $in_define = false;
+    $current = '';
+    $common = '';
+    $functions = array();
+    $namespace_begin = '';
+    $namespace_end = '';
+    foreach ($lines as $line) {
+        $add_depth = strpos($line, 'namespace ') === 0 ? 1 : (strpos($line, '}  // namespace') === 0 ? -1 : 0);
+        if ($add_depth) {
+            # namespace begin/end
+            $depth += $add_depth;
+            if ($depth <= $target_depth) {
+                if ($add_depth > 0) {
+                    $namespace_begin .= $line;
+                } else {
+                    $namespace_end .= $line;
+                }
+            }
+            if ($is_static) {
+                $common .= $current;
+            } else {
+                $functions[] = $current;
+            }
+            $common .= $line;
+            $current = '';
+            $is_static = false;
+            $in_define = false;
+            continue;
+        }
+
+        if (strpos($line, '#undef') === 0 && !trim($current)) {
+            continue;
+        }
+
+        if ($depth !== $target_depth) {
+            $common .= $line;
+            continue;
+        }
+         
+        if (strpos($line, 'static ') === 0 && $depth === $target_depth) {
+            $is_static = true;
+        }
+        if (!trim($current) && strpos($line, '#define ') === 0) {
+            $is_static = true;
+            $in_define = true;
+        }
+
+        $current .= $line;
+        if ((strpos($line, '}') === 0 || ($in_define && !trim($line)) || preg_match('/^[a-z].*;\s*$/i', $line)) && $depth === $target_depth) {
+            # block end
+            if ($is_static) {
+                $common .= $current;
+            } else {
+                $functions[] = $current;
+            }
+            $current = '';
+            $is_static = false;
+            $in_define = false;
+        }
+    }
+    if (!empty(trim($current))) {
+        fwrite(STDERR, "ERROR: $current".PHP_EOL);
+        exit();
+    }
+
+    if (count($functions) < $chunks) {
+        fwrite(STDERR, "ERROR: file is too small to be splitted more".PHP_EOL);
+        return;
+    }
+
+    $deps = array();  // all functions from the same subarray must be in the same file
+    $parents = array();
+    foreach ($functions as $i => $f) {
+        if (preg_match_all('/(?J)(create_handler|create_net_actor)<(?<name>[A-Z][A-Za-z]*)>|'.
+                           '(?<name>[A-Z][A-Za-z]*) : public (Td::ResultHandler|NetActor|Request)|'.
+                           '(CREATE_REQUEST|CREATE_NO_ARGS_REQUEST)[(](?<name>[A-Z][A-Za-z]*)|'.
+                           '(?<name>complete_pending_preauthentication_requests)|'.
+                           '(Up|Down)load[a-zA-Z]*C(?<name>allback)|(up|down)load_[a-z_]*_c(?<name>allback)_|'.
+                           '(?<name>V)(LOG.td_init|ERBOSITY_NAME)|'.
+                           '(?<name>LogEvent)[^sA]|'.
+                           '(?<name>parse)[(]|'.
+                           '(?<name>store)[(]/', $f, $matches, PREG_SET_ORDER)) {
+            foreach ($matches as $match) {
+                $name = $match['name'];
+                if ($name === 'parse' || $name === 'store') {
+                    if ($is_generated) {
+                        continue;
+                    }
+                    $name = 'LogEvent';
+                }
+                $deps[$name][] = $i;
+            }
+        }
+        $parents[$i] = $i;
+    }
+
+    foreach ($deps as $func_ids) {
+        foreach ($func_ids as $func_id) {
+            disjoint_set_union($parents, $func_ids[0], $func_id);
+        }
+    }
+    $sets = array();
+    $set_sizes = array();
+    foreach ($functions as $i => $f) {
+        $parent = disjoint_set_find($parents, $i);
+        if (!isset($sets[$parent])) {
+            $sets[$parent] = '';
+            $set_sizes[$parent] = 0;
+        }
+        $sets[$parent] .= $f;
+        $set_sizes[$parent] += strlen($f);
+    }
+    arsort($set_sizes);
+
+    $files = array_fill(0, $chunks, '');
+    $file_sizes = array_fill(0, $chunks, 0);
+    foreach ($set_sizes as $parent => $size) {
+      $file_id = array_search(min($file_sizes), $file_sizes);
+      $files[$file_id] .= $sets[$parent];
+      $file_sizes[$file_id] += $size;
+    }
+
+    foreach ($files as $n => $f) {
+        $new_content = $common.$namespace_begin.$f.$namespace_end;
+        if (!file_exists($new_files[$n]) || file_get_contents($new_files[$n]) !== $new_content) {
+            echo "Writing file ".$new_files[$n].PHP_EOL;
+            file_put_contents($new_files[$n], $new_content);
+        }
+    }
+}
+
+if (in_array('--help', $argv) || in_array('-h', $argv)) {
+    echo "Usage: php SplitSource.php [OPTION]...\nSplits some source files to reduce maximum RAM needed for compiling a single file.\n  -u, --undo Undo all source code changes.\n  -h, --help Show this help.\n";
+    exit(2);
+}
+
+$undo = in_array('--undo', $argv) || in_array('-u', $argv);
+$files = array('td/telegram/ContactsManager' => 10,
+               'td/telegram/MessagesManager' => 20,
+               'td/telegram/Td' => 20,
+               'td/telegram/StickersManager' => 10,
+               'td/generate/auto/td/telegram/td_api' => 10,
+               'td/generate/auto/td/telegram/td_api_json' => 10,
+               'td/generate/auto/td/telegram/telegram_api' => 10);
+
+foreach ($files as $file => $chunks) {
+    split_file($file, $chunks, $undo);
+}