#!/usr/bin/java --source 21 import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.FileVisitOption; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; import java.util.SequencedSet; import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.IntStream; public class UpdateMemoryManager { static final Set EXCLUDED_MANAGERS = Set.of("Memory", "Call", "DeviceToken", "LanguagePack", "Pts", "Password", "SecretChats", "Secure", "Config", "Storage", "FileLoad", "Parts", "FileGenerate", "Resource", "NetStats", "DcAuth", "State", "PhoneNumber" ); record Manager(Path directory, String name) { String includePath() { return directory.toString().substring(2) + "/" + name + "Manager.h"; } } public static void main(String[] args) throws Exception { if (args.length == 0) { System.err.println("Arguments: PATH"); System.exit(1); } var path = Path.of(args[0]); var telegramPath = path.resolve("td/telegram"); var memoryManager = new Manager(telegramPath, "Memory"); SequencedSet managers; try (var stream = Files.walk(telegramPath, 16)) { var endNamePattern = "Manager.h"; managers = stream .filter(Files::isRegularFile) .filter(p -> p.getFileName().toString().endsWith(endNamePattern)) .map(p -> new Manager(p.getParent(), p.getFileName().toString().substring(0, p.getFileName().toString().length() - endNamePattern.length()))) .filter(name -> !EXCLUDED_MANAGERS.contains(name.name)) .collect(Collectors.toCollection(LinkedHashSet::new)); } System.out.printf("Found %d managers%n", managers.size()); int fieldsFound = 0; int updatedManagers = 0; int totalManagers = 0; List invalidManagers = new ArrayList<>(); for (Manager manager : managers) { totalManagers++; var result = updateManagerJson(manager); if (result != null) { fieldsFound += result.fieldsFound(); if (result.changed) { updatedManagers++; } } else { invalidManagers.add(manager); } } updateMemoryManagerJson(memoryManager, managers); System.out.printf("%n%nDone.%n"); if (!invalidManagers.isEmpty()) { System.out.printf("%d invalid managers found:%n%s", invalidManagers.size(), invalidManagers.stream() .map(x -> "\t\"" + x.directory + "\": " + x.name + "\n") .collect(Collectors.joining(", "))); } System.out.printf("%d/%d managers updated, %d total fields%n", updatedManagers, totalManagers, fieldsFound); } enum FieldType { WaitFreeHashMap("WaitFreeHashMap", "calc_size"), WaitFreeHashSet("WaitFreeHashSet", "calc_size"), Vector("vector", "size"), FlatHashMap("FlatHashMap", "size"), FlatHashSet("FlatHashSet", "size") ; private final String fieldName; private final String sizeMethodName; public final Pattern pattern; FieldType(String fieldName, String sizeMethodName) { this.fieldName = fieldName; this.sizeMethodName = sizeMethodName; this.pattern = Pattern.compile("^ {2}(mutable )?" + fieldName + "(<([^ ]|, )+>)? +(?[a-zA-Z_]+);?[ \t/]*$"); } Pattern getPattern() { return pattern; } } record FoundField(FieldType type, String name) {} record UpdateResult(boolean changed, int fieldsFound) {} private static UpdateResult updateManagerJson(Manager manager) throws IOException { Path hFile = manager.directory.resolve(manager.name + "Manager.h"); Path cppFile = manager.directory.resolve(manager.name + "Manager.cpp"); System.out.printf("Updating manager \"%s\" files: [\"%s\", \"%s\"]%n", manager.name, hFile, cppFile); if (Files.notExists(hFile)) { System.out.printf("File not found, ignoring manager \"%s\": \"%s\"%n", manager.name, hFile); return null; } if (Files.notExists(cppFile)) { System.out.printf("File not found, ignoring manager \"%s\": \"%s\"%n", manager.name, cppFile); return null; } List fields = new ArrayList<>(); var hLines = normalizeSourceFile(readSourceFile(hFile)); boolean currentClass = false; for (String hLine : hLines) { FoundField field = null; if (hLine.startsWith("class ")) { currentClass = hLine.contains(" " + manager.name + "Manager"); } if (currentClass) { for (FieldType possibleFieldType : FieldType.values()) { var m = possibleFieldType.getPattern().matcher(hLine); if (m.matches()) { var fieldName = m.group("field"); field = new FoundField(possibleFieldType, fieldName); break; } } } if (field != null) { System.out.println("\tFound field: (%s) %s".formatted(field.type, field.name)); fields.add(field); } } StringBuilder memoryStatsMethod = new StringBuilder(); memoryStatsMethod.append("void %sManager::memory_stats(vector &output) {\n".formatted(manager.name)); memoryStatsMethod.append(fields.stream() .map(field -> " output.emplace_back(\"\\\"%s\\\":\"); output.emplace_back(std::to_string(this->%s.%s()));\n".formatted(field.name, field.name, field.type.sizeMethodName)) .collect(Collectors.joining(" output.emplace_back(\",\");\n"))); memoryStatsMethod.append("}\n"); List memoryStatsMethodLines = Arrays.asList(memoryStatsMethod.toString().split("\n")); var cppLines = readSourceFile(cppFile); var inputCppLines = new ArrayList<>(cppLines); // Remove the old memory_stats method var indexOfMemoryStatsStart = -1; var indexOfMemoryStatsEnd = -1; for (int i = 0; i < cppLines.size(); i++) { if (cppLines.get(i).contains("::memory_stats(")) { indexOfMemoryStatsStart = i; for (int j = i - 1; j >= 0; j--) { if (cppLines.get(j).isBlank()) { indexOfMemoryStatsStart = j; } else { break; } } break; } } if (indexOfMemoryStatsStart != -1) { for (int i = indexOfMemoryStatsStart + 1; i < cppLines.size(); i++) { if (cppLines.get(i).trim().equals("}")) { indexOfMemoryStatsEnd = i; break; } } if (indexOfMemoryStatsEnd == -1) { throw new IllegalStateException("memory_stats method end not found"); } cppLines.subList(indexOfMemoryStatsStart, indexOfMemoryStatsEnd + 1).clear(); } var last = cppLines.removeLast(); cppLines.addAll(memoryStatsMethodLines); cppLines.add(""); cppLines.addLast(last); boolean changed = !Objects.equals(inputCppLines, cppLines); if (changed) { System.out.printf("\tDone: %s.cpp file has been updated!%n", manager.name); Files.write(cppFile, cppLines, StandardCharsets.UTF_8); } else { System.out.printf("\tDone: %s.cpp file did not change.%n", manager.name); } return new UpdateResult(changed, fields.size()); } private static void updateMemoryManagerJson(Manager manager, SequencedSet managers) throws IOException { Path hFile = manager.directory.resolve(manager.name + "Manager.h"); Path cppFile = manager.directory.resolve(manager.name + "Manager.cpp"); System.out.printf("Updating memory manager \"%s\" files: [\"%s\", \"%s\"]%n", manager.name, hFile, cppFile); if (Files.notExists(hFile)) { System.out.printf("File not found for manager \"%s\": \"%s\"%n", manager.name, hFile); System.exit(1); return; } if (Files.notExists(cppFile)) { System.out.printf("File not found for manager \"%s\": \"%s\"%n", manager.name, cppFile); System.exit(1); return; } StringBuilder memoryStatsMethod = new StringBuilder(); memoryStatsMethod.append("void %sManager::print_managers_memory_stats(vector &output) const {\n".formatted(manager.name)); memoryStatsMethod.append(managers.stream() .map(m -> """ output.emplace_back("\\"%s_manager_\\":{"); td_->%s_manager_->memory_stats(output); output.emplace_back("}"); """.formatted(toSnakeCase(m.name), toSnakeCase(m.name))) .collect(Collectors.joining(" output.emplace_back(\",\");\n"))); memoryStatsMethod.append("}\n"); List memoryStatsMethodLines = Arrays.asList(memoryStatsMethod.toString().split("\n")); var cppLines = readSourceFile(cppFile); var inputCppLines = new ArrayList<>(cppLines); // Remove the old memory_stats method var indexOfMemoryStatsStart = -1; var indexOfMemoryStatsEnd = -1; for (int i = 0; i < cppLines.size(); i++) { if (cppLines.get(i).contains("::print_managers_memory_stats(")) { indexOfMemoryStatsStart = i; for (int j = i - 1; j >= 0; j--) { if (cppLines.get(j).isBlank()) { indexOfMemoryStatsStart = j; } else { break; } } break; } } if (indexOfMemoryStatsStart != -1) { for (int i = indexOfMemoryStatsStart + 1; i < cppLines.size(); i++) { if (cppLines.get(i).trim().equals("}")) { indexOfMemoryStatsEnd = i; break; } } if (indexOfMemoryStatsEnd == -1) { throw new IllegalStateException("print_managers_memory_stats method end not found"); } cppLines.subList(indexOfMemoryStatsStart, indexOfMemoryStatsEnd + 1).clear(); } var last = cppLines.removeLast(); cppLines.addAll(memoryStatsMethodLines); cppLines.add(""); cppLines.addLast(last); for (Manager m : managers) { var mInclude = "#include \"%s\"".formatted(m.includePath()); if (!cppLines.contains(mInclude)) { System.out.printf("\tMissing include, adding \"" + m.includePath() + "\"%n"); int includeInsertIndex = -1; for (int i = 0; i < cppLines.size(); i++) { if (cppLines.get(i).startsWith("#include")) { includeInsertIndex = i; } } if (includeInsertIndex == -1) { throw new IllegalStateException("Cannot find a place to put the include"); } cppLines.add(includeInsertIndex + 1, mInclude); } } boolean changed = !Objects.equals(inputCppLines, cppLines); if (changed) { System.out.printf("\tDone: %s.cpp file has been updated!%n", manager.name); Files.write(cppFile, cppLines, StandardCharsets.UTF_8); } else { System.out.printf("\tDone: %s.cpp file did not change.%n", manager.name); } } private static String toSnakeCase(String name) { var initialChar = name.codePoints() .limit(1) .map(Character::toLowerCase); var restOfString = name.codePoints() .skip(1) .flatMap(codePoint -> { if (Character.isUpperCase(codePoint)) { return IntStream.of('_', Character.toLowerCase(codePoint)); } else { return IntStream.of(codePoint); } }); var resultStream = IntStream.concat(initialChar, restOfString); return resultStream.collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append).toString(); } private static List readSourceFile(Path path) throws IOException { return Files.readAllLines(path, StandardCharsets.UTF_8); } private static List normalizeSourceFile(List lines) throws IOException { List srcLines = new ArrayList<>(lines); // Remove empty lines srcLines.removeIf(String::isBlank); // Remove unexpected newlines List srcLinesWithoutNewlines = new ArrayList<>(); StringBuilder buf = new StringBuilder(); for (String srcLine : srcLines) { if (!buf.isEmpty()) { buf.append(" "); } buf.append(srcLine); var trimmedLine = srcLine.trim(); if (!trimmedLine.endsWith(">")) { srcLinesWithoutNewlines.add(buf.toString()); buf.setLength(0); } } if (!buf.isEmpty()) { srcLinesWithoutNewlines.add(buf.toString()); } srcLinesWithoutNewlines.replaceAll(p -> { var commentStart = p.indexOf("//"); if (commentStart >= 0) { return p.substring(0, commentStart); } else { return p; } }); return srcLinesWithoutNewlines; } }