From 33131b65c306f4b5efdee06c80495a62d2eac19c Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Wed, 20 Mar 2024 23:22:10 +0100 Subject: [PATCH] Add memory manager json code generator --- UpdateMemoryManager.javash | 378 +++++++++++++++++++++++++++++++++++++ 1 file changed, 378 insertions(+) create mode 100755 UpdateMemoryManager.javash diff --git a/UpdateMemoryManager.javash b/UpdateMemoryManager.javash new file mode 100755 index 000000000..4f30fdd17 --- /dev/null +++ b/UpdateMemoryManager.javash @@ -0,0 +1,378 @@ +#!/usr/bin/java --source 21 +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; +import java.util.SequencedSet; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class UpdateMemoryManager { + + static final Set EXCLUDED_MANAGERS = Set.of("Memory", + "Call", + "DeviceToken", + "LanguagePack", + "Pts", + "Password", + "SecretChats", + "Secure", + "Config", + "Storage", + "FileLoad", + "Parts", + "FileGenerate", + "Resource", + "NetStats", + "DcAuth", + "State", + "PhoneNumber" + ); + + record Manager(Path directory, String name) { + String includePath() { + return directory.toString().substring(2) + "/" + name + "Manager.h"; + } + } + + public static void main(String[] args) throws Exception { + if (args.length == 0) { + System.err.println("Arguments: PATH"); + System.exit(1); + } + + var path = Path.of(args[0]); + var telegramPath = path.resolve("td/telegram"); + + var memoryManager = new Manager(telegramPath, "Memory"); + SequencedSet managers; + try (var stream = Files.walk(telegramPath, 16)) { + var endNamePattern = "Manager.h"; + managers = stream + .filter(Files::isRegularFile) + .filter(p -> p.getFileName().toString().endsWith(endNamePattern)) + .map(p -> new Manager(p.getParent(), p.getFileName().toString().substring(0, p.getFileName().toString().length() - endNamePattern.length()))) + .filter(name -> !EXCLUDED_MANAGERS.contains(name.name)) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + System.out.printf("Found %d managers%n", managers.size()); + + int fieldsFound = 0; + int updatedManagers = 0; + int totalManagers = 0; + List invalidManagers = new ArrayList<>(); + for (Manager manager : managers) { + totalManagers++; + var result = updateManagerJson(manager); + if (result != null) { + fieldsFound += result.fieldsFound(); + if (result.changed) { + updatedManagers++; + } + } else { + invalidManagers.add(manager); + } + } + + updateMemoryManagerJson(memoryManager, managers); + + System.out.printf("%n%nDone.%n"); + + if (!invalidManagers.isEmpty()) { + System.out.printf("%d invalid managers found:%n%s", + invalidManagers.size(), + invalidManagers.stream() + .map(x -> "\t\"" + x.directory + "\": " + x.name + "\n") + .collect(Collectors.joining(", "))); + } + System.out.printf("%d/%d managers updated, %d total fields%n", updatedManagers, totalManagers, fieldsFound); + } + + enum FieldType { + WaitFreeHashMap("WaitFreeHashMap", "calc_size"), + WaitFreeHashSet("WaitFreeHashSet", "calc_size"), + Vector("vector", "size"), + FlatHashMap("FlatHashMap", "size"), + FlatHashSet("FlatHashSet", "size") + ; + + private final String fieldName; + private final String sizeMethodName; + public final Pattern pattern; + + FieldType(String fieldName, String sizeMethodName) { + this.fieldName = fieldName; + this.sizeMethodName = sizeMethodName; + this.pattern = Pattern.compile("^ {2}(mutable )?" + fieldName + "(<([^ ]|, )+>)? +(?[a-zA-Z_]+);?[ \t/]*$"); + } + + Pattern getPattern() { + return pattern; + } + } + + record FoundField(FieldType type, String name) {} + record UpdateResult(boolean changed, int fieldsFound) {} + + private static UpdateResult updateManagerJson(Manager manager) throws IOException { + Path hFile = manager.directory.resolve(manager.name + "Manager.h"); + Path cppFile = manager.directory.resolve(manager.name + "Manager.cpp"); + + System.out.printf("Updating manager \"%s\" files: [\"%s\", \"%s\"]%n", manager.name, hFile, cppFile); + + if (Files.notExists(hFile)) { + System.out.printf("File not found, ignoring manager \"%s\": \"%s\"%n", manager.name, hFile); + return null; + } + if (Files.notExists(cppFile)) { + System.out.printf("File not found, ignoring manager \"%s\": \"%s\"%n", manager.name, cppFile); + return null; + } + + List fields = new ArrayList<>(); + + var hLines = normalizeSourceFile(readSourceFile(hFile)); + + boolean currentClass = false; + for (String hLine : hLines) { + FoundField field = null; + if (hLine.startsWith("class ")) { + currentClass = hLine.contains(" " + manager.name + "Manager"); + } + + if (currentClass) { + for (FieldType possibleFieldType : FieldType.values()) { + var m = possibleFieldType.getPattern().matcher(hLine); + if (m.matches()) { + var fieldName = m.group("field"); + field = new FoundField(possibleFieldType, fieldName); + break; + } + } + } + + if (field != null) { + System.out.println("\tFound field: (%s) %s".formatted(field.type, field.name)); + fields.add(field); + } + + } + + StringBuilder memoryStatsMethod = new StringBuilder(); + + memoryStatsMethod.append("void %sManager::memory_stats(vector &output) {\n".formatted(manager.name)); + memoryStatsMethod.append(fields.stream() + .map(field -> " output.emplace_back(\"\\\"%s\\\":\"); output.emplace_back(std::to_string(this->%s.%s()));\n".formatted(field.name, field.name, field.type.sizeMethodName)) + .collect(Collectors.joining(" output.emplace_back(\",\");\n"))); + memoryStatsMethod.append("}\n"); + + List memoryStatsMethodLines = Arrays.asList(memoryStatsMethod.toString().split("\n")); + + var cppLines = readSourceFile(cppFile); + var inputCppLines = new ArrayList<>(cppLines); + + // Remove the old memory_stats method + var indexOfMemoryStatsStart = -1; + var indexOfMemoryStatsEnd = -1; + for (int i = 0; i < cppLines.size(); i++) { + if (cppLines.get(i).contains("::memory_stats(")) { + indexOfMemoryStatsStart = i; + for (int j = i - 1; j >= 0; j--) { + if (cppLines.get(j).isBlank()) { + indexOfMemoryStatsStart = j; + } else { + break; + } + } + break; + } + } + if (indexOfMemoryStatsStart != -1) { + for (int i = indexOfMemoryStatsStart + 1; i < cppLines.size(); i++) { + if (cppLines.get(i).trim().equals("}")) { + indexOfMemoryStatsEnd = i; + break; + } + } + if (indexOfMemoryStatsEnd == -1) { + throw new IllegalStateException("memory_stats method end not found"); + } + cppLines.subList(indexOfMemoryStatsStart, indexOfMemoryStatsEnd + 1).clear(); + } + + var last = cppLines.removeLast(); + cppLines.addAll(memoryStatsMethodLines); + cppLines.add(""); + cppLines.addLast(last); + + boolean changed = !Objects.equals(inputCppLines, cppLines); + + if (changed) { + System.out.printf("\tDone: %s.cpp file has been updated!%n", manager.name); + Files.write(cppFile, cppLines, StandardCharsets.UTF_8); + } else { + System.out.printf("\tDone: %s.cpp file did not change.%n", manager.name); + } + + return new UpdateResult(changed, fields.size()); + } + + private static void updateMemoryManagerJson(Manager manager, SequencedSet managers) throws IOException { + Path hFile = manager.directory.resolve(manager.name + "Manager.h"); + Path cppFile = manager.directory.resolve(manager.name + "Manager.cpp"); + + System.out.printf("Updating memory manager \"%s\" files: [\"%s\", \"%s\"]%n", manager.name, hFile, cppFile); + + if (Files.notExists(hFile)) { + System.out.printf("File not found for manager \"%s\": \"%s\"%n", manager.name, hFile); + System.exit(1); + return; + } + if (Files.notExists(cppFile)) { + System.out.printf("File not found for manager \"%s\": \"%s\"%n", manager.name, cppFile); + System.exit(1); + return; + } + + StringBuilder memoryStatsMethod = new StringBuilder(); + + memoryStatsMethod.append("void %sManager::print_managers_memory_stats(vector &output) const {\n".formatted(manager.name)); + memoryStatsMethod.append(managers.stream() + .map(m -> """ + output.emplace_back("\\"%s_manager_\\":{"); td_->%s_manager_->memory_stats(output); output.emplace_back("}"); + """.formatted(toSnakeCase(m.name), toSnakeCase(m.name))) + .collect(Collectors.joining(" output.emplace_back(\",\");\n"))); + memoryStatsMethod.append("}\n"); + + List memoryStatsMethodLines = Arrays.asList(memoryStatsMethod.toString().split("\n")); + + var cppLines = readSourceFile(cppFile); + var inputCppLines = new ArrayList<>(cppLines); + + // Remove the old memory_stats method + var indexOfMemoryStatsStart = -1; + var indexOfMemoryStatsEnd = -1; + for (int i = 0; i < cppLines.size(); i++) { + if (cppLines.get(i).contains("::print_managers_memory_stats(")) { + indexOfMemoryStatsStart = i; + for (int j = i - 1; j >= 0; j--) { + if (cppLines.get(j).isBlank()) { + indexOfMemoryStatsStart = j; + } else { + break; + } + } + break; + } + } + if (indexOfMemoryStatsStart != -1) { + for (int i = indexOfMemoryStatsStart + 1; i < cppLines.size(); i++) { + if (cppLines.get(i).trim().equals("}")) { + indexOfMemoryStatsEnd = i; + break; + } + } + if (indexOfMemoryStatsEnd == -1) { + throw new IllegalStateException("print_managers_memory_stats method end not found"); + } + cppLines.subList(indexOfMemoryStatsStart, indexOfMemoryStatsEnd + 1).clear(); + } + + var last = cppLines.removeLast(); + cppLines.addAll(memoryStatsMethodLines); + cppLines.add(""); + cppLines.addLast(last); + + for (Manager m : managers) { + var mInclude = "#include \"%s\"".formatted(m.includePath()); + if (!cppLines.contains(mInclude)) { + System.out.printf("\tMissing include, adding \"" + m.includePath() + "\"%n"); + int includeInsertIndex = -1; + for (int i = 0; i < cppLines.size(); i++) { + if (cppLines.get(i).startsWith("#include")) { + includeInsertIndex = i; + } + } + if (includeInsertIndex == -1) { + throw new IllegalStateException("Cannot find a place to put the include"); + } + cppLines.add(includeInsertIndex + 1, mInclude); + } + } + + boolean changed = !Objects.equals(inputCppLines, cppLines); + + if (changed) { + System.out.printf("\tDone: %s.cpp file has been updated!%n", manager.name); + Files.write(cppFile, cppLines, StandardCharsets.UTF_8); + } else { + System.out.printf("\tDone: %s.cpp file did not change.%n", manager.name); + } + } + + private static String toSnakeCase(String name) { + var initialChar = name.codePoints() + .limit(1) + .map(Character::toLowerCase); + var restOfString = name.codePoints() + .skip(1) + .flatMap(codePoint -> { + if (Character.isUpperCase(codePoint)) { + return IntStream.of('_', Character.toLowerCase(codePoint)); + } else { + return IntStream.of(codePoint); + } + }); + var resultStream = IntStream.concat(initialChar, restOfString); + return resultStream.collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append).toString(); + } + + private static List readSourceFile(Path path) throws IOException { + return Files.readAllLines(path, StandardCharsets.UTF_8); + } + + private static List normalizeSourceFile(List lines) throws IOException { + List srcLines = new ArrayList<>(lines); + + // Remove empty lines + srcLines.removeIf(String::isBlank); + + // Remove unexpected newlines + List srcLinesWithoutNewlines = new ArrayList<>(); + StringBuilder buf = new StringBuilder(); + for (String srcLine : srcLines) { + if (!buf.isEmpty()) { + buf.append(" "); + } + buf.append(srcLine); + var trimmedLine = srcLine.trim(); + if (!trimmedLine.endsWith(">")) { + srcLinesWithoutNewlines.add(buf.toString()); + buf.setLength(0); + } + } + if (!buf.isEmpty()) { + srcLinesWithoutNewlines.add(buf.toString()); + } + + srcLinesWithoutNewlines.replaceAll(p -> { + var commentStart = p.indexOf("//"); + if (commentStart >= 0) { + return p.substring(0, commentStart); + } else { + return p; + } + }); + + return srcLinesWithoutNewlines; + } +}