Add binary strings

This commit is contained in:
Andrea Cavalli 2024-09-26 19:26:37 +02:00
parent 3b9a2479ca
commit 5d3a415afb
12 changed files with 353 additions and 70 deletions

View File

@ -29,6 +29,7 @@ public abstract class ClassGenerator {
protected final boolean deepCheckBeforeCreatingNewEqualInstances;
protected final boolean useRecordBuilders;
protected final boolean generateOldSerializers;
protected final boolean binaryStrings;
public ClassGenerator(ClassGeneratorParams params) {
this.generatedFilesToDelete = params.generatedFilesToDelete;
@ -38,6 +39,7 @@ public abstract class ClassGenerator {
this.deepCheckBeforeCreatingNewEqualInstances = params.deepCheckBeforeCreatingNewEqualInstances;
this.useRecordBuilders = params.useRecordBuilders;
this.generateOldSerializers = params.generateOldSerializers;
this.binaryStrings = params.binaryStrings;
}
public void run() throws IOException {
@ -84,10 +86,11 @@ public abstract class ClassGenerator {
public record GeneratedClass(String packageName, TypeSpec.Builder content) {}
public record ClassGeneratorParams(HashSet<Path> generatedFilesToDelete,
DataModel dataModel,
String basePackageName,
Path outPath,
boolean deepCheckBeforeCreatingNewEqualInstances,
boolean useRecordBuilders,
boolean generateOldSerializers) {}
DataModel dataModel,
String basePackageName,
Path outPath,
boolean deepCheckBeforeCreatingNewEqualInstances,
boolean useRecordBuilders,
boolean generateOldSerializers,
boolean binaryStrings) {}
}

View File

@ -4,17 +4,7 @@ import com.squareup.javapoet.ClassName;
import com.squareup.javapoet.ParameterizedTypeName;
import com.squareup.javapoet.TypeName;
import it.cavallium.datagen.NativeNullable;
import it.cavallium.datagen.nativedata.ArrayInt52Serializer;
import it.cavallium.datagen.nativedata.ArrayStringSerializer;
import it.cavallium.datagen.nativedata.ArraybooleanSerializer;
import it.cavallium.datagen.nativedata.ArraybyteSerializer;
import it.cavallium.datagen.nativedata.ArraycharSerializer;
import it.cavallium.datagen.nativedata.ArraydoubleSerializer;
import it.cavallium.datagen.nativedata.ArrayfloatSerializer;
import it.cavallium.datagen.nativedata.ArrayintSerializer;
import it.cavallium.datagen.nativedata.ArraylongSerializer;
import it.cavallium.datagen.nativedata.ArrayshortSerializer;
import it.cavallium.datagen.nativedata.Serializers;
import it.cavallium.datagen.nativedata.*;
import it.unimi.dsi.fastutil.booleans.BooleanList;
import it.unimi.dsi.fastutil.bytes.ByteList;
import it.unimi.dsi.fastutil.chars.CharList;
@ -30,13 +20,15 @@ import java.util.stream.Stream;
public final class ComputedTypeArrayNative implements ComputedTypeArray {
private final String baseType;
private final boolean byteStrings;
private ComputedTypeNative computedChild;
private final ComputedTypeSupplier computedTypeSupplier;
public ComputedTypeArrayNative(String baseType, ComputedTypeSupplier computedTypeSupplier) {
public ComputedTypeArrayNative(String baseType, ComputedTypeSupplier computedTypeSupplier, boolean byteStrings) {
this.baseType = baseType;
this.computedTypeSupplier = computedTypeSupplier;
this.byteStrings = byteStrings;
}
public ComputedType getBase() {
@ -113,7 +105,7 @@ public final class ComputedTypeArrayNative implements ComputedTypeArray {
case "long" -> ClassName.get(ArraylongSerializer.class);
case "float" -> ClassName.get(ArrayfloatSerializer.class);
case "double" -> ClassName.get(ArraydoubleSerializer.class);
case "String" -> ClassName.get(ArrayStringSerializer.class);
case "String" -> byteStrings ? ClassName.get(ArrayBinaryStringSerializer.class) : ClassName.get(ArrayStringSerializer.class);
case "Int52" -> ClassName.get(ArrayInt52Serializer.class);
default -> throw new UnsupportedOperationException();
};

View File

@ -2,10 +2,8 @@ package it.cavallium.datagen.plugin;
import com.squareup.javapoet.ClassName;
import com.squareup.javapoet.TypeName;
import it.cavallium.datagen.nativedata.Int52Serializer;
import it.cavallium.datagen.nativedata.Serializers;
import it.cavallium.datagen.nativedata.StringSerializer;
import it.cavallium.datagen.nativedata.Int52;
import it.cavallium.datagen.nativedata.*;
import java.util.List;
import java.util.Objects;
import java.util.Set;
@ -17,12 +15,14 @@ public final class ComputedTypeNative implements ComputedType {
private final String type;
private final ComputedTypeSupplier computedTypeSupplier;
private final boolean binaryStrings;
private boolean primitive;
public ComputedTypeNative(String type, ComputedTypeSupplier computedTypeSupplier) {
public ComputedTypeNative(String type, ComputedTypeSupplier computedTypeSupplier, boolean binaryStrings) {
this.type = type;
this.computedTypeSupplier = computedTypeSupplier;
this.primitive = PRIMITIVE_TYPES.contains(type);
this.binaryStrings = binaryStrings;
}
public String getName() {
@ -32,7 +32,7 @@ public final class ComputedTypeNative implements ComputedType {
@Override
public TypeName getJTypeName(String basePackageName) {
return switch (type) {
case "String" -> ClassName.get(String.class);
case "String" -> binaryStrings ? ClassName.get(BinaryString.class) : ClassName.get(String.class);
case "boolean" -> TypeName.BOOLEAN;
case "short" -> TypeName.SHORT;
case "char" -> TypeName.CHAR;
@ -54,7 +54,7 @@ public final class ComputedTypeNative implements ComputedType {
@Override
public TypeName getJSerializerName(String basePackageName) {
return switch (type) {
case "String" -> ClassName.get(StringSerializer.class);
case "String" -> binaryStrings ? ClassName.get(BinaryStringSerializer.class) : ClassName.get(StringSerializer.class);
case "boolean", "byte", "short", "char", "int", "long", "float", "double" ->
throw new UnsupportedOperationException("Type " + type
+ " is a native type, so it doesn't have a serializer");
@ -107,10 +107,10 @@ public final class ComputedTypeNative implements ComputedType {
return computedTypeSupplier.getDependents(getName());
}
public static List<ComputedTypeNative> get(ComputedTypeSupplier computedTypeSupplier) {
public static List<ComputedTypeNative> get(ComputedTypeSupplier computedTypeSupplier, boolean binaryStrings) {
return Stream
.of("String", "boolean", "short", "char", "int", "long", "float", "double", "byte", "Int52")
.map(name -> new ComputedTypeNative(name, computedTypeSupplier))
.map(name -> new ComputedTypeNative(name, computedTypeSupplier, binaryStrings))
.toList();
}

View File

@ -2,27 +2,8 @@ package it.cavallium.datagen.plugin;
import com.squareup.javapoet.ClassName;
import com.squareup.javapoet.TypeName;
import it.cavallium.datagen.nativedata.NullableInt52;
import it.cavallium.datagen.nativedata.NullableInt52Serializer;
import it.cavallium.datagen.nativedata.NullableString;
import it.cavallium.datagen.nativedata.NullableStringSerializer;
import it.cavallium.datagen.nativedata.Nullableboolean;
import it.cavallium.datagen.nativedata.NullablebooleanSerializer;
import it.cavallium.datagen.nativedata.Nullablebyte;
import it.cavallium.datagen.nativedata.NullablebyteSerializer;
import it.cavallium.datagen.nativedata.Nullablechar;
import it.cavallium.datagen.nativedata.NullablecharSerializer;
import it.cavallium.datagen.nativedata.Nullabledouble;
import it.cavallium.datagen.nativedata.NullabledoubleSerializer;
import it.cavallium.datagen.nativedata.Nullablefloat;
import it.cavallium.datagen.nativedata.NullablefloatSerializer;
import it.cavallium.datagen.nativedata.Nullableint;
import it.cavallium.datagen.nativedata.NullableintSerializer;
import it.cavallium.datagen.nativedata.Nullablelong;
import it.cavallium.datagen.nativedata.NullablelongSerializer;
import it.cavallium.datagen.nativedata.Nullableshort;
import it.cavallium.datagen.nativedata.NullableshortSerializer;
import it.cavallium.datagen.nativedata.Serializers;
import it.cavallium.datagen.nativedata.*;
import java.util.Objects;
import java.util.stream.Stream;
@ -30,14 +11,16 @@ public final class ComputedTypeNullableNative implements ComputedTypeNullable {
private final String baseType;
private final ComputedVersion latestVersion;
private final boolean binaryStrings;
private ComputedTypeNative computedChild;
private final ComputedTypeSupplier computedTypeSupplier;
public ComputedTypeNullableNative(String baseType, ComputedVersion latestVersion, ComputedTypeSupplier computedTypeSupplier) {
public ComputedTypeNullableNative(String baseType, ComputedVersion latestVersion, ComputedTypeSupplier computedTypeSupplier, boolean binaryStrings) {
this.baseType = baseType;
this.latestVersion = latestVersion;
this.computedTypeSupplier = computedTypeSupplier;
this.binaryStrings = binaryStrings;
}
public ComputedTypeNative getBase() {
@ -93,7 +76,7 @@ public final class ComputedTypeNullableNative implements ComputedTypeNullable {
case "long" -> ClassName.get(Nullablelong.class);
case "float" -> ClassName.get(Nullablefloat.class);
case "double" -> ClassName.get(Nullabledouble.class);
case "String" -> ClassName.get(NullableString.class);
case "String" -> binaryStrings ? ClassName.get(NullableBinaryString.class) : ClassName.get(NullableString.class);
case "Int52" -> ClassName.get(NullableInt52.class);
default -> ClassName.get(latestVersion.getDataNullablesPackage(basePackageName), "Nullable" + baseType);
};
@ -115,7 +98,7 @@ public final class ComputedTypeNullableNative implements ComputedTypeNullable {
case "long" -> ClassName.get(NullablelongSerializer.class);
case "float" -> ClassName.get(NullablefloatSerializer.class);
case "double" -> ClassName.get(NullabledoubleSerializer.class);
case "String" -> ClassName.get(NullableStringSerializer.class);
case "String" -> binaryStrings ? ClassName.get(NullableBinaryStringSerializer.class) : ClassName.get(NullableStringSerializer.class);
case "Int52" -> ClassName.get(NullableInt52Serializer.class);
default -> throw new UnsupportedOperationException();
};

View File

@ -58,12 +58,13 @@ public class DataModel {
private final Int2ObjectMap<Map<String, List<TransformationConfiguration>>> baseTypeDataChanges;
public DataModel(int hash,
String currentVersionKey,
Map<String, InterfaceDataConfiguration> interfacesData,
Map<String, ClassConfiguration> baseTypesData,
Map<String, Set<String>> superTypesData,
Map<String, CustomTypesConfiguration> customTypesData,
Map<String, VersionConfiguration> rawVersions) {
String currentVersionKey,
Map<String, InterfaceDataConfiguration> interfacesData,
Map<String, ClassConfiguration> baseTypesData,
Map<String, Set<String>> superTypesData,
Map<String, CustomTypesConfiguration> customTypesData,
Map<String, VersionConfiguration> rawVersions,
boolean binaryStrings) {
this.hash = hash;
@ -363,7 +364,7 @@ public class DataModel {
nullableRawTypes.stream()
.filter(NATIVE_TYPES::contains)
.map(baseType ->
new ComputedTypeNullableNative(baseType, computedVersions.get(latestVersion), computedTypeSupplier))
new ComputedTypeNullableNative(baseType, computedVersions.get(latestVersion), computedTypeSupplier, binaryStrings))
.forEach(versionBaseTypes::add);
}
// Compute array types
@ -393,7 +394,7 @@ public class DataModel {
.forEach(versionBaseTypes::add);
}
// Compute native types
versionBaseTypes.addAll(ComputedTypeNative.get(computedTypeSupplier));
versionBaseTypes.addAll(ComputedTypeNative.get(computedTypeSupplier, binaryStrings));
var allLatestTypes = versionBaseTypes.stream().distinct().collect(Collectors.toMap(ComputedType::getName, identity()));

View File

@ -69,14 +69,13 @@ public class SourcesGenerator {
private static final Logger logger = LoggerFactory.getLogger(SourcesGenerator.class);
private static final boolean OVERRIDE_ALL_NULLABLE_METHODS = false;
private static final String SERIAL_VERSION = "5";
private static final String SERIAL_VERSION = "6";
private final DataModel dataModel;
private final SourcesGeneratorConfiguration configuration;
private SourcesGenerator(InputStream yamlDataStream) {
Yaml yaml = new Yaml();
var configuration = yaml.loadAs(yamlDataStream, SourcesGeneratorConfiguration.class);
this.dataModel = configuration.buildDataModel();
this.configuration = yaml.loadAs(yamlDataStream, SourcesGeneratorConfiguration.class);
}
public static SourcesGenerator load(InputStream yamlData) {
@ -101,8 +100,15 @@ public class SourcesGenerator {
* @param useRecordBuilders if true, the data will have @RecordBuilder annotation
* @param force force overwrite
* @param deepCheckBeforeCreatingNewEqualInstances if true, use equals, if false, use ==
* @param binaryStrings use binary strings
*/
public void generateSources(String basePackageName, Path outPath, boolean useRecordBuilders, boolean force, boolean deepCheckBeforeCreatingNewEqualInstances, boolean generateOldSerializers) throws IOException {
public void generateSources(String basePackageName,
Path outPath,
boolean useRecordBuilders,
boolean force,
boolean deepCheckBeforeCreatingNewEqualInstances,
boolean generateOldSerializers,
boolean binaryStrings) throws IOException {
Path basePackageNamePath;
{
Path basePackageNamePathPartial = outPath;
@ -112,22 +118,25 @@ public class SourcesGenerator {
basePackageNamePath = basePackageNamePathPartial;
}
var hashPath = basePackageNamePath.resolve(".hash");
var dataModel = configuration.buildDataModel(binaryStrings);
var curHash = dataModel.computeHash();
if (Files.isRegularFile(hashPath) && Files.isReadable(hashPath)) {
var lines = Files.readAllLines(hashPath, StandardCharsets.UTF_8);
if (lines.size() >= 6) {
if (lines.size() >= 7) {
var prevBasePackageName = lines.get(0);
var prevRecordBuilders = lines.get(1);
var prevHash = lines.get(2);
var prevDeepCheckBeforeCreatingNewEqualInstances = lines.get(3);
var prevGenerateOldSerializers = lines.get(4);
var prevSerialVersion = lines.get(5);
var prevBinaryStrings = lines.get(6);
if (!force
&& prevBasePackageName.equals(basePackageName)
&& (prevRecordBuilders.equalsIgnoreCase("true") == useRecordBuilders)
&& (prevDeepCheckBeforeCreatingNewEqualInstances.equalsIgnoreCase("true") == deepCheckBeforeCreatingNewEqualInstances)
&& (prevGenerateOldSerializers.equalsIgnoreCase("true") == generateOldSerializers)
&& (prevBinaryStrings.equalsIgnoreCase("true") == binaryStrings)
&& (prevSerialVersion.equals(SERIAL_VERSION))
&& prevHash.equals(Integer.toString(curHash))) {
logger.info("Skipped sources generation because it didn't change");
@ -154,7 +163,8 @@ public class SourcesGenerator {
.collect(Collectors.toCollection(HashSet::new));
}
var genParams = new ClassGeneratorParams(generatedFilesToDelete, dataModel, basePackageName, outPath, deepCheckBeforeCreatingNewEqualInstances, useRecordBuilders, generateOldSerializers);
var genParams = new ClassGeneratorParams(generatedFilesToDelete, dataModel, basePackageName, outPath,
deepCheckBeforeCreatingNewEqualInstances, useRecordBuilders, generateOldSerializers, binaryStrings);
// Create the Versions class
new GenVersions(genParams).run();
@ -202,7 +212,13 @@ public class SourcesGenerator {
new GenUpgraderSuperX(genParams).run();
// Update the hash at the end
var newHashRaw = basePackageName + '\n' + useRecordBuilders + '\n' + deepCheckBeforeCreatingNewEqualInstances + '\n' + curHash + '\n';
var newHashRaw = basePackageName + '\n'
+ useRecordBuilders + '\n'
+ deepCheckBeforeCreatingNewEqualInstances + '\n'
+ generateOldSerializers + '\n'
+ binaryStrings + '\n'
+ SERIAL_VERSION + '\n'
+ curHash + '\n';
String oldHashRaw;
if (Files.exists(hashPath)) {
oldHashRaw = Files.readString(hashPath, StandardCharsets.UTF_8);

View File

@ -37,14 +37,15 @@ public class SourcesGeneratorConfiguration {
return hash;
}
public DataModel buildDataModel() {
public DataModel buildDataModel(boolean binaryStrings) {
return new DataModel(hashCode(),
currentVersion,
Objects.requireNonNullElse(interfacesData, Map.of()),
Objects.requireNonNullElse(baseTypesData, Map.of()),
Objects.requireNonNullElse(superTypesData, Map.of()),
Objects.requireNonNullElse(customTypesData, Map.of()),
Objects.requireNonNullElse(versions, Map.of())
Objects.requireNonNullElse(versions, Map.of()),
binaryStrings
);
}
}

View File

@ -0,0 +1,33 @@
package it.cavallium.datagen.nativedata;
import it.cavallium.datagen.DataSerializer;
import it.cavallium.stream.SafeDataInput;
import it.cavallium.stream.SafeDataOutput;
import org.jetbrains.annotations.NotNull;
import java.util.List;
public class ArrayBinaryStringSerializer implements DataSerializer<List<BinaryString>> {
@Override
public void serialize(SafeDataOutput dataOutput, @NotNull List<BinaryString> data) {
dataOutput.writeInt(data.size());
for (BinaryString item : data) {
dataOutput.writeShort(item.sizeBytes());
dataOutput.write(item.data());
}
}
@NotNull
@Override
public List<BinaryString> deserialize(SafeDataInput dataInput) {
var data = new BinaryString[dataInput.readInt()];
for (int i = 0; i < data.length; i++) {
var len = dataInput.readUnsignedShort();
byte[] stringData = new byte[len];
dataInput.readFully(stringData);
data[i] = new BinaryString(stringData);
}
return List.of(data);
}
}

View File

@ -0,0 +1,29 @@
package it.cavallium.datagen.nativedata;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Objects;
public record BinaryString(byte[] data) {
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
BinaryString that = (BinaryString) o;
return Objects.deepEquals(data, that.data);
}
@Override
public int hashCode() {
return Arrays.hashCode(data);
}
@Override
public String toString() {
return new String(data, StandardCharsets.UTF_8);
}
public int sizeBytes() {
return data.length;
}
}

View File

@ -0,0 +1,26 @@
package it.cavallium.datagen.nativedata;
import it.cavallium.datagen.DataSerializer;
import it.cavallium.stream.SafeDataInput;
import it.cavallium.stream.SafeDataOutput;
import org.jetbrains.annotations.NotNull;
public class BinaryStringSerializer implements DataSerializer<BinaryString> {
public static final BinaryStringSerializer INSTANCE = new BinaryStringSerializer();
@Override
public void serialize(SafeDataOutput dataOutput, @NotNull BinaryString data) {
dataOutput.writeInt(data.sizeBytes());
dataOutput.write(data.data());
}
@NotNull
@Override
public BinaryString deserialize(SafeDataInput dataInput) {
var size = dataInput.readInt();
byte[] bytes = new byte[size];
dataInput.readFully(bytes);
return new BinaryString(bytes);
}
}

View File

@ -0,0 +1,160 @@
package it.cavallium.datagen.nativedata;
import it.cavallium.datagen.NativeNullable;
import it.cavallium.datagen.TypedNullable;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.Serial;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
public class NullableBinaryString implements Serializable, INullable, TypedNullable<BinaryString> {
@Serial
private static final long serialVersionUID = 1L;
private static final NullableBinaryString NULL = new NullableBinaryString(null);
private final BinaryString value;
public NullableBinaryString(BinaryString value) {
this.value = value;
}
@SuppressWarnings("ConstantConditions")
public static NullableBinaryString of(@NotNull BinaryString value) {
if (value == null) {
throw new NullPointerException();
} else {
return new NullableBinaryString(value);
}
}
public static NullableBinaryString ofNullable(@Nullable BinaryString value) {
if (value == null) {
return NULL;
} else {
return new NullableBinaryString(value);
}
}
public static NullableBinaryString empty() {
return NULL;
}
@Override
public boolean isEmpty() {
return value == null;
}
@Override
public boolean isPresent() {
return value != null;
}
public boolean isContentful() {
return value != null && !isBlank();
}
public boolean isBlank() {
return value == null || value.sizeBytes() == 0;
}
@Override
@NotNull
public BinaryString get() {
if (value == null) {
throw new NullPointerException();
} else {
return value;
}
}
@Override
public @NotNull NullableBinaryString or(@NotNull NativeNullable<? extends BinaryString> fallback) {
if (value == null) {
if (fallback.getClass() == NullableBinaryString.class) {
return (NullableBinaryString) fallback;
} else {
return ofNullable(fallback.getNullable());
}
} else {
return this;
}
}
@NotNull
public NullableBinaryString or(NullableBinaryString fallback) {
if (value == null) {
return fallback;
} else {
return this;
}
}
public @NotNull NullableBinaryString orIfBlank(@NotNull NativeNullable<? extends BinaryString> fallback) {
if (isBlank()) {
if (fallback.getClass() == NullableBinaryString.class) {
return (NullableBinaryString) fallback;
} else {
return ofNullable(fallback.getNullable());
}
} else {
return this;
}
}
@NotNull
public NullableBinaryString orIfBlank(NullableBinaryString fallback) {
if (isBlank()) {
return fallback;
} else {
return this;
}
}
@Nullable
public BinaryString getNullable() {
return value;
}
@Override
@Nullable
public BinaryString getNullable(BinaryString defaultValue) {
return value == null ? defaultValue : value;
}
@NotNull
@Override
public NullableBinaryString clone() {
if (value != null) {
return NullableBinaryString.of(value);
} else {
return NullableBinaryString.empty();
}
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
var that = (NullableBinaryString) o;
return Objects.equals(value, that.value);
}
@Override
public int hashCode() {
return value == null ? 0 : value.hashCode();
}
@Override
public String toString() {
if (value == null) return "null";
return new String(value.data(), StandardCharsets.UTF_8);
}
}

View File

@ -0,0 +1,39 @@
package it.cavallium.datagen.nativedata;
import it.cavallium.datagen.DataSerializer;
import it.cavallium.stream.SafeDataInput;
import it.cavallium.stream.SafeDataOutput;
import org.jetbrains.annotations.NotNull;
import java.nio.charset.StandardCharsets;
public class NullableBinaryStringSerializer implements DataSerializer<NullableBinaryString> {
public static final NullableBinaryStringSerializer INSTANCE = new NullableBinaryStringSerializer();
@Override
public void serialize(SafeDataOutput dataOutput, @NotNull NullableBinaryString data) {
if (data.isEmpty()) {
dataOutput.writeBoolean(false);
} else {
dataOutput.writeBoolean(true);
BinaryString dataContent = data.get();
dataOutput.writeShort(dataContent.sizeBytes());
dataOutput.write(dataContent.data());
}
}
@NotNull
@Override
public NullableBinaryString deserialize(SafeDataInput dataInput) {
var isPresent = dataInput.readBoolean();
if (!isPresent) {
return NullableBinaryString.empty();
} else {
var size = dataInput.readUnsignedShort();
var data = new byte[size];
dataInput.readFully(data);
return NullableBinaryString.of(new BinaryString(data));
}
}
}