diff --git a/pom.xml b/pom.xml index 8b1b990..bd45dec 100644 --- a/pom.xml +++ b/pom.xml @@ -37,10 +37,17 @@ fastutil 8.2.2 + com.esotericsoftware kryo - 5.0.0-RC1 + 5.0.0-RC4 it.cavallium diff --git a/src/main/java/it/cavallium/strangedb/java/database/DatabaseDataInitializer.java b/src/main/java/it/cavallium/strangedb/java/database/DatabaseDataInitializer.java deleted file mode 100644 index 83f5498..0000000 --- a/src/main/java/it/cavallium/strangedb/java/database/DatabaseDataInitializer.java +++ /dev/null @@ -1,157 +0,0 @@ -package it.cavallium.strangedb.java.database; - -import it.cavallium.strangedb.java.annotations.*; -import org.apache.commons.lang3.reflect.FieldUtils; -import it.cavallium.strangedb.java.objects.EnhancedObject; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.util.LinkedHashMap; -import java.util.Map; - -public class DatabaseDataInitializer implements IDataInitializer { - - private final DatabaseObjectsIO objectsIO; - - public DatabaseDataInitializer(DatabaseObjectsIO objectsIO) { - this.objectsIO = objectsIO; - } - - @Override - public void initializeDbObject(EnhancedObject obj) throws IOException { - initializeDbObjectFields(obj); - initializeDbObjectPrimitiveFields(obj); - initializeDbObjectProperties(obj); - } - - private void initializeDbObjectFields(EnhancedObject obj) throws IOException { - // Declare the variables needed to getBlock the biggest field Id - Field[] unorderedFields = objectsIO.getFields(obj); - // Find the biggest field Id - int biggestFieldId = objectsIO.getBiggestFieldId(unorderedFields); - - // Allocate new UIDs - long[] fieldUIDs = objectsIO.allocateNewUIDs(biggestFieldId + 1); - - // Declare the other variables - Field[] fields = new Field[biggestFieldId + 1]; - DbDataType[] orderedFieldTypes = new DbDataType[biggestFieldId + 1]; - - // Load all fields metadata and load them - for (Field field : unorderedFields) { - DbField fieldAnnotation = field.getAnnotation(DbField.class); - int fieldId = fieldAnnotation.id(); - DbDataType fieldType = fieldAnnotation.type(); - objectsIO.loadField(obj, field, fieldType, fieldUIDs[fieldId]); - fields[fieldId] = field; - orderedFieldTypes[fieldId] = fieldType; - } - // Set fields metadata - obj.setFields(fields, orderedFieldTypes, fieldUIDs); - } - - private void initializeDbObjectPrimitiveFields(EnhancedObject obj) throws IOException { - // Declare the variables needed to getBlock the biggest field Id - Field[] unorderedFields = objectsIO.getPrimitiveFields(obj); - // Find the biggest field Id - int biggestFieldId = objectsIO.getBiggestPrimitiveFieldId(unorderedFields); - - // Allocate new UID - long fieldDataUID = objectsIO.newNullObject(); - - // Declare the other variables - Field[] fields = new Field[biggestFieldId + 1]; - DbPrimitiveType[] orderedFieldTypes = new DbPrimitiveType[biggestFieldId + 1]; - - // Load all fields metadata and load them - try { - for (Field field : unorderedFields) { - DbPrimitiveField fieldAnnotation = field.getAnnotation(DbPrimitiveField.class); - int fieldId = fieldAnnotation.id(); - DbPrimitiveType fieldType = fieldAnnotation.type(); - switch (fieldType) { - case BOOLEAN: - FieldUtils.writeField(field, obj, false, true); - break; - case BYTE: - FieldUtils.writeField(field, obj, (byte) 0, true); - break; - case CHAR: - FieldUtils.writeField(field, obj, (char) 0, true); - break; - case SHORT: - FieldUtils.writeField(field, obj, (short) 0, true); - break; - case INTEGER: - FieldUtils.writeField(field, obj, 0, true); - break; - case LONG: - FieldUtils.writeField(field, obj, (long) 0, true); - break; - case FLOAT: - FieldUtils.writeField(field, obj, (float) 0, true); - break; - case DOUBLE: - FieldUtils.writeField(field, obj, (double) 0, true); - break; - } - fields[fieldId] = field; - orderedFieldTypes[fieldId] = fieldType; - } - } catch (IllegalArgumentException | IllegalAccessException e) { - throw new IOException(e); - } - // Set fields metadata - obj.setPrimitiveFields(fields, orderedFieldTypes, fieldDataUID); - } - - private void initializeDbObjectProperties(EnhancedObject obj) throws IOException { - // Declare the variables needed to getBlock the biggest property Id - Method[] unorderedPropertyGetters = obj.getPropertyGetters(); - Method[] unorderedPropertySetters = obj.getPropertySetters(); - - // Find the biggest property Id - int biggestGetter = objectsIO.getBiggestPropertyGetterId(unorderedPropertyGetters); - int biggestSetter = objectsIO.getBiggestPropertySetterId(unorderedPropertySetters); - int biggestPropertyId = biggestGetter > biggestSetter ? biggestGetter : biggestSetter; - - // Allocate new UIDs - long[] propertyUIDs = objectsIO.allocateNewUIDs(biggestPropertyId + 1); - - for (Method property : unorderedPropertySetters) { - DbProperty fieldAnnotation = property.getAnnotation(DbProperty.class); - int propertyId = fieldAnnotation.id(); - if (propertyId > biggestPropertyId) { - biggestPropertyId = propertyId; - } - } - - // Declare the other variables - DbDataType[] propertyTypes = new DbDataType[biggestPropertyId + 1]; - Method[] propertyGetters = new Method[biggestPropertyId + 1]; - Method[] propertySetters = new Method[biggestPropertyId + 1]; - Map setterMethods = new LinkedHashMap<>(); - Map getterMethods = new LinkedHashMap<>(); - - // Load the properties metadata - for (Method property : unorderedPropertyGetters) { - DbProperty propertyAnnotation = property.getAnnotation(DbProperty.class); - int propertyId = propertyAnnotation.id(); - DbDataType propertyType = propertyAnnotation.type(); - propertyTypes[propertyId] = propertyType; - propertyGetters[propertyId] = property; - getterMethods.put(property.getName(), propertyAnnotation); - } - for (Method property : unorderedPropertySetters) { - DbProperty propertyAnnotation = property.getAnnotation(DbProperty.class); - int propertyId = propertyAnnotation.id(); - DbDataType propertyType = propertyAnnotation.type(); - propertyTypes[propertyId] = propertyType; - propertySetters[propertyId] = property; - setterMethods.put(property.getName(), propertyAnnotation); - } - // Set properties metadata - obj.setProperties(propertyGetters, propertySetters, propertyTypes, propertyUIDs, setterMethods, getterMethods); - } -} diff --git a/src/main/java/it/cavallium/strangedb/java/database/DatabaseJava.java b/src/main/java/it/cavallium/strangedb/java/database/DatabaseJava.java index f434515..7d429fa 100644 --- a/src/main/java/it/cavallium/strangedb/java/database/DatabaseJava.java +++ b/src/main/java/it/cavallium/strangedb/java/database/DatabaseJava.java @@ -5,12 +5,7 @@ import it.cavallium.strangedb.database.references.ReferenceInfo; import it.cavallium.strangedb.java.objects.EnhancedObject; import it.cavallium.strangedb.functionalinterfaces.FunctionWithIO; import it.cavallium.strangedb.java.objects.EnhancedObjectIndices; -import it.unimi.dsi.fastutil.ints.Int2ObjectLinkedOpenHashMap; -import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.longs.LongArrayList; -import it.unimi.dsi.fastutil.longs.LongArraySet; -import it.unimi.dsi.fastutil.longs.LongSet; -import it.unimi.dsi.fastutil.objects.ObjectIterator; import java.io.IOException; import java.nio.ByteBuffer; @@ -66,6 +61,26 @@ public class DatabaseJava extends DatabaseCore implements IDatabaseTools { return root; } + public T loadRoot(FunctionWithIO ifAbsent, Class forcedClassType) throws IOException { + if (hasLoadedRootObject) { + throw new RuntimeException("Root already set!"); + } + T root; + if (referencesMetadata.getFirstFreeReference() > 0) { + root = objectsIO.loadEnhancedObject(0, forcedClassType); + } else { + if (objectsIO.newNullObject() != 0) { + throw new IOException("Can't allocate root!"); + } else { + root = ifAbsent.apply(DatabaseJava.this); + objectsIO.setEnhancedObject(0, root); + } + } + loadedRootObject = root; + hasLoadedRootObject = true; + return root; + } + @Override public void closeAndClean() throws IOException { if (!this.closed) { @@ -97,7 +112,7 @@ public class DatabaseJava extends DatabaseCore implements IDatabaseTools { for (int referenceID = 0; referenceID < firstFreeReference; referenceID++) { try { - ReferenceInfo ref = databaseToClean.referencesMetadata.getReference(referenceID); + ReferenceInfo ref = databaseToClean.referencesMetadata.getCleanReference(referenceID); if (!NONEXISTENT_REFERENCE_INFO.equals(ref)) { referencesCount++; if (idsToKeep.contains(referenceID)) { @@ -123,7 +138,7 @@ public class DatabaseJava extends DatabaseCore implements IDatabaseTools { private void cleanRef(DatabaseJava db, LongArrayList idsToKeep, long ref) throws IOException { idsToKeep.add(ref); - ReferenceInfo refInfo = db.referencesMetadata.getReference(ref); + ReferenceInfo refInfo = db.referencesMetadata.getCleanReference(ref); if (!NONEXISTENT_REFERENCE_INFO.equals(refInfo)) { switch (refInfo.getCleanerId()) { case ENHANCED_OBJECT_METADATA_CLEANER: { diff --git a/src/main/java/it/cavallium/strangedb/java/database/DatabaseObjectsIO.java b/src/main/java/it/cavallium/strangedb/java/database/DatabaseObjectsIO.java index e630ffa..52c1620 100644 --- a/src/main/java/it/cavallium/strangedb/java/database/DatabaseObjectsIO.java +++ b/src/main/java/it/cavallium/strangedb/java/database/DatabaseObjectsIO.java @@ -1,8 +1,5 @@ package it.cavallium.strangedb.java.database; -import com.esotericsoftware.kryo.Kryo; -import com.esotericsoftware.kryo.io.Input; -import com.esotericsoftware.kryo.io.Output; import it.cavallium.strangedb.database.references.DatabaseReferencesIO; import it.cavallium.strangedb.java.annotations.*; import it.cavallium.strangedb.java.objects.EnhancedObject; @@ -20,15 +17,12 @@ import it.unimi.dsi.fastutil.longs.LongList; import it.unimi.dsi.fastutil.shorts.ShortArrayList; import org.apache.commons.lang3.reflect.FieldUtils; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.util.*; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import static it.cavallium.strangedb.database.references.DatabaseReferencesMetadata.BLANK_DATA_CLEANER; @@ -40,16 +34,14 @@ public class DatabaseObjectsIO implements IObjectsIO { private final IDatabaseTools databaseTools; private final DatabaseReferencesIO referencesIO; - private final Lock lock = new ReentrantLock(); private final DatabaseDataInitializer dataInitializer; - - private Kryo kryo = new Kryo(); + private final KryoSerializer serializer; public DatabaseObjectsIO(IDatabaseTools databaseTools, DatabaseReferencesIO referencesIO) { this.databaseTools = databaseTools; this.referencesIO = referencesIO; - this.dataInitializer = new DatabaseDataInitializer(this); - kryo.setRegistrationRequired(false); + this.serializer = new KryoSerializer(); + this.dataInitializer = new DatabaseDataInitializer(); int id = -90; registerClass(boolean[].class, id++); registerClass(byte[].class, id++); @@ -111,16 +103,19 @@ public class DatabaseObjectsIO implements IObjectsIO { @Override public T loadEnhancedObject(long reference) throws IOException { - lock.lock(); - try { - return loadEnhancedObject_(reference); - } finally { - lock.unlock(); - } + return loadEnhancedObject_(reference, null); + } + + @Override + public T loadEnhancedObject(long reference, Class forcedType) throws IOException { + if (forcedType == null) { + throw new NullPointerException("The class is null!"); + } + return loadEnhancedObject_(reference, forcedType); } @SuppressWarnings("unchecked") - private T loadEnhancedObject_(long reference) throws IOException { + private T loadEnhancedObject_(long reference, Class forcedType) throws IOException { ByteBuffer buffer = referencesIO.readFromReference(reference); if (buffer.limit() == 0) { return null; @@ -128,12 +123,17 @@ public class DatabaseObjectsIO implements IObjectsIO { int serializedVersion = Byte.toUnsignedInt(buffer.get()) - 5; if (serializedVersion < 0) { System.err.println("PLEASE UPGRADE THE DATABASE"); - throw new IllegalAccessError("PLEASE UPGRADE THE DATABASE"); + throw new IOException("PLEASE UPGRADE THE DATABASE"); } int serializedClassLength = Byte.toUnsignedInt(buffer.get()); byte[] serializedClassData = new byte[serializedClassLength]; buffer.get(serializedClassData); - Class objectType = kryo.readClass(new Input(serializedClassData)).getType(); + Class objectType; + try { + objectType = forcedType != null ? forcedType : serializer.readClassBytes(serializedClassData); + } catch (Exception | Error ex) { + throw new IOException(ex); + } int fieldsCount = buffer.getInt(); int methodsCount = buffer.getInt(); long[] fieldRefs = new long[fieldsCount]; @@ -152,8 +152,6 @@ public class DatabaseObjectsIO implements IObjectsIO { @SuppressWarnings("unchecked") @Override public EnhancedObjectIndices loadEnhancedObjectUids(long reference) throws IOException { - lock.lock(); - try { ByteBuffer buffer = referencesIO.readFromReference(reference); if (buffer.limit() == 0) { return null; @@ -163,7 +161,12 @@ public class DatabaseObjectsIO implements IObjectsIO { int serializedClassLength = Byte.toUnsignedInt(buffer.get()); byte[] serializedClassData = new byte[serializedClassLength]; buffer.get(serializedClassData); - Class objectType = (Class) kryo.readClass(new Input(serializedClassData)).getType(); + Class objectType; + try { + objectType = serializer.readClassBytes(serializedClassData); + } catch (Exception | Error ex) { + throw new IOException(ex); + } int fieldsCount = buffer.getInt(); int methodsCount = buffer.getInt(); long[] fieldRefs = new long[fieldsCount]; @@ -176,25 +179,17 @@ public class DatabaseObjectsIO implements IObjectsIO { } long nativeFieldsDataReference = buffer.getLong(); return new EnhancedObjectIndices(objectType, reference, fieldRefs, methodRefs, nativeFieldsDataReference); - } finally { - lock.unlock(); - } } public Object loadData(DbDataType propertyType, long dataReference) throws IOException { - lock.lock(); - try { - return loadData_(propertyType, dataReference); - } finally { - lock.unlock(); - } + return loadData_(propertyType, dataReference); } private Object loadData_(DbDataType propertyType, long dataReference) throws IOException { switch (propertyType) { case ENHANCED_OBJECT: - return loadEnhancedObject_(dataReference); + return loadEnhancedObject_(dataReference, null); case OBJECT: return loadObject_(dataReference); case REFERENCES_LIST: @@ -265,12 +260,7 @@ public class DatabaseObjectsIO implements IObjectsIO { @Override public void setEnhancedObject(long reference, T value) throws IOException { - lock.lock(); - try { - setEnhancedObject_(reference, value); - } finally { - lock.unlock(); - } + setEnhancedObject_(reference, value); } private void setEnhancedObject_(long reference, T value) throws IOException { @@ -291,9 +281,7 @@ public class DatabaseObjectsIO implements IObjectsIO { final long[] propertyReferences = objectFullInfo.getPropertyReferences(); final DbDataType[] propertyTypes = objectFullInfo.getPropertyTypes(); final Object[] propertyValues = objectFullInfo.getLoadedPropertyValues(); - Output serializedClassDataStream = new Output(1024, 8192); - kryo.writeClass(serializedClassDataStream, value.getClass()); - byte[] serializedClassData = serializedClassDataStream.toBytes(); + byte[] serializedClassData = serializer.writeClassBytes(value.getClass()); final int totalSize = Byte.BYTES + serializedClassData.length + Byte.BYTES + Integer.BYTES * 2 + fieldReferences.length * Long.BYTES + propertyReferences.length * Long.BYTES + Long.BYTES; ByteBuffer buffer = ByteBuffer.allocate(totalSize); @@ -340,12 +328,7 @@ public class DatabaseObjectsIO implements IObjectsIO { @Override public T loadObject(long reference) throws IOException { - lock.lock(); - try { - return loadObject_(reference); - } finally { - lock.unlock(); - } + return loadObject_(reference); } @SuppressWarnings("unchecked") @@ -356,17 +339,16 @@ public class DatabaseObjectsIO implements IObjectsIO { } buffer.rewind(); byte[] data = buffer.array(); - return (T) kryo.readClassAndObject(new Input(data)); + try { + return serializer.readClassAndObjectBytes(data); + } catch (Exception | Error ex) { + throw new IOException(ex); + } } @Override public LongArrayList loadReferencesList(long reference) throws IOException { - lock.lock(); - try { - return loadReferencesList_(reference); - } finally { - lock.unlock(); - } + return loadReferencesList_(reference); } private LongArrayList loadReferencesList_(long reference) throws IOException { @@ -384,12 +366,7 @@ public class DatabaseObjectsIO implements IObjectsIO { @Override public LongList loadPrimitiveData(long reference) throws IOException { - lock.lock(); - try { - return loadPrimitiveData_(reference); - } finally { - lock.unlock(); - } + return loadPrimitiveData_(reference); } private LongList loadPrimitiveData_(long reference) throws IOException { @@ -407,21 +384,12 @@ public class DatabaseObjectsIO implements IObjectsIO { @Override public void setObject(long reference, T value) throws IOException { - lock.lock(); - try { - setObject_(reference, value); - } finally { - lock.unlock(); - } + setObject_(reference, value); } private void setObject_(long reference, T value) throws IOException { if (value != null) { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - Output output = new Output(outputStream); - kryo.writeClassAndObject(output, value); - output.flush(); - byte[] data = outputStream.toByteArray(); + byte[] data = serializer.writeClassAndObjectBytes(value); ByteBuffer dataByteBuffer = ByteBuffer.wrap(data); referencesIO.writeToReference(reference, BLANK_DATA_CLEANER, data.length, dataByteBuffer); } else { @@ -431,12 +399,7 @@ public class DatabaseObjectsIO implements IObjectsIO { @Override public void setReferencesList(long reference, LongArrayList value) throws IOException { - lock.lock(); - try { - setReferencesList_(reference, value); - } finally { - lock.unlock(); - } + setReferencesList_(reference, value); } private void setReferencesList_(long reference, LongArrayList value) throws IOException { @@ -456,12 +419,7 @@ public class DatabaseObjectsIO implements IObjectsIO { @Override public long newNullObject() throws IOException { - lock.lock(); - try { - return newNullObject_(); - } finally { - lock.unlock(); - } + return newNullObject_(); } private long newNullObject_() throws IOException { @@ -480,7 +438,9 @@ public class DatabaseObjectsIO implements IObjectsIO { if (id < -100) { throw new IllegalArgumentException(); } - kryo.register(type, 100 + id); + final int realId = id + 100; + + serializer.registerClass(type, realId); } private void preloadEnhancedObjectProperties(T obj, long[] propertyReferences) { @@ -531,12 +491,7 @@ public class DatabaseObjectsIO implements IObjectsIO { } public int getBiggestPropertyGetterId(Method[] unorderedPropertyGetters) { - lock.lock(); - try { - return getBiggestPropertyGetterId_(unorderedPropertyGetters); - } finally { - lock.unlock(); - } + return getBiggestPropertyGetterId_(unorderedPropertyGetters); } private int getBiggestPropertyGetterId_(Method[] unorderedPropertyGetters) { @@ -552,12 +507,7 @@ public class DatabaseObjectsIO implements IObjectsIO { } public int getBiggestPropertySetterId(Method[] unorderedPropertySetters) { - lock.lock(); - try { - return getBiggestPropertySetterId_(unorderedPropertySetters); - } finally { - lock.unlock(); - } + return getBiggestPropertySetterId_(unorderedPropertySetters); } private int getBiggestPropertySetterId_(Method[] unorderedPropertySetters) { @@ -601,31 +551,33 @@ public class DatabaseObjectsIO implements IObjectsIO { Field field = fields[id]; DbPrimitiveType type = orderedFieldTypes[id]; - switch (type) { - case BOOLEAN: - FieldUtils.writeField(field, obj, (boolean) (buffer.getLong() % 2 == 1), true); - break; - case BYTE: - FieldUtils.writeField(field, obj, (byte) buffer.getLong(), true); - break; - case SHORT: - FieldUtils.writeField(field, obj, (short) buffer.getLong(), true); - break; - case CHAR: - FieldUtils.writeField(field, obj, (char) buffer.getLong(), true); - break; - case INTEGER: - FieldUtils.writeField(field, obj, (int) buffer.getLong(), true); - break; - case LONG: - FieldUtils.writeField(field, obj, (long) buffer.getLong(), true); - break; - case FLOAT: - FieldUtils.writeField(field, obj, Float.intBitsToFloat((int) buffer.getLong()), true); - break; - case DOUBLE: - FieldUtils.writeField(field, obj, Double.longBitsToDouble(buffer.getLong()), true); - break; + if (field != null) { + switch (type) { + case BOOLEAN: + FieldUtils.writeField(field, obj, (boolean) (buffer.getLong() % 2 == 1), true); + break; + case BYTE: + FieldUtils.writeField(field, obj, (byte) buffer.getLong(), true); + break; + case SHORT: + FieldUtils.writeField(field, obj, (short) buffer.getLong(), true); + break; + case CHAR: + FieldUtils.writeField(field, obj, (char) buffer.getLong(), true); + break; + case INTEGER: + FieldUtils.writeField(field, obj, (int) buffer.getLong(), true); + break; + case LONG: + FieldUtils.writeField(field, obj, (long) buffer.getLong(), true); + break; + case FLOAT: + FieldUtils.writeField(field, obj, Float.intBitsToFloat((int) buffer.getLong()), true); + break; + case DOUBLE: + FieldUtils.writeField(field, obj, Double.longBitsToDouble(buffer.getLong()), true); + break; + } } } } catch (IllegalArgumentException | IllegalAccessException e) { @@ -662,12 +614,7 @@ public class DatabaseObjectsIO implements IObjectsIO { public void loadField(T obj, Field field, DbDataType fieldType, long fieldReference) throws IOException { - lock.lock(); - try { - loadField_(obj, field, fieldType, fieldReference); - } finally { - lock.unlock(); - } + loadField_(obj, field, fieldType, fieldReference); } private void loadField_(T obj, Field field, DbDataType fieldType, long fieldReference) @@ -687,12 +634,7 @@ public class DatabaseObjectsIO implements IObjectsIO { } public Field[] getFields(T obj) { - lock.lock(); - try { - return getFields_(obj); - } finally { - lock.unlock(); - } + return getFields_(obj); } private Field[] getFields_(T obj) { @@ -700,12 +642,7 @@ public class DatabaseObjectsIO implements IObjectsIO { } public Field[] getPrimitiveFields(T obj) { - lock.lock(); - try { - return getPrimitiveFields_(obj); - } finally { - lock.unlock(); - } + return getPrimitiveFields_(obj); } private Field[] getPrimitiveFields_(T obj) { @@ -713,12 +650,7 @@ public class DatabaseObjectsIO implements IObjectsIO { } public int getBiggestFieldId(Field[] unorderedFields) { - lock.lock(); - try { - return getBiggestFieldId_(unorderedFields); - } finally { - lock.unlock(); - } + return getBiggestFieldId_(unorderedFields); } private int getBiggestFieldId_(Field[] unorderedFields) { @@ -734,12 +666,7 @@ public class DatabaseObjectsIO implements IObjectsIO { } public int getBiggestPrimitiveFieldId(Field[] unorderedFields) { - lock.lock(); - try { - return getBiggestPrimitiveFieldId_(unorderedFields); - } finally { - lock.unlock(); - } + return getBiggestPrimitiveFieldId_(unorderedFields); } private int getBiggestPrimitiveFieldId_(Field[] unorderedFields) { @@ -754,9 +681,10 @@ public class DatabaseObjectsIO implements IObjectsIO { return biggestFieldId; } - private T toInstance(Class type) throws IOException { + @SuppressWarnings("unchecked") + private T toInstance(Class type) throws IOException { try { - T obj = type.getConstructor().newInstance(); + T obj = (T) type.getConstructor().newInstance(); obj.setDatabaseTools(databaseTools); return obj; } catch (NoSuchMethodException e) { @@ -767,7 +695,7 @@ public class DatabaseObjectsIO implements IObjectsIO { } } - private T preloadEnhancedObject(Class objectType, int serializedVersion, long nativeFieldsRef, long[] fieldRefs, long[] methodRefs) throws IOException { + private T preloadEnhancedObject(Class objectType, int serializedVersion, long nativeFieldsRef, long[] fieldRefs, long[] methodRefs) throws IOException { // Instantiate the class to an object T obj = toInstance(objectType); @@ -777,22 +705,26 @@ public class DatabaseObjectsIO implements IObjectsIO { if (dbClass != null) { classVersion = dbClass.version(); } - if (classVersion > serializedVersion) { - DatabaseEnhancedObjectUpgrader enhancedObjectUpgrader = new DatabaseEnhancedObjectUpgrader(this, fieldRefs, methodRefs, nativeFieldsRef); - dataInitializer.initializeDbObject(obj); - obj.onUpgrade(serializedVersion, enhancedObjectUpgrader); - } else if (classVersion < serializedVersion) { - throw new IllegalStateException( - "The serialized class is more recent than the current version of that class!"); - } else { + if (classVersion == serializedVersion) { preloadEnhancedObjectPrimitiveFields(obj, nativeFieldsRef); preloadEnhancedObjectFields(obj, fieldRefs); preloadEnhancedObjectProperties(obj, methodRefs); + } else if (classVersion > serializedVersion) { + DatabaseEnhancedObjectUpgrader enhancedObjectUpgrader = new DatabaseEnhancedObjectUpgrader(this, fieldRefs, methodRefs, nativeFieldsRef); + dataInitializer.initializeDbObject_(obj); + obj.onUpgrade(serializedVersion, enhancedObjectUpgrader); + } else { + throw new IllegalStateException( + "The serialized class is more recent than the current version of that class!"); } return obj; } public long[] allocateNewUIDs(int quantity) throws IOException { + return allocateNewUIDs_(quantity); + } + + private long[] allocateNewUIDs_(int quantity) throws IOException { long[] ids = new long[quantity]; for (int i = 0; i < quantity; i++) { ids[i] = newNullObject_(); @@ -805,4 +737,151 @@ public class DatabaseObjectsIO implements IObjectsIO { return dataInitializer; } + private class DatabaseDataInitializer implements IDataInitializer { + + public DatabaseDataInitializer() { + } + + @Override + public void initializeDbObject(EnhancedObject obj) throws IOException { + initializeDbObject_(obj); + } + + private void initializeDbObject_(EnhancedObject obj) throws IOException { + initializeDbObjectFields(obj); + initializeDbObjectPrimitiveFields(obj); + initializeDbObjectProperties(obj); + } + + private void initializeDbObjectFields(EnhancedObject obj) throws IOException { + // Declare the variables needed to getBlock the biggest field Id + Field[] unorderedFields = getFields_(obj); + // Find the biggest field Id + int biggestFieldId = getBiggestFieldId_(unorderedFields); + + // Allocate new UIDs + long[] fieldUIDs = allocateNewUIDs_(biggestFieldId + 1); + + // Declare the other variables + Field[] fields = new Field[biggestFieldId + 1]; + DbDataType[] orderedFieldTypes = new DbDataType[biggestFieldId + 1]; + + // Load all fields metadata and load them + for (Field field : unorderedFields) { + DbField fieldAnnotation = field.getAnnotation(DbField.class); + int fieldId = fieldAnnotation.id(); + DbDataType fieldType = fieldAnnotation.type(); + loadField_(obj, field, fieldType, fieldUIDs[fieldId]); + fields[fieldId] = field; + orderedFieldTypes[fieldId] = fieldType; + } + // Set fields metadata + obj.setFields(fields, orderedFieldTypes, fieldUIDs); + } + + private void initializeDbObjectPrimitiveFields(EnhancedObject obj) throws IOException { + // Declare the variables needed to getBlock the biggest field Id + Field[] unorderedFields = getPrimitiveFields_(obj); + // Find the biggest field Id + int biggestFieldId = getBiggestPrimitiveFieldId_(unorderedFields); + + // Allocate new UID + long fieldDataUID = newNullObject_(); + + // Declare the other variables + Field[] fields = new Field[biggestFieldId + 1]; + DbPrimitiveType[] orderedFieldTypes = new DbPrimitiveType[biggestFieldId + 1]; + + // Load all fields metadata and load them + try { + for (Field field : unorderedFields) { + DbPrimitiveField fieldAnnotation = field.getAnnotation(DbPrimitiveField.class); + int fieldId = fieldAnnotation.id(); + DbPrimitiveType fieldType = fieldAnnotation.type(); + switch (fieldType) { + case BOOLEAN: + FieldUtils.writeField(field, obj, false, true); + break; + case BYTE: + FieldUtils.writeField(field, obj, (byte) 0, true); + break; + case CHAR: + FieldUtils.writeField(field, obj, (char) 0, true); + break; + case SHORT: + FieldUtils.writeField(field, obj, (short) 0, true); + break; + case INTEGER: + FieldUtils.writeField(field, obj, 0, true); + break; + case LONG: + FieldUtils.writeField(field, obj, (long) 0, true); + break; + case FLOAT: + FieldUtils.writeField(field, obj, (float) 0, true); + break; + case DOUBLE: + FieldUtils.writeField(field, obj, (double) 0, true); + break; + } + fields[fieldId] = field; + orderedFieldTypes[fieldId] = fieldType; + } + } catch (IllegalArgumentException | IllegalAccessException e) { + throw new IOException(e); + } + // Set fields metadata + obj.setPrimitiveFields(fields, orderedFieldTypes, fieldDataUID); + } + + private void initializeDbObjectProperties(EnhancedObject obj) throws IOException { + // Declare the variables needed to getBlock the biggest property Id + Method[] unorderedPropertyGetters = obj.getPropertyGetters(); + Method[] unorderedPropertySetters = obj.getPropertySetters(); + + // Find the biggest property Id + int biggestGetter = getBiggestPropertyGetterId_(unorderedPropertyGetters); + int biggestSetter = getBiggestPropertySetterId_(unorderedPropertySetters); + int biggestPropertyId = biggestGetter > biggestSetter ? biggestGetter : biggestSetter; + + // Allocate new UIDs + long[] propertyUIDs = allocateNewUIDs_(biggestPropertyId + 1); + + for (Method property : unorderedPropertySetters) { + DbProperty fieldAnnotation = property.getAnnotation(DbProperty.class); + int propertyId = fieldAnnotation.id(); + if (propertyId > biggestPropertyId) { + biggestPropertyId = propertyId; + } + } + + // Declare the other variables + DbDataType[] propertyTypes = new DbDataType[biggestPropertyId + 1]; + Method[] propertyGetters = new Method[biggestPropertyId + 1]; + Method[] propertySetters = new Method[biggestPropertyId + 1]; + Map setterMethods = new LinkedHashMap<>(); + Map getterMethods = new LinkedHashMap<>(); + + // Load the properties metadata + for (Method property : unorderedPropertyGetters) { + DbProperty propertyAnnotation = property.getAnnotation(DbProperty.class); + int propertyId = propertyAnnotation.id(); + DbDataType propertyType = propertyAnnotation.type(); + propertyTypes[propertyId] = propertyType; + propertyGetters[propertyId] = property; + getterMethods.put(property.getName(), propertyAnnotation); + } + for (Method property : unorderedPropertySetters) { + DbProperty propertyAnnotation = property.getAnnotation(DbProperty.class); + int propertyId = propertyAnnotation.id(); + DbDataType propertyType = propertyAnnotation.type(); + propertyTypes[propertyId] = propertyType; + propertySetters[propertyId] = property; + setterMethods.put(property.getName(), propertyAnnotation); + } + // Set properties metadata + obj.setProperties(propertyGetters, propertySetters, propertyTypes, propertyUIDs, setterMethods, getterMethods); + } + } + } diff --git a/src/main/java/it/cavallium/strangedb/java/database/FSTSerializer.java b/src/main/java/it/cavallium/strangedb/java/database/FSTSerializer.java new file mode 100644 index 0000000..d23e023 --- /dev/null +++ b/src/main/java/it/cavallium/strangedb/java/database/FSTSerializer.java @@ -0,0 +1,119 @@ +/** +package it.cavallium.strangedb.java.database; + +import org.nustaq.serialization.FSTConfiguration; +import org.nustaq.serialization.FSTObjectInput; +import org.nustaq.serialization.FSTObjectOutput; + +import java.io.*; + +public class FSTSerializer implements ISerializer { + + private FSTConfiguration conf = FSTConfiguration.createDefaultConfiguration(); + + public FSTSerializer() { + conf.setStructMode(true); + conf.setClassLoader(ClassLoader.getSystemClassLoader()); + conf.setCrossPlatform(false); + conf.setPreferSpeed(true); + } + + @SuppressWarnings("unchecked") + public T read(InputStream stream, Class type) throws IOException { + try { + FSTObjectInput in = conf.getObjectInput(stream); + T result = (T) in.readObject(type); + // DON'T: in.close(); here prevents reuse and will result in an exception + stream.close(); + return result; + } catch (Exception e) { + throw new IOException(e); + } + } + + public T readBytes(byte[] input, Class type) throws IOException { + return read(new ByteArrayInputStream(input), type); + } + + @SuppressWarnings("unchecked") + public Class readClass(InputStream stream) throws IOException { + try { + FSTObjectInput in = conf.getObjectInput(stream); + Class result = (Class) in.readObject(Class.class); + // DON'T: in.close(); here prevents reuse and will result in an exception + stream.close(); + return result; + } catch (Exception e) { + throw new IOException(e); + } + } + + @Override + public Class readClassBytes(byte[] input) throws IOException { + return readClass(new ByteArrayInputStream(input)); + } + + public void write(OutputStream stream, T toWrite, Class type) throws IOException { + FSTObjectOutput out = conf.getObjectOutput(stream); + out.writeObject(toWrite, type); + // DON'T out.close() when using factory method; + out.flush(); + stream.close(); + } + + public byte[] writeBytes(T toWrite, Class type) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + write(baos, toWrite, type); + return baos.toByteArray(); + } + + public void writeClass(OutputStream stream, Class toWrite) throws IOException { + FSTObjectOutput out = conf.getObjectOutput(stream); + out.writeObject(toWrite, Class.class); + // DON'T out.close() when using factory method; + out.flush(); + stream.close(); + } + + @Override + public byte[] writeClassBytes(Class toWrite) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + writeClass(baos, toWrite); + return baos.toByteArray(); + } + + public void registerClass(Class type, int id) { + conf.registerClass(type); + } + + @Override + public byte[] writeClassAndObjectBytes(T value) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + FSTObjectOutput out = conf.getObjectOutput(baos); + out.writeObject(value); + // DON'T out.close() when using factory method; + out.flush(); + baos.close(); + + return baos.toByteArray(); + } + + @Override + @SuppressWarnings("unchecked") + public T readClassAndObjectBytes(byte[] input) throws IOException { + try { + ByteArrayInputStream bais = new ByteArrayInputStream(input); + FSTObjectInput in = conf.getObjectInput(bais); + T result; + result = (T) in.readObject(); + // DON'T: in.close(); here prevents reuse and will result in an exception + bais.close(); + return result; + } catch (Exception e) { + throw new IOException(e); + } + } +} + + **/ \ No newline at end of file diff --git a/src/main/java/it/cavallium/strangedb/java/database/IObjectsIO.java b/src/main/java/it/cavallium/strangedb/java/database/IObjectsIO.java index 542a1a8..a60494a 100644 --- a/src/main/java/it/cavallium/strangedb/java/database/IObjectsIO.java +++ b/src/main/java/it/cavallium/strangedb/java/database/IObjectsIO.java @@ -13,6 +13,9 @@ public interface IObjectsIO { @SuppressWarnings("unchecked") T loadEnhancedObject(long reference) throws IOException; + @SuppressWarnings("unchecked") + T loadEnhancedObject(long reference, Class forcedType) throws IOException; + EnhancedObjectIndices loadEnhancedObjectUids(long reference) throws IOException; T loadObject(long reference) throws IOException; diff --git a/src/main/java/it/cavallium/strangedb/java/database/ISerializer.java b/src/main/java/it/cavallium/strangedb/java/database/ISerializer.java new file mode 100644 index 0000000..1fb1331 --- /dev/null +++ b/src/main/java/it/cavallium/strangedb/java/database/ISerializer.java @@ -0,0 +1,16 @@ +package it.cavallium.strangedb.java.database; + +import java.io.IOException; + +public interface ISerializer { + Class readClassBytes(byte[] input) throws IOException; + + byte[] writeClassBytes(Class toWrite) throws IOException; + + byte[] writeClassAndObjectBytes(T value) throws IOException; + + @SuppressWarnings("unchecked") + T readClassAndObjectBytes(byte[] input) throws IOException; + + void registerClass(Class type, int id); +} diff --git a/src/main/java/it/cavallium/strangedb/java/database/KryoSerializer.java b/src/main/java/it/cavallium/strangedb/java/database/KryoSerializer.java new file mode 100644 index 0000000..61102f3 --- /dev/null +++ b/src/main/java/it/cavallium/strangedb/java/database/KryoSerializer.java @@ -0,0 +1,97 @@ +package it.cavallium.strangedb.java.database; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.ByteBufferOutput; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.KryoDataOutput; +import com.esotericsoftware.kryo.io.Output; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; + +public class KryoSerializer implements ISerializer { + + private static final int KRYO_INSTANCES = 20; + + private ReentrantLock[] locks = new ReentrantLock[KRYO_INSTANCES]; + private final Kryo[] kryo = new Kryo[KRYO_INSTANCES]; + private AtomicInteger current = new AtomicInteger(0); + + public KryoSerializer() { + for (int i = 0; i < KRYO_INSTANCES; i++) { + locks[i] = new ReentrantLock(false); + kryo[i] = new Kryo(); + kryo[i].setRegistrationRequired(false); + kryo[i].setWarnUnregisteredClasses(true); + } + } + + @SuppressWarnings("unchecked") + @Override + public Class readClassBytes(byte[] input) { + int i = current.getAndUpdate((currentInt) -> currentInt + 1 == KRYO_INSTANCES ? 0 : currentInt + 1); + locks[i].lock(); + try { + return (Class) kryo[i].readClass(new Input(input)).getType(); + } finally { + locks[i].unlock(); + } + } + + @Override + public byte[] writeClassBytes(Class value) { + int i = current.getAndUpdate((currentInt) -> currentInt + 1 == KRYO_INSTANCES ? 0 : currentInt + 1); + locks[i].lock(); + try { + Output out = new Output(1024, Integer.MAX_VALUE); + kryo[i].writeClass(out, value); + out.flush(); + out.close(); + return out.toBytes(); + } finally { + locks[i].unlock(); + } + } + + @Override + public byte[] writeClassAndObjectBytes(T value) { + int i = current.getAndUpdate((currentInt) -> currentInt + 1 == KRYO_INSTANCES ? 0 : currentInt + 1); + locks[i].lock(); + try { + Output out = new Output(1024, Integer.MAX_VALUE); + kryo[i].writeClassAndObject(out, value); + out.flush(); + out.close(); + return out.toBytes(); + } finally { + locks[i].unlock(); + } + } + + @SuppressWarnings("unchecked") + @Override + public T readClassAndObjectBytes(byte[] input) { + int i = current.getAndUpdate((currentInt) -> currentInt + 1 == KRYO_INSTANCES ? 0 : currentInt + 1); + locks[i].lock(); + try { + return (T) kryo[i].readClassAndObject(new Input(input)); + } finally { + locks[i].unlock(); + } + } + + @Override + public void registerClass(Class type, int id) { + for (int i = 0; i < KRYO_INSTANCES; i++) { + locks[i].lock(); + try { + kryo[i].register(type, id); + } finally { + locks[i].unlock(); + } + } + } +} diff --git a/src/main/java/it/cavallium/strangedb/java/objects/lists/ElementsArrayList.java b/src/main/java/it/cavallium/strangedb/java/objects/lists/ElementsArrayList.java index afd22c7..0ce7c9f 100644 --- a/src/main/java/it/cavallium/strangedb/java/objects/lists/ElementsArrayList.java +++ b/src/main/java/it/cavallium/strangedb/java/objects/lists/ElementsArrayList.java @@ -1,11 +1,18 @@ package it.cavallium.strangedb.java.objects.lists; +import it.cavallium.strangedb.functionalinterfaces.ConsumerWithIO; + +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.concurrent.CompletionException; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class ElementsArrayList implements ElementsList { + ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(false); + private final ArrayList list; public ElementsArrayList() { @@ -25,9 +32,34 @@ public class ElementsArrayList implements ElementsList { return list.get(index); } + @Override + public void forEachParallelUnsorted(ConsumerWithIO action) throws IOException { + readWriteLock.readLock().lock(); + try { + try { + list.parallelStream().forEach((item) -> { + try { + action.accept(item); + } catch (IOException ex) { + throw new CompletionException(ex); + } + }); + } catch (CompletionException ex) { + throw new IOException(ex.getCause()); + } + } finally { + readWriteLock.readLock().unlock(); + } + } + @Override public void add(T value) { - list.add(value); + readWriteLock.writeLock().lock(); + try { + list.add(value); + } finally { + readWriteLock.writeLock().unlock(); + } } @Override @@ -37,34 +69,69 @@ public class ElementsArrayList implements ElementsList { @Override public void set(int index, T value) { - list.set(index, value); + readWriteLock.writeLock().lock(); + try { + list.set(index, value); + } finally { + readWriteLock.writeLock().unlock(); + } } @Override public void add(int index, T value) { - list.add(index, value); + readWriteLock.writeLock().lock(); + try { + list.add(index, value); + } finally { + readWriteLock.writeLock().unlock(); + } } @Override public T getLast() { - return list.get(list.size() - 1); + readWriteLock.readLock().lock(); + try { + return list.get(list.size() - 1); + } finally { + readWriteLock.readLock().unlock(); + } } @Override public boolean isEmpty() { - return list.isEmpty(); + readWriteLock.readLock().lock(); + try { + return list.isEmpty(); + } finally { + readWriteLock.readLock().unlock(); + } } @Override public int size() { - return list.size(); + readWriteLock.readLock().lock(); + try { + return list.size(); + } finally { + readWriteLock.readLock().unlock(); + } } public boolean contains(Object o) { - return list.contains(o); + readWriteLock.readLock().lock(); + try { + return list.contains(o); + } finally { + readWriteLock.readLock().unlock(); + } } public ArrayList asList() { - return list; + readWriteLock.readLock().lock(); + try { + return list; + } finally { + readWriteLock.readLock().unlock(); + } } } diff --git a/src/main/java/it/cavallium/strangedb/java/objects/lists/ElementsList.java b/src/main/java/it/cavallium/strangedb/java/objects/lists/ElementsList.java index 1ffcd41..f320fe2 100644 --- a/src/main/java/it/cavallium/strangedb/java/objects/lists/ElementsList.java +++ b/src/main/java/it/cavallium/strangedb/java/objects/lists/ElementsList.java @@ -1,9 +1,12 @@ package it.cavallium.strangedb.java.objects.lists; +import it.cavallium.strangedb.functionalinterfaces.ConsumerWithIO; + import java.io.IOException; interface ElementsList { T get(int index) throws IOException; + void forEachParallelUnsorted(ConsumerWithIO action) throws IOException; void add(T value) throws IOException; void update(int index, T value) throws IOException; void set(int index, T value) throws IOException; diff --git a/src/main/java/it/cavallium/strangedb/java/objects/lists/EnhancedObjectStrangeDbList.java b/src/main/java/it/cavallium/strangedb/java/objects/lists/EnhancedObjectStrangeDbList.java index 2637530..ae9f558 100644 --- a/src/main/java/it/cavallium/strangedb/java/objects/lists/EnhancedObjectStrangeDbList.java +++ b/src/main/java/it/cavallium/strangedb/java/objects/lists/EnhancedObjectStrangeDbList.java @@ -15,9 +15,6 @@ public class EnhancedObjectStrangeDbList extends Stran @DbField(id = 0, type = DbDataType.REFERENCES_LIST) private LongArrayList indices; - @DbField(id = 1, type = DbDataType.OBJECT) - private Class type; - @Override protected LongArrayList getIndices() { return indices; @@ -27,9 +24,13 @@ public class EnhancedObjectStrangeDbList extends Stran super(); } + @Deprecated public EnhancedObjectStrangeDbList(IDatabaseTools databaseTools, Class type) throws IOException { + this(databaseTools); + } + + public EnhancedObjectStrangeDbList(IDatabaseTools databaseTools) throws IOException { super(databaseTools); - this.type = type; indices = new LongArrayList(); } @@ -51,10 +52,7 @@ public class EnhancedObjectStrangeDbList extends Stran public ElementsArrayList query(ListQuery query) throws IOException { ElementsArrayList uids = queryUids(query); ElementsArrayList elements = new ElementsArrayList<>(uids.size()); - for (int i = 0; i < uids.size(); i++) { - T element = databaseTools.getObjectsIO().loadEnhancedObject(uids.get(i).objectUid); - elements.add(element); - } + uids.forEachParallelUnsorted((uid) -> elements.add(databaseTools.getObjectsIO().loadEnhancedObject(uid.objectUid))); return elements; } @@ -96,6 +94,12 @@ public class EnhancedObjectStrangeDbList extends Stran } } + /** + * This method is slow + * @param elementsList + * @return + */ + @Deprecated @SuppressWarnings("unchecked") static ElementsArrayList toElementsArrayList(ElementsList elementsList) { if (elementsList instanceof EnhancedObjectStrangeDbList) { @@ -112,10 +116,7 @@ public class EnhancedObjectStrangeDbList extends Stran ElementsArrayList results = new ElementsArrayList<>(); if (inputList instanceof EnhancedObjectStrangeDbList) { EnhancedObjectStrangeDbList dbList = ((EnhancedObjectStrangeDbList) inputList); - final int listSize = inputList.size(); - final LongArrayList indices = dbList.getIndices(); - for (int i = 0; i < listSize; i++) { - Long elementUid = indices.get(i); + dbList.forEachIndexParallelUnsorted((Long elementUid) -> { EnhancedObjectIndices elementUids = dbio.loadEnhancedObjectUids(elementUid); // check if the parent object is the declared type Class declaredRootType = query.valuePointer.getRootType(); Class obtainedRootType = elementUids.type; @@ -128,22 +129,20 @@ public class EnhancedObjectStrangeDbList extends Stran //todo: use logging api System.err.println(obtainedRootType.getSimpleName() + " is not instance of " + declaredRootType.getSimpleName()); } - } + }); } else if (inputList instanceof ElementsArrayList) { - final int listSize = inputList.size(); ElementsArrayList elementsUids = ((ElementsArrayList) inputList); - for (int i = 0; i < listSize; i++) { - EnhancedObjectIndices elementUid = elementsUids.get(i); + elementsUids.forEachParallelUnsorted((elementUid) -> { Object result = resolveItemFromDb(query.valuePointer, dbio, elementUid); if (query.valueOperation.evaluate(result)) { results.add(elementUid); } - } + }); } return results; } - static Object resolveItemFromDb(ValuePointer pointer, IObjectsIO objectsIO, EnhancedObjectIndices element) throws IOException { + private static Object resolveItemFromDb(ValuePointer pointer, IObjectsIO objectsIO, EnhancedObjectIndices element) throws IOException { EnhancedObjectIndices currentElement = element; boolean isLastElement; for (int i = 0; i < pointer.size(); i++) { diff --git a/src/main/java/it/cavallium/strangedb/java/objects/lists/StrangeDbList.java b/src/main/java/it/cavallium/strangedb/java/objects/lists/StrangeDbList.java index 1d4f9e6..5ba40a4 100644 --- a/src/main/java/it/cavallium/strangedb/java/objects/lists/StrangeDbList.java +++ b/src/main/java/it/cavallium/strangedb/java/objects/lists/StrangeDbList.java @@ -1,15 +1,21 @@ package it.cavallium.strangedb.java.objects.lists; +import it.cavallium.strangedb.VariableWrapper; +import it.cavallium.strangedb.functionalinterfaces.ConsumerWithIO; import it.unimi.dsi.fastutil.longs.LongArrayList; import it.cavallium.strangedb.java.objects.EnhancedObject; import it.cavallium.strangedb.java.database.IDatabaseTools; +import java.io.IOError; import java.io.IOException; import java.util.StringJoiner; +import java.util.concurrent.*; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Consumer; public abstract class StrangeDbList extends EnhancedObject implements ElementsList { - private final Object indicesAccessLock = new Object(); + private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(false); protected abstract LongArrayList getIndices(); @@ -23,18 +29,104 @@ public abstract class StrangeDbList extends EnhancedObject implements Element @Override public T get(int index) throws IOException { - synchronized (indicesAccessLock) { + readWriteLock.readLock().lock(); + try { long uid = getIndices().getLong(index); return loadItem(uid); + } finally { + readWriteLock.readLock().unlock(); } } @Override public void add(T value) throws IOException { long uid = databaseTools.getObjectsIO().newNullObject(); - synchronized (indicesAccessLock) { + readWriteLock.writeLock().lock(); + try { getIndices().add(uid); writeItemToDisk(uid, value); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + @Override + public void forEachParallelUnsorted(ConsumerWithIO action) throws IOException { + readWriteLock.readLock().lock(); + try { + forEachParallelUnsorted_(action); + } finally { + readWriteLock.readLock().unlock(); + } + } + + protected void forEachParallelUnsorted_(ConsumerWithIO action) throws IOException { + try { + int size = size(); + ExecutorService executorService = Executors.newFixedThreadPool(16, (r) -> new Thread(r, "StrangeDbList.forEachParallelUnsorted worker")); + VariableWrapper exceptionVariableWrapper = new VariableWrapper<>(null); + for (int i = 0; i < size; i++) { + final int index = i; + executorService.execute(() -> { + try { + T t = get(index); + action.accept(t); + } catch (IOException e) { + if (exceptionVariableWrapper.var == null) exceptionVariableWrapper.var = e; + } + }); + } + executorService.shutdown(); + executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); + if (exceptionVariableWrapper.var != null) { + throw exceptionVariableWrapper.var; + } + executorService.shutdownNow(); + } catch (InterruptedException e) { + throw new IOException(e); + } catch (CompletionException e) { + throw new IOException(e.getCause()); + } + } + + public void forEachIndexParallelUnsorted(ConsumerWithIO action) throws IOException { + readWriteLock.readLock().lock(); + try { + forEachIndexParallelUnsorted_(action); + } finally { + readWriteLock.readLock().unlock(); + } + } + + protected void forEachIndexParallelUnsorted_(ConsumerWithIO action) throws IOException { + try { + this.getIndices().parallelStream().forEach((id) -> { + try { + action.accept(id); + } catch (IOException e) { + throw new CompletionException(e); + } + }); + } catch (CompletionException ex) { + throw new IOException(ex.getCause()); + } + } + + public void forEach(ConsumerWithIO action) throws IOException { + readWriteLock.readLock().lock(); + try { + forEach_(action); + } finally { + readWriteLock.readLock().unlock(); + } + } + + protected void forEach_(ConsumerWithIO action) throws IOException { + int size = size(); + for (int i = 0; i < size; i++) { + final int index = i; + T value = get(index); + action.accept(value); } } @@ -46,40 +138,55 @@ public abstract class StrangeDbList extends EnhancedObject implements Element @Override public void set(int index, T value) throws IOException { long uid = databaseTools.getObjectsIO().newNullObject(); - synchronized (indicesAccessLock) { + readWriteLock.writeLock().lock(); + try { getIndices().set(index, uid); writeItemToDisk(uid, value); + } finally { + readWriteLock.writeLock().unlock(); } } @Override public void add(int index, T value) throws IOException { long uid = databaseTools.getObjectsIO().newNullObject(); - synchronized (indicesAccessLock) { + readWriteLock.writeLock().lock(); + try { getIndices().add(index, uid); writeItemToDisk(uid, value); + } finally { + readWriteLock.writeLock().unlock(); } } public T getLast() throws IOException { - synchronized (indicesAccessLock) { + readWriteLock.readLock().lock(); + try { if (getIndices().size() > 0) { return get(getIndices().size() - 1); } else { return null; } + } finally { + readWriteLock.readLock().unlock(); } } public boolean isEmpty() { - synchronized (indicesAccessLock) { + readWriteLock.readLock().lock(); + try { return getIndices().size() <= 0; + } finally { + readWriteLock.readLock().unlock(); } } public int size() { - synchronized (indicesAccessLock) { + readWriteLock.readLock().lock(); + try { return getIndices().size(); + } finally { + readWriteLock.readLock().unlock(); } } diff --git a/src/test/java/it/cavallium/strangedb/tests/EnhancedClassUpdate.java b/src/test/java/it/cavallium/strangedb/tests/EnhancedClassUpdate.java index bb6b272..4f0cfde 100644 --- a/src/test/java/it/cavallium/strangedb/tests/EnhancedClassUpdate.java +++ b/src/test/java/it/cavallium/strangedb/tests/EnhancedClassUpdate.java @@ -35,7 +35,7 @@ public class EnhancedClassUpdate { @Test public void shouldUpdateClass() throws IOException { db = new DatabaseJava(path1, path2, path3); - V2Class root = db.loadRoot(V2Class::new); + V2Class root = db.loadRoot(V2Class::new, V2Class.class); assertEquals(root.field4, "Abc"); assertEquals(root.field2, 12); assertEquals(root.field1, 13L); diff --git a/src/test/java/it/cavallium/strangedb/tests/Performance.java b/src/test/java/it/cavallium/strangedb/tests/Performance.java index 1f82833..e34d032 100644 --- a/src/test/java/it/cavallium/strangedb/tests/Performance.java +++ b/src/test/java/it/cavallium/strangedb/tests/Performance.java @@ -4,17 +4,24 @@ import it.cavallium.strangedb.functionalinterfaces.RunnableWithIO; import it.cavallium.strangedb.java.annotations.*; import it.cavallium.strangedb.java.database.DatabaseJava; import it.cavallium.strangedb.java.objects.lists.EnhancedObjectStrangeDbList; +import it.cavallium.strangedb.java.objects.lists.ListQuery; import it.cavallium.strangedb.java.objects.lists.ObjectStrangeDbList; +import it.cavallium.strangedb.java.objects.lists.ValuePointer; +import it.cavallium.strangedb.java.objects.lists.operations.ContainsIgnoreCase; +import it.cavallium.strangedb.tests.query.*; import it.unimi.dsi.fastutil.longs.LongArrayList; import it.cavallium.strangedb.java.objects.EnhancedObject; import it.cavallium.strangedb.java.database.IDatabaseTools; import it.cavallium.strangedb.VariableWrapper; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.List; +import java.util.Random; public class Performance { private static boolean FAST_TESTS; @@ -24,6 +31,7 @@ public class Performance { private static Path dbBlocksFile; private static Path dbReferencesFile; private static DatabaseJava db; + private static boolean tempDirectory; /** * @@ -40,18 +48,21 @@ public class Performance { } catch (Exception ex) { } + tempDirectory = false; } else { rootDirectory = Files.createTempDirectory("performance-tests"); + tempDirectory = true; } generateDb(); System.out.println("Performance test started."); System.out.println("-------------------------------------------------------+-----------------------------------------------------------------"); System.out.println("Test name Total Time | Time at 1 Time at 10 Time at 100 Time at 1K Time at 10K"); System.out.println("-------------------------------------------------------+-----------------------------------------------------------------"); - testS("DatabaseCore creation", 3000, Performance::deleteDb, Performance::generateDb, () -> {}); - testS("DatabaseCore root creation", 3000, Performance::regenDb, () -> db.loadRoot(PreloadedListContainer::new), () -> {}); + testS("DatabaseCore creation", 300, Performance::deleteDb, Performance::generateDb, () -> {}); + testS("DatabaseCore root creation", 300, Performance::regenDb, () -> db.loadRoot(PreloadedListContainer::new), () -> {}); final VariableWrapper preloadedListContainer = new VariableWrapper<>(null); final VariableWrapper simpleEnhancedObjectContainer = new VariableWrapper<>(null); + /* testS("ObjectStrangeDbList creation", 3000, () -> { regenDb(); preloadedListContainer.var = db.loadRoot(PreloadedListContainer::new); @@ -107,9 +118,7 @@ public class Performance { preloadedListContainer.var.list.add(1000); } }, () -> { - for (int i = 0; i < 1000; i++) { - preloadedListContainer.var.list.get(i); - } + preloadedListContainer.var.list.forEachParallelUnsorted((i) -> {}); }, () -> {}); testS("ObjectStrangeDbList: Loading with 1000 items", 100, () -> { regenDb(); @@ -126,9 +135,7 @@ public class Performance { preloadedListContainer.var.listOfEnhancedObj.add(simpleEnhancedObjectContainer.var); } }, () -> { - for (int i = 0; i < 1000; i++) { - preloadedListContainer.var.listOfEnhancedObj.get(i); - } + preloadedListContainer.var.listOfEnhancedObj.forEachParallelUnsorted((i) -> {}); }, () -> {}); testS("ObjectStrangeDbList: Loading 10000 items", 10, () -> { regenDb(); @@ -138,9 +145,7 @@ public class Performance { preloadedListContainer.var.list.add(1000); } }, () -> { - for (int i = 0; i < 10000; i++) { - preloadedListContainer.var.list.get(i); - } + preloadedListContainer.var.list.forEachParallelUnsorted((i) -> {}); }, () -> {}); testS("ObjectStrangeDbList: getLast() with 1000 items", 100, () -> { regenDb(); @@ -179,10 +184,44 @@ public class Performance { }, () -> { preloadedListContainer.var.list.size(); }, () -> {}); + */ + for (int items = 1000; items <= 100000; items *= 10) { + final int itemsF = items; + testS("ListQuery: query with " + items + " items", 100 / (items / 1000), () -> { + regenDb(); + preloadedListContainer.var = db.loadRoot(PreloadedListContainer::new); + preloadedListContainer.var.listOfMessages = new EnhancedObjectStrangeDbList<>(db); + + Random random = new Random(); + for (int i = 0; i < itemsF; i++) { + EMessageContent content; + if (random.nextBoolean()) { + List stringList = new ArrayList<>(); + for (int j = 0; j < 10; j++) { + stringList.add("[entity]"); + } + byte[] stringBytes = new byte[200]; + random.nextBytes(stringBytes); + content = new EMessageText(db, new EFormattedText(db, new String(stringBytes, StandardCharsets.UTF_8) + (random.nextBoolean() ? "not found" : " text to find!"), stringList.toArray(new String[0]))); + } else { + content = new EMessageOtherContent(db, "EMPTY ABCDEFG"); + } + EMessage message = new EMessage(db, content); + preloadedListContainer.var.listOfMessages.add(message); + } + }, () -> { + ListQuery query = ListQuery.create( + ValuePointer.ofField(EMessage.class, "content").field(EMessageText.class, "text").field(EFormattedText.class, "text"), + ContainsIgnoreCase.containsValue("text to find")); + ArrayList results = preloadedListContainer.var.listOfMessages.query(query).asList(); + }, () -> {}); + } System.out.println("-------------------------------------------------------+-----------------------------------------------------------------"); System.out.println("Performance test finished."); deleteDb(); - Files.deleteIfExists(rootDirectory); + if (tempDirectory) { + Files.deleteIfExists(rootDirectory); + } } private static void NtestS(String description, int times, RunnableWithIO beforeAction, RunnableWithIO action, RunnableWithIO afterAction) throws IOException, InterruptedException { @@ -272,6 +311,15 @@ public class Performance { Files.createFile(dbBlocksFile); Files.createFile(dbReferencesFile); db = new DatabaseJava(dbDataFile, dbBlocksFile, dbReferencesFile); + int i = 0; + db.getObjectsIO().registerClass(SimpleEnhancedObject.class, i++); + db.getObjectsIO().registerClass(PreloadedListContainer.class, i++); + db.getObjectsIO().registerClass(DynamicListContainer.class, i++); + db.getObjectsIO().registerClass(EMessage.class, i++); + db.getObjectsIO().registerClass(EMessageContent.class, i++); + db.getObjectsIO().registerClass(EMessageText.class, i++); + db.getObjectsIO().registerClass(EMessageOtherContent.class, i++); + db.getObjectsIO().registerClass(EFormattedText.class, i++); } public static void deleteDb() throws IOException { @@ -298,6 +346,9 @@ public class Performance { @DbField(id = 1, type = DbDataType.ENHANCED_OBJECT) public EnhancedObjectStrangeDbList listOfEnhancedObj; + @DbField(id = 2, type = DbDataType.ENHANCED_OBJECT) + public EnhancedObjectStrangeDbList listOfMessages; + public PreloadedListContainer() { } diff --git a/src/test/java/it/cavallium/strangedb/tests/query/EFormattedText.java b/src/test/java/it/cavallium/strangedb/tests/query/EFormattedText.java new file mode 100644 index 0000000..c6bbf05 --- /dev/null +++ b/src/test/java/it/cavallium/strangedb/tests/query/EFormattedText.java @@ -0,0 +1,32 @@ +package it.cavallium.strangedb.tests.query; + +import it.cavallium.strangedb.java.annotations.DbDataType; +import it.cavallium.strangedb.java.annotations.DbField; +import it.cavallium.strangedb.java.database.IDatabaseTools; + +import java.io.IOException; + +public class EFormattedText extends EMessageContent { + + /** + * The text. + */ + @DbField(id = 0, type = DbDataType.OBJECT, name = "text") + public String text; + /** + * Entities contained in the text. + */ + @DbField(id = 1, type = DbDataType.OBJECT, name = "entities") + public String[] entities; + + @Deprecated + public EFormattedText() { + + } + + public EFormattedText(IDatabaseTools tools, String text, String[] entities) throws IOException { + super(tools); + this.text = text; + this.entities =entities; + } +} diff --git a/src/test/java/it/cavallium/strangedb/tests/query/EMessage.java b/src/test/java/it/cavallium/strangedb/tests/query/EMessage.java new file mode 100644 index 0000000..04ad38d --- /dev/null +++ b/src/test/java/it/cavallium/strangedb/tests/query/EMessage.java @@ -0,0 +1,138 @@ +package it.cavallium.strangedb.tests.query; + +import it.cavallium.strangedb.java.annotations.DbDataType; +import it.cavallium.strangedb.java.annotations.DbField; +import it.cavallium.strangedb.java.annotations.DbPrimitiveField; +import it.cavallium.strangedb.java.annotations.DbPrimitiveType; +import it.cavallium.strangedb.java.database.IDatabaseTools; +import it.cavallium.strangedb.java.objects.EnhancedObject; + +import java.io.IOException; + +public class EMessage extends EnhancedObject { + /** + * Message identifier, unique for the chat to which the message belongs. + */ + @DbPrimitiveField(id = 0, type = DbPrimitiveType.LONG, name = "id") + public long id; + /** + * Identifier of the user who sent the message; 0 if unknown. It is unknown for channel posts. + */ + @DbPrimitiveField(id = 1, type = DbPrimitiveType.INTEGER, name = "senderUserId") + public int senderUserId; + /** + * Chat identifier. + */ + @DbPrimitiveField(id = 2, type = DbPrimitiveType.LONG, name = "chatId") + public long chatId; + /** + * Information about the sending state of the message; may be null. + */ + @DbField(id = 0, type = DbDataType.OBJECT, name = "sendingState") + public Object sendingState; + /** + * True, if the message is outgoing. + */ + @DbPrimitiveField(id = 3, type = DbPrimitiveType.BOOLEAN, name = "isOutgoing") + public boolean isOutgoing; + /** + * True, if the message can be edited. + */ + @DbPrimitiveField(id = 4, type = DbPrimitiveType.BOOLEAN, name = "canBeEdited") + public boolean canBeEdited; + /** + * True, if the message can be forwarded. + */ + @DbPrimitiveField(id = 5, type = DbPrimitiveType.BOOLEAN, name = "canBeForwarded") + public boolean canBeForwarded; + /** + * True, if the message can be deleted only for the current user while other users will continue to see it. + */ + @DbPrimitiveField(id = 6, type = DbPrimitiveType.BOOLEAN, name = "canBeDeletedOnlyForSelf") + public boolean canBeDeletedOnlyForSelf; + /** + * True, if the message can be deleted for all users. + */ + @DbPrimitiveField(id = 7, type = DbPrimitiveType.BOOLEAN, name = "canBeDeletedForAllUsers") + public boolean canBeDeletedForAllUsers; + /** + * True, if the message is a channel post. All messages to channels are channel posts, all other messages are not channel posts. + */ + @DbPrimitiveField(id = 8, type = DbPrimitiveType.BOOLEAN, name = "isChannelPost") + public boolean isChannelPost; + /** + * True, if the message contains an unread mention for the current user. + */ + @DbPrimitiveField(id = 9, type = DbPrimitiveType.BOOLEAN, name = "containsUnreadMention") + public boolean containsUnreadMention; + /** + * Point in time (Unix timestamp) when the message was sent. + */ + @DbPrimitiveField(id = 10, type = DbPrimitiveType.INTEGER, name = "date") + public int date; + /** + * Point in time (Unix timestamp) when the message was last edited. + */ + @DbPrimitiveField(id = 11, type = DbPrimitiveType.INTEGER, name = "editDate") + public int editDate; + /** + * Information about the initial message sender; may be null. + */ + @DbField(id = 1, type = DbDataType.OBJECT, name = "forwardInfo") + public Object forwardInfo; + /** + * If non-zero, the identifier of the message this message is replying to; can be the identifier of a deleted message. + */ + @DbPrimitiveField(id = 12, type = DbPrimitiveType.LONG, name = "replyToMessageId") + public long replyToMessageId; + /** + * For self-destructing messages, the message's TTL (Time To Live), in seconds; 0 if none. TDLib will send updateDeleteMessages or updateMessageContent once the TTL expires. + */ + @DbPrimitiveField(id = 13, type = DbPrimitiveType.INTEGER, name = "ttl") + public int ttl; + /** + * Time left before the message expires, in seconds. + */ + @DbPrimitiveField(id = 14, type = DbPrimitiveType.DOUBLE, name = "ttlExpiresIn") + public double ttlExpiresIn; + /** + * If non-zero, the user identifier of the bot through which this message was sent. + */ + @DbPrimitiveField(id = 15, type = DbPrimitiveType.INTEGER, name = "viaBotUserId") + public int viaBotUserId; + /** + * For channel posts, optional author signature. + */ + @DbField(id = 2, type = DbDataType.OBJECT, name = "authorSignature") + public String authorSignature; + /** + * Number of times this message was viewed. + */ + @DbPrimitiveField(id = 16, type = DbPrimitiveType.INTEGER, name = "views") + public int views; + /** + * Unique identifier of an album this message belongs to. Only photos and videos can be grouped together in albums. + */ + @DbPrimitiveField(id = 17, type = DbPrimitiveType.LONG, name = "mediaAlbumId") + public long mediaAlbumId; + /** + * Content of the message. + */ + @DbField(id = 3, type = DbDataType.ENHANCED_OBJECT, name = "content") + public EMessageContent content; + /** + * Reply markup for the message; may be null. + */ + @DbField(id = 4, type = DbDataType.OBJECT, name = "replyMarkup") + public Object replyMarkup; + + @Deprecated + public EMessage() { + + } + + public EMessage(IDatabaseTools tools, EMessageContent content) throws IOException { + super(tools); + this.content = content; + } +} diff --git a/src/test/java/it/cavallium/strangedb/tests/query/EMessageContent.java b/src/test/java/it/cavallium/strangedb/tests/query/EMessageContent.java new file mode 100644 index 0000000..fd619e0 --- /dev/null +++ b/src/test/java/it/cavallium/strangedb/tests/query/EMessageContent.java @@ -0,0 +1,18 @@ +package it.cavallium.strangedb.tests.query; + +import it.cavallium.strangedb.java.database.IDatabaseTools; +import it.cavallium.strangedb.java.objects.EnhancedObject; + +import java.io.IOException; + +public abstract class EMessageContent extends EnhancedObject { + + @Deprecated + public EMessageContent() { + + } + + public EMessageContent(IDatabaseTools tools) throws IOException { + super(tools); + } +} diff --git a/src/test/java/it/cavallium/strangedb/tests/query/EMessageOtherContent.java b/src/test/java/it/cavallium/strangedb/tests/query/EMessageOtherContent.java new file mode 100644 index 0000000..8081248 --- /dev/null +++ b/src/test/java/it/cavallium/strangedb/tests/query/EMessageOtherContent.java @@ -0,0 +1,23 @@ +package it.cavallium.strangedb.tests.query; + +import it.cavallium.strangedb.java.annotations.DbDataType; +import it.cavallium.strangedb.java.annotations.DbField; +import it.cavallium.strangedb.java.database.IDatabaseTools; + +import java.io.IOException; + +public class EMessageOtherContent extends EMessageContent { + + @DbField(id = 0, type = DbDataType.OBJECT, name = "content") + public Object content; + + @Deprecated + public EMessageOtherContent() { + + } + + public EMessageOtherContent(IDatabaseTools tools, Object content) throws IOException { + super(tools); + this.content = content; + } +} diff --git a/src/test/java/it/cavallium/strangedb/tests/query/EMessageText.java b/src/test/java/it/cavallium/strangedb/tests/query/EMessageText.java new file mode 100644 index 0000000..7acd3f9 --- /dev/null +++ b/src/test/java/it/cavallium/strangedb/tests/query/EMessageText.java @@ -0,0 +1,31 @@ +package it.cavallium.strangedb.tests.query; + +import it.cavallium.strangedb.java.annotations.DbDataType; +import it.cavallium.strangedb.java.annotations.DbField; +import it.cavallium.strangedb.java.database.IDatabaseTools; + +import java.io.IOException; + +public class EMessageText extends EMessageContent { + + /** + * Text of the message. + */ + @DbField(id = 0, type = DbDataType.ENHANCED_OBJECT, name = "text") + public EFormattedText text; + /** + * A preview of the web page that's mentioned in the text; may be null. + */ + @DbField(id = 1, type = DbDataType.OBJECT, name = "webPage") + public Object webPage; + + @Deprecated + public EMessageText() { + + } + + public EMessageText(IDatabaseTools tools, EFormattedText text) throws IOException { + super(tools); + this.text = text; + } +} diff --git a/src/test/java/it/cavallium/strangedb/utils/NTestUtils.java b/src/test/java/it/cavallium/strangedb/utils/NTestUtils.java index f34246b..3634f9e 100644 --- a/src/test/java/it/cavallium/strangedb/utils/NTestUtils.java +++ b/src/test/java/it/cavallium/strangedb/utils/NTestUtils.java @@ -233,7 +233,7 @@ public class NTestUtils { @DbField(id = 0, type = DbDataType.OBJECT) public String field7; - @DbField(id = 1, type = DbDataType.REFERENCES_LIST) + @DbField(id = 1, type = DbDataType.OBJECT) public LongArrayList field8; @DbField(id = 2, type = DbDataType.ENHANCED_OBJECT) @@ -253,7 +253,7 @@ public class NTestUtils { return getProperty(); } - @DbProperty(id = 1, type = DbDataType.REFERENCES_LIST) + @DbProperty(id = 1, type = DbDataType.OBJECT) @DbPropertyGetter public LongArrayList get8() { return getProperty(); @@ -271,7 +271,7 @@ public class NTestUtils { setProperty(val); } - @DbProperty(id = 1, type = DbDataType.REFERENCES_LIST) + @DbProperty(id = 1, type = DbDataType.OBJECT) @DbPropertySetter public void set8(LongArrayList val) { setProperty(val);