Cleaner, queries and locks

This commit is contained in:
Andrea Cavalli 2019-04-19 02:55:47 +02:00
parent a1624bf91e
commit e0a7c2f6c0
9 changed files with 539 additions and 227 deletions

View File

@ -26,7 +26,7 @@ public class DatabaseDataInitializer implements IDataInitializer {
}
private void initializeDbObjectFields(EnhancedObject obj) throws IOException {
// Declare the variables needed to get the biggest field Id
// Declare the variables needed to getBlock the biggest field Id
Field[] unorderedFields = objectsIO.getFields(obj);
// Find the biggest field Id
int biggestFieldId = objectsIO.getBiggestFieldId(unorderedFields);
@ -52,7 +52,7 @@ public class DatabaseDataInitializer implements IDataInitializer {
}
private void initializeDbObjectPrimitiveFields(EnhancedObject obj) throws IOException {
// Declare the variables needed to get the biggest field Id
// Declare the variables needed to getBlock the biggest field Id
Field[] unorderedFields = objectsIO.getPrimitiveFields(obj);
// Find the biggest field Id
int biggestFieldId = objectsIO.getBiggestPrimitiveFieldId(unorderedFields);
@ -107,7 +107,7 @@ public class DatabaseDataInitializer implements IDataInitializer {
}
private void initializeDbObjectProperties(EnhancedObject obj) throws IOException {
// Declare the variables needed to get the biggest property Id
// Declare the variables needed to getBlock the biggest property Id
Method[] unorderedPropertyGetters = obj.getPropertyGetters();
Method[] unorderedPropertySetters = obj.getPropertySetters();

View File

@ -1,21 +1,38 @@
package it.cavallium.strangedb.java.database;
import it.cavallium.strangedb.database.DatabaseCore;
import it.cavallium.strangedb.database.references.ReferenceInfo;
import it.cavallium.strangedb.java.objects.EnhancedObject;
import it.cavallium.strangedb.functionalinterfaces.FunctionWithIO;
import it.cavallium.strangedb.java.objects.EnhancedObjectIndices;
import it.unimi.dsi.fastutil.ints.Int2ObjectLinkedOpenHashMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongArraySet;
import it.unimi.dsi.fastutil.longs.LongSet;
import it.unimi.dsi.fastutil.objects.ObjectIterator;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import static it.cavallium.strangedb.database.references.DatabaseReferencesMetadata.*;
import static it.cavallium.strangedb.java.database.DatabaseObjectsIO.ENHANCED_OBJECT_METADATA_CLEANER;
import static it.cavallium.strangedb.java.database.DatabaseObjectsIO.REFERENCES_LIST_CLEANER;
public class DatabaseJava extends DatabaseCore implements IDatabaseTools {
private final IDatabaseTools databaseTools;
private final DatabaseObjectsIO objectsIO;
private EnhancedObject loadedRootObject;
private boolean hasLoadedRootObject;
public DatabaseJava(Path dataFile, Path blocksMetaFile, Path referencesMetaFile) throws IOException {
super(dataFile, blocksMetaFile, referencesMetaFile);
this.databaseTools = this;
this.objectsIO = new DatabaseObjectsIO(databaseTools, referencesIO);
this.hasLoadedRootObject = false;
}
@Override
@ -23,12 +40,14 @@ public class DatabaseJava extends DatabaseCore implements IDatabaseTools {
if (this.closed) {
throw new IOException("The database has been already closed!");
}
this.objectsIO.setEnhancedObject(0, loadedRootObject);
if (hasLoadedRootObject) {
this.objectsIO.setEnhancedObject(0, loadedRootObject);
}
super.close();
}
public <T extends EnhancedObject> T loadRoot(FunctionWithIO<IDatabaseTools, T> ifAbsent) throws IOException {
if (loadedRootObject != null) {
if (hasLoadedRootObject) {
throw new RuntimeException("Root already set!");
}
T root;
@ -43,13 +62,110 @@ public class DatabaseJava extends DatabaseCore implements IDatabaseTools {
}
}
loadedRootObject = root;
hasLoadedRootObject = true;
return root;
}
@Override
public void closeAndClean() throws IOException {
if (!this.closed) {
this.close();
}
Path newDataFile = dataFile.resolveSibling("compressed-data-file.tmp");
Path newBlocksFile = blocksMetaFile.resolveSibling("compressed-blocks-file.tmp");
Path newReferencesFile = referencesMetaFile.resolveSibling("compressed-references-file.tmp");
Path backupDataFile = dataFile.resolveSibling("backup-data.db.bak");
Path backupBlocksFile = blocksMetaFile.resolveSibling("backup-blocks.dat.bak");
Path backupReferencesFile = referencesMetaFile.resolveSibling("backup-references.dat.bak");
Files.copy(dataFile, backupDataFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
Files.copy(blocksMetaFile, backupBlocksFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
Files.copy(referencesMetaFile, backupReferencesFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
Files.move(dataFile, newDataFile, StandardCopyOption.REPLACE_EXISTING);
Files.move(blocksMetaFile, newBlocksFile, StandardCopyOption.REPLACE_EXISTING);
Files.move(referencesMetaFile, newReferencesFile, StandardCopyOption.REPLACE_EXISTING);
DatabaseJava databaseToClean = instantiateNewDatabase(newDataFile, newBlocksFile, newReferencesFile);
DatabaseJava newDatabase = instantiateNewDatabase(dataFile, blocksMetaFile, referencesMetaFile);
long firstFreeReference = databaseToClean.referencesMetadata.getFirstFreeReference();
long referencesCount = 0;
long blocksCount = databaseToClean.blocksMetadata.getTotalBlocksCount();
long writtenReferences = 0;
long writtenBlocks = 0;
LongArrayList idsToKeep = new LongArrayList();
cleanRef(databaseToClean, idsToKeep, 0);
for (int referenceID = 0; referenceID < firstFreeReference; referenceID++) {
try {
ReferenceInfo ref = databaseToClean.referencesMetadata.getReference(referenceID);
if (!NONEXISTENT_REFERENCE_INFO.equals(ref)) {
referencesCount++;
if (idsToKeep.contains(referenceID)) {
ByteBuffer buffer = databaseToClean.referencesIO.readFromReference(referenceID);
newDatabase.referencesIO.writeToReference(referenceID, ref.getCleanerId(), buffer.limit(), buffer);
writtenReferences++;
if (buffer.limit() > 0) {
writtenBlocks++;
}
}
}
} catch (IOException ex) {
System.out.println("Error while reading reference " + referenceID + ". References written: " + writtenReferences);
}
}
System.out.println("[Java Cleaner] References written: " + writtenReferences + ". Removed " + (blocksCount - writtenBlocks) + " blocks. Removed " + (referencesCount - writtenReferences) + " references.");
databaseToClean.close();
newDatabase.close();
Files.deleteIfExists(newDataFile);
Files.deleteIfExists(newBlocksFile);
Files.deleteIfExists(newReferencesFile);
}
private void cleanRef(DatabaseJava db, LongArrayList idsToKeep, long ref) throws IOException {
idsToKeep.add(ref);
ReferenceInfo refInfo = db.referencesMetadata.getReference(ref);
if (!NONEXISTENT_REFERENCE_INFO.equals(refInfo)) {
switch (refInfo.getCleanerId()) {
case ENHANCED_OBJECT_METADATA_CLEANER: {
EnhancedObjectIndices enhancedObjectUids = db.objectsIO.loadEnhancedObjectUids(ref);
for (long fieldUid : enhancedObjectUids.fieldUids) {
cleanRef(db, idsToKeep, fieldUid);
}
for (long propUid : enhancedObjectUids.propertyUids) {
cleanRef(db, idsToKeep, propUid);
}
cleanRef(db, idsToKeep, enhancedObjectUids.nativeDataUid);
break;
}
case ERRORED_CLEANER: {
System.err.println("Errored cleaner found! Skipping...");
break;
}
case REFERENCES_LIST_CLEANER: {
LongArrayList refList = db.objectsIO.loadReferencesList(ref);
for (long elemRef : refList) {
cleanRef(db, idsToKeep, elemRef);
}
break;
}
case BLANK_DATA_CLEANER: {
break;
}
default: {
System.err.println("Unrecognized cleaner found! Skipping...");
}
}
}
}
protected void registerClass(Class<?> type, int id) {
this.objectsIO.registerClass(type, id);
}
protected DatabaseJava instantiateNewDatabase(Path dataFile, Path blocksMetaFile, Path referencesMetaFile) throws IOException {
return new DatabaseJava(dataFile, blocksMetaFile, referencesMetaFile);
}
@Override
public void initializeEnhancedObject(EnhancedObject enhancedObject) throws IOException {
this.objectsIO.getDataInitializer().initializeDbObject(enhancedObject);

View File

@ -3,8 +3,6 @@ package it.cavallium.strangedb.java.database;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import it.cavallium.strangedb.java.objects.EnhancedObject;
import it.cavallium.strangedb.java.objects.EnhancedObjectFullInfo;
import it.cavallium.strangedb.database.references.DatabaseReferencesIO;
import it.cavallium.strangedb.java.annotations.*;
import it.cavallium.strangedb.java.objects.EnhancedObject;
@ -29,14 +27,20 @@ import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.function.Supplier;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import static it.cavallium.strangedb.database.references.DatabaseReferencesMetadata.BLANK_DATA_CLEANER;
public class DatabaseObjectsIO implements IObjectsIO {
public static final byte ENHANCED_OBJECT_METADATA_CLEANER = 1;
public static final byte REFERENCES_LIST_CLEANER = 2;
private final IDatabaseTools databaseTools;
private final DatabaseReferencesIO referencesIO;
private final Object accessLock = new Object();
private final Lock lock = new ReentrantLock();
private final DatabaseDataInitializer dataInitializer;
private Kryo kryo = new Kryo();
@ -105,49 +109,61 @@ public class DatabaseObjectsIO implements IObjectsIO {
registerClass(StrangeDbList.class, id++);
}
@SuppressWarnings("unchecked")
@Override
public <T extends EnhancedObject> T loadEnhancedObject(long reference, Class<T> objectType) throws IOException {
synchronized (accessLock) {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return null;
}
int serializedClassLength = 0xFF & buffer.get();
byte[] serializedClassData = new byte[serializedClassLength];
buffer.get(serializedClassData);
Class<T> objectType = kryo.readClass(new Input(serializedClassData)).getType();
int serializedVersion = Byte.toUnsignedInt(buffer.get());
int fieldsCount = buffer.getInt();
int methodsCount = buffer.getInt();
long[] fieldRefs = new long[fieldsCount];
long[] methodRefs = new long[methodsCount];
for (int i = 0; i < fieldsCount; i++) {
fieldRefs[i] = buffer.getLong();
}
for (int i = 0; i < methodsCount; i++) {
methodRefs[i] = buffer.getLong();
}
long nativeFieldsDataReference = buffer.getLong();
return preloadEnhancedObject(objectType, serializedVersion, nativeFieldsDataReference, fieldRefs,
methodRefs);
public <T extends EnhancedObject> T loadEnhancedObject(long reference) throws IOException {
lock.lock();
try {
return loadEnhancedObject_(reference);
} finally {
lock.unlock();
}
}
@SuppressWarnings("unchecked")
private <T extends EnhancedObject> T loadEnhancedObject_(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return null;
}
int serializedVersion = Byte.toUnsignedInt(buffer.get()) - 5;
if (serializedVersion < 0) {
System.err.println("PLEASE UPGRADE THE DATABASE");
throw new IllegalAccessError("PLEASE UPGRADE THE DATABASE");
}
int serializedClassLength = Byte.toUnsignedInt(buffer.get());
byte[] serializedClassData = new byte[serializedClassLength];
buffer.get(serializedClassData);
Class<T> objectType = kryo.readClass(new Input(serializedClassData)).getType();
int fieldsCount = buffer.getInt();
int methodsCount = buffer.getInt();
long[] fieldRefs = new long[fieldsCount];
long[] methodRefs = new long[methodsCount];
for (int i = 0; i < fieldsCount; i++) {
fieldRefs[i] = buffer.getLong();
}
for (int i = 0; i < methodsCount; i++) {
methodRefs[i] = buffer.getLong();
}
long nativeFieldsDataReference = buffer.getLong();
return preloadEnhancedObject(objectType, serializedVersion, nativeFieldsDataReference, fieldRefs,
methodRefs);
}
@SuppressWarnings("unchecked")
@Override
public EnhancedObjectIndices loadEnhancedObjectUids(long reference) throws IOException {
lock.readLock().lock();
readLock.lock();
lock.lock();
try {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return null;
}
//noinspection unused
int serializedClassLength = 0xFF & buffer.get();
int serializedVersion = Byte.toUnsignedInt(buffer.get()) - 5;
int serializedClassLength = Byte.toUnsignedInt(buffer.get());
byte[] serializedClassData = new byte[serializedClassLength];
buffer.get(serializedClassData);
int serializedVersion = Byte.toUnsignedInt(buffer.get());
Class<? extends EnhancedObject> objectType = (Class<? extends EnhancedObject>) kryo.readClass(new Input(serializedClassData)).getType();
int fieldsCount = buffer.getInt();
int methodsCount = buffer.getInt();
long[] fieldRefs = new long[fieldsCount];
@ -159,42 +175,50 @@ public class DatabaseObjectsIO implements IObjectsIO {
methodRefs[i] = buffer.getLong();
}
long nativeFieldsDataReference = buffer.getLong();
return new EnhancedObjectIndices(reference, fieldRefs, methodRefs, nativeFieldsDataReference);
} finally {
readLock.unlock();
lock.readLock().unlock();
return new EnhancedObjectIndices(objectType, reference, fieldRefs, methodRefs, nativeFieldsDataReference);
} finally {
lock.unlock();
}
}
}
public Object loadData(DbDataType propertyType, long dataReference) throws IOException {
lock.lock();
try {
return loadData_(propertyType, dataReference);
} finally {
lock.unlock();
}
}
private Object loadData_(DbDataType propertyType, long dataReference) throws IOException {
Object loadData(DbDataType propertyType, long dataReference) throws IOException {
switch (propertyType) {
case ENHANCED_OBJECT:
return loadEnhancedObject(dataReference);
return loadEnhancedObject_(dataReference);
case OBJECT:
return loadObject(dataReference);
return loadObject_(dataReference);
case REFERENCES_LIST:
return loadReferencesList(dataReference);
return loadReferencesList_(dataReference);
default:
throw new NullPointerException("Unknown data type");
}
}
<T> void setData(long reference, DbDataType propertyType, T loadedPropertyValue) throws IOException {
private <T> void setData(long reference, DbDataType propertyType, T loadedPropertyValue) throws IOException {
switch (propertyType) {
case OBJECT:
setObject(reference, loadedPropertyValue);
setObject_(reference, loadedPropertyValue);
break;
case REFERENCES_LIST:
setReferencesList(reference, (LongArrayList) loadedPropertyValue);
setReferencesList_(reference, (LongArrayList) loadedPropertyValue);
break;
case ENHANCED_OBJECT:
setEnhancedObject(reference, (EnhancedObject) loadedPropertyValue);
setEnhancedObject_(reference, (EnhancedObject) loadedPropertyValue);
break;
}
}
<T extends EnhancedObject> void setPrimitives(T enhancedObject, long reference, DbPrimitiveType[] types, Field[] fields)
private <T extends EnhancedObject> void setPrimitives(T enhancedObject, long reference, DbPrimitiveType[] types, Field[] fields)
throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * fields.length);
try {
@ -236,169 +260,219 @@ public class DatabaseObjectsIO implements IObjectsIO {
throw new IOException(e);
}
buffer.flip();
referencesIO.writeToReference(reference, buffer.limit(), buffer);
referencesIO.writeToReference(reference, BLANK_DATA_CLEANER, buffer.limit(), buffer);
}
@Override
public <T extends EnhancedObject> void setEnhancedObject(long reference, T value) throws IOException {
synchronized (accessLock) {
if (value != null) {
EnhancedObjectFullInfo objectFullInfo = value.getAllInfo();
if (objectFullInfo == null || objectFullInfo.getFieldReferences() == null
|| objectFullInfo.getPropertyReferences() == null) {
throw new NullPointerException(
"An EnhancedObject has been initialized using the empty constructor!");
}
lock.lock();
try {
setEnhancedObject_(reference, value);
} finally {
lock.unlock();
}
}
final long[] fieldReferences = objectFullInfo.getFieldReferences();
final DbDataType[] fieldTypes = objectFullInfo.getFieldTypes();
final Field[] fields = objectFullInfo.getFields();
final long nativeFieldDataReference = objectFullInfo.getPrimitiveFieldDataReference();
final DbPrimitiveType[] nativeFieldTypes = objectFullInfo.getPrimitiveFieldTypes();
final Field[] nativeFields = objectFullInfo.getPrimitiveFields();
final long[] propertyReferences = objectFullInfo.getPropertyReferences();
final DbDataType[] propertyTypes = objectFullInfo.getPropertyTypes();
final Object[] propertyValues = objectFullInfo.getLoadedPropertyValues();
Output serializedClassDataStream = new Output(1024, 8192);
kryo.writeClass(serializedClassDataStream, value.getClass());
byte[] serializedClassData = serializedClassDataStream.toBytes();
final int totalSize = Byte.BYTES + serializedClassData.length + Byte.BYTES + Integer.BYTES * 2 + fieldReferences.length * Long.BYTES
+ propertyReferences.length * Long.BYTES + Long.BYTES;
ByteBuffer buffer = ByteBuffer.allocate(totalSize);
if (serializedClassData.length > 255) {
throw new IndexOutOfBoundsException("The class name is too long!");
}
buffer.put((byte) serializedClassData.length);
buffer.put(serializedClassData);
buffer.put((byte) objectFullInfo.getVersion());
buffer.putInt(fieldReferences.length);
buffer.putInt(propertyReferences.length);
for (int i = 0; i < fieldReferences.length; i++) {
buffer.putLong(fieldReferences[i]);
}
for (int i = 0; i < propertyReferences.length; i++) {
buffer.putLong(propertyReferences[i]);
}
buffer.putLong(nativeFieldDataReference);
buffer.flip();
for (int i = 0; i < fieldReferences.length; i++) {
if (fields[i] != null) {
try {
setData(fieldReferences[i], fieldTypes[i], fields[i].get(value));
} catch (IllegalAccessException e) {
throw new IOException(e);
}
}
}
for (int i = 0; i < propertyReferences.length; i++) {
if (propertyValues[i] != null) {
setData(propertyReferences[i], propertyTypes[i], propertyValues[i]);
}
}
setPrimitives(value, nativeFieldDataReference, nativeFieldTypes, nativeFields);
referencesIO.writeToReference(reference, totalSize, buffer);
} else {
referencesIO.writeToReference(reference, 0, null);
private <T extends EnhancedObject> void setEnhancedObject_(long reference, T value) throws IOException {
if (value != null) {
EnhancedObjectFullInfo objectFullInfo = value.getAllInfo();
if (objectFullInfo == null || objectFullInfo.getFieldReferences() == null
|| objectFullInfo.getPropertyReferences() == null) {
throw new NullPointerException(
"An EnhancedObject has been initialized using the empty constructor!");
}
final long[] fieldReferences = objectFullInfo.getFieldReferences();
final DbDataType[] fieldTypes = objectFullInfo.getFieldTypes();
final Field[] fields = objectFullInfo.getFields();
final long nativeFieldDataReference = objectFullInfo.getPrimitiveFieldDataReference();
final DbPrimitiveType[] nativeFieldTypes = objectFullInfo.getPrimitiveFieldTypes();
final Field[] nativeFields = objectFullInfo.getPrimitiveFields();
final long[] propertyReferences = objectFullInfo.getPropertyReferences();
final DbDataType[] propertyTypes = objectFullInfo.getPropertyTypes();
final Object[] propertyValues = objectFullInfo.getLoadedPropertyValues();
Output serializedClassDataStream = new Output(1024, 8192);
kryo.writeClass(serializedClassDataStream, value.getClass());
byte[] serializedClassData = serializedClassDataStream.toBytes();
final int totalSize = Byte.BYTES + serializedClassData.length + Byte.BYTES + Integer.BYTES * 2 + fieldReferences.length * Long.BYTES
+ propertyReferences.length * Long.BYTES + Long.BYTES;
ByteBuffer buffer = ByteBuffer.allocate(totalSize);
if (serializedClassData.length > 255) {
throw new IndexOutOfBoundsException("The class name is too long!");
}
if (objectFullInfo.getVersion() > 250) {
throw new IndexOutOfBoundsException("The class version is too long!");
}
buffer.put((byte) (objectFullInfo.getVersion() + 5));
buffer.put((byte) serializedClassData.length);
buffer.put(serializedClassData);
buffer.putInt(fieldReferences.length);
buffer.putInt(propertyReferences.length);
for (int i = 0; i < fieldReferences.length; i++) {
buffer.putLong(fieldReferences[i]);
}
for (int i = 0; i < propertyReferences.length; i++) {
buffer.putLong(propertyReferences[i]);
}
buffer.putLong(nativeFieldDataReference);
buffer.flip();
for (int i = 0; i < fieldReferences.length; i++) {
if (fields[i] != null) {
try {
setData(fieldReferences[i], fieldTypes[i], fields[i].get(value));
} catch (IllegalAccessException e) {
throw new IOException(e);
}
}
}
for (int i = 0; i < propertyReferences.length; i++) {
if (propertyValues[i] != null) {
setData(propertyReferences[i], propertyTypes[i], propertyValues[i]);
}
}
setPrimitives(value, nativeFieldDataReference, nativeFieldTypes, nativeFields);
referencesIO.writeToReference(reference, ENHANCED_OBJECT_METADATA_CLEANER, totalSize, buffer);
} else {
referencesIO.writeToReference(reference, BLANK_DATA_CLEANER, 0, null);
}
}
@Override
public <T> T loadObject(long reference) throws IOException {
lock.lock();
try {
return loadObject_(reference);
} finally {
lock.unlock();
}
}
@SuppressWarnings("unchecked")
@Override
public <T> T loadObject(long reference) throws IOException {
synchronized (accessLock) {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return null;
}
buffer.rewind();
byte[] data = buffer.array();
return (T) kryo.readClassAndObject(new Input(data));
private <T> T loadObject_(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return null;
}
buffer.rewind();
byte[] data = buffer.array();
return (T) kryo.readClassAndObject(new Input(data));
}
@Override
public LongArrayList loadReferencesList(long reference) throws IOException {
synchronized (accessLock) {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return null;
}
int itemsCount = buffer.getInt();
LongArrayList arrayList = new LongArrayList();
for (int i = 0; i < itemsCount; i++) {
arrayList.add(buffer.getLong());
}
return arrayList;
lock.lock();
try {
return loadReferencesList_(reference);
} finally {
lock.unlock();
}
}
private LongArrayList loadReferencesList_(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return null;
}
int itemsCount = buffer.getInt();
LongArrayList arrayList = new LongArrayList();
for (int i = 0; i < itemsCount; i++) {
arrayList.add(buffer.getLong());
}
return arrayList;
}
@Override
public LongList loadPrimitiveData(long reference) throws IOException {
synchronized (accessLock) {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return null;
}
int size = buffer.limit() / Long.BYTES;
LongArrayList result = new LongArrayList(size);
for (int i = 0; i < size; i++) {
result.add(buffer.getLong());
}
return result;
lock.lock();
try {
return loadPrimitiveData_(reference);
} finally {
lock.unlock();
}
}
private LongList loadPrimitiveData_(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return null;
}
int size = buffer.limit() / Long.BYTES;
LongArrayList result = new LongArrayList(size);
for (int i = 0; i < size; i++) {
result.add(buffer.getLong());
}
return result;
}
@Override
public <T> void setObject(long reference, T value) throws IOException {
synchronized (accessLock) {
if (value != null) {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
Output output = new Output(outputStream);
kryo.writeClassAndObject(output, value);
output.flush();
byte[] data = outputStream.toByteArray();
ByteBuffer dataByteBuffer = ByteBuffer.wrap(data);
referencesIO.writeToReference(reference, data.length, dataByteBuffer);
} else {
referencesIO.writeToReference(reference, 0, null);
}
lock.lock();
try {
setObject_(reference, value);
} finally {
lock.unlock();
}
}
private <T> void setObject_(long reference, T value) throws IOException {
if (value != null) {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
Output output = new Output(outputStream);
kryo.writeClassAndObject(output, value);
output.flush();
byte[] data = outputStream.toByteArray();
ByteBuffer dataByteBuffer = ByteBuffer.wrap(data);
referencesIO.writeToReference(reference, BLANK_DATA_CLEANER, data.length, dataByteBuffer);
} else {
referencesIO.writeToReference(reference, BLANK_DATA_CLEANER, 0, null);
}
}
@Override
public void setReferencesList(long reference, LongArrayList value) throws IOException {
synchronized (accessLock) {
if (value != null) {
int items = value.size();
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * items + Integer.BYTES);
buffer.putInt(items);
for (int i = 0; i < items; i++) {
buffer.putLong(value.getLong(i));
}
buffer.flip();
referencesIO.writeToReference(reference, buffer.limit(), buffer);
} else {
referencesIO.writeToReference(reference, 0, null);
lock.lock();
try {
setReferencesList_(reference, value);
} finally {
lock.unlock();
}
}
private void setReferencesList_(long reference, LongArrayList value) throws IOException {
if (value != null) {
int items = value.size();
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * items + Integer.BYTES);
buffer.putInt(items);
for (int i = 0; i < items; i++) {
buffer.putLong(value.getLong(i));
}
buffer.flip();
referencesIO.writeToReference(reference, REFERENCES_LIST_CLEANER, buffer.limit(), buffer);
} else {
referencesIO.writeToReference(reference, REFERENCES_LIST_CLEANER, 0, null);
}
}
@Override
public long newNullObject() throws IOException {
synchronized (accessLock) {
return referencesIO.allocateReference();
lock.lock();
try {
return newNullObject_();
} finally {
lock.unlock();
}
}
private long newNullObject_() throws IOException {
return referencesIO.allocateReference();
}
@Override
public void loadProperty(EnhancedObject obj, int propertyId, DbDataType propertyType,
long propertyUID) throws IOException {
synchronized (accessLock) {
obj.setProperty(propertyId, loadData(propertyType, propertyUID, property::getReturnType));
obj.setProperty(propertyId, loadData(propertyType, propertyUID));
}
obj.setProperty(propertyId, loadData_(propertyType, propertyUID));
obj.setProperty(propertyId, loadData_(propertyType, propertyUID));
}
@Override
@ -410,13 +484,13 @@ public class DatabaseObjectsIO implements IObjectsIO {
}
private <T extends EnhancedObject> void preloadEnhancedObjectProperties(T obj, long[] propertyReferences) {
// Declare the variables needed to get the biggest property Id
// Declare the variables needed to getBlock the biggest property Id
Method[] unorderedPropertyGetters = obj.getPropertyGetters();
Method[] unorderedPropertySetters = obj.getPropertySetters();
// Find the biggest property Id
int biggestGetter = getBiggestPropertyGetterId(unorderedPropertyGetters);
int biggestSetter = getBiggestPropertySetterId(unorderedPropertySetters);
int biggestGetter = getBiggestPropertyGetterId_(unorderedPropertyGetters);
int biggestSetter = getBiggestPropertySetterId_(unorderedPropertySetters);
int biggestPropertyId = biggestGetter > biggestSetter ? biggestGetter : biggestSetter;
for (Method property : unorderedPropertySetters) {
@ -456,7 +530,16 @@ public class DatabaseObjectsIO implements IObjectsIO {
getterMethods);
}
int getBiggestPropertyGetterId(Method[] unorderedPropertyGetters) {
public int getBiggestPropertyGetterId(Method[] unorderedPropertyGetters) {
lock.lock();
try {
return getBiggestPropertyGetterId_(unorderedPropertyGetters);
} finally {
lock.unlock();
}
}
private int getBiggestPropertyGetterId_(Method[] unorderedPropertyGetters) {
int biggestPropertyId = -1;
for (Method property : unorderedPropertyGetters) {
DbProperty fieldAnnotation = property.getAnnotation(DbProperty.class);
@ -468,7 +551,16 @@ public class DatabaseObjectsIO implements IObjectsIO {
return biggestPropertyId;
}
int getBiggestPropertySetterId(Method[] unorderedPropertySetters) {
public int getBiggestPropertySetterId(Method[] unorderedPropertySetters) {
lock.lock();
try {
return getBiggestPropertySetterId_(unorderedPropertySetters);
} finally {
lock.unlock();
}
}
private int getBiggestPropertySetterId_(Method[] unorderedPropertySetters) {
int biggestPropertyId = -1;
for (Method property : unorderedPropertySetters) {
DbProperty fieldAnnotation = property.getAnnotation(DbProperty.class);
@ -482,10 +574,10 @@ public class DatabaseObjectsIO implements IObjectsIO {
private <T extends EnhancedObject> void preloadEnhancedObjectPrimitiveFields(T obj, long nativeFieldsDataReference)
throws IOException {
// Declare the variables needed to get the biggest field Id
Field[] unorderedFields = getPrimitiveFields(obj);
// Declare the variables needed to getBlock the biggest field Id
Field[] unorderedFields = getPrimitiveFields_(obj);
// Find the biggest field Id
int biggestFieldId = getBiggestPrimitiveFieldId(unorderedFields);
int biggestFieldId = getBiggestPrimitiveFieldId_(unorderedFields);
// Declare the other variables
Field[] fields = new Field[biggestFieldId + 1];
@ -546,8 +638,8 @@ public class DatabaseObjectsIO implements IObjectsIO {
private <T extends EnhancedObject> void preloadEnhancedObjectFields(T obj, long[] fieldReferences)
throws IOException {
// Declare the variables needed to get the biggest field Id
Field[] unorderedFields = getFields(obj);
// Declare the variables needed to getBlock the biggest field Id
Field[] unorderedFields = getFields_(obj);
// Find the biggest field Id
int biggestFieldId = getBiggestFieldId(unorderedFields);
@ -560,7 +652,7 @@ public class DatabaseObjectsIO implements IObjectsIO {
DbField fieldAnnotation = field.getAnnotation(DbField.class);
int fieldId = fieldAnnotation.id();
DbDataType fieldType = fieldAnnotation.type();
loadField(obj, field, fieldType, fieldReferences[fieldId]);
loadField_(obj, field, fieldType, fieldReferences[fieldId]);
fields[fieldId] = field;
orderedFieldTypes[fieldId] = fieldType;
}
@ -568,12 +660,22 @@ public class DatabaseObjectsIO implements IObjectsIO {
obj.setFields(fields, orderedFieldTypes, fieldReferences);
}
<T extends EnhancedObject> void loadField(T obj, Field field, DbDataType fieldType, long fieldReference)
public <T extends EnhancedObject> void loadField(T obj, Field field, DbDataType fieldType, long fieldReference)
throws IOException {
lock.readLock().lock();
lock.lock();
try {
Object data = loadData(fieldType, fieldReference);
try {
loadField_(obj, field, fieldType, fieldReference);
} finally {
lock.unlock();
}
}
private <T extends EnhancedObject> void loadField_(T obj, Field field, DbDataType fieldType, long fieldReference)
throws IOException {
Object data = loadData_(fieldType, fieldReference);
try {
if (fieldType == DbDataType.OBJECT && data != null) {
if (!field.getType().isInstance(data)) {
throw new IOException("There is an attempt to load an object of type " + data.getClass()
+ " into a field of type " + field.getType());
}
@ -584,15 +686,42 @@ public class DatabaseObjectsIO implements IObjectsIO {
}
}
<T extends EnhancedObject> Field[] getFields(T obj) {
public <T extends EnhancedObject> Field[] getFields(T obj) {
lock.lock();
try {
return getFields_(obj);
} finally {
lock.unlock();
}
}
private <T extends EnhancedObject> Field[] getFields_(T obj) {
return FieldUtils.getFieldsWithAnnotation(obj.getClass(), DbField.class);
}
<T extends EnhancedObject> Field[] getPrimitiveFields(T obj) {
public <T extends EnhancedObject> Field[] getPrimitiveFields(T obj) {
lock.lock();
try {
return getPrimitiveFields_(obj);
} finally {
lock.unlock();
}
}
private <T extends EnhancedObject> Field[] getPrimitiveFields_(T obj) {
return FieldUtils.getFieldsWithAnnotation(obj.getClass(), DbPrimitiveField.class);
}
int getBiggestFieldId(Field[] unorderedFields) {
public int getBiggestFieldId(Field[] unorderedFields) {
lock.lock();
try {
return getBiggestFieldId_(unorderedFields);
} finally {
lock.unlock();
}
}
private int getBiggestFieldId_(Field[] unorderedFields) {
int biggestFieldId = -1;
for (Field field : unorderedFields) {
DbField fieldAnnotation = field.getAnnotation(DbField.class);
@ -604,7 +733,16 @@ public class DatabaseObjectsIO implements IObjectsIO {
return biggestFieldId;
}
int getBiggestPrimitiveFieldId(Field[] unorderedFields) {
public int getBiggestPrimitiveFieldId(Field[] unorderedFields) {
lock.lock();
try {
return getBiggestPrimitiveFieldId_(unorderedFields);
} finally {
lock.unlock();
}
}
private int getBiggestPrimitiveFieldId_(Field[] unorderedFields) {
int biggestFieldId = -1;
for (Field field : unorderedFields) {
DbPrimitiveField fieldAnnotation = field.getAnnotation(DbPrimitiveField.class);
@ -629,8 +767,7 @@ public class DatabaseObjectsIO implements IObjectsIO {
}
}
private <T extends EnhancedObject> T preloadEnhancedObject(Class<T> objectType, int serializedVersion,
long nativeFieldsRef, long[] fieldRefs, long[] methodRefs) throws IOException {
private <T extends EnhancedObject> T preloadEnhancedObject(Class<T> objectType, int serializedVersion, long nativeFieldsRef, long[] fieldRefs, long[] methodRefs) throws IOException {
// Instantiate the class to an object
T obj = toInstance(objectType);
@ -658,7 +795,7 @@ public class DatabaseObjectsIO implements IObjectsIO {
public long[] allocateNewUIDs(int quantity) throws IOException {
long[] ids = new long[quantity];
for (int i = 0; i < quantity; i++) {
ids[i] = newNullObject();
ids[i] = newNullObject_();
}
return ids;
}

View File

@ -1,18 +1,16 @@
package it.cavallium.strangedb.java.objects;
import it.cavallium.strangedb.java.annotations.*;
import it.cavallium.strangedb.java.database.IDatabaseTools;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.commons.lang3.reflect.MethodUtils;
import it.cavallium.strangedb.java.annotations.DbClass;
import it.cavallium.strangedb.java.annotations.DbDataType;
import it.cavallium.strangedb.java.annotations.DbPrimitiveType;
import it.cavallium.strangedb.java.annotations.DbPropertyGetter;
import it.cavallium.strangedb.java.annotations.DbPropertySetter;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.Map;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
public abstract class EnhancedObject {
protected final int version;
@ -62,7 +60,7 @@ public abstract class EnhancedObject {
return MethodUtils.getMethodsWithAnnotation(this.getClass(), DbPropertySetter.class);
}
public void setProperties(Method[] propertyGetters, Method[] propertySetters, DbDataType[] propertyTypes, long[] propertyReferences, Map<String, DbPropertySetter> setterMethods, Map<String, DbPropertyGetter> getterMethods) {
public void setProperties(Method[] propertyGetters, Method[] propertySetters, DbDataType[] propertyTypes, long[] propertyReferences, Map<String, DbProperty> setterMethods, Map<String, DbProperty> getterMethods) {
this.propertyGetters = propertyGetters;
this.propertySetters = propertySetters;
this.propertyTypes = propertyTypes;

View File

@ -5,8 +5,10 @@ public class EnhancedObjectIndices {
public final long[] fieldUids;
public final long[] propertyUids;
public final long nativeDataUid;
public final Class<? extends EnhancedObject> type;
public EnhancedObjectIndices(long objectUid, long[] fieldUids, long[] propertyUids, long nativeDataUid) {
public EnhancedObjectIndices(Class<? extends EnhancedObject> type, long objectUid, long[] fieldUids, long[] propertyUids, long nativeDataUid) {
this.type = type;
this.objectUid = objectUid;
this.fieldUids = fieldUids;
this.propertyUids = propertyUids;

View File

@ -1,6 +1,6 @@
package it.cavallium.strangedb.java.objects.lists;
public enum ValueType {
public enum ClassPosition {
PROPERTY,
FIELD,
PRIMITIVE_FIELD

View File

@ -1,6 +1,7 @@
package it.cavallium.strangedb.java.objects.lists;
import it.cavallium.strangedb.java.database.IObjectsIO;
import it.cavallium.strangedb.java.objects.EnhancedObjectIndices;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.cavallium.strangedb.java.objects.EnhancedObject;
import it.cavallium.strangedb.java.database.IDatabaseTools;
@ -115,10 +116,17 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
final LongArrayList indices = dbList.getIndices();
for (int i = 0; i < listSize; i++) {
Long elementUid = indices.get(i);
EnhancedObjectIndices elementUids = dbio.loadEnhancedObjectUids(elementUid);
Object result = resolveItemFromDb(query.valuePointer, dbio, elementUids);
if (query.valueOperation.evaluate(result)) {
results.add(elementUids);
EnhancedObjectIndices elementUids = dbio.loadEnhancedObjectUids(elementUid); // check if the parent object is the declared type
Class<?> declaredRootType = query.valuePointer.getRootType();
Class<?> obtainedRootType = elementUids.type;
if (isInstanceOf(obtainedRootType, declaredRootType)) {
Object result = resolveItemFromDb(query.valuePointer, dbio, elementUids);
if (query.valueOperation.evaluate(result)) {
results.add(elementUids);
}
} else {
//todo: use logging api
System.err.println(obtainedRootType.getSimpleName() + " is not instance of " + declaredRootType.getSimpleName());
}
}
} else if (inputList instanceof ElementsArrayList<?>) {
@ -143,13 +151,13 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
ValuePointer currentPointer = pointer.at(i);
int pathNumber = currentPointer.getPathNumber();
ValueType valueType = currentPointer.getPathType();
ClassPosition valueType = currentPointer.getPathType();
Object value;
switch (valueType) {
case FIELD: {
if (isLastElement) {
//TODO: get field data type. it can be an enhancedObject or an object
//TODO: getBlock field data type. it can be an enhancedObject or an object
value = objectsIO.loadObject(currentElement.fieldUids[pathNumber]);
} else {
value = objectsIO.loadEnhancedObjectUids(currentElement.fieldUids[pathNumber]);
@ -158,7 +166,7 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
}
case PRIMITIVE_FIELD: {
if (isLastElement) {
//TODO: get primitive type
//TODO: getBlock primitive type
value = objectsIO.loadPrimitiveData(currentElement.nativeDataUid).getLong(pathNumber);
} else {
throw new IllegalArgumentException("You can access to a type field only in the last pointer");
@ -167,7 +175,7 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
}
case PROPERTY: {
if (isLastElement) {
//TODO: get field data type. it can be an enhancedObject or an object
//TODO: getBlock field data type. it can be an enhancedObject or an object
value = objectsIO.loadObject(currentElement.fieldUids[pathNumber]);
} else {
value = objectsIO.loadEnhancedObjectUids(currentElement.propertyUids[pathNumber]);
@ -177,13 +185,25 @@ public class EnhancedObjectStrangeDbList<T extends EnhancedObject> extends Stran
throw new IllegalArgumentException("Not implemented");
}
}
if (isLastElement) {
return value;
} else {
currentElement = (EnhancedObjectIndices) value;
// check if the object that we obtained is the declared type
Class<?> declaredType = currentPointer.getAdditionalData();
Class<?> obtainedType = currentElement.type;
if (!isInstanceOf(obtainedType, declaredType)) {
return null;
}
}
}
throw new IOException("The pointer is empty");
}
private static boolean isInstanceOf(Class<?> clazz, Class<?> obj){
return obj.isAssignableFrom(clazz);
}
}

View File

@ -40,11 +40,7 @@ public abstract class StrangeDbList<T> extends EnhancedObject implements Element
@Override
public void update(int index, T value) throws IOException {
lock.writeLock().lock();
try {
set(index, value);
} finally {
}
set(index, value);
}
@Override

View File

@ -13,14 +13,24 @@ import java.util.Arrays;
public class ValuePointer {
private static ValuePointer base = new ValuePointer(new int[0], new ValueType[0]);
private static ValuePointer base = new ValuePointer(new int[0], new ClassPosition[0], new Object[0], null);
private final int[] pathNumbers;
private final ValueType[] pathTypes;
private final ClassPosition[] pathTypes;
/**
* PropertyType
*/
private final Object[] additionalData;
/**
* ParentType
*/
private final Class<?> rootType;
private ValuePointer(int[] pathNumbers, ValueType[] pathTypes) {
private ValuePointer(int[] pathNumbers, ClassPosition[] pathTypes, Object[] additionalData, Class<?> rootType) {
this.pathNumbers = pathNumbers;
this.pathTypes = pathTypes;
this.additionalData = additionalData;
this.rootType = rootType;
}
public static <T extends EnhancedObject> ValuePointer ofField(Class<T> parentType, String field) {
@ -36,13 +46,15 @@ public class ValuePointer {
public <T extends EnhancedObject> ValuePointer prop(Class<T> parentType, String propertyName) {
ValueType[] newPathTypes = append(this.pathTypes, ValueType.FIELD);
ClassPosition[] newPathTypes = append(this.pathTypes, ClassPosition.FIELD);
Method[] methods = MethodUtils.getMethodsWithAnnotation(parentType, DbProperty.class);
DbProperty dbProperty = null;
Class<?> propType = null;
for (Method method : methods) {
DbProperty annotation = method.getAnnotation(DbProperty.class);
if (annotation.name().equals(propertyName)) {
propType = method.getReturnType();
dbProperty = annotation;
break;
}
@ -52,17 +64,24 @@ public class ValuePointer {
}
int[] newPathNumbers = append(this.pathNumbers, dbProperty.id());
return new ValuePointer(newPathNumbers, newPathTypes);
Object[] newAdditionalData = append(this.additionalData, propType);
Class<?> rootType = this.rootType == null ? parentType : this.rootType;
if (newAdditionalData.length > 1) {
newAdditionalData[newAdditionalData.length - 2] = parentType;
}
return new ValuePointer(newPathNumbers, newPathTypes, newAdditionalData, rootType);
}
public <T extends EnhancedObject> ValuePointer field(Class<T> parentType, String fieldName) {
ValueType[] newPathTypes = append(this.pathTypes, ValueType.FIELD);
ClassPosition[] newPathTypes = append(this.pathTypes, ClassPosition.FIELD);
Field[] fields = FieldUtils.getFieldsWithAnnotation(parentType, DbField.class);
DbField dbField = null;
Class<?> fieldType = null;
for (Field field : fields) {
DbField annotation = field.getAnnotation(DbField.class);
if (annotation.name().equals(fieldName)) {
fieldType = field.getType();
dbField = annotation;
break;
}
@ -72,17 +91,24 @@ public class ValuePointer {
}
int[] newPathNumbers = append(this.pathNumbers, dbField.id());
return new ValuePointer(newPathNumbers, newPathTypes);
Object[] newAdditionalData = append(this.additionalData, fieldType);
if (newAdditionalData.length > 1) {
newAdditionalData[newAdditionalData.length - 2] = parentType;
}
Class<?> rootType = this.rootType == null ? parentType : this.rootType;
return new ValuePointer(newPathNumbers, newPathTypes, newAdditionalData, rootType);
}
public <T extends EnhancedObject> ValuePointer primitiveField(Class<T> parentType, String primitiveFieldName) {
ValueType[] newPathTypes = append(this.pathTypes, ValueType.PRIMITIVE_FIELD);
ClassPosition[] newPathTypes = append(this.pathTypes, ClassPosition.PRIMITIVE_FIELD);
Field[] fields = FieldUtils.getFieldsWithAnnotation(parentType, DbPrimitiveField.class);
DbPrimitiveField dbField = null;
Class<?> fieldType = null;
for (Field field : fields) {
DbPrimitiveField annotation = field.getAnnotation(DbPrimitiveField.class);
if (annotation.name().equals(primitiveFieldName)) {
fieldType = field.getType();
dbField = annotation;
break;
}
@ -92,7 +118,12 @@ public class ValuePointer {
}
int[] newPathNumbers = append(this.pathNumbers, dbField.id());
return new ValuePointer(newPathNumbers, newPathTypes);
Object[] newAdditionalData = append(this.additionalData, fieldType);
if (newAdditionalData.length > 1) {
newAdditionalData[newAdditionalData.length - 2] = parentType;
}
Class<?> rootType = this.rootType == null ? parentType : this.rootType;
return new ValuePointer(newPathNumbers, newPathTypes, newAdditionalData, rootType);
}
public ValuePointer at(int index) {
@ -102,7 +133,10 @@ public class ValuePointer {
if (index == pathNumbers.length - 1) {
return this;
}
return new ValuePointer(Arrays.copyOf(this.pathNumbers, index + 1), Arrays.copyOf(this.pathTypes, index + 1));
return new ValuePointer(Arrays.copyOf(this.pathNumbers, index + 1),
Arrays.copyOf(this.pathTypes, index + 1),
Arrays.copyOf(this.additionalData, index + 1),
rootType);
}
public int size() {
@ -133,7 +167,16 @@ public class ValuePointer {
return pathNumbers[pathNumbers.length - 1];
}
public ValueType getPathType() {
public ClassPosition getPathType() {
return pathTypes[pathTypes.length - 1];
}
@SuppressWarnings("unchecked")
public <T> T getAdditionalData() {
return (T) additionalData[additionalData.length - 1];
}
public Class<?> getRootType() {
return rootType;
}
}