Finished new implementation with annotations

This commit is contained in:
Andrea Cavalli 2019-01-18 02:35:55 +01:00
parent 7123228027
commit 41b2bb77a2
22 changed files with 1402 additions and 514 deletions

16
pom.xml
View File

@ -6,7 +6,7 @@
<groupId>org.warp</groupId>
<artifactId>jcwdb</artifactId>
<version>1.0-SNAPSHOT</version>
<version>1.1-SNAPSHOT</version>
<name>jcwdb</name>
<!-- FIXME change it to the project's website -->
@ -14,10 +14,18 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<maven.compiler.source>10</maven.compiler.source>
<maven.compiler.target>10</maven.compiler.target>
</properties>
<repositories>
<repository>
<id>sonatype-snapshots</id>
<name>sonatype snapshots repo</name>
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>junit</groupId>
@ -33,7 +41,7 @@
<dependency>
<groupId>com.esotericsoftware</groupId>
<artifactId>kryo</artifactId>
<version>5.0.0-RC1</version>
<version>5.0.0-RC2-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>net.openhft</groupId>

View File

@ -29,7 +29,7 @@ public class FileAllocator implements AutoCloseable {
this.fileSize = this.dataFileChannel.size();
}
public FileAllocator(SeekableByteChannel dataFileChannel, long fileSize, Long2IntMap freeBytes) throws IOException {
public FileAllocator(SeekableByteChannel dataFileChannel, long fileSize, Long2IntMap freeBytes) {
this.dataFileChannel = dataFileChannel;
this.fileSize = fileSize;
this.freeBytes.putAll(freeBytes);
@ -44,16 +44,20 @@ public class FileAllocator implements AutoCloseable {
public long allocate(int size) {
checkClosed();
synchronized (allocateLock) {
long offset = allocateIntoUnusedParts(size);
if (offset == -1) {
return allocateToEnd(size);
} else {
long offset;
if ((offset = allocateIntoUnusedParts(size)) != -1) {
if (offset + size > fileSize) {
fileSize = offset + size;
}
return offset;
} else {
return allocateToEnd(size);
}
}
}
private long allocateIntoUnusedParts(int size) {
if (FileIndexManager.ALWAYS_ALLOCATE_NEW) return -1;
Stream<Map.Entry<Long,Integer>> sorted =
freeBytes.entrySet().stream()
.sorted(Collections.reverseOrder(Map.Entry.comparingByValue()));
@ -104,6 +108,7 @@ public class FileAllocator implements AutoCloseable {
*/
public void markFree(long startPosition, int length) {
checkClosed();
if (FileIndexManager.ALWAYS_ALLOCATE_NEW) return;
if (freeBytes.containsKey(startPosition + length)) {
int secondLength = freeBytes.remove(startPosition + length);

View File

@ -4,7 +4,7 @@ import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import it.unimi.dsi.fastutil.longs.*;
import it.unimi.dsi.fastutil.objects.ObjectIterator;
import org.warp.jcwdb.ann.Database;
import org.warp.jcwdb.ann.DatabaseManager;
import java.io.IOException;
import java.nio.ByteBuffer;
@ -13,14 +13,14 @@ import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.function.Consumer;
public class FileIndexManager implements IndexManager {
public static final boolean ALWAYS_ALLOCATE_NEW = false;
private final SeekableByteChannel dataFileChannel, metadataFileChannel;
private volatile long metadataFileChannelSize;
private final FileAllocator fileAllocator;
private final ByteBuffer metadataByteBuffer = ByteBuffer.allocateDirect(IndexDetails.TOTAL_BYTES);
private final ByteBuffer maskByteBuffer = ByteBuffer.allocateDirect(Integer.BYTES);
private final ByteBuffer maskByteBuffer = ByteBuffer.allocateDirect(Long.BYTES);
private volatile boolean closed;
private final Object closeLock = new Object();
private final Object metadataByteBufferLock = new Object();
@ -36,19 +36,17 @@ public class FileIndexManager implements IndexManager {
/**
* Edit this using editIndex()
*/
private final LongSet dirtyLoadedIndices, flushingAllowedIndices, removedIndices;
private final LongSet dirtyLoadedIndices, removedIndices;
private long firstAllocableIndex;
public FileIndexManager(Path dataFile, Path metadataFile) throws IOException {
if (Cleaner.DISABLE_CLEANER) {
loadedIndices = new Long2ObjectOpenHashMap<>();
dirtyLoadedIndices = new LongOpenHashSet();
flushingAllowedIndices = new LongOpenHashSet();
removedIndices = new LongOpenHashSet();
} else {
loadedIndices = new Long2ObjectLinkedOpenHashMap<>();
dirtyLoadedIndices = new LongLinkedOpenHashSet();
flushingAllowedIndices = new LongLinkedOpenHashSet();
removedIndices = new LongLinkedOpenHashSet();
}
if (Files.notExists(dataFile)) {
@ -59,27 +57,30 @@ public class FileIndexManager implements IndexManager {
}
dataFileChannel = Files.newByteChannel(dataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
metadataFileChannel = Files.newByteChannel(metadataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
fileAllocator = createFileAllocator(dataFileChannel, metadataFileChannel.position(0));
metadataFileChannelSize = metadataFileChannel.size();
fileAllocator = createFileAllocator(dataFileChannel, metadataFileChannel.position(0));
firstAllocableIndex = getMetadataFileChannelSize() / (long) IndexDetails.TOTAL_BYTES;
if (firstAllocableIndex == 0) {
firstAllocableIndex = 1;
}
}
private long getMetadataFileChannelSize() throws IOException {
private long getMetadataFileChannelSize() {
return metadataFileChannelSize;
}
private FileAllocator createFileAllocator(final SeekableByteChannel dataFileChannel, final SeekableByteChannel metadataFileChannel) throws IOException {
if (ALWAYS_ALLOCATE_NEW) {
return new FileAllocator(dataFileChannel);
} else {
Long2IntMap freeBytes = new Long2IntRBTreeMap();
Long2IntMap usedBytes = new Long2IntRBTreeMap();
long firstOffset = 0;
metadataFileChannel.position(0);
while (metadataFileChannel.position() + IndexDetails.TOTAL_BYTES <= getMetadataFileChannelSize()) {
IndexDetails indexDetails = readIndexDetailsAt(metadataFileChannel);
if (indexDetails != null) {
long offset = indexDetails.getOffset();
if (!usedBytes.containsKey(offset) || indexDetails.getSize() > usedBytes.get(offset)) {
usedBytes.put(offset, indexDetails.getSize());
}
if (offset < firstOffset) {
firstOffset = offset;
}
@ -107,6 +108,7 @@ public class FileIndexManager implements IndexManager {
return new FileAllocator(dataFileChannel, fileSize, freeBytes);
}
}
@Override
public <T> T get(long index, DBReader<T> reader) throws IOException {
@ -121,7 +123,7 @@ public class FileIndexManager implements IndexManager {
public IndexDetails set(long index, int size, DBWriter data) throws IOException {
checkClosed();
IndexDetails indexDetails = getIndexMetadataUnsafe(index);
if (indexDetails == null || indexDetails.getSize() < size) {
if (ALWAYS_ALLOCATE_NEW || indexDetails == null || indexDetails.getSize() < size) {
// Allocate new space
IndexDetails newDetails = allocateAndWrite(index, size, data);
if (indexDetails != null) {
@ -144,16 +146,6 @@ public class FileIndexManager implements IndexManager {
}
}
@Override
public void setFlushingAllowed(long index, boolean isUnloadingAllowed) {
checkClosed();
if (isUnloadingAllowed) {
flushingAllowedIndices.add(index);
} else {
flushingAllowedIndices.remove(index);
}
}
@Override
public long add(int size) {
checkClosed();
@ -192,7 +184,6 @@ public class FileIndexManager implements IndexManager {
throw new IOException("Unable to write " + size + " in a space of " + indexDetails.getSize());
}
final long offset = indexDetails.getOffset();
final Output o = new Output(Channels.newOutputStream(dataFileChannel.position(offset)), size);
data.write(o);
o.flush();
@ -214,7 +205,6 @@ public class FileIndexManager implements IndexManager {
}
synchronized (indicesMapsAccessLock) {
dirtyLoadedIndices.remove(index);
flushingAllowedIndices.remove(index);
loadedIndices.remove(index);
removedIndices.add(index);
}
@ -225,7 +215,6 @@ public class FileIndexManager implements IndexManager {
synchronized (indicesMapsAccessLock) {
removedIndices.remove(index);
dirtyLoadedIndices.remove(index);
flushingAllowedIndices.remove(index);
loadedIndices.remove(index);
}
// Update indices metadata
@ -238,7 +227,6 @@ public class FileIndexManager implements IndexManager {
if (dirtyLoadedIndices.contains(index)) {
indexDetails = loadedIndices.get(index);
dirtyLoadedIndices.remove(index);
flushingAllowedIndices.remove(index);
}
}
if (isDirty) {
@ -302,7 +290,6 @@ public class FileIndexManager implements IndexManager {
synchronized (indicesMapsAccessLock) {
loadedIndices.put(index, details);
dirtyLoadedIndices.add(index);
flushingAllowedIndices.remove(index);
}
}
@ -311,7 +298,6 @@ public class FileIndexManager implements IndexManager {
long newIndex = firstAllocableIndex++;
loadedIndices.put(newIndex, indexDetails);
dirtyLoadedIndices.add(newIndex);
flushingAllowedIndices.remove(newIndex);
removedIndices.remove(newIndex);
return newIndex;
}
@ -442,7 +428,6 @@ public class FileIndexManager implements IndexManager {
long lastIndex = -2;
synchronized (indicesMapsAccessLock) {
for (long index : dirtyLoadedIndices) {
if (!flushingAllowedIndices.contains(index)) {
IndexDetails indexDetails = loadedIndices.get(index);
long position = index * IndexDetails.TOTAL_BYTES;
resizeMetadataFileChannel(position);
@ -453,10 +438,7 @@ public class FileIndexManager implements IndexManager {
lastIndex = index;
flushedIndices++;
}
}
dirtyLoadedIndices.clear();
dirtyLoadedIndices.addAll(flushingAllowedIndices);
flushingAllowedIndices.clear();
}
return flushedIndices;
}
@ -484,12 +466,12 @@ public class FileIndexManager implements IndexManager {
long removedIndices = 0;
LongArrayList toUnload = new LongArrayList();
synchronized (indicesMapsAccessLock) {
if (loadedIndices.size() > Database.MAX_LOADED_INDICES) {
if (loadedIndices.size() > DatabaseManager.MAX_LOADED_INDICES) {
long count = loadedIndices.size();
LongIterator it = loadedIndices.keySet().iterator();
while (it.hasNext()) {
long loadedIndex = it.nextLong();
if (count < Database.MAX_LOADED_INDICES * 3l / 2l) {
if (count < DatabaseManager.MAX_LOADED_INDICES * 3l / 2l) {
break;
}
toUnload.add(loadedIndex);

View File

@ -13,7 +13,6 @@ public interface IndexManager extends Cleanable {
long add(int size, DBWriter writer) throws IOException;
FullIndexDetails addAndGetDetails(int size, DBWriter writer) throws IOException;
IndexDetails set(long index, int size, DBWriter writer) throws IOException;
void setFlushingAllowed(long index, boolean isUnloadingAllowed);
void delete(long index) throws IOException;
boolean has(long index);
void close() throws IOException;

View File

@ -1,9 +0,0 @@
package org.warp.jcwdb.ann;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@Retention(RetentionPolicy.RUNTIME)
public @interface DBClass {
int classTypeId();
}

View File

@ -0,0 +1,35 @@
package org.warp.jcwdb.ann;
import java.io.IOException;
import java.util.LinkedList;
public class DBDBObjectList<T extends DBObject> extends DBList<T> {
@DBField(id = 2, type = DBDataType.OBJECT)
public Class<T> type;
public DBDBObjectList(JCWDatabase db, Class<T> type) {
super(db, 10);
this.type = type;
}
public DBDBObjectList(JCWDatabase db, Class<T> type, int maxLoadedItems) {
super(db, maxLoadedItems);
this.type = type;
}
public DBDBObjectList(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
super(database, objectInfo);
}
@Override
protected T loadObjectFromDatabase(long uid) throws IOException {
return database.loadDBObject(type, uid);
}
@Override
protected long saveObjectToDatabase(long uid, T value) throws IOException {
database.writeObjectProperty(uid, DBDataType.DATABASE_OBJECT, value);
return uid;
}
}

View File

@ -3,5 +3,6 @@ package org.warp.jcwdb.ann;
public enum DBDataType {
DATABASE_OBJECT,
OBJECT,
INTEGER
INTEGER,
UID_LIST
}

View File

@ -0,0 +1,171 @@
package org.warp.jcwdb.ann;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import java.io.IOError;
import java.io.IOException;
import java.util.LinkedList;
import java.util.StringJoiner;
public abstract class DBList<T> extends DBObject {
@DBField(id = 0, type = DBDataType.INTEGER)
private int maxLoadedItems;
@DBField(id = 1, type = DBDataType.UID_LIST)
private LongArrayList itemIndices;
public DBList(JCWDatabase db, int maxLoadedItems) {
super(db);
this.maxLoadedItems = maxLoadedItems;
this.itemIndices = new LongArrayList();
}
public DBList(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
super(database, objectInfo);
}
public int getMaxLoadedItems() {
return maxLoadedItems;
}
public void add(T obj) {
try {
itemIndices.add(saveObjectToDatabase(database.allocateNullValue(), obj));
} catch (IOException e) {
throw new IOError(e);
}
}
public int size() {
return this.itemIndices.size();
}
private T loadObjectAt(int i) throws IOException {
return loadObjectFromDatabase(itemIndices.getLong(i));
}
protected abstract T loadObjectFromDatabase(long uid) throws IOException;
protected abstract long saveObjectToDatabase(long uid, T value) throws IOException;
public DBListIterator<T> iterator() {
return new DBListIterator<>() {
private int position = itemIndices.size();
private LinkedList<T> cachedItems;
private int objectToSavePosition;
private T objectToSave;
@Override
public boolean hasNext() {
if (position > 0) {
return true;
} else {
try {
saveObservedObject();
} catch (IOException e) {
throw new IOError(e);
}
return false;
}
}
@Override
public T next() {
position--;
if (position < 0) {
throw new NullPointerException("Position < 0");
}
if (cachedItems == null || cachedItems.size() == 0) {
try {
cachedItems = fillCache(position);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
try {
return switchObservedObject(position, cachedItems.removeFirst());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public void remove() {
try {
saveObservedObject();
} catch (IOException e) {
throw new RuntimeException(e);
}
itemIndices.removeLong(position);
}
@Override
public void insert(T obj) {
try {
itemIndices.add(position, saveObjectToDatabase(database.allocateNullValue(), obj));
} catch (IOException e) {
throw new IOError(e);
}
}
@Override
public void set(T obj) {
try {
itemIndices.set(position, saveObjectToDatabase(database.allocateNullValue(), obj));
} catch (IOException e) {
throw new IOError(e);
}
}
private LinkedList<T> fillCache(int position) throws IOException {
LinkedList<T> cachedItems = new LinkedList<>();
int firstObjectIndex = position;
int lastObjectIndex = firstObjectIndex - maxLoadedItems;
if (lastObjectIndex < 0) {
lastObjectIndex = 0;
}
for (int i = firstObjectIndex; i >= lastObjectIndex; i--) {
cachedItems.addLast(loadObjectAt(i));
}
return cachedItems;
}
private void saveObservedObject() throws IOException {
if (objectToSave != null) {
itemIndices.set(objectToSavePosition, saveObjectToDatabase(database.allocateNullValue(), objectToSave));
objectToSave = null;
objectToSavePosition = 0;
}
}
private T switchObservedObject(int position, T obj) throws IOException {
saveObservedObject();
objectToSave = obj;
objectToSavePosition = position;
return obj;
}
};
}
public interface DBListIterator<T> {
boolean hasNext();
T next();
void remove();
void insert(T obj);
void set(T obj);
}
@Override
public String toString() {
return new StringJoiner(", ", DBList.class.getSimpleName() + "[", "]")
.add("size=" + size())
.add("maxLoadedItems=" + maxLoadedItems)
.add("itemIndices=" + (itemIndices.size() < 100 ? itemIndices : "["+itemIndices.size()+" items]"))
.toString();
}
}

View File

@ -1,12 +1,16 @@
package org.warp.jcwdb.ann;
import org.apache.commons.lang3.reflect.FieldUtils;
import java.io.IOError;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.LinkedHashMap;
import java.util.Map;
public abstract class DBObject {
private final Database database;
private final long uid;
protected final DatabaseManager database;
private Field[] fields;
private DBDataType[] fieldTypes;
private long[] fieldUIDs;
@ -17,17 +21,103 @@ public abstract class DBObject {
private long[] propertyUIDs;
private boolean[] loadedProperties;
private Object[] loadedPropertyValues;
private Map<String, DBPropertySetter> setterMethods;
private Map<String, DBPropertyGetter> getterMethods;
private final Object fieldsAccessLock = new Object();
private final Object propertiesAccessLock = new Object();
public DBObject(Database database) {
this.database = database;
this.uid = database.newDBObject(this);
database.preloadDBObject(this);
public DBObject(JCWDatabase database) {
this.database = database.getDatabaseManager();
try {
initializeDBObject();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public DBObject(Database database, long uid) {
this.database = database;
this.uid = uid;
database.preloadDBObject(this);
private void initializeDBObject() throws IOException {
initializeDBObjectFields();
initializeDBObjectProperties();
}
private void initializeDBObjectFields() throws IOException {
// Declare the variables needed to get the biggest field Id
Field[] unorderedFields = database.getFields(this);
// Find the biggest field Id
int biggestFieldId = database.getBiggestFieldId(unorderedFields);
// Allocate new UIDs
fieldUIDs = database.allocateNewUIDs(biggestFieldId + 1);
// Declare the other variables
Field[] fields = new Field[biggestFieldId + 1];
DBDataType[] orderedFieldTypes = new DBDataType[biggestFieldId + 1];
// Load all fields metadata and load them
for (Field field : unorderedFields) {
DBField fieldAnnotation = field.getAnnotation(DBField.class);
int fieldId = fieldAnnotation.id();
DBDataType fieldType = fieldAnnotation.type();
database.loadField(this, field, fieldType, fieldUIDs[fieldId]);
fields[fieldId] = field;
orderedFieldTypes[fieldId] = fieldType;
}
// Set fields metadata
setFields(fields, orderedFieldTypes, fieldUIDs);
}
private void initializeDBObjectProperties() {
// Declare the variables needed to get the biggest property Id
Method[] unorderedPropertyGetters = database.getPropertyGetters(this);
Method[] unorderedPropertySetters = database.getPropertySetters(this);
// Find the biggest property Id
int biggestGetter = database.getBiggestPropertyGetterId(unorderedPropertyGetters);
int biggestSetter = database.getBiggestPropertySetterId(unorderedPropertySetters);
int biggestPropertyId = biggestGetter > biggestSetter ? biggestGetter : biggestSetter;
// Allocate new UIDs
propertyUIDs = database.allocateNewUIDs(biggestPropertyId + 1);
for (Method property : unorderedPropertySetters) {
DBPropertySetter fieldAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
// Declare the other variables
DBDataType[] propertyTypes = new DBDataType[biggestPropertyId + 1];
Method[] propertyGetters = new Method[biggestPropertyId + 1];
Method[] propertySetters = new Method[biggestPropertyId + 1];
Map<String, DBPropertySetter> setterMethods = new LinkedHashMap<>();
Map<String, DBPropertyGetter> getterMethods = new LinkedHashMap<>();
// Load the properties metadata
for (Method property : unorderedPropertyGetters) {
DBPropertyGetter propertyAnnotation = property.getAnnotation(DBPropertyGetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertyGetters[propertyId] = property;
getterMethods.put(property.getName(), propertyAnnotation);
}
for (Method property : unorderedPropertySetters) {
DBPropertySetter propertyAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertySetters[propertyId] = property;
setterMethods.put(property.getName(), propertyAnnotation);
}
// Set properties metadata
setProperties(propertyGetters, propertySetters, propertyTypes, propertyUIDs, setterMethods, getterMethods);
}
public DBObject(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
this.database = database.getDatabaseManager();
this.database.preloadDBObject(this, objectInfo);
}
public <T> T getProperty() {
@ -36,71 +126,86 @@ public abstract class DBObject {
try {
int propertyId = stackFrame.getDeclaringClass().getDeclaredMethod(stackFrame.getMethodName()).getAnnotation(DBPropertyGetter.class).id();
return getProperty(propertyId);
} catch (NoSuchMethodException e) {
} catch (IOException | NoSuchMethodException e) {
throw new RuntimeException(e);
}
}
<T> void setLoadedProperty(int propertyId, T value) {
loadedPropertyValues[propertyId] = value;
loadedProperties[propertyId] = true;
}
@SuppressWarnings("unchecked")
private <T> T getProperty(int propertyId) {
private <T> T getProperty(int propertyId) throws IOException {
synchronized (propertiesAccessLock) {
if (!loadedProperties[propertyId]) {
try {
database.loadProperty(this, propertyId, propertyGetters[propertyId], propertyTypes[propertyId], propertyUIDs[propertyId]);
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException e) {
throw new RuntimeException(e);
}
long propertyUID = propertyUIDs[propertyId];
database.loadProperty(this, propertyId, propertyGetters[propertyId], propertyTypes[propertyId], propertyUID);
}
return (T) loadedPropertyValues[propertyId];
}
}
public <T> void setProperty(T value) {
StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
StackWalker.StackFrame stackFrame = walker.walk(f -> f.skip(1).findFirst().orElse(null));
try {
int propertyId = stackFrame.getDeclaringClass().getDeclaredMethod(stackFrame.getMethodName(), value.getClass()).getAnnotation(DBPropertySetter.class).id();
setProperty(propertyId, value);
} catch (NoSuchMethodException e) {
throw new RuntimeException(e);
}
DBPropertySetter propertyAnnotation = setterMethods.get(stackFrame.getMethodName());
setProperty(propertyAnnotation.id(), propertyAnnotation.type(), value);
}
public <T> void setProperty(int propertyId, T value) {
public <T> void setProperty(int propertyId, DBDataType propertyType, T value) {
synchronized (propertiesAccessLock) {
loadedPropertyValues[propertyId] = value;
loadedProperties[propertyId] = true;
}
}
public void close() {
database.saveObject(this);
public void writeToDisk(long uid) {
//System.err.println("Saving object " + uid + ":" + this);
try {
synchronized (propertiesAccessLock) {
synchronized (fieldsAccessLock) {
database.writeObjectInfo(uid, fieldUIDs, propertyUIDs);
}
}
synchronized (fieldsAccessLock) {
for (int i = 0; i < fieldUIDs.length; i++) {
try {
database.writeObjectProperty(fieldUIDs[i], fieldTypes[i], fields[i].get(this));
} catch (IllegalAccessException e) {
throw new IOError(e);
}
}
}
synchronized (propertiesAccessLock) {
for (int i = 0; i < propertyUIDs.length; i++) {
database.writeObjectProperty(propertyUIDs[i], propertyTypes[i], loadedPropertyValues[i]);
}
}
} catch (IOException e) {
throw new IOError(e);
}
}
public final void setFields(Field[] fields, DBDataType[] fieldTypes, long[] fieldUIDs) {
synchronized (fieldsAccessLock) {
this.fields = fields;
this.fieldTypes = fieldTypes;
this.fieldUIDs = fieldUIDs;
}
}
public final void setProperties(Method[] propertyGetters, Method[] propertySetters, DBDataType[] propertyTypes, long[] propertyUIDs) {
public final void setProperties(Method[] propertyGetters, Method[] propertySetters, DBDataType[] propertyTypes, long[] propertyUIDs, Map<String, DBPropertySetter> setterMethods, Map<String, DBPropertyGetter> getterMethods) {
synchronized (propertiesAccessLock) {
this.propertyGetters = propertyGetters;
this.propertySetters = propertySetters;
this.propertyTypes = propertyTypes;
this.propertyUIDs = propertyUIDs;
this.loadedProperties = new boolean[this.propertyUIDs.length];
this.loadedPropertyValues = new Object[this.propertyUIDs.length];
this.setterMethods = setterMethods;
this.getterMethods = getterMethods;
}
public final long getUID() {
return uid;
}
public long[] getAllFieldUIDs() {
return fieldUIDs;
}
public long[] getAllPropertyUIDs() {
return propertyUIDs;
}
}

View File

@ -8,12 +8,18 @@ import java.io.IOException;
public class DBObjectIndicesManager {
private final FileIndexManager indices;
public DBObjectIndicesManager(FileIndexManager indices) {
DBObjectIndicesManager(FileIndexManager indices) {
this.indices = indices;
}
public long allocate(int fieldsCount, int propertiesCount) {
return indices.add(calculateObjectSize(fieldsCount, propertiesCount));
long uid = indices.add(calculateObjectSize(fieldsCount, propertiesCount));
//System.err.println("ALLOCATED UID " + uid);
return uid;
}
public void setNull(long uid) throws IOException {
indices.set(uid, 0, (w) -> w.write(new byte[0]));
}
public void set(long uid, long[] fields, long[] properties) throws IOException {
@ -23,7 +29,7 @@ public class DBObjectIndicesManager {
for (int i = 0; i < fields.length; i++) {
w.writeLong(fields[i]);
}
for (int i = 0; i < fields.length; i++) {
for (int i = 0; i < properties.length; i++) {
w.writeLong(properties[i]);
}
});
@ -31,6 +37,9 @@ public class DBObjectIndicesManager {
public DBObjectInfo get(long uid) throws IOException {
return indices.get(uid, (i, size) -> {
if (size < Integer.BYTES * 2) {
return null;
}
long[] indices = new long[i.readInt()];
long[] properties = new long[i.readInt()];
if (size != calculateObjectSize(indices, properties)) {
@ -46,6 +55,10 @@ public class DBObjectIndicesManager {
});
}
public boolean has(long uid) {
return indices.has(uid);
}
private int calculateObjectSize(long[] fields, long[] properties) {
return calculateObjectSize(fields.length, properties.length);
}

View File

@ -0,0 +1,25 @@
package org.warp.jcwdb.ann;
import java.io.IOException;
public class DBObjectList<T> extends DBList<T> {
public DBObjectList(JCWDatabase db, int maxLoadedItems) {
super(db, maxLoadedItems);
}
public DBObjectList(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
super(database, objectInfo);
}
@Override
protected T loadObjectFromDatabase(long uid) throws IOException {
return database.loadObject(uid);
}
@Override
protected long saveObjectToDatabase(long uid, T value) throws IOException {
database.writeObjectProperty(uid, DBDataType.OBJECT, value);
return uid;
}
}

View File

@ -1,275 +0,0 @@
package org.warp.jcwdb.ann;
import com.esotericsoftware.kryo.Kryo;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.commons.lang3.reflect.MethodUtils;
import org.warp.jcwdb.FileIndexManager;
import java.io.IOError;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.file.Path;
import java.util.LinkedList;
public class Database {
public static final long MAX_LOADED_INDICES = 100000;
private final DBObjectIndicesManager objectIndicesManager;
private final FileIndexManager indices;
private final LinkedList<WeakReference<DBObject>> loadedObjects = new LinkedList<>();
private static final Kryo kryo = new Kryo();
public Database(Path dataFile, Path metadataFile) throws IOException {
this.indices = new FileIndexManager(dataFile, metadataFile);
this.objectIndicesManager = new DBObjectIndicesManager(this.indices);
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
Database.this.close();
} catch (Exception e) {
e.printStackTrace();
}
}));
}
private void close() throws IOException {
indices.close();
for (WeakReference<DBObject> loadedObjectReference : loadedObjects) {
DBObject loadedObject = loadedObjectReference.get();
if (loadedObject != null) {
loadedObject.close();
}
}
}
public void preloadDBObject(DBObject obj) {
DBObjectIndicesManager.DBObjectInfo UIDs = readUIDs(obj.getUID());
preloadDBObjectFields(obj, UIDs.getFields());
preloadDBObjectProperties(obj, UIDs.getProperties());
}
private void preloadDBObjectFields(DBObject obj, long[] fieldUIDs) {
// Declare the variables needed to get the biggest field Id
Field[] unorderedFields = getFields(obj);
// Find the biggest field Id
int biggestFieldId = getBiggestFieldId(unorderedFields);
// Declare the other variables
Field[] fields = new Field[biggestFieldId + 1];
DBDataType[] orderedFieldTypes = new DBDataType[biggestFieldId + 1];
// Load all fields metadata and load them
for (Field field : unorderedFields) {
DBField fieldAnnotation = field.getAnnotation(DBField.class);
int fieldId = fieldAnnotation.id();
DBDataType fieldType = fieldAnnotation.type();
try {
loadField(obj, field, fieldType, fieldUIDs[fieldId]);
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException e) {
throw new RuntimeException(e);
}
fields[fieldId] = field;
orderedFieldTypes[fieldId] = fieldType;
}
// Set fields metadata
obj.setFields(fields, orderedFieldTypes, fieldUIDs);
}
private void preloadDBObjectProperties(DBObject obj, long[] propertyUIDs) {
// Declare the variables needed to get the biggest property Id
Method[] unorderedPropertyGetters = getPropertyGetters(obj);
Method[] unorderedPropertySetters = getPropertySetters(obj);
// Find the biggest property Id
int biggestGetter = getBiggestPropertyGetterId(unorderedPropertyGetters);
int biggestSetter = getBiggestPropertySetterId(unorderedPropertySetters);
int biggestPropertyId = biggestGetter > biggestSetter ? biggestGetter : biggestSetter;
for (Method property : unorderedPropertySetters) {
DBPropertySetter fieldAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
// Declare the other variables
DBDataType[] propertyTypes = new DBDataType[biggestPropertyId + 1];
Method[] propertyGetters = new Method[biggestPropertyId + 1];
Method[] propertySetters = new Method[biggestPropertyId + 1];
// Load the properties metadata
for (Method property : unorderedPropertyGetters) {
DBPropertyGetter propertyAnnotation = property.getAnnotation(DBPropertyGetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertyGetters[propertyId] = property;
}
for (Method property : unorderedPropertySetters) {
DBPropertySetter propertyAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertySetters[propertyId] = property;
}
// Set properties metadata
obj.setProperties(propertyGetters, propertySetters, propertyTypes, propertyUIDs);
}
private Method[] getPropertyGetters(DBObject obj) {
return MethodUtils.getMethodsWithAnnotation(obj.getClass(), DBPropertyGetter.class);
}
private Method[] getPropertySetters(DBObject obj) {
return MethodUtils.getMethodsWithAnnotation(obj.getClass(), DBPropertySetter.class);
}
private Field[] getFields(DBObject obj) {
return FieldUtils.getFieldsWithAnnotation(obj.getClass(), DBField.class);
}
private int getBiggestPropertyGetterId(Method[] unorderedPropertyGetters) {
int biggestPropertyId = -1;
for (Method property : unorderedPropertyGetters) {
DBPropertyGetter fieldAnnotation = property.getAnnotation(DBPropertyGetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
return biggestPropertyId;
}
private int getBiggestPropertySetterId(Method[] unorderedPropertySetters) {
int biggestPropertyId = -1;
for (Method property : unorderedPropertySetters) {
DBPropertySetter fieldAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
return biggestPropertyId;
}
private int getBiggestFieldId(Field[] unorderedFields) {
int biggestFieldId = -1;
for (Field field : unorderedFields) {
DBField fieldAnnotation = field.getAnnotation(DBField.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestFieldId) {
biggestFieldId = propertyId;
}
}
return biggestFieldId;
}
public void loadProperty(DBObject obj, int propertyId, Method property, DBDataType propertyType, long propertyUID) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
switch (propertyType) {
case DATABASE_OBJECT:
DBObject fieldDBObjectValue = (DBObject) property.getDeclaringClass().getConstructor(Database.class, long.class).newInstance(this, propertyUID);
obj.setLoadedProperty(propertyId, fieldDBObjectValue);
break;
case OBJECT:
Object fieldObjectValue = loadObject(propertyUID);
obj.setLoadedProperty(propertyId, fieldObjectValue);
break;
case INTEGER:
int fieldIntValue = loadInt(propertyUID);
obj.setLoadedProperty(propertyId, fieldIntValue);
break;
default:
throw new NullPointerException("Unknown Field Type");
}
}
public void loadField(DBObject obj, Field field, DBDataType fieldType, long fieldUID) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
switch (fieldType) {
case DATABASE_OBJECT:
DBObject fieldDBObjectValue = (DBObject) field.getDeclaringClass().getConstructor(Database.class, long.class).newInstance(this, fieldUID);
field.set(obj, fieldDBObjectValue);
break;
case OBJECT:
Object fieldObjectValue = loadObject(fieldUID);
field.set(obj, fieldObjectValue);
break;
case INTEGER:
int fieldIntValue = loadInt(fieldUID);
field.setInt(obj, fieldIntValue);
break;
default:
throw new NullPointerException("Unknown Field Type");
}
}
@SuppressWarnings("unchecked")
public <T> T loadObject(long uid) {
try {
return (T) indices.get(uid, (i, size) -> size == 0 ? null : kryo.readClassAndObject(i));
} catch (IOException ex) {
throw new IOError(ex);
}
}
public int loadInt(long uid) {
try {
return indices.get(uid, (i, size) -> size == 0 ? 0 : i.readInt());
} catch (IOException ex) {
throw new IOError(ex);
}
}
public void watchObject(DBObject obj) {
loadedObjects.add(new WeakReference<>(obj));
}
/**
*
* @param uid
* @return
*/
private DBObjectIndicesManager.DBObjectInfo readUIDs(long uid) {
try {
return objectIndicesManager.get(uid);
} catch (IOException e) {
throw new IOError(e);
}
}
public void saveObject(DBObject obj) {
System.out.println("Saving object " + obj.getUID());
try {
objectIndicesManager.set(obj.getUID(), obj.getAllFieldUIDs(), obj.getAllPropertyUIDs());
} catch (IOException e) {
throw new IOError(e);
}
}
public long newDBObject(DBObject obj) {
int fieldsCount = getBiggestFieldId(getFields(obj)) + 1;
int biggestGetter = getBiggestPropertyGetterId(getPropertyGetters(obj));
int biggestSetter = getBiggestPropertySetterId(getPropertySetters(obj));
int propertiesCount = (biggestGetter > biggestSetter ? biggestGetter : biggestSetter) + 1;
long uid = objectIndicesManager.allocate(fieldsCount, propertiesCount);
long[] fields = new long[fieldsCount];
for (int i = 0; i < fieldsCount; i++) {
fields[i] = indices.add(0);
}
long[] properties = new long[propertiesCount];
for (int i = 0; i < propertiesCount; i++) {
properties[i] = indices.add(0);
}
try {
objectIndicesManager.set(uid, fields, properties);
} catch (IOException e) {
throw new IOError(e);
}
return uid;
}
}

View File

@ -0,0 +1,447 @@
package org.warp.jcwdb.ann;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Registration;
import com.esotericsoftware.kryo.io.Output;
import it.unimi.dsi.fastutil.booleans.BooleanArrayList;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.chars.CharArrayList;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.shorts.ShortArrayList;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.commons.lang3.reflect.MethodUtils;
import org.warp.jcwdb.Cleanable;
import org.warp.jcwdb.Cleaner;
import org.warp.jcwdb.FileIndexManager;
import org.warp.jcwdb.ann.exampleimpl.ClassWithList;
import java.io.ByteArrayOutputStream;
import java.io.IOError;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.file.Path;
import java.util.*;
public class DatabaseManager implements Cleanable {
public static final long MAX_LOADED_INDICES = 100000;
private final DBObjectIndicesManager objectIndicesManager;
private final FileIndexManager indices;
private final Cleaner cleaner;
private final JCWDatabase jcwDatabase;
private DBObject loadedRootObject = null;
private final Kryo kryo = new Kryo();
private volatile boolean closed;
DatabaseManager(JCWDatabase jcwDatabase, Path dataFile, Path metadataFile) throws IOException {
this.jcwDatabase = jcwDatabase;
kryo.setRegistrationRequired(true);
registerDefaultClasses();
this.indices = new FileIndexManager(dataFile, metadataFile);
if (!indices.has(0)) {
allocateNullValue();
}
this.objectIndicesManager = new DBObjectIndicesManager(this.indices);
this.cleaner = new Cleaner(this);
this.cleaner.start();
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
DatabaseManager.this.close();
} catch (Exception e) {
e.printStackTrace();
}
}));
}
private void registerDefaultClasses() {
int id = -90;
registerClass(boolean[].class, id++);
registerClass(byte[].class, id++);
registerClass(short[].class, id++);
registerClass(char[].class, id++);
registerClass(int[].class, id++);
registerClass(long[].class, id++);
registerClass(Boolean[].class, id++);
registerClass(Byte[].class, id++);
registerClass(Short[].class, id++);
registerClass(Character[].class, id++);
registerClass(Integer[].class, id++);
registerClass(Long[].class, id++);
registerClass(String.class, id++);
registerClass(String[].class, id++);
registerClass(Boolean.class, id++);
registerClass(Byte.class, id++);
registerClass(Short.class, id++);
registerClass(Character.class, id++);
registerClass(Integer.class, id++);
registerClass(Class.class, id++);
registerClass(Object.class, id++);
registerClass(Object[].class, id++);
registerClass(Long.class, id++);
registerClass(String.class, id++);
registerClass(String[].class, id++);
registerClass(boolean[][].class, id++);
registerClass(byte[][].class, id++);
registerClass(short[][].class, id++);
registerClass(char[][].class, id++);
registerClass(int[][].class, id++);
registerClass(long[][].class, id++);
registerClass(String[][].class, id++);
registerClass(List.class, id++);
registerClass(ArrayList.class, id++);
registerClass(LinkedList.class, id++);
registerClass(Set.class, id++);
registerClass(HashSet.class, id++);
registerClass(LinkedHashSet.class, id++);
registerClass(Map.class, id++);
registerClass(HashMap.class, id++);
registerClass(LinkedHashMap.class, id++);
registerClass(TreeMap.class, id++);
registerClass(BooleanArrayList.class, id++);
registerClass(ByteArrayList.class, id++);
registerClass(ShortArrayList.class, id++);
registerClass(CharArrayList.class, id++);
registerClass(IntArrayList.class, id++);
registerClass(LongArrayList.class, id++);
}
@SuppressWarnings("unchecked")
public <T extends DBObject> T loadRoot(Class<T> rootType) throws IOException {
if (loadedRootObject != null) {
throw new RuntimeException("Root already set!");
}
if (isDBObjectNull(0)) {
try {
T root = rootType.getConstructor(JCWDatabase.class).newInstance(this.jcwDatabase);
loadedRootObject = root;
return root;
} catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
throw new IOError(e);
}
} else {
T root = (T) loadDBObject(rootType, 0);
loadedRootObject = root;
return root;
}
}
public void close() throws IOException {
if (!closed) {
closed = true;
DatabaseManager.this.cleaner.stop();
if (loadedRootObject != null) {
loadedRootObject.writeToDisk(0);
}
indices.close();
}
}
public void preloadDBObject(DBObject obj, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
preloadDBObjectFields(obj, objectInfo.getFields());
preloadDBObjectProperties(obj, objectInfo.getProperties());
}
private void preloadDBObjectFields(DBObject obj, long[] fieldUIDs) throws IOException {
// Declare the variables needed to get the biggest field Id
Field[] unorderedFields = getFields(obj);
// Find the biggest field Id
int biggestFieldId = getBiggestFieldId(unorderedFields);
// Declare the other variables
Field[] fields = new Field[biggestFieldId + 1];
DBDataType[] orderedFieldTypes = new DBDataType[biggestFieldId + 1];
// Load all fields metadata and load them
for (Field field : unorderedFields) {
DBField fieldAnnotation = field.getAnnotation(DBField.class);
int fieldId = fieldAnnotation.id();
DBDataType fieldType = fieldAnnotation.type();
loadField(obj, field, fieldType, fieldUIDs[fieldId]);
fields[fieldId] = field;
orderedFieldTypes[fieldId] = fieldType;
}
// Set fields metadata
obj.setFields(fields, orderedFieldTypes, fieldUIDs);
}
private void preloadDBObjectProperties(DBObject obj, long[] propertyUIDs) {
// Declare the variables needed to get the biggest property Id
Method[] unorderedPropertyGetters = getPropertyGetters(obj);
Method[] unorderedPropertySetters = getPropertySetters(obj);
// Find the biggest property Id
int biggestGetter = getBiggestPropertyGetterId(unorderedPropertyGetters);
int biggestSetter = getBiggestPropertySetterId(unorderedPropertySetters);
int biggestPropertyId = biggestGetter > biggestSetter ? biggestGetter : biggestSetter;
for (Method property : unorderedPropertySetters) {
DBPropertySetter fieldAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
// Declare the other variables
DBDataType[] propertyTypes = new DBDataType[biggestPropertyId + 1];
Method[] propertyGetters = new Method[biggestPropertyId + 1];
Method[] propertySetters = new Method[biggestPropertyId + 1];
Map<String, DBPropertySetter> setterMethods = new LinkedHashMap<>();
Map<String, DBPropertyGetter> getterMethods = new LinkedHashMap<>();
// Load the properties metadata
for (Method property : unorderedPropertyGetters) {
DBPropertyGetter propertyAnnotation = property.getAnnotation(DBPropertyGetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertyGetters[propertyId] = property;
getterMethods.put(property.getName(), propertyAnnotation);
}
for (Method property : unorderedPropertySetters) {
DBPropertySetter propertyAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertySetters[propertyId] = property;
setterMethods.put(property.getName(), propertyAnnotation);
}
// Set properties metadata
obj.setProperties(propertyGetters, propertySetters, propertyTypes, propertyUIDs, setterMethods, getterMethods);
}
Method[] getPropertyGetters(DBObject obj) {
return MethodUtils.getMethodsWithAnnotation(obj.getClass(), DBPropertyGetter.class);
}
Method[] getPropertySetters(DBObject obj) {
return MethodUtils.getMethodsWithAnnotation(obj.getClass(), DBPropertySetter.class);
}
protected Field[] getFields(DBObject obj) {
return FieldUtils.getFieldsWithAnnotation(obj.getClass(), DBField.class);
}
int getBiggestPropertyGetterId(Method[] unorderedPropertyGetters) {
int biggestPropertyId = -1;
for (Method property : unorderedPropertyGetters) {
DBPropertyGetter fieldAnnotation = property.getAnnotation(DBPropertyGetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
return biggestPropertyId;
}
int getBiggestPropertySetterId(Method[] unorderedPropertySetters) {
int biggestPropertyId = -1;
for (Method property : unorderedPropertySetters) {
DBPropertySetter fieldAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
return biggestPropertyId;
}
protected int getBiggestFieldId(Field[] unorderedFields) {
int biggestFieldId = -1;
for (Field field : unorderedFields) {
DBField fieldAnnotation = field.getAnnotation(DBField.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestFieldId) {
biggestFieldId = propertyId;
}
}
return biggestFieldId;
}
@SuppressWarnings("unchecked")
public void loadProperty(DBObject obj, int propertyId, Method property, DBDataType propertyType, long propertyUID) throws IOException {
switch (propertyType) {
case DATABASE_OBJECT:
DBObject fieldDBObjectValue = loadDBObject((Class<? extends DBObject>) property.getReturnType(), propertyUID);
//System.err.println("Loading prop. DBObj " + propertyUID + ":" + fieldDBObjectValue);
obj.setLoadedProperty(propertyId, fieldDBObjectValue);
break;
case OBJECT:
Object fieldObjectValue = loadObject(propertyUID);
//System.err.println("Loading prop. Obj " + propertyUID + ":" + fieldObjectValue);
obj.setLoadedProperty(propertyId, fieldObjectValue);
break;
case UID_LIST:
LongArrayList fieldListObjectValue = loadListObject(propertyUID);
//System.err.println("Loading prop. LOb " + propertyUID + ":" + fieldListObjectValue);
obj.setLoadedProperty(propertyId, fieldListObjectValue);
break;
case INTEGER:
int fieldIntValue = loadInt(propertyUID);
//System.err.println("Loading prop. Int " + propertyUID + ":" + fieldIntValue);
obj.setLoadedProperty(propertyId, fieldIntValue);
break;
default:
throw new NullPointerException("Unknown Field Type");
}
}
@SuppressWarnings("unchecked")
public void loadField(DBObject obj, Field field, DBDataType fieldType, long fieldUID) throws IOException {
try {
switch (fieldType) {
case DATABASE_OBJECT:
DBObject fieldDBObjectValue = loadDBObject((Class<? extends DBObject>) field.getType(), fieldUID);
//System.err.println("Loading field DBObj " + fieldUID + ":" + fieldDBObjectValue);
FieldUtils.writeField(field, obj, fieldDBObjectValue, true);
break;
case OBJECT:
Object fieldObjectValue = loadObject(fieldUID);
//System.err.println("Loading field Obj " + fieldUID + ":" + fieldObjectValue);
FieldUtils.writeField(field, obj, fieldObjectValue, true);
break;
case UID_LIST:
LongArrayList fieldListObjectValue = loadListObject(fieldUID);
//System.err.println("Loading field LOb " + fieldUID + ":" + fieldObjectValue);
FieldUtils.writeField(field, obj, fieldListObjectValue, true);
break;
case INTEGER:
int fieldIntValue = loadInt(fieldUID);
//System.err.println("Loading field Int " + fieldUID + ":" + fieldIntValue);
FieldUtils.writeField(field, obj, fieldIntValue, true);
break;
default:
throw new NullPointerException("Unknown Field Type");
}
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
}
public <T extends DBObject> T loadDBObject(Class<T> type, long propertyUID) throws IOException {
try {
DBObjectIndicesManager.DBObjectInfo objectInfo = readUIDs(propertyUID);
if (objectInfo == null) return null;
return type.getDeclaredConstructor(JCWDatabase.class, DBObjectIndicesManager.DBObjectInfo.class).newInstance(jcwDatabase, objectInfo);
} catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
throw new IOException(e);
}
}
private boolean isDBObjectNull(long uid) {
try {
return !objectIndicesManager.has(uid) || objectIndicesManager.get(uid) == null;
} catch (IOException ex) {
throw new IOError(ex);
}
}
@SuppressWarnings("unchecked")
public <T> T loadObject(long uid) throws IOException {
return indices.get(uid, (i, size) -> size == 0 ? null : (T) kryo.readClassAndObject(i));
}
@SuppressWarnings("unchecked")
private LongArrayList loadListObject(long uid) throws IOException {
return indices.get(uid, (i, size) -> {
if (size == 0) return null;
LongArrayList list = new LongArrayList();
int listSize = i.readVarInt(true);
for (int li = 0; li < listSize; li++) {
list.add(i.readVarLong(true));
}
return list;
});
}
public int loadInt(long uid) throws IOException {
return indices.get(uid, (i, size) -> size == 0 ? 0 : i.readInt());
}
/**
*
* @param uid
* @return
*/
public DBObjectIndicesManager.DBObjectInfo readUIDs(long uid) {
try {
return objectIndicesManager.get(uid);
} catch (IOException e) {
throw new IOError(e);
}
}
public boolean exists(long uid) {
return objectIndicesManager.has(uid);
}
public void writeObjectInfo(long uid, long[] fieldUIDs, long[] propertyUIDs) throws IOException {
//System.err.println("Saving obj. " + uid);
this.objectIndicesManager.set(uid, fieldUIDs, propertyUIDs);
}
private void writeObjectInfoNull(long uid) throws IOException {
this.objectIndicesManager.setNull(uid);
}
public <T> void writeObjectProperty(long uid, DBDataType propertyType, T loadedPropertyValue) throws IOException {
switch (propertyType) {
case INTEGER:
indices.set(uid, Integer.BYTES, (o) -> o.writeInt((int) loadedPropertyValue));
//System.err.println("Saving prop. Int " + uid + ":" + loadedPropertyValue);
break;
case OBJECT:
Output baosOutput = new Output(new ByteArrayOutputStream());
kryo.writeClassAndObject(baosOutput, loadedPropertyValue);
//System.err.println("Saving prop. Obj " + uid + ":" + loadedPropertyValue);
byte[] out = baosOutput.toBytes();
indices.set(uid, out.length, o -> o.write(out, 0, out.length));
break;
case UID_LIST:
LongArrayList list = (LongArrayList) loadedPropertyValue;
final int listSize = list.size();
Output baosListOutput = new Output(Long.BYTES * 100, Long.BYTES * listSize);
baosListOutput.writeVarInt(listSize, true);
for (int i = 0; i < listSize; i++) {
baosListOutput.writeVarLong(list.getLong(i), true);
}
//System.err.println("Saving prop. LOb " + uid + ":" + loadedPropertyValue);
byte[] outList = baosListOutput.toBytes();
indices.set(uid, outList.length, o -> o.write(outList, 0, outList.length));
break;
case DATABASE_OBJECT:
//System.err.println("Saving prop. DBObj " + uid + ":" + loadedPropertyValue);
if (loadedPropertyValue == null) {
writeObjectInfoNull(uid);
} else {
((DBObject) loadedPropertyValue).writeToDisk(uid);
}
break;
}
}
public void registerClass(Class<?> clazz, int id) {
kryo.register(clazz, 100 + id);
}
public long allocateNullValue() {
return indices.add(0);
}
@Override
public long clean() {
return 0;//indices.clean();
}
public long[] allocateNewUIDs(int quantity) {
long[] ids = new long[quantity];
for (int i = 0; i < quantity; i++) {
ids[i] = allocateNullValue();
}
return ids;
}
}

View File

@ -0,0 +1,32 @@
package org.warp.jcwdb.ann;
import java.io.IOException;
import java.nio.file.Path;
public class JCWDatabase {
private final DatabaseManager database;
public JCWDatabase(Path dataFile, Path metadataFile) throws IOException {
this.database = new DatabaseManager(this, dataFile, metadataFile);
}
public <T extends DBObject> T loadRoot(Class<T> rootClass) {
try {
return database.loadRoot(rootClass);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
DatabaseManager getDatabaseManager() {
return database;
}
public void registerClass(Class<?> clazz, int id) {
database.registerClass(clazz, id);
}
public void close() throws IOException {
database.close();
}
}

View File

@ -2,14 +2,17 @@ package org.warp.jcwdb.ann.exampleimpl;
import org.warp.jcwdb.ann.*;
@DBClass(classTypeId = 0)
import java.io.IOException;
import java.util.StringJoiner;
public class Class1 extends DBObject {
public Class1(Database database) {
public Class1(JCWDatabase database) {
super(database);
}
public Class1(Database database, long uid) {
super(database, uid);
public Class1(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
super(database, objectInfo);
}
@DBField(id = 0, type = DBDataType.OBJECT)
@ -18,6 +21,9 @@ public class Class1 extends DBObject {
@DBField(id = 1, type = DBDataType.INTEGER)
public int value2;
@DBField(id = 2, type = DBDataType.INTEGER)
public int value3;
@DBPropertyGetter(id = 0, type = DBDataType.OBJECT)
public String getValue3() {
return getProperty();
@ -37,4 +43,26 @@ public class Class1 extends DBObject {
public void setValue4(Class1 value) {
setProperty(value);
}
@DBPropertyGetter(id = 2, type = DBDataType.OBJECT)
public String getValueStr() {
return getProperty();
}
@DBPropertySetter(id = 2, type = DBDataType.OBJECT)
public void setValueStr(String value) {
setProperty(value);
}
@Override
public String toString() {
return new StringJoiner(", ", Class1.class.getSimpleName() + "[", "]")
.add("value1='" + value1 + "'")
.add("value2=" + value2)
.add("value3=" + value3)
.add("getValue3=" + getValue3())
.add("getValue4=" + getValue4())
.add("getValueStr=" + getValueStr())
.toString();
}
}

View File

@ -0,0 +1,40 @@
package org.warp.jcwdb.ann.exampleimpl;
import org.warp.jcwdb.ann.*;
import java.io.IOException;
import java.util.StringJoiner;
public class Class2 extends Class1 {
//@DBField(id = 3, type = DBDataType.DATABASE_OBJECT)
//public DBDBObjectList<Class1> value4;
@DBField(id = 3, type = DBDataType.DATABASE_OBJECT)
public Class1 value4;
@DBField(id = 4, type = DBDataType.INTEGER)
public int value5;
public Class2(JCWDatabase database) {
super(database);
}
public Class2(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
super(database, objectInfo);
}
@Override
public String toString() {
return new StringJoiner(", ", Class1.class.getSimpleName() + "[", "]")
.add("value1='" + value1 + "'")
.add("value2=" + value2)
.add("value3=" + value3)
.add("value4=" + value4)
.add("value5=" + value5)
.add("getValue3=" + getValue3())
.add("getValue4=" + getValue4())
.add("getValueStr=" + getValueStr())
.toString();
}
}

View File

@ -0,0 +1,48 @@
package org.warp.jcwdb.ann.exampleimpl;
import org.warp.jcwdb.ann.*;
import java.io.IOError;
import java.io.IOException;
import java.util.StringJoiner;
public class ClassWithList extends DBObject {
public ClassWithList(JCWDatabase database) {
super(database);
}
public ClassWithList(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
super(database, objectInfo);
}
@DBField(id = 0, type = DBDataType.DATABASE_OBJECT)
public DBDBObjectList<IntClass> valueList;
@DBPropertySetter(id = 0, type = DBDataType.DATABASE_OBJECT)
public void setList(DBDBObjectList<IntClass> value) {
setProperty(value);
}
@DBPropertyGetter(id = 0, type = DBDataType.DATABASE_OBJECT)
public DBDBObjectList<IntClass> getList() {
return getProperty();
}
@Override
public String toString() {
DBDBObjectList<IntClass> list;
boolean listErrored = false;
try {
list = getList();
} catch (IOError | Exception ex) {
list = null;
listErrored = true;
}
return new StringJoiner(", ", ClassWithList.class.getSimpleName() + "[", "]")
.add("valueList=" + valueList)
.add("getList=" + (listErrored ? "{error}" : (list == null ? list : list.size() < 100 ? list : "[" + list.size() + " items])")))
.toString();
}
}

View File

@ -0,0 +1,27 @@
package org.warp.jcwdb.ann.exampleimpl;
import org.warp.jcwdb.ann.*;
import java.io.IOException;
import java.util.StringJoiner;
public class IntClass extends DBObject {
public IntClass(JCWDatabase database) {
super(database);
}
public IntClass(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
super(database, objectInfo);
}
@DBField(id = 0, type = DBDataType.INTEGER)
public int value;
@Override
public String toString() {
return new StringJoiner(", ", IntClass.class.getSimpleName() + "[", "]")
.add("value=" + value)
.toString();
}
}

View File

@ -1,6 +1,7 @@
package org.warp.jcwdb.ann.exampleimpl;
import org.warp.jcwdb.ann.Database;
import org.warp.jcwdb.ann.DBDBObjectList;
import org.warp.jcwdb.ann.JCWDatabase;
import java.io.IOException;
import java.nio.file.Paths;
@ -8,25 +9,57 @@ import java.nio.file.Paths;
public class Main {
public static void main(String[] args) throws IOException {
Database db = new Database(Paths.get("database_temp.db"), Paths.get("database_temp.idx"));
Class1 class1 = new Class1(db, 0);
class1.value1 = "ciao";
long t0 = System.currentTimeMillis();
JCWDatabase db = new JCWDatabase(Paths.get("N:\\TimedTemp\\database_temp.db"), Paths.get("N:\\TimedTemp\\database_temp.idx"));
db.registerClass(Class1.class, 0);
db.registerClass(Class2.class, 1);
Class2 class1 = db.loadRoot(Class2.class);
long t1 = System.currentTimeMillis();
System.err.println("Loading took " + (t1-t0)/1000d + " seconds");
t0 = System.currentTimeMillis();
System.err.println("[MAIN] class1="+class1);
class1.value1 = "ciaoooooooooooooooooooooo";
class1.value2 = 3;
System.out.println("value3="+class1.getValue3());
class1.value5 = 5;
System.err.println("[MAIN] value3="+class1.getValue3());
class1.setValue3("Ciao 3");
System.out.println("value3="+class1.getValue3());
System.err.println("[MAIN] value3="+class1.getValue3());
Class1 nested = new Class1(db);
class1.setValue4(nested);
nested.setValue3("Ciao nested 3");
try {
class1.close();
nested.close();
System.out.println(class1.getValue4().getValue3());
} catch (Exception ex) {
ex.printStackTrace();
} catch (Throwable throwable) {
throwable.printStackTrace();
System.err.println("[MAIN] propString="+class1.getValueStr());
class1.setValueStr("Ciao String");
System.err.println("[MAIN] propString="+class1.getValueStr());
System.err.println("[MAIN] getValue4="+class1.getValue4());
t1 = System.currentTimeMillis();
System.err.println("Post-loading took " + (t1-t0)/1000d + " seconds");
t0 = System.currentTimeMillis();
for (int i = 0; i < 200; i++) {
Class1 nested;
if ((nested = class1.getValue4()) == null) {
//System.err.println("[MAIN] Created nested class");
class1.setValue4(nested = new Class1(db));
}
nested.getValue3();
//System.err.println("[MAIN] value4="+class1.getValue4());
//System.err.println("[MAIN] nested value3="+nested.getValue3());
nested.setValue3("Ciao nested 3");
//System.err.println("[MAIN] nested value3=" + class1.getValue4().getValue3());
}
t1 = System.currentTimeMillis();
System.err.println("Took " + (t1-t0)/1000d + " seconds");
/*
if (class1.value4 == null) {
class1.value4 = new DBDBObjectList<>(db, 100, Class1.class);
}
for (int i = 0; i < 15; i++) {
Class1 c1 = new Class1(db);
c1.value1 = "" + i;
c1.value2 = i;
c1.setValueStr("" + i);
class1.value4.add(c1);
}*/
class1.value4 = new Class1(db);
}
}

View File

@ -0,0 +1,52 @@
package org.warp.jcwdb.ann.exampleimpl;
import org.warp.jcwdb.ann.DBDBObjectList;
import org.warp.jcwdb.ann.DBList;
import org.warp.jcwdb.ann.JCWDatabase;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
public class MainSingleClass {
public static void main(String[] args) throws IOException {
long t0 = System.currentTimeMillis();
Files.deleteIfExists(Paths.get("N:\\TimedTemp\\database_t.db"));
Files.deleteIfExists(Paths.get("N:\\TimedTemp\\database_t.idx"));
for (int i = 0; i < 18; i++) {
doIt();
}
}
public static void doIt() throws IOException {
System.err.println("doIt");
JCWDatabase db = new JCWDatabase(Paths.get("N:\\TimedTemp\\database_t.db"), Paths.get("N:\\TimedTemp\\database_t.idx"));
db.registerClass(ClassWithList.class, 0);
db.registerClass(IntClass.class, 1);
ClassWithList classWithList = db.loadRoot(ClassWithList.class);
System.err.println("[MAIN init] classWithList="+classWithList);
if (classWithList.valueList == null) {
System.out.println("Get list was null");
classWithList.valueList = new DBDBObjectList<>(db, IntClass.class, 10000);
}
DBDBObjectList<IntClass> list = classWithList.valueList;
for (int i = 0; i < 1000000; i++) {
IntClass intClass = new IntClass(db);
intClass.value = i+0xFF00;
//System.err.println("[WRITE]" + intClass.value);
list.add(intClass);
}
DBList.DBListIterator<IntClass> it = list.iterator();
while (it.hasNext()) {
IntClass intClass = it.next();
//System.err.println("[READ]" + intClass.value);
}
System.err.println("[MAIN end.] singleClass="+classWithList);
db.close();
}
}

View File

@ -0,0 +1,68 @@
package org.warp.jcwdb.ann.exampleimpl;
import org.warp.jcwdb.ann.*;
import java.io.IOException;
import java.util.StringJoiner;
public class SingleClass extends DBObject {
@DBField(id = 0, type = DBDataType.INTEGER)
public int valueInt;
@DBField(id = 1, type = DBDataType.OBJECT)
public String valueObj;
@DBField(id = 2, type = DBDataType.DATABASE_OBJECT)
public IntClass valueDB;
@DBPropertySetter(id = 0, type = DBDataType.INTEGER)
public void setInt(int value) {
setProperty(value);
}
@DBPropertySetter(id = 1, type = DBDataType.OBJECT)
public void setObj(String value) {
setProperty(value);
}
@DBPropertySetter(id = 2, type = DBDataType.DATABASE_OBJECT)
public void setDB(IntClass value) {
setProperty(value);
}
@DBPropertyGetter(id = 0, type = DBDataType.INTEGER)
public int getInt() {
return getProperty();
}
@DBPropertyGetter(id = 1, type = DBDataType.OBJECT)
public String getObj() {
return getProperty();
}
@DBPropertyGetter(id = 2, type = DBDataType.DATABASE_OBJECT)
public IntClass getDB() {
return getProperty();
}
public SingleClass(JCWDatabase database) {
super(database);
}
public SingleClass(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
super(database, objectInfo);
}
@Override
public String toString() {
return new StringJoiner(", ", SingleClass.class.getSimpleName() + "[", "]")
.add("valueInt=" + valueInt)
.add("valueObj='" + valueObj + "'")
.add("valueDB=" + valueDB)
.add("getInt=" + getInt())
.add("getObj='" + getObj() + "'")
.add("getDB=" + getDB())
.toString();
}
}

View File

@ -1,6 +1,13 @@
package org.warp.jcwdb;
import org.junit.Test;
import org.warp.jcwdb.ann.JCWDatabase;
import org.warp.jcwdb.ann.exampleimpl.IntClass;
import org.warp.jcwdb.ann.exampleimpl.SingleClass;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import static org.junit.Assert.assertTrue;
@ -15,4 +22,50 @@ public class AppTest {
public void shouldAnswerWithTrue() {
assertTrue(true);
}
@Test
public void simpleClassTest() throws IOException {
long t0 = System.currentTimeMillis();
Files.deleteIfExists(Paths.get("N:\\TimedTemp\\database_s.db"));
Files.deleteIfExists(Paths.get("N:\\TimedTemp\\database_s.idx"));
for (int i = 0; i < 3; i++) {
doSingleClassTest();
}
}
public static void doSingleClassTest() throws IOException {
System.err.println("doSingleClassTest");
JCWDatabase db = new JCWDatabase(Paths.get("N:\\TimedTemp\\database_s.db"), Paths.get("N:\\TimedTemp\\database_s.idx"));
db.registerClass(SingleClass.class, 0);
db.registerClass(IntClass.class, 1);
SingleClass singleClass = db.loadRoot(SingleClass.class);
System.err.println("[MAIN init] singleClass="+singleClass);
singleClass.getInt();
singleClass.setInt((int) (Math.random() * 100));
singleClass.getObj();
singleClass.setObj(String.valueOf(Math.random() * 100));
if (singleClass.getDB() == null) {
System.out.println("Get db was null");
singleClass.setDB(new IntClass(db));
}
IntClass intClass = singleClass.getDB();
intClass.value = (int) (Math.random() * 100);
singleClass.valueInt = (int) (Math.random() * 100);
singleClass.valueObj = String.valueOf(Math.random() * 100);
if (singleClass.valueDB == null) {
System.out.println("Value db was null");
singleClass.valueDB = new IntClass(db);
}
singleClass.valueDB.value = (int) (Math.random() * 100);
System.err.println("[MAIN end.] singleClass="+singleClass);
db.close();
}
}