andreacavalli/strangedb
1
1
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Finished new implementation with annotations

Browse Source
This commit is contained in:
Andrea Cavalli 2019-01-18 02:35:55 +01:00
parent 7123228027
commit 41b2bb77a2
22 changed files with 1402 additions and 514 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
pom.xml
View File

@ -6,7 +6,7 @@
<groupId>org.warp</groupId>
<artifactId>jcwdb</artifactId>
<version>1.0-SNAPSHOT</version>
<version>1.1-SNAPSHOT</version>
<name>jcwdb</name>
<!-- FIXME change it to the project's website -->
@ -14,10 +14,18 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<maven.compiler.source>10</maven.compiler.source>
<maven.compiler.target>10</maven.compiler.target>
</properties>
<repositories>
<repository>
<id>sonatype-snapshots</id>
<name>sonatype snapshots repo</name>
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>junit</groupId>
@ -33,7 +41,7 @@
<dependency>
<groupId>com.esotericsoftware</groupId>
<artifactId>kryo</artifactId>
<version>5.0.0-RC1</version>
<version>5.0.0-RC2-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>net.openhft</groupId>

15
src/main/java/org/warp/jcwdb/FileAllocator.java
View File

@ -29,7 +29,7 @@ public class FileAllocator implements AutoCloseable {
this.fileSize = this.dataFileChannel.size();
}
public FileAllocator(SeekableByteChannel dataFileChannel, long fileSize, Long2IntMap freeBytes) throws IOException {
public FileAllocator(SeekableByteChannel dataFileChannel, long fileSize, Long2IntMap freeBytes) {
this.dataFileChannel = dataFileChannel;
this.fileSize = fileSize;
this.freeBytes.putAll(freeBytes);
@ -44,16 +44,20 @@ public class FileAllocator implements AutoCloseable {
public long allocate(int size) {
checkClosed();
synchronized (allocateLock) {
long offset = allocateIntoUnusedParts(size);
if (offset == -1) {
return allocateToEnd(size);
} else {
long offset;
if ((offset = allocateIntoUnusedParts(size)) != -1) {
if (offset + size > fileSize) {
fileSize = offset + size;
}
return offset;
} else {
return allocateToEnd(size);
}
}
}
private long allocateIntoUnusedParts(int size) {
if (FileIndexManager.ALWAYS_ALLOCATE_NEW) return -1;
Stream<Map.Entry<Long,Integer>> sorted =
freeBytes.entrySet().stream()
.sorted(Collections.reverseOrder(Map.Entry.comparingByValue()));
@ -104,6 +108,7 @@ public class FileAllocator implements AutoCloseable {
*/
public void markFree(long startPosition, int length) {
checkClosed();
if (FileIndexManager.ALWAYS_ALLOCATE_NEW) return;
if (freeBytes.containsKey(startPosition + length)) {
int secondLength = freeBytes.remove(startPosition + length);

50
src/main/java/org/warp/jcwdb/FileIndexManager.java
View File

@ -4,7 +4,7 @@ import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import it.unimi.dsi.fastutil.longs.*;
import it.unimi.dsi.fastutil.objects.ObjectIterator;
import org.warp.jcwdb.ann.Database;
import org.warp.jcwdb.ann.DatabaseManager;
import java.io.IOException;
import java.nio.ByteBuffer;
@ -13,14 +13,14 @@ import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.function.Consumer;
public class FileIndexManager implements IndexManager {
public static final boolean ALWAYS_ALLOCATE_NEW = false;
private final SeekableByteChannel dataFileChannel, metadataFileChannel;
private volatile long metadataFileChannelSize;
private final FileAllocator fileAllocator;
private final ByteBuffer metadataByteBuffer = ByteBuffer.allocateDirect(IndexDetails.TOTAL_BYTES);
private final ByteBuffer maskByteBuffer = ByteBuffer.allocateDirect(Integer.BYTES);
private final ByteBuffer maskByteBuffer = ByteBuffer.allocateDirect(Long.BYTES);
private volatile boolean closed;
private final Object closeLock = new Object();
private final Object metadataByteBufferLock = new Object();
@ -36,19 +36,17 @@ public class FileIndexManager implements IndexManager {
/**
* Edit this using editIndex()
*/
private final LongSet dirtyLoadedIndices, flushingAllowedIndices, removedIndices;
private final LongSet dirtyLoadedIndices, removedIndices;
private long firstAllocableIndex;
public FileIndexManager(Path dataFile, Path metadataFile) throws IOException {
if (Cleaner.DISABLE_CLEANER) {
loadedIndices = new Long2ObjectOpenHashMap<>();
dirtyLoadedIndices = new LongOpenHashSet();
flushingAllowedIndices = new LongOpenHashSet();
removedIndices = new LongOpenHashSet();
} else {
loadedIndices = new Long2ObjectLinkedOpenHashMap<>();
dirtyLoadedIndices = new LongLinkedOpenHashSet();
flushingAllowedIndices = new LongLinkedOpenHashSet();
removedIndices = new LongLinkedOpenHashSet();
}
if (Files.notExists(dataFile)) {
@ -59,27 +57,30 @@ public class FileIndexManager implements IndexManager {
}
dataFileChannel = Files.newByteChannel(dataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
metadataFileChannel = Files.newByteChannel(metadataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
fileAllocator = createFileAllocator(dataFileChannel, metadataFileChannel.position(0));
metadataFileChannelSize = metadataFileChannel.size();
fileAllocator = createFileAllocator(dataFileChannel, metadataFileChannel.position(0));
firstAllocableIndex = getMetadataFileChannelSize() / (long) IndexDetails.TOTAL_BYTES;
if (firstAllocableIndex == 0) {
firstAllocableIndex = 1;
}
}
private long getMetadataFileChannelSize() throws IOException {
private long getMetadataFileChannelSize() {
return metadataFileChannelSize;
}
private FileAllocator createFileAllocator(final SeekableByteChannel dataFileChannel, final SeekableByteChannel metadataFileChannel) throws IOException {
if (ALWAYS_ALLOCATE_NEW) {
return new FileAllocator(dataFileChannel);
} else {
Long2IntMap freeBytes = new Long2IntRBTreeMap();
Long2IntMap usedBytes = new Long2IntRBTreeMap();
long firstOffset = 0;
metadataFileChannel.position(0);
while (metadataFileChannel.position() + IndexDetails.TOTAL_BYTES <= getMetadataFileChannelSize()) {
IndexDetails indexDetails = readIndexDetailsAt(metadataFileChannel);
if (indexDetails != null) {
long offset = indexDetails.getOffset();
if (!usedBytes.containsKey(offset) || indexDetails.getSize() > usedBytes.get(offset)) {
usedBytes.put(offset, indexDetails.getSize());
}
if (offset < firstOffset) {
firstOffset = offset;
}
@ -107,6 +108,7 @@ public class FileIndexManager implements IndexManager {
return new FileAllocator(dataFileChannel, fileSize, freeBytes);
}
}
@Override
public <T> T get(long index, DBReader<T> reader) throws IOException {
@ -121,7 +123,7 @@ public class FileIndexManager implements IndexManager {
public IndexDetails set(long index, int size, DBWriter data) throws IOException {
checkClosed();
IndexDetails indexDetails = getIndexMetadataUnsafe(index);
if (indexDetails == null || indexDetails.getSize() < size) {
if (ALWAYS_ALLOCATE_NEW || indexDetails == null || indexDetails.getSize() < size) {
// Allocate new space
IndexDetails newDetails = allocateAndWrite(index, size, data);
if (indexDetails != null) {
@ -144,16 +146,6 @@ public class FileIndexManager implements IndexManager {
}
}
@Override
public void setFlushingAllowed(long index, boolean isUnloadingAllowed) {
checkClosed();
if (isUnloadingAllowed) {
flushingAllowedIndices.add(index);
} else {
flushingAllowedIndices.remove(index);
}
}
@Override
public long add(int size) {
checkClosed();
@ -192,7 +184,6 @@ public class FileIndexManager implements IndexManager {
throw new IOException("Unable to write " + size + " in a space of " + indexDetails.getSize());
}
final long offset = indexDetails.getOffset();
final Output o = new Output(Channels.newOutputStream(dataFileChannel.position(offset)), size);
data.write(o);
o.flush();
@ -214,7 +205,6 @@ public class FileIndexManager implements IndexManager {
}
synchronized (indicesMapsAccessLock) {
dirtyLoadedIndices.remove(index);
flushingAllowedIndices.remove(index);
loadedIndices.remove(index);
removedIndices.add(index);
}
@ -225,7 +215,6 @@ public class FileIndexManager implements IndexManager {
synchronized (indicesMapsAccessLock) {
removedIndices.remove(index);
dirtyLoadedIndices.remove(index);
flushingAllowedIndices.remove(index);
loadedIndices.remove(index);
}
// Update indices metadata
@ -238,7 +227,6 @@ public class FileIndexManager implements IndexManager {
if (dirtyLoadedIndices.contains(index)) {
indexDetails = loadedIndices.get(index);
dirtyLoadedIndices.remove(index);
flushingAllowedIndices.remove(index);
}
}
if (isDirty) {
@ -302,7 +290,6 @@ public class FileIndexManager implements IndexManager {
synchronized (indicesMapsAccessLock) {
loadedIndices.put(index, details);
dirtyLoadedIndices.add(index);
flushingAllowedIndices.remove(index);
}
}
@ -311,7 +298,6 @@ public class FileIndexManager implements IndexManager {
long newIndex = firstAllocableIndex++;
loadedIndices.put(newIndex, indexDetails);
dirtyLoadedIndices.add(newIndex);
flushingAllowedIndices.remove(newIndex);
removedIndices.remove(newIndex);
return newIndex;
}
@ -442,7 +428,6 @@ public class FileIndexManager implements IndexManager {
long lastIndex = -2;
synchronized (indicesMapsAccessLock) {
for (long index : dirtyLoadedIndices) {
if (!flushingAllowedIndices.contains(index)) {
IndexDetails indexDetails = loadedIndices.get(index);
long position = index * IndexDetails.TOTAL_BYTES;
resizeMetadataFileChannel(position);
@ -453,10 +438,7 @@ public class FileIndexManager implements IndexManager {
lastIndex = index;
flushedIndices++;
}
}
dirtyLoadedIndices.clear();
dirtyLoadedIndices.addAll(flushingAllowedIndices);
flushingAllowedIndices.clear();
}
return flushedIndices;
}
@ -484,12 +466,12 @@ public class FileIndexManager implements IndexManager {
long removedIndices = 0;
LongArrayList toUnload = new LongArrayList();
synchronized (indicesMapsAccessLock) {
if (loadedIndices.size() > Database.MAX_LOADED_INDICES) {
if (loadedIndices.size() > DatabaseManager.MAX_LOADED_INDICES) {
long count = loadedIndices.size();
LongIterator it = loadedIndices.keySet().iterator();
while (it.hasNext()) {
long loadedIndex = it.nextLong();
if (count < Database.MAX_LOADED_INDICES * 3l / 2l) {
if (count < DatabaseManager.MAX_LOADED_INDICES * 3l / 2l) {
break;
}
toUnload.add(loadedIndex);

1
src/main/java/org/warp/jcwdb/IndexManager.java
View File

@ -13,7 +13,6 @@ public interface IndexManager extends Cleanable {
long add(int size, DBWriter writer) throws IOException;
FullIndexDetails addAndGetDetails(int size, DBWriter writer) throws IOException;
IndexDetails set(long index, int size, DBWriter writer) throws IOException;
void setFlushingAllowed(long index, boolean isUnloadingAllowed);
void delete(long index) throws IOException;
boolean has(long index);
void close() throws IOException;

9
src/main/java/org/warp/jcwdb/ann/DBClass.java
View File

@ -1,9 +0,0 @@
package org.warp.jcwdb.ann;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@Retention(RetentionPolicy.RUNTIME)
public @interface DBClass {
int classTypeId();
}

35
src/main/java/org/warp/jcwdb/ann/DBDBObjectList.java Normal file
View File

@ -0,0 +1,35 @@
package org.warp.jcwdb.ann;
import java.io.IOException;
import java.util.LinkedList;
public class DBDBObjectList<T extends DBObject> extends DBList<T> {
@DBField(id = 2, type = DBDataType.OBJECT)
public Class<T> type;
public DBDBObjectList(JCWDatabase db, Class<T> type) {
super(db, 10);
this.type = type;
}
public DBDBObjectList(JCWDatabase db, Class<T> type, int maxLoadedItems) {
super(db, maxLoadedItems);
this.type = type;
}
public DBDBObjectList(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
super(database, objectInfo);
}
@Override
protected T loadObjectFromDatabase(long uid) throws IOException {
return database.loadDBObject(type, uid);
}
@Override
protected long saveObjectToDatabase(long uid, T value) throws IOException {
database.writeObjectProperty(uid, DBDataType.DATABASE_OBJECT, value);
return uid;
}
}

3
src/main/java/org/warp/jcwdb/ann/DBDataType.java
View File

@ -3,5 +3,6 @@ package org.warp.jcwdb.ann;
public enum DBDataType {
DATABASE_OBJECT,
OBJECT,
INTEGER
INTEGER,
UID_LIST
}

171
src/main/java/org/warp/jcwdb/ann/DBList.java Normal file
View File

@ -0,0 +1,171 @@
package org.warp.jcwdb.ann;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import java.io.IOError;
import java.io.IOException;
import java.util.LinkedList;
import java.util.StringJoiner;
public abstract class DBList<T> extends DBObject {
@DBField(id = 0, type = DBDataType.INTEGER)
private int maxLoadedItems;
@DBField(id = 1, type = DBDataType.UID_LIST)
private LongArrayList itemIndices;
public DBList(JCWDatabase db, int maxLoadedItems) {
super(db);
this.maxLoadedItems = maxLoadedItems;
this.itemIndices = new LongArrayList();
}
public DBList(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
super(database, objectInfo);
}
public int getMaxLoadedItems() {
return maxLoadedItems;
}
public void add(T obj) {
try {
itemIndices.add(saveObjectToDatabase(database.allocateNullValue(), obj));
} catch (IOException e) {
throw new IOError(e);
}
}
public int size() {
return this.itemIndices.size();
}
private T loadObjectAt(int i) throws IOException {
return loadObjectFromDatabase(itemIndices.getLong(i));
}
protected abstract T loadObjectFromDatabase(long uid) throws IOException;
protected abstract long saveObjectToDatabase(long uid, T value) throws IOException;
public DBListIterator<T> iterator() {
return new DBListIterator<>() {
private int position = itemIndices.size();
private LinkedList<T> cachedItems;
private int objectToSavePosition;
private T objectToSave;
@Override
public boolean hasNext() {
if (position > 0) {
return true;
} else {
try {
saveObservedObject();
} catch (IOException e) {
throw new IOError(e);
}
return false;
}
}
@Override
public T next() {
position--;
if (position < 0) {
throw new NullPointerException("Position < 0");
}
if (cachedItems == null || cachedItems.size() == 0) {
try {
cachedItems = fillCache(position);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
try {
return switchObservedObject(position, cachedItems.removeFirst());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public void remove() {
try {
saveObservedObject();
} catch (IOException e) {
throw new RuntimeException(e);
}
itemIndices.removeLong(position);
}
@Override
public void insert(T obj) {
try {
itemIndices.add(position, saveObjectToDatabase(database.allocateNullValue(), obj));
} catch (IOException e) {
throw new IOError(e);
}
}
@Override
public void set(T obj) {
try {
itemIndices.set(position, saveObjectToDatabase(database.allocateNullValue(), obj));
} catch (IOException e) {
throw new IOError(e);
}
}
private LinkedList<T> fillCache(int position) throws IOException {
LinkedList<T> cachedItems = new LinkedList<>();
int firstObjectIndex = position;
int lastObjectIndex = firstObjectIndex - maxLoadedItems;
if (lastObjectIndex < 0) {
lastObjectIndex = 0;
}
for (int i = firstObjectIndex; i >= lastObjectIndex; i--) {
cachedItems.addLast(loadObjectAt(i));
}
return cachedItems;
}
private void saveObservedObject() throws IOException {
if (objectToSave != null) {
itemIndices.set(objectToSavePosition, saveObjectToDatabase(database.allocateNullValue(), objectToSave));
objectToSave = null;
objectToSavePosition = 0;
}
}
private T switchObservedObject(int position, T obj) throws IOException {
saveObservedObject();
objectToSave = obj;
objectToSavePosition = position;
return obj;
}
};
}
public interface DBListIterator<T> {
boolean hasNext();
T next();
void remove();
void insert(T obj);
void set(T obj);
}
@Override
public String toString() {
return new StringJoiner(", ", DBList.class.getSimpleName() + "[", "]")
.add("size=" + size())
.add("maxLoadedItems=" + maxLoadedItems)
.add("itemIndices=" + (itemIndices.size() < 100 ? itemIndices : "["+itemIndices.size()+" items]"))
.toString();
}
}

183
src/main/java/org/warp/jcwdb/ann/DBObject.java
View File

@ -1,12 +1,16 @@
package org.warp.jcwdb.ann;
import org.apache.commons.lang3.reflect.FieldUtils;
import java.io.IOError;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.LinkedHashMap;
import java.util.Map;
public abstract class DBObject {
private final Database database;
private final long uid;
protected final DatabaseManager database;
private Field[] fields;
private DBDataType[] fieldTypes;
private long[] fieldUIDs;
@ -17,17 +21,103 @@ public abstract class DBObject {
private long[] propertyUIDs;
private boolean[] loadedProperties;
private Object[] loadedPropertyValues;
private Map<String, DBPropertySetter> setterMethods;
private Map<String, DBPropertyGetter> getterMethods;
private final Object fieldsAccessLock = new Object();
private final Object propertiesAccessLock = new Object();
public DBObject(Database database) {
this.database = database;
this.uid = database.newDBObject(this);
database.preloadDBObject(this);
public DBObject(JCWDatabase database) {
this.database = database.getDatabaseManager();
try {
initializeDBObject();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public DBObject(Database database, long uid) {
this.database = database;
this.uid = uid;
database.preloadDBObject(this);
private void initializeDBObject() throws IOException {
initializeDBObjectFields();
initializeDBObjectProperties();
}
private void initializeDBObjectFields() throws IOException {
// Declare the variables needed to get the biggest field Id
Field[] unorderedFields = database.getFields(this);
// Find the biggest field Id
int biggestFieldId = database.getBiggestFieldId(unorderedFields);
// Allocate new UIDs
fieldUIDs = database.allocateNewUIDs(biggestFieldId + 1);
// Declare the other variables
Field[] fields = new Field[biggestFieldId + 1];
DBDataType[] orderedFieldTypes = new DBDataType[biggestFieldId + 1];
// Load all fields metadata and load them
for (Field field : unorderedFields) {
DBField fieldAnnotation = field.getAnnotation(DBField.class);
int fieldId = fieldAnnotation.id();
DBDataType fieldType = fieldAnnotation.type();
database.loadField(this, field, fieldType, fieldUIDs[fieldId]);
fields[fieldId] = field;
orderedFieldTypes[fieldId] = fieldType;
}
// Set fields metadata
setFields(fields, orderedFieldTypes, fieldUIDs);
}
private void initializeDBObjectProperties() {
// Declare the variables needed to get the biggest property Id
Method[] unorderedPropertyGetters = database.getPropertyGetters(this);
Method[] unorderedPropertySetters = database.getPropertySetters(this);
// Find the biggest property Id
int biggestGetter = database.getBiggestPropertyGetterId(unorderedPropertyGetters);
int biggestSetter = database.getBiggestPropertySetterId(unorderedPropertySetters);
int biggestPropertyId = biggestGetter > biggestSetter ? biggestGetter : biggestSetter;
// Allocate new UIDs
propertyUIDs = database.allocateNewUIDs(biggestPropertyId + 1);
for (Method property : unorderedPropertySetters) {
DBPropertySetter fieldAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
// Declare the other variables
DBDataType[] propertyTypes = new DBDataType[biggestPropertyId + 1];
Method[] propertyGetters = new Method[biggestPropertyId + 1];
Method[] propertySetters = new Method[biggestPropertyId + 1];
Map<String, DBPropertySetter> setterMethods = new LinkedHashMap<>();
Map<String, DBPropertyGetter> getterMethods = new LinkedHashMap<>();
// Load the properties metadata
for (Method property : unorderedPropertyGetters) {
DBPropertyGetter propertyAnnotation = property.getAnnotation(DBPropertyGetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertyGetters[propertyId] = property;
getterMethods.put(property.getName(), propertyAnnotation);
}
for (Method property : unorderedPropertySetters) {
DBPropertySetter propertyAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertySetters[propertyId] = property;
setterMethods.put(property.getName(), propertyAnnotation);
}
// Set properties metadata
setProperties(propertyGetters, propertySetters, propertyTypes, propertyUIDs, setterMethods, getterMethods);
}
public DBObject(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
this.database = database.getDatabaseManager();
this.database.preloadDBObject(this, objectInfo);
}
public <T> T getProperty() {
@ -36,71 +126,86 @@ public abstract class DBObject {
try {
int propertyId = stackFrame.getDeclaringClass().getDeclaredMethod(stackFrame.getMethodName()).getAnnotation(DBPropertyGetter.class).id();
return getProperty(propertyId);
} catch (NoSuchMethodException e) {
} catch (IOException | NoSuchMethodException e) {
throw new RuntimeException(e);
}
}
<T> void setLoadedProperty(int propertyId, T value) {
loadedPropertyValues[propertyId] = value;
loadedProperties[propertyId] = true;
}
@SuppressWarnings("unchecked")
private <T> T getProperty(int propertyId) {
private <T> T getProperty(int propertyId) throws IOException {
synchronized (propertiesAccessLock) {
if (!loadedProperties[propertyId]) {
try {
database.loadProperty(this, propertyId, propertyGetters[propertyId], propertyTypes[propertyId], propertyUIDs[propertyId]);
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException e) {
throw new RuntimeException(e);
}
long propertyUID = propertyUIDs[propertyId];
database.loadProperty(this, propertyId, propertyGetters[propertyId], propertyTypes[propertyId], propertyUID);
}
return (T) loadedPropertyValues[propertyId];
}
}
public <T> void setProperty(T value) {
StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
StackWalker.StackFrame stackFrame = walker.walk(f -> f.skip(1).findFirst().orElse(null));
try {
int propertyId = stackFrame.getDeclaringClass().getDeclaredMethod(stackFrame.getMethodName(), value.getClass()).getAnnotation(DBPropertySetter.class).id();
setProperty(propertyId, value);
} catch (NoSuchMethodException e) {
throw new RuntimeException(e);
}
DBPropertySetter propertyAnnotation = setterMethods.get(stackFrame.getMethodName());
setProperty(propertyAnnotation.id(), propertyAnnotation.type(), value);
}
public <T> void setProperty(int propertyId, T value) {
public <T> void setProperty(int propertyId, DBDataType propertyType, T value) {
synchronized (propertiesAccessLock) {
loadedPropertyValues[propertyId] = value;
loadedProperties[propertyId] = true;
}
}
public void close() {
database.saveObject(this);
public void writeToDisk(long uid) {
//System.err.println("Saving object " + uid + ":" + this);
try {
synchronized (propertiesAccessLock) {
synchronized (fieldsAccessLock) {
database.writeObjectInfo(uid, fieldUIDs, propertyUIDs);
}
}
synchronized (fieldsAccessLock) {
for (int i = 0; i < fieldUIDs.length; i++) {
try {
database.writeObjectProperty(fieldUIDs[i], fieldTypes[i], fields[i].get(this));
} catch (IllegalAccessException e) {
throw new IOError(e);
}
}
}
synchronized (propertiesAccessLock) {
for (int i = 0; i < propertyUIDs.length; i++) {
database.writeObjectProperty(propertyUIDs[i], propertyTypes[i], loadedPropertyValues[i]);
}
}
} catch (IOException e) {
throw new IOError(e);
}
}
public final void setFields(Field[] fields, DBDataType[] fieldTypes, long[] fieldUIDs) {
synchronized (fieldsAccessLock) {
this.fields = fields;
this.fieldTypes = fieldTypes;
this.fieldUIDs = fieldUIDs;
}
}
public final void setProperties(Method[] propertyGetters, Method[] propertySetters, DBDataType[] propertyTypes, long[] propertyUIDs) {
public final void setProperties(Method[] propertyGetters, Method[] propertySetters, DBDataType[] propertyTypes, long[] propertyUIDs, Map<String, DBPropertySetter> setterMethods, Map<String, DBPropertyGetter> getterMethods) {
synchronized (propertiesAccessLock) {
this.propertyGetters = propertyGetters;
this.propertySetters = propertySetters;
this.propertyTypes = propertyTypes;
this.propertyUIDs = propertyUIDs;
this.loadedProperties = new boolean[this.propertyUIDs.length];
this.loadedPropertyValues = new Object[this.propertyUIDs.length];
this.setterMethods = setterMethods;
this.getterMethods = getterMethods;
}
public final long getUID() {
return uid;
}
public long[] getAllFieldUIDs() {
return fieldUIDs;
}
public long[] getAllPropertyUIDs() {
return propertyUIDs;
}
}

19
src/main/java/org/warp/jcwdb/ann/DBObjectIndicesManager.java
View File

@ -8,12 +8,18 @@ import java.io.IOException;
public class DBObjectIndicesManager {
private final FileIndexManager indices;
public DBObjectIndicesManager(FileIndexManager indices) {
DBObjectIndicesManager(FileIndexManager indices) {
this.indices = indices;
}
public long allocate(int fieldsCount, int propertiesCount) {
return indices.add(calculateObjectSize(fieldsCount, propertiesCount));
long uid = indices.add(calculateObjectSize(fieldsCount, propertiesCount));
//System.err.println("ALLOCATED UID " + uid);
return uid;
}
public void setNull(long uid) throws IOException {
indices.set(uid, 0, (w) -> w.write(new byte[0]));
}
public void set(long uid, long[] fields, long[] properties) throws IOException {
@ -23,7 +29,7 @@ public class DBObjectIndicesManager {
for (int i = 0; i < fields.length; i++) {
w.writeLong(fields[i]);
}
for (int i = 0; i < fields.length; i++) {
for (int i = 0; i < properties.length; i++) {
w.writeLong(properties[i]);
}
});
@ -31,6 +37,9 @@ public class DBObjectIndicesManager {
public DBObjectInfo get(long uid) throws IOException {
return indices.get(uid, (i, size) -> {
if (size < Integer.BYTES * 2) {
return null;
}
long[] indices = new long[i.readInt()];
long[] properties = new long[i.readInt()];
if (size != calculateObjectSize(indices, properties)) {
@ -46,6 +55,10 @@ public class DBObjectIndicesManager {
});
}
public boolean has(long uid) {
return indices.has(uid);
}
private int calculateObjectSize(long[] fields, long[] properties) {
return calculateObjectSize(fields.length, properties.length);
}

25
src/main/java/org/warp/jcwdb/ann/DBObjectList.java Normal file
View File

@ -0,0 +1,25 @@
package org.warp.jcwdb.ann;
import java.io.IOException;
public class DBObjectList<T> extends DBList<T> {
public DBObjectList(JCWDatabase db, int maxLoadedItems) {
super(db, maxLoadedItems);
}
public DBObjectList(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
super(database, objectInfo);
}
@Override
protected T loadObjectFromDatabase(long uid) throws IOException {
return database.loadObject(uid);
}
@Override
protected long saveObjectToDatabase(long uid, T value) throws IOException {
database.writeObjectProperty(uid, DBDataType.OBJECT, value);
return uid;
}
}

275
src/main/java/org/warp/jcwdb/ann/Database.java
View File

@ -1,275 +0,0 @@
package org.warp.jcwdb.ann;
import com.esotericsoftware.kryo.Kryo;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.commons.lang3.reflect.MethodUtils;
import org.warp.jcwdb.FileIndexManager;
import java.io.IOError;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.file.Path;
import java.util.LinkedList;
public class Database {
public static final long MAX_LOADED_INDICES = 100000;
private final DBObjectIndicesManager objectIndicesManager;
private final FileIndexManager indices;
private final LinkedList<WeakReference<DBObject>> loadedObjects = new LinkedList<>();
private static final Kryo kryo = new Kryo();
public Database(Path dataFile, Path metadataFile) throws IOException {
this.indices = new FileIndexManager(dataFile, metadataFile);
this.objectIndicesManager = new DBObjectIndicesManager(this.indices);
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
Database.this.close();
} catch (Exception e) {
e.printStackTrace();
}
}));
}
private void close() throws IOException {
indices.close();
for (WeakReference<DBObject> loadedObjectReference : loadedObjects) {
DBObject loadedObject = loadedObjectReference.get();
if (loadedObject != null) {
loadedObject.close();
}
}
}
public void preloadDBObject(DBObject obj) {
DBObjectIndicesManager.DBObjectInfo UIDs = readUIDs(obj.getUID());
preloadDBObjectFields(obj, UIDs.getFields());
preloadDBObjectProperties(obj, UIDs.getProperties());
}
private void preloadDBObjectFields(DBObject obj, long[] fieldUIDs) {
// Declare the variables needed to get the biggest field Id
Field[] unorderedFields = getFields(obj);
// Find the biggest field Id
int biggestFieldId = getBiggestFieldId(unorderedFields);
// Declare the other variables
Field[] fields = new Field[biggestFieldId + 1];
DBDataType[] orderedFieldTypes = new DBDataType[biggestFieldId + 1];
// Load all fields metadata and load them
for (Field field : unorderedFields) {
DBField fieldAnnotation = field.getAnnotation(DBField.class);
int fieldId = fieldAnnotation.id();
DBDataType fieldType = fieldAnnotation.type();
try {
loadField(obj, field, fieldType, fieldUIDs[fieldId]);
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException e) {
throw new RuntimeException(e);
}
fields[fieldId] = field;
orderedFieldTypes[fieldId] = fieldType;
}
// Set fields metadata
obj.setFields(fields, orderedFieldTypes, fieldUIDs);
}
private void preloadDBObjectProperties(DBObject obj, long[] propertyUIDs) {
// Declare the variables needed to get the biggest property Id
Method[] unorderedPropertyGetters = getPropertyGetters(obj);
Method[] unorderedPropertySetters = getPropertySetters(obj);
// Find the biggest property Id
int biggestGetter = getBiggestPropertyGetterId(unorderedPropertyGetters);
int biggestSetter = getBiggestPropertySetterId(unorderedPropertySetters);
int biggestPropertyId = biggestGetter > biggestSetter ? biggestGetter : biggestSetter;
for (Method property : unorderedPropertySetters) {
DBPropertySetter fieldAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
// Declare the other variables
DBDataType[] propertyTypes = new DBDataType[biggestPropertyId + 1];
Method[] propertyGetters = new Method[biggestPropertyId + 1];
Method[] propertySetters = new Method[biggestPropertyId + 1];
// Load the properties metadata
for (Method property : unorderedPropertyGetters) {
DBPropertyGetter propertyAnnotation = property.getAnnotation(DBPropertyGetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertyGetters[propertyId] = property;
}
for (Method property : unorderedPropertySetters) {
DBPropertySetter propertyAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertySetters[propertyId] = property;
}
// Set properties metadata
obj.setProperties(propertyGetters, propertySetters, propertyTypes, propertyUIDs);
}
private Method[] getPropertyGetters(DBObject obj) {
return MethodUtils.getMethodsWithAnnotation(obj.getClass(), DBPropertyGetter.class);
}
private Method[] getPropertySetters(DBObject obj) {
return MethodUtils.getMethodsWithAnnotation(obj.getClass(), DBPropertySetter.class);
}
private Field[] getFields(DBObject obj) {
return FieldUtils.getFieldsWithAnnotation(obj.getClass(), DBField.class);
}
private int getBiggestPropertyGetterId(Method[] unorderedPropertyGetters) {
int biggestPropertyId = -1;
for (Method property : unorderedPropertyGetters) {
DBPropertyGetter fieldAnnotation = property.getAnnotation(DBPropertyGetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
return biggestPropertyId;
}
private int getBiggestPropertySetterId(Method[] unorderedPropertySetters) {
int biggestPropertyId = -1;
for (Method property : unorderedPropertySetters) {
DBPropertySetter fieldAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
return biggestPropertyId;
}
private int getBiggestFieldId(Field[] unorderedFields) {
int biggestFieldId = -1;
for (Field field : unorderedFields) {
DBField fieldAnnotation = field.getAnnotation(DBField.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestFieldId) {
biggestFieldId = propertyId;
}
}
return biggestFieldId;
}
public void loadProperty(DBObject obj, int propertyId, Method property, DBDataType propertyType, long propertyUID) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
switch (propertyType) {
case DATABASE_OBJECT:
DBObject fieldDBObjectValue = (DBObject) property.getDeclaringClass().getConstructor(Database.class, long.class).newInstance(this, propertyUID);
obj.setLoadedProperty(propertyId, fieldDBObjectValue);
break;
case OBJECT:
Object fieldObjectValue = loadObject(propertyUID);
obj.setLoadedProperty(propertyId, fieldObjectValue);
break;
case INTEGER:
int fieldIntValue = loadInt(propertyUID);
obj.setLoadedProperty(propertyId, fieldIntValue);
break;
default:
throw new NullPointerException("Unknown Field Type");
}
}
public void loadField(DBObject obj, Field field, DBDataType fieldType, long fieldUID) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException {
switch (fieldType) {
case DATABASE_OBJECT:
DBObject fieldDBObjectValue = (DBObject) field.getDeclaringClass().getConstructor(Database.class, long.class).newInstance(this, fieldUID);
field.set(obj, fieldDBObjectValue);
break;
case OBJECT:
Object fieldObjectValue = loadObject(fieldUID);
field.set(obj, fieldObjectValue);
break;
case INTEGER:
int fieldIntValue = loadInt(fieldUID);
field.setInt(obj, fieldIntValue);
break;
default:
throw new NullPointerException("Unknown Field Type");
}
}
@SuppressWarnings("unchecked")
public <T> T loadObject(long uid) {
try {
return (T) indices.get(uid, (i, size) -> size == 0 ? null : kryo.readClassAndObject(i));
} catch (IOException ex) {
throw new IOError(ex);
}
}
public int loadInt(long uid) {
try {
return indices.get(uid, (i, size) -> size == 0 ? 0 : i.readInt());
} catch (IOException ex) {
throw new IOError(ex);
}
}
public void watchObject(DBObject obj) {
loadedObjects.add(new WeakReference<>(obj));
}
/**
*
* @param uid
* @return
*/
private DBObjectIndicesManager.DBObjectInfo readUIDs(long uid) {
try {
return objectIndicesManager.get(uid);
} catch (IOException e) {
throw new IOError(e);
}
}
public void saveObject(DBObject obj) {
System.out.println("Saving object " + obj.getUID());
try {
objectIndicesManager.set(obj.getUID(), obj.getAllFieldUIDs(), obj.getAllPropertyUIDs());
} catch (IOException e) {
throw new IOError(e);
}
}
public long newDBObject(DBObject obj) {
int fieldsCount = getBiggestFieldId(getFields(obj)) + 1;
int biggestGetter = getBiggestPropertyGetterId(getPropertyGetters(obj));
int biggestSetter = getBiggestPropertySetterId(getPropertySetters(obj));
int propertiesCount = (biggestGetter > biggestSetter ? biggestGetter : biggestSetter) + 1;
long uid = objectIndicesManager.allocate(fieldsCount, propertiesCount);
long[] fields = new long[fieldsCount];
for (int i = 0; i < fieldsCount; i++) {
fields[i] = indices.add(0);
}
long[] properties = new long[propertiesCount];
for (int i = 0; i < propertiesCount; i++) {
properties[i] = indices.add(0);
}
try {
objectIndicesManager.set(uid, fields, properties);
} catch (IOException e) {
throw new IOError(e);
}
return uid;
}
}

447
src/main/java/org/warp/jcwdb/ann/DatabaseManager.java Normal file
View File

@ -0,0 +1,447 @@
package org.warp.jcwdb.ann;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Registration;
import com.esotericsoftware.kryo.io.Output;
import it.unimi.dsi.fastutil.booleans.BooleanArrayList;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.chars.CharArrayList;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.shorts.ShortArrayList;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.commons.lang3.reflect.MethodUtils;
import org.warp.jcwdb.Cleanable;
import org.warp.jcwdb.Cleaner;
import org.warp.jcwdb.FileIndexManager;
import org.warp.jcwdb.ann.exampleimpl.ClassWithList;
import java.io.ByteArrayOutputStream;
import java.io.IOError;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.file.Path;
import java.util.*;
public class DatabaseManager implements Cleanable {
public static final long MAX_LOADED_INDICES = 100000;
private final DBObjectIndicesManager objectIndicesManager;
private final FileIndexManager indices;
private final Cleaner cleaner;
private final JCWDatabase jcwDatabase;
private DBObject loadedRootObject = null;
private final Kryo kryo = new Kryo();
private volatile boolean closed;
DatabaseManager(JCWDatabase jcwDatabase, Path dataFile, Path metadataFile) throws IOException {
this.jcwDatabase = jcwDatabase;
kryo.setRegistrationRequired(true);
registerDefaultClasses();
this.indices = new FileIndexManager(dataFile, metadataFile);
if (!indices.has(0)) {
allocateNullValue();
}
this.objectIndicesManager = new DBObjectIndicesManager(this.indices);
this.cleaner = new Cleaner(this);
this.cleaner.start();
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
DatabaseManager.this.close();
} catch (Exception e) {
e.printStackTrace();
}
}));
}
private void registerDefaultClasses() {
int id = -90;
registerClass(boolean[].class, id++);
registerClass(byte[].class, id++);
registerClass(short[].class, id++);
registerClass(char[].class, id++);
registerClass(int[].class, id++);
registerClass(long[].class, id++);
registerClass(Boolean[].class, id++);
registerClass(Byte[].class, id++);
registerClass(Short[].class, id++);
registerClass(Character[].class, id++);
registerClass(Integer[].class, id++);
registerClass(Long[].class, id++);
registerClass(String.class, id++);
registerClass(String[].class, id++);
registerClass(Boolean.class, id++);
registerClass(Byte.class, id++);
registerClass(Short.class, id++);
registerClass(Character.class, id++);
registerClass(Integer.class, id++);
registerClass(Class.class, id++);
registerClass(Object.class, id++);
registerClass(Object[].class, id++);
registerClass(Long.class, id++);
registerClass(String.class, id++);
registerClass(String[].class, id++);
registerClass(boolean[][].class, id++);
registerClass(byte[][].class, id++);
registerClass(short[][].class, id++);
registerClass(char[][].class, id++);
registerClass(int[][].class, id++);
registerClass(long[][].class, id++);
registerClass(String[][].class, id++);
registerClass(List.class, id++);
registerClass(ArrayList.class, id++);
registerClass(LinkedList.class, id++);
registerClass(Set.class, id++);
registerClass(HashSet.class, id++);
registerClass(LinkedHashSet.class, id++);
registerClass(Map.class, id++);
registerClass(HashMap.class, id++);
registerClass(LinkedHashMap.class, id++);
registerClass(TreeMap.class, id++);
registerClass(BooleanArrayList.class, id++);
registerClass(ByteArrayList.class, id++);
registerClass(ShortArrayList.class, id++);
registerClass(CharArrayList.class, id++);
registerClass(IntArrayList.class, id++);
registerClass(LongArrayList.class, id++);
}
@SuppressWarnings("unchecked")
public <T extends DBObject> T loadRoot(Class<T> rootType) throws IOException {
if (loadedRootObject != null) {
throw new RuntimeException("Root already set!");
}
if (isDBObjectNull(0)) {
try {
T root = rootType.getConstructor(JCWDatabase.class).newInstance(this.jcwDatabase);
loadedRootObject = root;
return root;
} catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
throw new IOError(e);
}
} else {
T root = (T) loadDBObject(rootType, 0);
loadedRootObject = root;
return root;
}
}
public void close() throws IOException {
if (!closed) {
closed = true;
DatabaseManager.this.cleaner.stop();
if (loadedRootObject != null) {
loadedRootObject.writeToDisk(0);
}
indices.close();
}
}
public void preloadDBObject(DBObject obj, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
preloadDBObjectFields(obj, objectInfo.getFields());
preloadDBObjectProperties(obj, objectInfo.getProperties());
}
private void preloadDBObjectFields(DBObject obj, long[] fieldUIDs) throws IOException {
// Declare the variables needed to get the biggest field Id
Field[] unorderedFields = getFields(obj);
// Find the biggest field Id
int biggestFieldId = getBiggestFieldId(unorderedFields);
// Declare the other variables
Field[] fields = new Field[biggestFieldId + 1];
DBDataType[] orderedFieldTypes = new DBDataType[biggestFieldId + 1];
// Load all fields metadata and load them
for (Field field : unorderedFields) {
DBField fieldAnnotation = field.getAnnotation(DBField.class);
int fieldId = fieldAnnotation.id();
DBDataType fieldType = fieldAnnotation.type();
loadField(obj, field, fieldType, fieldUIDs[fieldId]);
fields[fieldId] = field;
orderedFieldTypes[fieldId] = fieldType;
}
// Set fields metadata
obj.setFields(fields, orderedFieldTypes, fieldUIDs);
}
private void preloadDBObjectProperties(DBObject obj, long[] propertyUIDs) {
// Declare the variables needed to get the biggest property Id
Method[] unorderedPropertyGetters = getPropertyGetters(obj);
Method[] unorderedPropertySetters = getPropertySetters(obj);
// Find the biggest property Id
int biggestGetter = getBiggestPropertyGetterId(unorderedPropertyGetters);
int biggestSetter = getBiggestPropertySetterId(unorderedPropertySetters);
int biggestPropertyId = biggestGetter > biggestSetter ? biggestGetter : biggestSetter;
for (Method property : unorderedPropertySetters) {
DBPropertySetter fieldAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
// Declare the other variables
DBDataType[] propertyTypes = new DBDataType[biggestPropertyId + 1];
Method[] propertyGetters = new Method[biggestPropertyId + 1];
Method[] propertySetters = new Method[biggestPropertyId + 1];
Map<String, DBPropertySetter> setterMethods = new LinkedHashMap<>();
Map<String, DBPropertyGetter> getterMethods = new LinkedHashMap<>();
// Load the properties metadata
for (Method property : unorderedPropertyGetters) {
DBPropertyGetter propertyAnnotation = property.getAnnotation(DBPropertyGetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertyGetters[propertyId] = property;
getterMethods.put(property.getName(), propertyAnnotation);
}
for (Method property : unorderedPropertySetters) {
DBPropertySetter propertyAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertySetters[propertyId] = property;
setterMethods.put(property.getName(), propertyAnnotation);
}
// Set properties metadata
obj.setProperties(propertyGetters, propertySetters, propertyTypes, propertyUIDs, setterMethods, getterMethods);
}
Method[] getPropertyGetters(DBObject obj) {
return MethodUtils.getMethodsWithAnnotation(obj.getClass(), DBPropertyGetter.class);
}
Method[] getPropertySetters(DBObject obj) {
return MethodUtils.getMethodsWithAnnotation(obj.getClass(), DBPropertySetter.class);
}
protected Field[] getFields(DBObject obj) {
return FieldUtils.getFieldsWithAnnotation(obj.getClass(), DBField.class);
}
int getBiggestPropertyGetterId(Method[] unorderedPropertyGetters) {
int biggestPropertyId = -1;
for (Method property : unorderedPropertyGetters) {
DBPropertyGetter fieldAnnotation = property.getAnnotation(DBPropertyGetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
return biggestPropertyId;
}
int getBiggestPropertySetterId(Method[] unorderedPropertySetters) {
int biggestPropertyId = -1;
for (Method property : unorderedPropertySetters) {
DBPropertySetter fieldAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
return biggestPropertyId;
}
protected int getBiggestFieldId(Field[] unorderedFields) {
int biggestFieldId = -1;
for (Field field : unorderedFields) {
DBField fieldAnnotation = field.getAnnotation(DBField.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestFieldId) {
biggestFieldId = propertyId;
}
}
return biggestFieldId;
}
@SuppressWarnings("unchecked")
public void loadProperty(DBObject obj, int propertyId, Method property, DBDataType propertyType, long propertyUID) throws IOException {
switch (propertyType) {
case DATABASE_OBJECT:
DBObject fieldDBObjectValue = loadDBObject((Class<? extends DBObject>) property.getReturnType(), propertyUID);
//System.err.println("Loading prop. DBObj " + propertyUID + ":" + fieldDBObjectValue);
obj.setLoadedProperty(propertyId, fieldDBObjectValue);
break;
case OBJECT:
Object fieldObjectValue = loadObject(propertyUID);
//System.err.println("Loading prop. Obj " + propertyUID + ":" + fieldObjectValue);
obj.setLoadedProperty(propertyId, fieldObjectValue);
break;
case UID_LIST:
LongArrayList fieldListObjectValue = loadListObject(propertyUID);
//System.err.println("Loading prop. LOb " + propertyUID + ":" + fieldListObjectValue);
obj.setLoadedProperty(propertyId, fieldListObjectValue);
break;
case INTEGER:
int fieldIntValue = loadInt(propertyUID);
//System.err.println("Loading prop. Int " + propertyUID + ":" + fieldIntValue);
obj.setLoadedProperty(propertyId, fieldIntValue);
break;
default:
throw new NullPointerException("Unknown Field Type");
}
}
@SuppressWarnings("unchecked")
public void loadField(DBObject obj, Field field, DBDataType fieldType, long fieldUID) throws IOException {
try {
switch (fieldType) {
case DATABASE_OBJECT:
DBObject fieldDBObjectValue = loadDBObject((Class<? extends DBObject>) field.getType(), fieldUID);
//System.err.println("Loading field DBObj " + fieldUID + ":" + fieldDBObjectValue);
FieldUtils.writeField(field, obj, fieldDBObjectValue, true);
break;
case OBJECT:
Object fieldObjectValue = loadObject(fieldUID);
//System.err.println("Loading field Obj " + fieldUID + ":" + fieldObjectValue);
FieldUtils.writeField(field, obj, fieldObjectValue, true);
break;
case UID_LIST:
LongArrayList fieldListObjectValue = loadListObject(fieldUID);
//System.err.println("Loading field LOb " + fieldUID + ":" + fieldObjectValue);
FieldUtils.writeField(field, obj, fieldListObjectValue, true);
break;
case INTEGER:
int fieldIntValue = loadInt(fieldUID);
//System.err.println("Loading field Int " + fieldUID + ":" + fieldIntValue);
FieldUtils.writeField(field, obj, fieldIntValue, true);
break;
default:
throw new NullPointerException("Unknown Field Type");
}
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
}
public <T extends DBObject> T loadDBObject(Class<T> type, long propertyUID) throws IOException {
try {
DBObjectIndicesManager.DBObjectInfo objectInfo = readUIDs(propertyUID);
if (objectInfo == null) return null;
return type.getDeclaredConstructor(JCWDatabase.class, DBObjectIndicesManager.DBObjectInfo.class).newInstance(jcwDatabase, objectInfo);
} catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
throw new IOException(e);
}
}
private boolean isDBObjectNull(long uid) {
try {
return !objectIndicesManager.has(uid) || objectIndicesManager.get(uid) == null;
} catch (IOException ex) {
throw new IOError(ex);
}
}
@SuppressWarnings("unchecked")
public <T> T loadObject(long uid) throws IOException {
return indices.get(uid, (i, size) -> size == 0 ? null : (T) kryo.readClassAndObject(i));
}
@SuppressWarnings("unchecked")
private LongArrayList loadListObject(long uid) throws IOException {
return indices.get(uid, (i, size) -> {
if (size == 0) return null;
LongArrayList list = new LongArrayList();
int listSize = i.readVarInt(true);
for (int li = 0; li < listSize; li++) {
list.add(i.readVarLong(true));
}
return list;
});
}
public int loadInt(long uid) throws IOException {
return indices.get(uid, (i, size) -> size == 0 ? 0 : i.readInt());
}
/**
*
* @param uid
* @return
*/
public DBObjectIndicesManager.DBObjectInfo readUIDs(long uid) {
try {
return objectIndicesManager.get(uid);
} catch (IOException e) {
throw new IOError(e);
}
}
public boolean exists(long uid) {
return objectIndicesManager.has(uid);
}
public void writeObjectInfo(long uid, long[] fieldUIDs, long[] propertyUIDs) throws IOException {
//System.err.println("Saving obj. " + uid);
this.objectIndicesManager.set(uid, fieldUIDs, propertyUIDs);
}
private void writeObjectInfoNull(long uid) throws IOException {
this.objectIndicesManager.setNull(uid);
}
public <T> void writeObjectProperty(long uid, DBDataType propertyType, T loadedPropertyValue) throws IOException {
switch (propertyType) {
case INTEGER:
indices.set(uid, Integer.BYTES, (o) -> o.writeInt((int) loadedPropertyValue));
//System.err.println("Saving prop. Int " + uid + ":" + loadedPropertyValue);
break;
case OBJECT:
Output baosOutput = new Output(new ByteArrayOutputStream());
kryo.writeClassAndObject(baosOutput, loadedPropertyValue);
//System.err.println("Saving prop. Obj " + uid + ":" + loadedPropertyValue);
byte[] out = baosOutput.toBytes();
indices.set(uid, out.length, o -> o.write(out, 0, out.length));
break;
case UID_LIST:
LongArrayList list = (LongArrayList) loadedPropertyValue;
final int listSize = list.size();
Output baosListOutput = new Output(Long.BYTES * 100, Long.BYTES * listSize);
baosListOutput.writeVarInt(listSize, true);
for (int i = 0; i < listSize; i++) {
baosListOutput.writeVarLong(list.getLong(i), true);
}
//System.err.println("Saving prop. LOb " + uid + ":" + loadedPropertyValue);
byte[] outList = baosListOutput.toBytes();
indices.set(uid, outList.length, o -> o.write(outList, 0, outList.length));
break;
case DATABASE_OBJECT:
//System.err.println("Saving prop. DBObj " + uid + ":" + loadedPropertyValue);
if (loadedPropertyValue == null) {
writeObjectInfoNull(uid);
} else {
((DBObject) loadedPropertyValue).writeToDisk(uid);
}
break;
}
}
public void registerClass(Class<?> clazz, int id) {
kryo.register(clazz, 100 + id);
}
public long allocateNullValue() {
return indices.add(0);
}
@Override
public long clean() {
return 0;//indices.clean();
}
public long[] allocateNewUIDs(int quantity) {
long[] ids = new long[quantity];
for (int i = 0; i < quantity; i++) {
ids[i] = allocateNullValue();
}
return ids;
}
}

32
src/main/java/org/warp/jcwdb/ann/JCWDatabase.java Normal file
View File

@ -0,0 +1,32 @@
package org.warp.jcwdb.ann;
import java.io.IOException;
import java.nio.file.Path;
public class JCWDatabase {
private final DatabaseManager database;
public JCWDatabase(Path dataFile, Path metadataFile) throws IOException {
this.database = new DatabaseManager(this, dataFile, metadataFile);
}
public <T extends DBObject> T loadRoot(Class<T> rootClass) {
try {
return database.loadRoot(rootClass);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
DatabaseManager getDatabaseManager() {
return database;
}
public void registerClass(Class<?> clazz, int id) {
database.registerClass(clazz, id);
}
public void close() throws IOException {
database.close();
}
}

36
src/main/java/org/warp/jcwdb/ann/exampleimpl/Class1.java
View File

@ -2,14 +2,17 @@ package org.warp.jcwdb.ann.exampleimpl;
import org.warp.jcwdb.ann.*;
@DBClass(classTypeId = 0)
import java.io.IOException;
import java.util.StringJoiner;
public class Class1 extends DBObject {
public Class1(Database database) {
public Class1(JCWDatabase database) {
super(database);
}
public Class1(Database database, long uid) {
super(database, uid);
public Class1(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
super(database, objectInfo);
}
@DBField(id = 0, type = DBDataType.OBJECT)
@ -18,6 +21,9 @@ public class Class1 extends DBObject {
@DBField(id = 1, type = DBDataType.INTEGER)
public int value2;
@DBField(id = 2, type = DBDataType.INTEGER)
public int value3;
@DBPropertyGetter(id = 0, type = DBDataType.OBJECT)
public String getValue3() {
return getProperty();
@ -37,4 +43,26 @@ public class Class1 extends DBObject {
public void setValue4(Class1 value) {
setProperty(value);
}
@DBPropertyGetter(id = 2, type = DBDataType.OBJECT)
public String getValueStr() {
return getProperty();
}
@DBPropertySetter(id = 2, type = DBDataType.OBJECT)
public void setValueStr(String value) {
setProperty(value);
}
@Override
public String toString() {
return new StringJoiner(", ", Class1.class.getSimpleName() + "[", "]")
.add("value1='" + value1 + "'")
.add("value2=" + value2)
.add("value3=" + value3)
.add("getValue3=" + getValue3())
.add("getValue4=" + getValue4())
.add("getValueStr=" + getValueStr())
.toString();
}
}

40
src/main/java/org/warp/jcwdb/ann/exampleimpl/Class2.java Normal file
View File

@ -0,0 +1,40 @@
package org.warp.jcwdb.ann.exampleimpl;
import org.warp.jcwdb.ann.*;
import java.io.IOException;
import java.util.StringJoiner;
public class Class2 extends Class1 {
//@DBField(id = 3, type = DBDataType.DATABASE_OBJECT)
//public DBDBObjectList<Class1> value4;
@DBField(id = 3, type = DBDataType.DATABASE_OBJECT)
public Class1 value4;
@DBField(id = 4, type = DBDataType.INTEGER)
public int value5;
public Class2(JCWDatabase database) {
super(database);
}
public Class2(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
super(database, objectInfo);
}
@Override
public String toString() {
return new StringJoiner(", ", Class1.class.getSimpleName() + "[", "]")
.add("value1='" + value1 + "'")
.add("value2=" + value2)
.add("value3=" + value3)
.add("value4=" + value4)
.add("value5=" + value5)
.add("getValue3=" + getValue3())
.add("getValue4=" + getValue4())
.add("getValueStr=" + getValueStr())
.toString();
}
}

48
src/main/java/org/warp/jcwdb/ann/exampleimpl/ClassWithList.java Normal file
View File

@ -0,0 +1,48 @@
package org.warp.jcwdb.ann.exampleimpl;
import org.warp.jcwdb.ann.*;
import java.io.IOError;
import java.io.IOException;
import java.util.StringJoiner;
public class ClassWithList extends DBObject {
public ClassWithList(JCWDatabase database) {
super(database);
}
public ClassWithList(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
super(database, objectInfo);
}
@DBField(id = 0, type = DBDataType.DATABASE_OBJECT)
public DBDBObjectList<IntClass> valueList;
@DBPropertySetter(id = 0, type = DBDataType.DATABASE_OBJECT)
public void setList(DBDBObjectList<IntClass> value) {
setProperty(value);
}
@DBPropertyGetter(id = 0, type = DBDataType.DATABASE_OBJECT)
public DBDBObjectList<IntClass> getList() {
return getProperty();
}
@Override
public String toString() {
DBDBObjectList<IntClass> list;
boolean listErrored = false;
try {
list = getList();
} catch (IOError | Exception ex) {
list = null;
listErrored = true;
}
return new StringJoiner(", ", ClassWithList.class.getSimpleName() + "[", "]")
.add("valueList=" + valueList)
.add("getList=" + (listErrored ? "{error}" : (list == null ? list : list.size() < 100 ? list : "[" + list.size() + " items])")))
.toString();
}
}

27
src/main/java/org/warp/jcwdb/ann/exampleimpl/IntClass.java Normal file
View File

@ -0,0 +1,27 @@
package org.warp.jcwdb.ann.exampleimpl;
import org.warp.jcwdb.ann.*;
import java.io.IOException;
import java.util.StringJoiner;
public class IntClass extends DBObject {
public IntClass(JCWDatabase database) {
super(database);
}
public IntClass(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
super(database, objectInfo);
}
@DBField(id = 0, type = DBDataType.INTEGER)
public int value;
@Override
public String toString() {
return new StringJoiner(", ", IntClass.class.getSimpleName() + "[", "]")
.add("value=" + value)
.toString();
}
}

67
src/main/java/org/warp/jcwdb/ann/exampleimpl/Main.java
View File

@ -1,6 +1,7 @@
package org.warp.jcwdb.ann.exampleimpl;
import org.warp.jcwdb.ann.Database;
import org.warp.jcwdb.ann.DBDBObjectList;
import org.warp.jcwdb.ann.JCWDatabase;
import java.io.IOException;
import java.nio.file.Paths;
@ -8,25 +9,57 @@ import java.nio.file.Paths;
public class Main {
public static void main(String[] args) throws IOException {
Database db = new Database(Paths.get("database_temp.db"), Paths.get("database_temp.idx"));
Class1 class1 = new Class1(db, 0);
class1.value1 = "ciao";
long t0 = System.currentTimeMillis();
JCWDatabase db = new JCWDatabase(Paths.get("N:\\TimedTemp\\database_temp.db"), Paths.get("N:\\TimedTemp\\database_temp.idx"));
db.registerClass(Class1.class, 0);
db.registerClass(Class2.class, 1);
Class2 class1 = db.loadRoot(Class2.class);
long t1 = System.currentTimeMillis();
System.err.println("Loading took " + (t1-t0)/1000d + " seconds");
t0 = System.currentTimeMillis();
System.err.println("[MAIN] class1="+class1);
class1.value1 = "ciaoooooooooooooooooooooo";
class1.value2 = 3;
System.out.println("value3="+class1.getValue3());
class1.value5 = 5;
System.err.println("[MAIN] value3="+class1.getValue3());
class1.setValue3("Ciao 3");
System.out.println("value3="+class1.getValue3());
System.err.println("[MAIN] value3="+class1.getValue3());
Class1 nested = new Class1(db);
class1.setValue4(nested);
nested.setValue3("Ciao nested 3");
try {
class1.close();
nested.close();
System.out.println(class1.getValue4().getValue3());
} catch (Exception ex) {
ex.printStackTrace();
} catch (Throwable throwable) {
throwable.printStackTrace();
System.err.println("[MAIN] propString="+class1.getValueStr());
class1.setValueStr("Ciao String");
System.err.println("[MAIN] propString="+class1.getValueStr());
System.err.println("[MAIN] getValue4="+class1.getValue4());
t1 = System.currentTimeMillis();
System.err.println("Post-loading took " + (t1-t0)/1000d + " seconds");
t0 = System.currentTimeMillis();
for (int i = 0; i < 200; i++) {
Class1 nested;
if ((nested = class1.getValue4()) == null) {
//System.err.println("[MAIN] Created nested class");
class1.setValue4(nested = new Class1(db));
}
nested.getValue3();
//System.err.println("[MAIN] value4="+class1.getValue4());
//System.err.println("[MAIN] nested value3="+nested.getValue3());
nested.setValue3("Ciao nested 3");
//System.err.println("[MAIN] nested value3=" + class1.getValue4().getValue3());
}
t1 = System.currentTimeMillis();
System.err.println("Took " + (t1-t0)/1000d + " seconds");
/*
if (class1.value4 == null) {
class1.value4 = new DBDBObjectList<>(db, 100, Class1.class);
}
for (int i = 0; i < 15; i++) {
Class1 c1 = new Class1(db);
c1.value1 = "" + i;
c1.value2 = i;
c1.setValueStr("" + i);
class1.value4.add(c1);
}*/
class1.value4 = new Class1(db);
}
}

52
src/main/java/org/warp/jcwdb/ann/exampleimpl/MainSingleClass.java Normal file
View File

@ -0,0 +1,52 @@
package org.warp.jcwdb.ann.exampleimpl;
import org.warp.jcwdb.ann.DBDBObjectList;
import org.warp.jcwdb.ann.DBList;
import org.warp.jcwdb.ann.JCWDatabase;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
public class MainSingleClass {
public static void main(String[] args) throws IOException {
long t0 = System.currentTimeMillis();
Files.deleteIfExists(Paths.get("N:\\TimedTemp\\database_t.db"));
Files.deleteIfExists(Paths.get("N:\\TimedTemp\\database_t.idx"));
for (int i = 0; i < 18; i++) {
doIt();
}
}
public static void doIt() throws IOException {
System.err.println("doIt");
JCWDatabase db = new JCWDatabase(Paths.get("N:\\TimedTemp\\database_t.db"), Paths.get("N:\\TimedTemp\\database_t.idx"));
db.registerClass(ClassWithList.class, 0);
db.registerClass(IntClass.class, 1);
ClassWithList classWithList = db.loadRoot(ClassWithList.class);
System.err.println("[MAIN init] classWithList="+classWithList);
if (classWithList.valueList == null) {
System.out.println("Get list was null");
classWithList.valueList = new DBDBObjectList<>(db, IntClass.class, 10000);
}
DBDBObjectList<IntClass> list = classWithList.valueList;
for (int i = 0; i < 1000000; i++) {
IntClass intClass = new IntClass(db);
intClass.value = i+0xFF00;
//System.err.println("[WRITE]" + intClass.value);
list.add(intClass);
}
DBList.DBListIterator<IntClass> it = list.iterator();
while (it.hasNext()) {
IntClass intClass = it.next();
//System.err.println("[READ]" + intClass.value);
}
System.err.println("[MAIN end.] singleClass="+classWithList);
db.close();
}
}

68
src/main/java/org/warp/jcwdb/ann/exampleimpl/SingleClass.java Normal file
View File

@ -0,0 +1,68 @@
package org.warp.jcwdb.ann.exampleimpl;
import org.warp.jcwdb.ann.*;
import java.io.IOException;
import java.util.StringJoiner;
public class SingleClass extends DBObject {
@DBField(id = 0, type = DBDataType.INTEGER)
public int valueInt;
@DBField(id = 1, type = DBDataType.OBJECT)
public String valueObj;
@DBField(id = 2, type = DBDataType.DATABASE_OBJECT)
public IntClass valueDB;
@DBPropertySetter(id = 0, type = DBDataType.INTEGER)
public void setInt(int value) {
setProperty(value);
}
@DBPropertySetter(id = 1, type = DBDataType.OBJECT)
public void setObj(String value) {
setProperty(value);
}
@DBPropertySetter(id = 2, type = DBDataType.DATABASE_OBJECT)
public void setDB(IntClass value) {
setProperty(value);
}
@DBPropertyGetter(id = 0, type = DBDataType.INTEGER)
public int getInt() {
return getProperty();
}
@DBPropertyGetter(id = 1, type = DBDataType.OBJECT)
public String getObj() {
return getProperty();
}
@DBPropertyGetter(id = 2, type = DBDataType.DATABASE_OBJECT)
public IntClass getDB() {
return getProperty();
}
public SingleClass(JCWDatabase database) {
super(database);
}
public SingleClass(JCWDatabase database, DBObjectIndicesManager.DBObjectInfo objectInfo) throws IOException {
super(database, objectInfo);
}
@Override
public String toString() {
return new StringJoiner(", ", SingleClass.class.getSimpleName() + "[", "]")
.add("valueInt=" + valueInt)
.add("valueObj='" + valueObj + "'")
.add("valueDB=" + valueDB)
.add("getInt=" + getInt())
.add("getObj='" + getObj() + "'")
.add("getDB=" + getDB())
.toString();
}
}

53
src/test/java/org/warp/jcwdb/AppTest.java
View File

@ -1,6 +1,13 @@
package org.warp.jcwdb;
import org.junit.Test;
import org.warp.jcwdb.ann.JCWDatabase;
import org.warp.jcwdb.ann.exampleimpl.IntClass;
import org.warp.jcwdb.ann.exampleimpl.SingleClass;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import static org.junit.Assert.assertTrue;
@ -15,4 +22,50 @@ public class AppTest {
public void shouldAnswerWithTrue() {
assertTrue(true);
}
@Test
public void simpleClassTest() throws IOException {
long t0 = System.currentTimeMillis();
Files.deleteIfExists(Paths.get("N:\\TimedTemp\\database_s.db"));
Files.deleteIfExists(Paths.get("N:\\TimedTemp\\database_s.idx"));
for (int i = 0; i < 3; i++) {
doSingleClassTest();
}
}
public static void doSingleClassTest() throws IOException {
System.err.println("doSingleClassTest");
JCWDatabase db = new JCWDatabase(Paths.get("N:\\TimedTemp\\database_s.db"), Paths.get("N:\\TimedTemp\\database_s.idx"));
db.registerClass(SingleClass.class, 0);
db.registerClass(IntClass.class, 1);
SingleClass singleClass = db.loadRoot(SingleClass.class);
System.err.println("[MAIN init] singleClass="+singleClass);
singleClass.getInt();
singleClass.setInt((int) (Math.random() * 100));
singleClass.getObj();
singleClass.setObj(String.valueOf(Math.random() * 100));
if (singleClass.getDB() == null) {
System.out.println("Get db was null");
singleClass.setDB(new IntClass(db));
}
IntClass intClass = singleClass.getDB();
intClass.value = (int) (Math.random() * 100);
singleClass.valueInt = (int) (Math.random() * 100);
singleClass.valueObj = String.valueOf(Math.random() * 100);
if (singleClass.valueDB == null) {
System.out.println("Value db was null");
singleClass.valueDB = new IntClass(db);
}
singleClass.valueDB.value = (int) (Math.random() * 100);
System.err.println("[MAIN end.] singleClass="+singleClass);
db.close();
}
}