andreacavalli/strangedb
1
1
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Merge branch 'new-impl'

Browse Source
This commit is contained in:
Andrea Cavalli 2019-02-01 00:05:13 +01:00
commit e68d29e64e
66 changed files with 2314 additions and 3361 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
pom.xml
View File

@ -1,80 +1,100 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.warp</groupId>
<artifactId>jcwdb</artifactId>
<version>1.0-SNAPSHOT</version>
<groupId>org.warp</groupId>
<artifactId>jcwdb</artifactId>
<version>1.3</version>
<name>jcwdb</name>
<!-- FIXME change it to the project's website -->
<url>http://www.example.com</url>
<name>jcwdb</name>
<!-- FIXME change it to the project's website -->
<url>http://www.example.com</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
</properties>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>10</maven.compiler.source>
<maven.compiler.target>10</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
<version>8.2.2</version>
</dependency>
<dependency>
<groupId>com.esotericsoftware</groupId>
<artifactId>kryo</artifactId>
<version>5.0.0-RC1</version>
</dependency>
<dependency>
<groupId>net.openhft</groupId>
<artifactId>zero-allocation-hashing</artifactId>
<version>0.8</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>sonatype-snapshots</id>
<name>sonatype snapshots repo</name>
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
</repository>
</repositories>
<build>
<pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
<plugins>
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>3.0.0</version>
</plugin>
<!-- see http://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.1</version>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
</plugins>
</pluginManagement>
</build>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
<version>8.2.2</version>
</dependency>
<dependency>
<groupId>com.esotericsoftware</groupId>
<artifactId>kryo</artifactId>
<version>5.0.0-RC1</version>
</dependency>
<dependency>
<groupId>net.openhft</groupId>
<artifactId>zero-allocation-hashing</artifactId>
<version>0.8</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.5</version>
</dependency>
<dependency>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
<version>RELEASE</version>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
<plugins>
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>3.0.0</version>
</plugin>
<!-- see http://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.1</version>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
</plugins>
</pluginManagement>
</build>
</project>

44
src/main/java/org/warp/cowdb/BlockInfo.java Normal file
View File

@ -0,0 +1,44 @@
package org.warp.cowdb;
import java.util.Objects;
import java.util.StringJoiner;
public class BlockInfo {
private final long index;
private final int size;
public BlockInfo(long index, int size) {
this.index = index;
this.size = size;
}
public long getIndex() {
return index;
}
public int getSize() {
return size;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
BlockInfo blockInfo = (BlockInfo) o;
return index == blockInfo.index &&
size == blockInfo.size;
}
@Override
public int hashCode() {
return Objects.hash(index, size);
}
@Override
public String toString() {
return new StringJoiner(", ", BlockInfo.class.getSimpleName() + "[", "]")
.add("index=" + index)
.add("size=" + size)
.toString();
}
}

33
src/main/java/org/warp/cowdb/ByteBufferBackedInputStream.java Normal file
View File

@ -0,0 +1,33 @@
package org.warp.cowdb;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
public class ByteBufferBackedInputStream extends InputStream {
ByteBuffer buf;
public ByteBufferBackedInputStream(ByteBuffer buf) {
this.buf = buf;
}
public int read() throws IOException {
if (!buf.hasRemaining()) {
return -1;
}
return buf.get() & 0xFF;
}
public int read(byte[] bytes, int off, int len)
throws IOException {
if (!buf.hasRemaining()) {
return -1;
}
len = Math.min(len, buf.remaining());
buf.get(bytes, off, len);
return len;
}
}

906
src/main/java/org/warp/cowdb/Database.java Normal file
View File

@ -0,0 +1,906 @@
package org.warp.cowdb;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.ByteBufferInput;
import com.esotericsoftware.kryo.io.ByteBufferInputStream;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import it.unimi.dsi.fastutil.booleans.BooleanArrayList;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.chars.CharArrayList;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.shorts.ShortArrayList;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.warp.jcwdb.ann.*;
import java.io.ByteArrayOutputStream;
import java.io.IOError;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.*;
import java.util.function.Supplier;
import static org.warp.cowdb.IBlocksMetadata.EMPTY_BLOCK_ID;
import static org.warp.cowdb.IBlocksMetadata.EMPTY_BLOCK_INFO;
public class Database implements IDatabase {
private final DatabaseFileIO fileIO;
private final DatabaseBlocksIO blocksIO;
private final DatabaseBlocksMetadata blocksMetadata;
private final DatabaseReferencesIO referencesIO;
private final DatabaseReferencesMetadata referencesMetadata;
private final DatabaseObjectsIO objectsIO;
private final DatabaseDataInitializer dataInitializer;
private EnhancedObject loadedRootObject;
public Database(Path dataFile, Path blocksMetaFile, Path referencesMetaFile) throws IOException {
if (Files.notExists(dataFile)) {
Files.createFile(dataFile);
}
if (Files.notExists(blocksMetaFile)) {
Files.createFile(blocksMetaFile);
}
if (Files.notExists(referencesMetaFile)) {
Files.createFile(referencesMetaFile);
}
this.fileIO = new DatabaseFileIO(dataFile);
this.blocksMetadata = new DatabaseBlocksMetadata(blocksMetaFile);
this.blocksIO = new DatabaseBlocksIO(fileIO, blocksMetadata);
this.referencesMetadata = new DatabaseReferencesMetadata(referencesMetaFile);
this.referencesIO = new DatabaseReferencesIO(blocksIO, referencesMetadata);
this.objectsIO = new DatabaseObjectsIO(this, referencesIO);
this.dataInitializer = new DatabaseDataInitializer(objectsIO);
}
@Override
public IDataInitializer getDataInitializer() {
return dataInitializer;
}
@Override
public IObjectsIO getObjectsIO() {
return objectsIO;
}
@Override
public void close() throws IOException {
this.objectsIO.setEnhancedObject(0, loadedRootObject);
this.referencesMetadata.close();
this.blocksMetadata.close();
this.fileIO.close();
}
public <T extends EnhancedObject> T loadRoot(Class<T> type) throws IOException {
return loadRoot(type, () -> {
T obj = objectsIO.instantiateEnhancedObject(type);
dataInitializer.initializeDBObject(obj);
return obj;
});
}
public <T extends EnhancedObject> T loadRoot(Class<T> type, SupplierWithIO<T> ifAbsent) throws IOException {
if (loadedRootObject != null) {
throw new RuntimeException("Root already set!");
}
T root;
if (referencesMetadata.firstFreeReference > 0) {
root = objectsIO.loadEnhancedObject(0, type);
} else {
if (objectsIO.newNullObject() != 0) {
throw new IOException("Can't allocate root!");
} else {
root = ifAbsent.getWithIO();
objectsIO.setEnhancedObject(0, root);
}
}
loadedRootObject = root;
return root;
}
protected void registerClass(Class<?> type, int id) {
this.objectsIO.registerClass(type, id);
}
public static class DatabaseDataInitializer implements IDataInitializer {
private final DatabaseObjectsIO objectsIO;
public DatabaseDataInitializer(DatabaseObjectsIO objectsIO) {
this.objectsIO = objectsIO;
}
@Override
public void initializeDBObject(EnhancedObject obj) throws IOException {
initializeDBObjectFields(obj);
initializeDBObjectProperties(obj);
obj.initialize();
}
private void initializeDBObjectFields(EnhancedObject obj) throws IOException {
// Declare the variables needed to get the biggest field Id
Field[] unorderedFields = objectsIO.getFields(obj);
// Find the biggest field Id
int biggestFieldId = objectsIO.getBiggestFieldId(unorderedFields);
// Allocate new UIDs
long[] fieldUIDs = objectsIO.allocateNewUIDs(biggestFieldId + 1);
// Declare the other variables
Field[] fields = new Field[biggestFieldId + 1];
DBDataType[] orderedFieldTypes = new DBDataType[biggestFieldId + 1];
// Load all fields metadata and load them
for (Field field : unorderedFields) {
DBField fieldAnnotation = field.getAnnotation(DBField.class);
int fieldId = fieldAnnotation.id();
DBDataType fieldType = fieldAnnotation.type();
objectsIO.loadField(obj, field, fieldType, fieldUIDs[fieldId]);
fields[fieldId] = field;
orderedFieldTypes[fieldId] = fieldType;
}
// Set fields metadata
obj.setFields(fields, orderedFieldTypes, fieldUIDs);
}
private void initializeDBObjectProperties(EnhancedObject obj) throws IOException {
// Declare the variables needed to get the biggest property Id
Method[] unorderedPropertyGetters = obj.getPropertyGetters();
Method[] unorderedPropertySetters = obj.getPropertySetters();
// Find the biggest property Id
int biggestGetter = objectsIO.getBiggestPropertyGetterId(unorderedPropertyGetters);
int biggestSetter = objectsIO.getBiggestPropertySetterId(unorderedPropertySetters);
int biggestPropertyId = biggestGetter > biggestSetter ? biggestGetter : biggestSetter;
// Allocate new UIDs
long[] propertyUIDs = objectsIO.allocateNewUIDs(biggestPropertyId + 1);
for (Method property : unorderedPropertySetters) {
DBPropertySetter fieldAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
// Declare the other variables
DBDataType[] propertyTypes = new DBDataType[biggestPropertyId + 1];
Method[] propertyGetters = new Method[biggestPropertyId + 1];
Method[] propertySetters = new Method[biggestPropertyId + 1];
Map<String, DBPropertySetter> setterMethods = new LinkedHashMap<>();
Map<String, DBPropertyGetter> getterMethods = new LinkedHashMap<>();
// Load the properties metadata
for (Method property : unorderedPropertyGetters) {
DBPropertyGetter propertyAnnotation = property.getAnnotation(DBPropertyGetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertyGetters[propertyId] = property;
getterMethods.put(property.getName(), propertyAnnotation);
}
for (Method property : unorderedPropertySetters) {
DBPropertySetter propertyAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertySetters[propertyId] = property;
setterMethods.put(property.getName(), propertyAnnotation);
}
// Set properties metadata
obj.setProperties(propertyGetters, propertySetters, propertyTypes, propertyUIDs, setterMethods, getterMethods);
}
}
public static class DatabaseObjectsIO implements IObjectsIO {
private final IDatabase database;
private final DatabaseReferencesIO referencesIO;
private Kryo kryo = new Kryo();
private DatabaseObjectsIO(IDatabase database, DatabaseReferencesIO referencesIO) {
this.database = database;
this.referencesIO = referencesIO;
kryo.setRegistrationRequired(false);
int id = -90;
registerClass(boolean[].class, id++);
registerClass(byte[].class, id++);
registerClass(short[].class, id++);
registerClass(char[].class, id++);
registerClass(int[].class, id++);
registerClass(long[].class, id++);
registerClass(Boolean[].class, id++);
registerClass(Byte[].class, id++);
registerClass(Short[].class, id++);
registerClass(Character[].class, id++);
registerClass(Integer[].class, id++);
registerClass(Long[].class, id++);
registerClass(String.class, id++);
registerClass(String[].class, id++);
registerClass(Boolean.class, id++);
registerClass(Byte.class, id++);
registerClass(Short.class, id++);
registerClass(Character.class, id++);
registerClass(Integer.class, id++);
registerClass(Class.class, id++);
registerClass(Object.class, id++);
registerClass(Object[].class, id++);
registerClass(Long.class, id++);
registerClass(String.class, id++);
registerClass(String[].class, id++);
registerClass(boolean[][].class, id++);
registerClass(byte[][].class, id++);
registerClass(short[][].class, id++);
registerClass(char[][].class, id++);
registerClass(int[][].class, id++);
registerClass(long[][].class, id++);
registerClass(String[][].class, id++);
registerClass(List.class, id++);
registerClass(ArrayList.class, id++);
registerClass(LinkedList.class, id++);
registerClass(Set.class, id++);
registerClass(HashSet.class, id++);
registerClass(LinkedHashSet.class, id++);
registerClass(Map.class, id++);
registerClass(HashMap.class, id++);
registerClass(LinkedHashMap.class, id++);
registerClass(TreeMap.class, id++);
registerClass(BooleanArrayList.class, id++);
registerClass(ByteArrayList.class, id++);
registerClass(ShortArrayList.class, id++);
registerClass(CharArrayList.class, id++);
registerClass(IntArrayList.class, id++);
registerClass(LongArrayList.class, id++);
registerClass(TreeSet.class, id++);
registerClass(SortedSet.class, id++);
registerClass(SortedMap.class, id++);
}
@Override
public <T extends EnhancedObject> T loadEnhancedObject(long reference, Class<T> objectType) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return null;
}
int fieldsCount = buffer.getInt();
int methodsCount = buffer.getInt();
long[] fieldRefs = new long[fieldsCount];
long[] methodRefs = new long[methodsCount];
for (int i = 0; i < fieldsCount; i++) {
fieldRefs[i] = buffer.getLong();
}
for (int i = 0; i < methodsCount; i++) {
methodRefs[i] = buffer.getLong();
}
return preloadEnhancedObject(objectType, fieldRefs, methodRefs);
}
@SuppressWarnings("unchecked")
private Object loadData(DBDataType propertyType, long dataReference, Supplier<Class<?>> returnType) throws IOException {
switch (propertyType) {
case DATABASE_OBJECT:
return loadEnhancedObject(dataReference, (Class<? extends EnhancedObject>) returnType.get());
case OBJECT:
return loadObject(dataReference);
case REFERENCES_LIST:
return loadReferencesList(dataReference);
case BOOLEAN:
return loadBoolean(dataReference);
case BYTE:
return loadByte(dataReference);
case SHORT:
return loadShort(dataReference);
case CHAR:
return loadChar(dataReference);
case INTEGER:
return loadInt(dataReference);
case LONG:
return loadLong(dataReference);
default:
throw new NullPointerException("Unknown data type");
}
}
private <T> void setData(long reference, DBDataType propertyType, T loadedPropertyValue) throws IOException {
switch (propertyType) {
case BOOLEAN:
setBoolean(reference, loadedPropertyValue != null && (boolean) loadedPropertyValue);
break;
case BYTE:
setByte(reference, loadedPropertyValue == null ? 0 : (byte) loadedPropertyValue);
break;
case SHORT:
setShort(reference, loadedPropertyValue == null ? 0 : (short) loadedPropertyValue);
break;
case CHAR:
setChar(reference, loadedPropertyValue == null ? 0 : (char) loadedPropertyValue);
break;
case INTEGER:
setInt(reference, loadedPropertyValue == null ? 0 : (int) loadedPropertyValue);
break;
case LONG:
setLong(reference, loadedPropertyValue == null ? 0 : (long) loadedPropertyValue);
break;
case OBJECT:
setObject(reference, loadedPropertyValue);
break;
case REFERENCES_LIST:
setReferencesList(reference, (LongArrayList) loadedPropertyValue);
break;
case DATABASE_OBJECT:
setEnhancedObject(reference, (EnhancedObject) loadedPropertyValue);
break;
}
}
@Override
public <T extends EnhancedObject> void setEnhancedObject(long reference, T value) throws IOException {
if (value != null) {
EnhancedObjectFullInfo objectFullInfo = value.getAllInfo();
int totalSize = Integer.BYTES * 2 + objectFullInfo.getFieldReferences().length * Long.BYTES + objectFullInfo.getPropertyReferences().length * Long.BYTES;
ByteBuffer buffer = ByteBuffer.allocate(totalSize);
buffer.putInt(objectFullInfo.getFieldReferences().length);
buffer.putInt(objectFullInfo.getPropertyReferences().length);
for (int i = 0; i < objectFullInfo.getFieldReferences().length; i++) {
buffer.putLong(objectFullInfo.getFieldReferences()[i]);
}
for (int i = 0; i < objectFullInfo.getPropertyReferences().length; i++) {
buffer.putLong(objectFullInfo.getPropertyReferences()[i]);
}
for (int i = 0; i < objectFullInfo.getFieldReferences().length; i++) {
try {
setData(objectFullInfo.getFieldReferences()[i], objectFullInfo.getFieldTypes()[i], objectFullInfo.getFields()[i].get(value));
} catch (IllegalAccessException e) {
throw new IOException(e);
}
}
for (int i = 0; i < objectFullInfo.getPropertyReferences().length; i++) {
setData(objectFullInfo.getPropertyReferences()[i], objectFullInfo.getPropertyTypes()[i], objectFullInfo.getLoadedPropertyValues()[i]);
}
buffer.flip();
referencesIO.writeToReference(reference, totalSize, buffer);
} else {
referencesIO.writeToReference(reference, 0, null);
}
}
@SuppressWarnings("unchecked")
@Override
public <T> T loadObject(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return null;
}
buffer.rewind();
byte[] data = buffer.array();
return (T) kryo.readClassAndObject(new Input(data));
}
@Override
public LongArrayList loadReferencesList(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return null;
}
int itemsCount = buffer.getInt();
LongArrayList arrayList = new LongArrayList();
for (int i = 0; i < itemsCount; i++) {
arrayList.add(buffer.getLong());
}
return arrayList;
}
@Override
public boolean loadBoolean(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return false;
}
return buffer.get() == 1;
}
@Override
public byte loadByte(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return 0;
}
return buffer.get();
}
@Override
public short loadShort(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return 0;
}
return buffer.getShort();
}
@Override
public char loadChar(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return 0;
}
return buffer.getChar();
}
@Override
public int loadInt(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return 0;
}
return buffer.getInt();
}
@Override
public long loadLong(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return 0;
}
return buffer.getLong();
}
@Override
public <T> void setObject(long reference, T value) throws IOException {
if (value != null) {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
Output output = new Output(outputStream);
kryo.writeClassAndObject(output, value);
output.flush();
byte[] data = outputStream.toByteArray();
ByteBuffer dataByteBuffer = ByteBuffer.wrap(data);
referencesIO.writeToReference(reference, data.length, dataByteBuffer);
} else {
referencesIO.writeToReference(reference, 0, null);
}
}
@Override
public void setReferencesList(long reference, LongArrayList value) throws IOException {
if (value != null) {
int items = value.size();
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * items + Integer.BYTES);
buffer.putInt(items);
for (int i = 0; i < items; i++) {
buffer.putLong(value.getLong(i));
}
buffer.flip();
referencesIO.writeToReference(reference, buffer.limit(), buffer);
} else {
referencesIO.writeToReference(reference, 0, null);
}
}
@Override
public void setBoolean(long reference, boolean value) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(Byte.BYTES);
buffer.put(value ? (byte) 1 : (byte) 0);
buffer.flip();
referencesIO.writeToReference(reference, Byte.BYTES, buffer);
}
@Override
public void setByte(long reference, byte value) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(Byte.BYTES);
buffer.put(value);
buffer.flip();
referencesIO.writeToReference(reference, Byte.BYTES, buffer);
}
@Override
public void setShort(long reference, short value) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(Short.BYTES);
buffer.putShort(value);
buffer.flip();
referencesIO.writeToReference(reference, Short.BYTES, buffer);
}
@Override
public void setChar(long reference, char value) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(Character.BYTES);
buffer.putChar(value);
buffer.flip();
referencesIO.writeToReference(reference, Character.BYTES, buffer);
}
@Override
public void setInt(long reference, int value) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
buffer.putInt(value);
buffer.flip();
referencesIO.writeToReference(reference, Integer.BYTES, buffer);
}
@Override
public void setLong(long reference, long value) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
buffer.putLong(value);
buffer.flip();
referencesIO.writeToReference(reference, Long.BYTES, buffer);
}
@Override
public long newNullObject() throws IOException {
return referencesIO.allocateReference();
}
@Override
public void loadProperty(EnhancedObject obj, int propertyId, Method property, DBDataType propertyType, long propertyUID) throws IOException {
obj.setProperty(propertyId, loadData(propertyType, propertyUID, property::getReturnType));
}
@Override
public void registerClass(Class<?> type, int id) {
if (id < -100) {
throw new IllegalArgumentException();
}
kryo.register(type, 100 + id);
}
private <T extends EnhancedObject> void preloadEnhancedObjectProperties(T obj, long[] propertyReferences) {
// Declare the variables needed to get the biggest property Id
Method[] unorderedPropertyGetters = obj.getPropertyGetters();
Method[] unorderedPropertySetters = obj.getPropertySetters();
// Find the biggest property Id
int biggestGetter = getBiggestPropertyGetterId(unorderedPropertyGetters);
int biggestSetter = getBiggestPropertySetterId(unorderedPropertySetters);
int biggestPropertyId = biggestGetter > biggestSetter ? biggestGetter : biggestSetter;
for (Method property : unorderedPropertySetters) {
DBPropertySetter fieldAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
// Declare the other variables
DBDataType[] propertyTypes = new DBDataType[biggestPropertyId + 1];
Method[] propertyGetters = new Method[biggestPropertyId + 1];
Method[] propertySetters = new Method[biggestPropertyId + 1];
Map<String, DBPropertySetter> setterMethods = new LinkedHashMap<>();
Map<String, DBPropertyGetter> getterMethods = new LinkedHashMap<>();
// Load the properties metadata
for (Method property : unorderedPropertyGetters) {
DBPropertyGetter propertyAnnotation = property.getAnnotation(DBPropertyGetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertyGetters[propertyId] = property;
getterMethods.put(property.getName(), propertyAnnotation);
}
for (Method property : unorderedPropertySetters) {
DBPropertySetter propertyAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertySetters[propertyId] = property;
setterMethods.put(property.getName(), propertyAnnotation);
}
// Set properties metadata
obj.setProperties(propertyGetters, propertySetters, propertyTypes, propertyReferences, setterMethods, getterMethods);
}
private int getBiggestPropertyGetterId(Method[] unorderedPropertyGetters) {
int biggestPropertyId = -1;
for (Method property : unorderedPropertyGetters) {
DBPropertyGetter fieldAnnotation = property.getAnnotation(DBPropertyGetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
return biggestPropertyId;
}
private int getBiggestPropertySetterId(Method[] unorderedPropertySetters) {
int biggestPropertyId = -1;
for (Method property : unorderedPropertySetters) {
DBPropertySetter fieldAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
return biggestPropertyId;
}
private <T extends EnhancedObject> void preloadEnhancedObjectFields(T obj, long[] fieldReferences) throws IOException {
// Declare the variables needed to get the biggest field Id
Field[] unorderedFields = getFields(obj);
// Find the biggest field Id
int biggestFieldId = getBiggestFieldId(unorderedFields);
// Declare the other variables
Field[] fields = new Field[biggestFieldId + 1];
DBDataType[] orderedFieldTypes = new DBDataType[biggestFieldId + 1];
// Load all fields metadata and load them
for (Field field : unorderedFields) {
DBField fieldAnnotation = field.getAnnotation(DBField.class);
int fieldId = fieldAnnotation.id();
DBDataType fieldType = fieldAnnotation.type();
loadField(obj, field, fieldType, fieldReferences[fieldId]);
fields[fieldId] = field;
orderedFieldTypes[fieldId] = fieldType;
}
// Set fields metadata
obj.setFields(fields, orderedFieldTypes, fieldReferences);
}
private <T extends EnhancedObject> void loadField(T obj, Field field, DBDataType fieldType, long fieldReference) throws IOException {
Object data = loadData(fieldType, fieldReference, field::getType);
try {
if (fieldType == DBDataType.OBJECT && data != null) {
if (!field.getType().isInstance(data)) {
throw new IOException("There is an attempt to load an object of type " + data.getClass() + " into a field of type " + field.getType());
}
}
FieldUtils.writeField(field, obj, data, true);
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
}
private <T extends EnhancedObject> Field[] getFields(T obj) {
return FieldUtils.getFieldsWithAnnotation(obj.getClass(), DBField.class);
}
private int getBiggestFieldId(Field[] unorderedFields) {
int biggestFieldId = -1;
for (Field field : unorderedFields) {
DBField fieldAnnotation = field.getAnnotation(DBField.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestFieldId) {
biggestFieldId = propertyId;
}
}
return biggestFieldId;
}
private <T extends EnhancedObject> T instantiateEnhancedObject(Class<T> type) throws IOException {
try {
T obj = type.getConstructor().newInstance();
obj.database = database;
return obj;
} catch (NoSuchMethodException e) {
throw new IOException("You must declare a public empty constructor in class " + type + ": public " + type.getSimpleName() + "()", e);
} catch (IllegalAccessException | InvocationTargetException | InstantiationException e) {
throw new IOException(e);
}
}
private <T extends EnhancedObject> T preloadEnhancedObject(Class<T> objectType, long[] fieldRefs, long[] methodRefs) throws IOException {
T obj = instantiateEnhancedObject(objectType);
preloadEnhancedObjectFields(obj, fieldRefs);
preloadEnhancedObjectProperties(obj, methodRefs);
return obj;
}
public long[] allocateNewUIDs(int quantity) throws IOException {
long[] ids = new long[quantity];
for (int i = 0; i < quantity; i++) {
ids[i] = newNullObject();
}
return ids;
}
}
public static class DatabaseReferencesIO implements IReferencesIO {
private final DatabaseBlocksIO blocksIO;
private final DatabaseReferencesMetadata referencesMetadata;
public DatabaseReferencesIO(DatabaseBlocksIO blocksIO, DatabaseReferencesMetadata referencesMetadata) {
this.blocksIO = blocksIO;
this.referencesMetadata = referencesMetadata;
}
@Override
public long allocateReference() throws IOException {
return referencesMetadata.newReference(EMPTY_BLOCK_ID);
}
@Override
public long allocateReference(int size, ByteBuffer data) throws IOException {
long blockId = (size == 0) ? EMPTY_BLOCK_ID : blocksIO.newBlock(size, data);
return referencesMetadata.newReference(blockId);
}
@Override
public void writeToReference(long reference, int size, ByteBuffer data) throws IOException {
long blockId = (size == 0) ? EMPTY_BLOCK_ID : blocksIO.newBlock(size, data);
referencesMetadata.editReference(reference, blockId);
}
@Override
public ByteBuffer readFromReference(long reference) throws IOException {
long blockId = referencesMetadata.getReference(reference);
return blocksIO.readBlock(blockId);
}
}
public static class DatabaseReferencesMetadata implements IReferencesMetadata {
private final SeekableByteChannel metaFileChannel;
private final int REF_META_BYTES_COUNT = Long.BYTES;
private long firstFreeReference;
private DatabaseReferencesMetadata(Path refMetaFile) throws IOException {
metaFileChannel = Files.newByteChannel(refMetaFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
firstFreeReference = metaFileChannel.size() / REF_META_BYTES_COUNT;
}
@Override
public long getReference(long reference) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(REF_META_BYTES_COUNT);
if (reference >= firstFreeReference) {
return EMPTY_BLOCK_ID;
}
SeekableByteChannel currentFileChannel = metaFileChannel.position(reference * REF_META_BYTES_COUNT);
currentFileChannel.read(buffer);
buffer.flip();
return buffer.getLong();
}
@Override
public long newReference(long blockId) throws IOException {
long newReference = firstFreeReference++;
editReference(newReference, blockId);
return newReference;
}
@Override
public void editReference(long reference, long blockId) throws IOException {
ByteBuffer data = ByteBuffer.allocate(REF_META_BYTES_COUNT);
data.putLong(blockId);
SeekableByteChannel currentFileChannel = metaFileChannel.position(reference * REF_META_BYTES_COUNT);
data.flip();
currentFileChannel.write(data);
}
@Override
public void close() throws IOException {
metaFileChannel.close();
}
}
public static class DatabaseBlocksIO implements IBlocksIO {
private final DatabaseFileIO fileIO;
private final IBlocksMetadata blocksMetadata;
private DatabaseBlocksIO(DatabaseFileIO fileIO, IBlocksMetadata blocksMetadata) {
this.fileIO = fileIO;
this.blocksMetadata = blocksMetadata;
}
@Override
public long newBlock(int size, ByteBuffer data) throws IOException {
long index = fileIO.writeAtEnd(size, data);
return blocksMetadata.newBlock(index, size);
}
@Override
public ByteBuffer readBlock(long blockId) throws IOException {
if (blockId == EMPTY_BLOCK_ID) {
return ByteBuffer.wrap(new byte[0]);
}
BlockInfo blockInfo = blocksMetadata.getBlockInfo(blockId);
return fileIO.readAt(blockInfo.getIndex(), blockInfo.getSize());
}
@Override
public void close() {
}
}
public static class DatabaseBlocksMetadata implements IBlocksMetadata {
private final SeekableByteChannel metaFileChannel;
private final int BLOCK_META_BYTES_COUNT = Long.BYTES + Integer.BYTES;
private long firstFreeBlock;
private DatabaseBlocksMetadata(Path metaFile) throws IOException {
metaFileChannel = Files.newByteChannel(metaFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
firstFreeBlock = metaFileChannel.size() / BLOCK_META_BYTES_COUNT;
}
@Override
public BlockInfo getBlockInfo(long blockId) throws IOException {
if (blockId == EMPTY_BLOCK_ID) {
return EMPTY_BLOCK_INFO;
}
ByteBuffer buffer = ByteBuffer.allocate(BLOCK_META_BYTES_COUNT);
metaFileChannel.position(blockId * BLOCK_META_BYTES_COUNT).read(buffer);
buffer.flip();
long index = buffer.getLong();
int size = buffer.getInt();
return new BlockInfo(index, size);
}
@Override
public long newBlock(long index, int size) throws IOException {
long newBlockId = firstFreeBlock++;
ByteBuffer data = ByteBuffer.allocate(BLOCK_META_BYTES_COUNT);
data.putLong(index);
data.putInt(size);
data.flip();
metaFileChannel.position(newBlockId * BLOCK_META_BYTES_COUNT).write(data);
return newBlockId;
}
@Override
public void close() throws IOException {
metaFileChannel.close();
}
}
public static class DatabaseFileIO implements IFileIO {
private final SeekableByteChannel dataFileChannel;
private final Object dataAccessLock = new Object();
private long firstFreeIndex;
private DatabaseFileIO(Path dataFile) throws IOException {
synchronized (dataAccessLock) {
dataFileChannel = Files.newByteChannel(dataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
firstFreeIndex = dataFileChannel.size();
}
}
@Override
public ByteBuffer readAt(long index, int length) throws IOException {
ByteBuffer dataBuffer = ByteBuffer.allocate(length);
dataFileChannel.position(index).read(dataBuffer);
dataBuffer.flip();
return dataBuffer;
}
@Override
public void writeAt(long index, int length, ByteBuffer data) throws IOException {
synchronized (dataAccessLock) {
if (data.position() != 0) {
throw new IOException("You didn't flip the ByteBuffer!");
}
if (firstFreeIndex < index + length) {
firstFreeIndex = index + length;
}
dataFileChannel.position(index).write(data);
}
}
@Override
public long writeAtEnd(int length, ByteBuffer data) throws IOException {
synchronized (dataAccessLock) {
long index = firstFreeIndex;
firstFreeIndex += length;
writeAt(index, length, data);
return index;
}
}
@Override
public void close() throws IOException {
synchronized (dataAccessLock) {
dataFileChannel.close();
}
}
}
}

101
src/main/java/org/warp/cowdb/EnhancedObject.java Normal file
View File

@ -0,0 +1,101 @@
package org.warp.cowdb;
import org.apache.commons.lang3.reflect.MethodUtils;
import org.warp.jcwdb.ann.DBDataType;
import org.warp.jcwdb.ann.DBPropertyGetter;
import org.warp.jcwdb.ann.DBPropertySetter;
import java.io.IOError;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.Map;
public abstract class EnhancedObject {
protected IDatabase database;
private Field[] fields;
private DBDataType[] fieldTypes;
private long[] fieldReferences;
private Method[] propertyGetters;
private Method[] propertySetters;
private DBDataType[] propertyTypes;
private long[] propertyReferences;
private boolean[] loadedProperties;
private Object[] loadedPropertyValues;
private Map<String, DBPropertySetter> setterMethods;
private Map<String, DBPropertyGetter> getterMethods;
public EnhancedObject() {
}
public EnhancedObject(IDatabase database) throws IOException {
this.database = database;
database.getDataInitializer().initializeDBObject(this);
}
public abstract void initialize() throws IOException;
public void setFields(Field[] fields, DBDataType[] fieldTypes, long[] fieldReferences) {
this.fields = fields;
this.fieldTypes = fieldTypes;
this.fieldReferences = fieldReferences;
}
public Method[] getPropertyGetters() {
return MethodUtils.getMethodsWithAnnotation(this.getClass(), DBPropertyGetter.class);
}
public Method[] getPropertySetters() {
return MethodUtils.getMethodsWithAnnotation(this.getClass(), DBPropertySetter.class);
}
public void setProperties(Method[] propertyGetters, Method[] propertySetters, DBDataType[] propertyTypes, long[] propertyReferences, Map<String, DBPropertySetter> setterMethods, Map<String, DBPropertyGetter> getterMethods) {
this.propertyGetters = propertyGetters;
this.propertySetters = propertySetters;
this.propertyTypes = propertyTypes;
this.propertyReferences = propertyReferences;
this.loadedProperties = new boolean[this.propertyReferences.length];
this.loadedPropertyValues = new Object[this.propertyReferences.length];
this.setterMethods = setterMethods;
this.getterMethods = getterMethods;
}
public EnhancedObjectFullInfo getAllInfo() {
return new EnhancedObjectFullInfo(fieldReferences, fieldTypes, fields, propertyReferences, propertyTypes, loadedPropertyValues);
}
public <T> T getProperty() {
StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
StackWalker.StackFrame stackFrame = walker.walk(f -> f.skip(1).findFirst().orElse(null));
try {
int propertyId = stackFrame.getDeclaringClass().getDeclaredMethod(stackFrame.getMethodName()).getAnnotation(DBPropertyGetter.class).id();
return getProperty(propertyId);
} catch (IOException | NoSuchMethodException e) {
throw new RuntimeException(e);
}
}
@SuppressWarnings("unchecked")
private <T> T getProperty(int propertyId) throws IOException {
if (!loadedProperties[propertyId]) {
long propertyUID = propertyReferences[propertyId];
database.getObjectsIO().loadProperty(this, propertyId, propertyGetters[propertyId], propertyTypes[propertyId], propertyUID);
}
return (T) loadedPropertyValues[propertyId];
}
public <T> void setProperty(T value) {
StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
StackWalker.StackFrame stackFrame = walker.walk(f -> f.skip(1).findFirst().orElse(null));
DBPropertySetter propertyAnnotation = setterMethods.get(stackFrame.getMethodName());
setProperty(propertyAnnotation.id(), value);
}
public <T> void setProperty(int propertyId, T value) {
loadedPropertyValues[propertyId] = value;
loadedProperties[propertyId] = true;
}
}

47
src/main/java/org/warp/cowdb/EnhancedObjectFullInfo.java Normal file
View File

@ -0,0 +1,47 @@
package org.warp.cowdb;
import org.warp.jcwdb.ann.DBDataType;
import java.lang.reflect.Field;
public class EnhancedObjectFullInfo {
private final long[] fieldReferences;
private final DBDataType[] fieldTypes;
private final Field[] fields;
private final long[] propertyReferences;
private final DBDataType[] propertyTypes;
private final Object[] loadedPropertyValues;
EnhancedObjectFullInfo(long[] fieldReferences, DBDataType[] fieldTypes, Field[] fields, long[] propertyReferences, DBDataType[] propertyTypes, Object[] loadedPropertyValues) {
this.fieldReferences = fieldReferences;
this.fieldTypes = fieldTypes;
this.fields = fields;
this.propertyReferences = propertyReferences;
this.propertyTypes = propertyTypes;
this.loadedPropertyValues = loadedPropertyValues;
}
long[] getFieldReferences() {
return fieldReferences;
}
DBDataType[] getFieldTypes() {
return fieldTypes;
}
public Field[] getFields() {
return fields;
}
long[] getPropertyReferences() {
return propertyReferences;
}
DBDataType[] getPropertyTypes() {
return propertyTypes;
}
Object[] getLoadedPropertyValues() {
return loadedPropertyValues;
}
}

28
src/main/java/org/warp/cowdb/IBlocksIO.java Normal file
View File

@ -0,0 +1,28 @@
package org.warp.cowdb;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
public interface IBlocksIO {
/**
* Allocate a block
* @param size block size
* @param data block data
* @return the block id
*/
long newBlock(int size, ByteBuffer data) throws IOException;
/**
* Read a block
* @param blockId block id
* @return block data
*/
ByteBuffer readBlock(long blockId) throws IOException;
/**
* Close file
*/
void close();
}

44
src/main/java/org/warp/cowdb/IBlocksMetadata.java Normal file
View File

@ -0,0 +1,44 @@
package org.warp.cowdb;
import java.io.IOException;
public interface IBlocksMetadata {
long EMPTY_BLOCK_ID = -1;
BlockInfo EMPTY_BLOCK_INFO = new BlockInfo(0, 0);
/**
* Get block info
* @param blockId block id
* @return block metadata
*/
BlockInfo getBlockInfo(long blockId) throws IOException;
/**
* New empty block info
* @return block id
*/
default long newBlock() {
return EMPTY_BLOCK_ID;
}
/**
* Set block info
* @param index block index
* @param size block size
* @return block id
*/
long newBlock(long index, int size) throws IOException;
/**
* Set block info
* @param blockInfo block info
* @return block id
*/
default long newBlock(BlockInfo blockInfo) throws IOException {
return this.newBlock(blockInfo.getIndex(), blockInfo.getSize());
}
/**
* Close file
*/
void close() throws IOException;
}

7
src/main/java/org/warp/cowdb/IDataInitializer.java Normal file
View File

@ -0,0 +1,7 @@
package org.warp.cowdb;
import java.io.IOException;
public interface IDataInitializer {
void initializeDBObject(EnhancedObject obj) throws IOException;
}

10
src/main/java/org/warp/cowdb/IDatabase.java Normal file
View File

@ -0,0 +1,10 @@
package org.warp.cowdb;
import java.io.IOException;
public interface IDatabase {
void close() throws IOException;
IDataInitializer getDataInitializer();
IObjectsIO getObjectsIO();
}

35
src/main/java/org/warp/cowdb/IFileIO.java Normal file
View File

@ -0,0 +1,35 @@
package org.warp.cowdb;
import java.io.*;
import java.nio.ByteBuffer;
public interface IFileIO {
/**
* Read *length* bytes in position *index*
* @param index index
* @param length length
* @return bytes
*/
ByteBuffer readAt(long index, int length) throws IOException;
/**
* Write *length* bytes in position *index*
* @param index index
* @param length length
* @param data bytes
*/
void writeAt(long index, int length, ByteBuffer data) throws IOException;
/**
* Write *length* bytes in position *index*
* @param length length
* @param data bytes
* @return index
*/
long writeAtEnd(int length, ByteBuffer data) throws IOException;
/**
* Close the file
*/
void close() throws IOException;
}

111
src/main/java/org/warp/cowdb/IObjectsIO.java Normal file
View File

@ -0,0 +1,111 @@
package org.warp.cowdb;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.warp.jcwdb.ann.DBDataType;
import java.io.IOException;
import java.lang.reflect.Method;
public interface IObjectsIO {
<T extends EnhancedObject> T loadEnhancedObject(long reference, Class<T> objectType) throws IOException;
<T> T loadObject(long reference) throws IOException;
LongArrayList loadReferencesList(long reference) throws IOException;
boolean loadBoolean(long reference) throws IOException;
byte loadByte(long reference) throws IOException;
short loadShort(long reference) throws IOException;
char loadChar(long reference) throws IOException;
int loadInt(long reference) throws IOException;
long loadLong(long reference) throws IOException;
<T extends EnhancedObject> void setEnhancedObject(long reference, T value) throws IOException;
<T> void setObject(long reference, T value) throws IOException;
void setReferencesList(long reference, LongArrayList value) throws IOException;
void setBoolean(long reference, boolean value) throws IOException;
void setByte(long reference, byte value) throws IOException;
void setShort(long reference, short value) throws IOException;
void setChar(long reference, char value) throws IOException;
void setInt(long reference, int value) throws IOException;
void setLong(long reference, long value) throws IOException;
long newNullObject() throws IOException;
default <T extends EnhancedObject> long newEnhancedObject(T value) throws IOException {
long reference = newNullObject();
if (value != null) {
setEnhancedObject(reference, value);
}
return reference;
}
default <T> long newObject(T value) throws IOException {
long reference = newNullObject();
if (value != null) {
setObject(reference, value);
}
return reference;
}
default long newReferencesList(LongArrayList value) throws IOException {
long reference = newNullObject();
if (value != null) {
setReferencesList(reference, value);
}
return reference;
}
default long newBoolean(boolean value) throws IOException {
long reference = newNullObject();
setBoolean(reference, value);
return reference;
}
default long newByte(byte value) throws IOException {
long reference = newNullObject();
setByte(reference, value);
return reference;
}
default long newShort(short value) throws IOException {
long reference = newNullObject();
setShort(reference, value);
return reference;
}
default long newChar(char value) throws IOException {
long reference = newNullObject();
setChar(reference, value);
return reference;
}
default long newInt(int value) throws IOException {
long reference = newNullObject();
setInt(reference, value);
return reference;
}
default long newLong(long value) throws IOException {
long reference = newNullObject();
setLong(reference, value);
return reference;
}
void loadProperty(EnhancedObject enhancedObject, int propertyId, Method propertyGetter, DBDataType propertyType, long propertyUID) throws IOException;
void registerClass(Class<?> type, int id);
}

37
src/main/java/org/warp/cowdb/IReferencesIO.java Normal file
View File

@ -0,0 +1,37 @@
package org.warp.cowdb;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
public interface IReferencesIO {
/**
* Allocate a new empty reference
* @return the new reference
*/
long allocateReference() throws IOException;
/**
* Allocate a new reference with that data
* @param size data size
* @param data bytes
* @return the new reference
*/
long allocateReference(int size, ByteBuffer data) throws IOException;
/**
* Write some data to the reference
* @param reference reference
* @param size data size
* @param data bytes
*/
void writeToReference(long reference, int size, ByteBuffer data) throws IOException;
/**
* Read data from the reference
* @param reference reference
* @return bytes
*/
ByteBuffer readFromReference(long reference) throws IOException;
}

31
src/main/java/org/warp/cowdb/IReferencesMetadata.java Normal file
View File

@ -0,0 +1,31 @@
package org.warp.cowdb;
import java.io.IOException;
public interface IReferencesMetadata {
/**
* Get block of reference
* @param reference reference
* @return block id
*/
long getReference(long reference) throws IOException;
/**
* Allocate a block for a new reference
* @param blockId block id
* @return reference
*/
long newReference(long blockId) throws IOException;
/**
* Change reference size
* @param reference reference
* @param blockId block id
*/
void editReference(long reference, long blockId) throws IOException;
/**
* Close file
*/
void close() throws IOException;
}

101
src/main/java/org/warp/cowdb/lists/CowList.java Normal file
View File

@ -0,0 +1,101 @@
package org.warp.cowdb.lists;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.warp.cowdb.EnhancedObject;
import org.warp.cowdb.IDatabase;
import org.warp.jcwdb.ann.DBDataType;
import org.warp.jcwdb.ann.DBField;
import java.io.IOException;
import java.util.StringJoiner;
public abstract class CowList<T> extends EnhancedObject {
private final Object indicesAccessLock = new Object();
@DBField(id = 0, type = DBDataType.REFERENCES_LIST)
private LongArrayList indices;
public CowList() {
}
public CowList(IDatabase database) throws IOException {
super(database);
}
@Override
public void initialize() throws IOException {
indices = new LongArrayList();
}
public T get(int index) throws IOException {
synchronized (indicesAccessLock) {
long uid = indices.getLong(index);
return loadItem(uid);
}
}
public void add(T value) throws IOException {
long uid = database.getObjectsIO().newNullObject();
synchronized (indicesAccessLock) {
indices.add(uid);
writeItemToDisk(uid, value);
}
}
public void update(int index, T value) throws IOException {
synchronized (indicesAccessLock) {
set(index, value);
}
}
public void set(int index, T value) throws IOException {
long uid = database.getObjectsIO().newNullObject();
synchronized (indicesAccessLock) {
indices.set(index, uid);
writeItemToDisk(uid, value);
}
}
public void add(int index, T value) throws IOException {
long uid = database.getObjectsIO().newNullObject();
synchronized (indicesAccessLock) {
indices.add(index, uid);
writeItemToDisk(uid, value);
}
}
public T getLast() throws IOException {
synchronized (indicesAccessLock) {
if (indices.size() > 0) {
return get(indices.size() - 1);
} else {
return null;
}
}
}
public boolean isEmpty() {
synchronized (indicesAccessLock) {
return indices.size() <= 0;
}
}
public int size() {
synchronized (indicesAccessLock) {
return indices.size();
}
}
protected abstract T loadItem(long uid) throws IOException;
protected abstract void writeItemToDisk(long uid, T item) throws IOException;
@Override
public String toString() {
return new StringJoiner(", ", CowList.class.getSimpleName() + "[", "]")
.add(indices.size() + " items")
.toString();
}
}

33
src/main/java/org/warp/cowdb/lists/EnhancedObjectCowList.java Normal file
View File

@ -0,0 +1,33 @@
package org.warp.cowdb.lists;
import org.warp.cowdb.EnhancedObject;
import org.warp.cowdb.IDatabase;
import org.warp.jcwdb.ann.DBDataType;
import org.warp.jcwdb.ann.DBField;
import java.io.IOException;
public class EnhancedObjectCowList<T extends EnhancedObject> extends CowList<T> {
@DBField(id = 1, type = DBDataType.OBJECT)
private Class<T> type;
public EnhancedObjectCowList() {
super();
}
public EnhancedObjectCowList(IDatabase database, Class<T> type) throws IOException {
super(database);
this.type = type;
}
@Override
protected T loadItem(long uid) throws IOException {
return database.getObjectsIO().loadEnhancedObject(uid, type);
}
@Override
protected void writeItemToDisk(long uid, T item) throws IOException {
database.getObjectsIO().setEnhancedObject(uid, item);
}
}

29
src/main/java/org/warp/cowdb/lists/ObjectCowList.java Normal file
View File

@ -0,0 +1,29 @@
package org.warp.cowdb.lists;
import org.warp.cowdb.EnhancedObject;
import org.warp.cowdb.IDatabase;
import org.warp.jcwdb.ann.DBDataType;
import org.warp.jcwdb.ann.DBField;
import java.io.IOException;
public class ObjectCowList<T> extends CowList<T> {
public ObjectCowList() {
super();
}
public ObjectCowList(IDatabase database) throws IOException {
super(database);
}
@Override
protected T loadItem(long uid) throws IOException {
return database.getObjectsIO().loadObject(uid);
}
@Override
protected void writeItemToDisk(long uid, T item) throws IOException {
database.getObjectsIO().setObject(uid, item);
}
}

73
src/main/java/org/warp/jcwdb/CacheIndexManager.java
View File

@ -1,73 +0,0 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
public class CacheIndexManager implements IndexManager {
public CacheIndexManager() {
}
@Override
public <T> T get(long index, DBReader<T> reader) {
// TODO: implement
return null;
}
@Override
public int getType(long index) {
// TODO: implement
return 0;
}
@Override
public long getHash(long index) {
// TODO: implement
return 0;
}
@Override
public <T> long add(DBDataOutput<T> writer) {
// TODO: implement
return 0;
}
@Override
public <T> FullIndexDetails addAndGetDetails(DBDataOutput<T> writer) {
// TODO: implement
return null;
}
@Override
public <T> IndexDetails set(long index, DBDataOutput<T> writer) {
// TODO: implement
return null;
}
@Override
public void setFlushingAllowed(long index, boolean isUnloadingAllowed) {
// TODO: implement
}
@Override
public void delete(long index) {
// TODO: implement
}
@Override
public boolean has(long index) {
// TODO: implement
return false;
}
@Override
public void close() {
// TODO: implement
}
@Override
public long clean() {
return 0;
}
}

5
src/main/java/org/warp/jcwdb/Castable.java
View File

@ -1,5 +0,0 @@
package org.warp.jcwdb;
public interface Castable {
<T> T cast();
}

9
src/main/java/org/warp/jcwdb/Cleanable.java
View File

@ -1,9 +0,0 @@
package org.warp.jcwdb;
public interface Cleanable {
/**
* Clean the object
* @return the approximated number of cleaned items
*/
public long clean();
}

122
src/main/java/org/warp/jcwdb/Cleaner.java
View File

@ -1,122 +0,0 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.nio.channels.ClosedChannelException;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
public class Cleaner {
public static final boolean DISABLE_CLEANER = false;
public static final boolean ENABLE_CLEANER_LOGGING = false;
private static final double MAXIMUM_SLEEP_INTERVAL = 8d * 1000d; // 8 seconds
private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second
private static final double NORMAL_REMOVED_ITEMS = 2500l;
private static final double REMOVED_ITEMS_RATIO = 2.5d; // 250%
private final Cleanable[] objectsToClean;
private final Thread cleanerThread;
private int sleepInterval = (int) MINIMUM_SLEEP_INTERVAL;
private volatile boolean stopRequest = false;
public Cleaner(Cleanable... objectsToClean) {
this.objectsToClean = objectsToClean;
this.cleanerThread = new Thread(new CleanLoop());
this.cleanerThread.setName("Cleaner thread");
this.cleanerThread.setDaemon(true);
}
public void start() {
if (!DISABLE_CLEANER) {
this.cleanerThread.start();
}
}
/**
* Clean
* @return number of removed items
*/
private long clean() {
long cleanedItems = 0;
for (Cleanable cleanable : objectsToClean) {
cleanedItems += cleanable.clean();
}
//System.gc();
return cleanedItems;
}
public void stop() {
if (cleanerThread != null) {
stopRequest = true;
while (cleanerThread.isAlive()) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
private class CleanLoop implements Runnable {
@Override
public void run() {
while(!stopRequest) {
try {
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Waiting " + sleepInterval + "ms.");
sleepFor(sleepInterval);
final long time1 = System.currentTimeMillis();
final double removedItems = clean();
final long time2 = System.currentTimeMillis();
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1));
double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2;
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems);
if (removedItems > 0) {
final double removedItemsRatio = removedItems / NORMAL_REMOVED_ITEMS;
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] REMOVED_ITEMS_RATIO: " + removedItemsRatio);
if (removedItemsRatio < 1d / REMOVED_ITEMS_RATIO || removedItemsRatio >= REMOVED_ITEMS_RATIO) {
suggestedExecutionTimeByItemsCalculations = sleepInterval / removedItemsRatio;
}
}
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Items: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + suggestedExecutionTimeByItemsCalculations + "ms");
double newSleepInterval = suggestedExecutionTimeByItemsCalculations;
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Total: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
if (newSleepInterval > MAXIMUM_SLEEP_INTERVAL) {
sleepInterval = (int) MAXIMUM_SLEEP_INTERVAL;
} else if (newSleepInterval < MINIMUM_SLEEP_INTERVAL) {
sleepInterval = (int) MINIMUM_SLEEP_INTERVAL;
} else {
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] CHANGED SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
sleepInterval = (int) newSleepInterval;
}
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Cleaned " + removedItems + " items.");
}catch (InterruptedException e) {
}
}
}
private void sleepFor(int sleepInterval) throws InterruptedException {
int lastI = (int) Math.ceil(((double) sleepInterval) / 1000d);
for (int i = 0; i < lastI; i++) {
if (stopRequest) {
return;
}
if (i == lastI) {
Thread.sleep(sleepInterval % 1000);
} else {
Thread.sleep(lastI);
}
Thread.sleep(sleepInterval);
}
}
}
}

34
src/main/java/org/warp/jcwdb/DBDataOutput.java
View File

@ -1,34 +0,0 @@
package org.warp.jcwdb;
public interface DBDataOutput<T> {
int getSize();
int getType();
long calculateHash();
DBWriter<T> getWriter();
static <T> DBDataOutput<T> create(DBWriter<T> writer, int type, int size, long hash) {
return new DBDataOutput<T>() {
@Override
public int getSize() {
return size;
}
@Override
public int getType() {
return type;
}
@Override
public long calculateHash() {
return hash;
}
@Override
public DBWriter<T> getWriter() {
return writer;
}
};
}
}

58
src/main/java/org/warp/jcwdb/DBGenericObjectParser.java
View File

@ -1,58 +0,0 @@
package org.warp.jcwdb;
import java.io.ByteArrayOutputStream;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Output;
import net.openhft.hashing.LongHashFunction;
public class DBGenericObjectParser extends DBTypeParserImpl<Object> implements DBTypedObjectParser<Object> {
private static final LongHashFunction hashFunction = net.openhft.hashing.LongHashFunction.xx();
private static final Kryo kryo = new Kryo();
static {
kryo.setRegistrationRequired(false);
}
private static final DBReader<Object> defaultReader = (i, size) -> {
return kryo.readClassAndObject(i);
};
public DBReader<Object> getReader() {
return defaultReader;
}
public DBDataOutput<Object> getWriter(final Object value) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
Output tmpO = new Output(baos);
kryo.writeClassAndObject(tmpO, value);
tmpO.flush();
final byte[] bytes = baos.toByteArray();
final long hash = hashFunction.hashBytes(bytes);
tmpO.close();
return DBDataOutput.create((o) -> {
o.write(bytes);
}, DBStandardTypes.GENERIC_OBJECT, bytes.length, hash);
}
@Override
public long calculateHash(Object value) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
Output tmpO = new Output(baos);
kryo.writeClassAndObject(tmpO, value);
tmpO.flush();
final byte[] bytes = baos.toByteArray();
final long hash = hashFunction.hashBytes(bytes);
tmpO.close();
return hash;
}
@Override
public <U> void registerClass(Class<U> clazz, int id) {
if (id >= Integer.MAX_VALUE - 100) {
throw new IndexOutOfBoundsException();
}
kryo.register(clazz, id + 100);
}
}

45
src/main/java/org/warp/jcwdb/DBLightArrayListParser.java
View File

@ -1,45 +0,0 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.longs.LongArrayList;
public class DBLightArrayListParser<T> extends DBTypeParserImpl<LightArrayList<T>> {
private final JCWDatabase db;
public DBLightArrayListParser(JCWDatabase db) {
this.db = db;
}
public DBReader<LightArrayList<T>> getReader() {
return (i, size) -> {
LongArrayList internalList = new LongArrayList();
long max = size / Long.BYTES;
for (int item = 0; item < max; item++) {
long itm = i.readLong();
internalList.add(itm);
}
return new LightArrayList<T>(db, internalList);
};
}
public DBDataOutput<LightArrayList<T>> getWriter(final LightArrayList<T> value) {
final int elementsCount = value.size();
return DBDataOutput.create((o) -> {
LongArrayList list = value.internalList;
for (int i = 0; i < elementsCount; i++) {
o.writeLong(list.getLong(i));
}
}, DBStandardTypes.LIGHT_LIST_ARRAY, elementsCount * Long.BYTES, calculateHash(value));
}
@Override
public long calculateHash(LightArrayList<T> value) {
return value.internalList.hashCode();
}
@Override
public String toString() {
return "DBLightArrayListParser{" +
"db=" + db +
'}';
}
}

44
src/main/java/org/warp/jcwdb/DBLightBigListParser.java
View File

@ -1,44 +0,0 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
public class DBLightBigListParser<T> extends DBTypeParserImpl<LightBigList<T>> {
private final JCWDatabase db;
public DBLightBigListParser(JCWDatabase db) {
this.db = db;
}
public DBReader<LightBigList<T>> getReader() {
return (i, size) -> {
LongArrayList chunks = new LongArrayList();
IntArrayList chunkSizes = new IntArrayList();
long max = size / (Long.BYTES + Integer.BYTES);
for (int item = 0; item < max; item++) {
long itm = i.readLong();
int itm2 = i.readInt();
chunks.add(itm);
chunkSizes.add(itm2);
}
return new LightBigList<>(db, chunks, chunkSizes);
};
}
public DBDataOutput<LightBigList<T>> getWriter(final LightBigList<T> value) {
final int elementsCount = value.chunksCount();
return DBDataOutput.create((o) -> {
LongArrayList list = value.chunks;
IntArrayList list2 = value.chunkSizes;
for (int i = 0; i < elementsCount; i++) {
o.writeLong(list.getLong(i));
o.writeInt(list2.getInt(i));
}
}, DBStandardTypes.LIGHT_LIST_BIG, elementsCount * (Long.BYTES + Integer.BYTES), calculateHash(value));
}
@Override
public long calculateHash(LightBigList<T> value) {
return (((long)value.chunks.hashCode()) << 32) | (value.chunkSizes.hashCode() & 0xffffffffL);
}
}

7
src/main/java/org/warp/jcwdb/DBReader.java
View File

@ -1,7 +0,0 @@
package org.warp.jcwdb;
import com.esotericsoftware.kryo.io.Input;
public interface DBReader<T> {
T read(Input i, int size);
}

24
src/main/java/org/warp/jcwdb/DBStandardTypes.java
View File

@ -1,24 +0,0 @@
package org.warp.jcwdb;
public class DBStandardTypes {
private static final int STD = 0xFFFFF000;
public static final int BOOLEAN = STD| 0x000;
public static final int BYTE = STD| 0x001;
public static final int SHORT = STD| 0x002;
public static final int CHAR = STD| 0x003;
public static final int INTEGER = STD| 0x004;
public static final int FLOAT = STD| 0x005;
public static final int DOUBLE = STD| 0x006;
public static final int STRING = STD| 0x007;
public static final int BYTE_ARRAY = STD| 0x008;
public static final int LIGHT_LIST_ARRAY = STD| 0x009;
public static final int LIGHT_LIST_BIG = STD| 0x00A;
public static final int GENERIC_OBJECT = STD| 0x00B;
public static void registerStandardTypes(JCWDatabase db, TypesManager typesManager) {
typesManager.registerType(String.class, STRING, new DBStringParser());
typesManager.registerType(LightArrayList.class, LIGHT_LIST_ARRAY, new DBLightArrayListParser(db));
typesManager.registerType(LightBigList.class, LIGHT_LIST_BIG, new DBLightBigListParser(db));
typesManager.registerTypeFallback(new DBGenericObjectParser());
}
}

27
src/main/java/org/warp/jcwdb/DBStringParser.java
View File

@ -1,27 +0,0 @@
package org.warp.jcwdb;
import java.nio.charset.StandardCharsets;
import net.openhft.hashing.LongHashFunction;
public class DBStringParser extends DBTypeParserImpl<String> {
private static final LongHashFunction hashFunction = net.openhft.hashing.LongHashFunction.xx();
private static final DBReader<String> defaultReader = (i, size) -> {
return new String(i.readBytes(size), StandardCharsets.UTF_16LE);
};
public DBReader<String> getReader() {
return defaultReader;
}
public DBDataOutput<String> getWriter(final String value) {
return DBDataOutput.create((o) -> {
o.write(value.getBytes(StandardCharsets.UTF_16LE));
}, DBStandardTypes.STRING, value.length() * 2, calculateHash(value));
}
@Override
public long calculateHash(String value) {
return hashFunction.hashBytes(value.getBytes(StandardCharsets.UTF_16LE));
}
}

7
src/main/java/org/warp/jcwdb/DBTypeParser.java
View File

@ -1,7 +0,0 @@
package org.warp.jcwdb;
public interface DBTypeParser<T> extends Castable {
DBReader<T> getReader();
DBDataOutput<T> getWriter(final T value);
long calculateHash(final T value);
}

9
src/main/java/org/warp/jcwdb/DBTypeParserImpl.java
View File

@ -1,9 +0,0 @@
package org.warp.jcwdb;
public abstract class DBTypeParserImpl<T> implements DBTypeParser<T> {
@SuppressWarnings("unchecked")
@Override
public T cast() {
return (T) this;
}
}

5
src/main/java/org/warp/jcwdb/DBTypedObjectParser.java
View File

@ -1,5 +0,0 @@
package org.warp.jcwdb;
public interface DBTypedObjectParser<T> extends DBTypeParser<T> {
public <U> void registerClass(Class<U> clazz, int type);
}

7
src/main/java/org/warp/jcwdb/DBWriter.java
View File

@ -1,7 +0,0 @@
package org.warp.jcwdb;
import com.esotericsoftware.kryo.io.Output;
public interface DBWriter<T> {
void write(Output o);
}

60
src/main/java/org/warp/jcwdb/Editable.java
View File

@ -1,60 +0,0 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
public interface Editable<T> {
/**
* Reccomended way to edit the value
*
* @param editFunction
* @throws IOException
*/
void editValue(BiFunction<T, Saveable, T> editFunction);
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
void editValue(Function<T, T> editFunction);
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
void editValue(BiConsumer<T, Saveable> editFunction);
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
void editValue(Consumer<T> editFunction);
/**
* Reccomended way to view the value
* @param viewFunction
* @throws IOException
*/
void viewValue(Consumer<T> viewFunction);
/**
* Substitute the old value with a new one
* @param val
* @throws IOException
*/
void setValue(T val);
/**
* DO NOT ATTEMPT TO MODIFY THE VALUE RETURNED
* @return
*/
T getValueReadOnlyUnsafe();
}

266
src/main/java/org/warp/jcwdb/EntryReference.java
View File

@ -1,266 +0,0 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
/**
* You must have only a maximum of 1 reference for each index
* @param <T>
*/
public class EntryReference<T> implements Editable<T>, Saveable, Castable {
private final JCWDatabase.EntryReferenceTools db;
private final long entryIndex;
private final DBTypeParser<T> parser;
private T value;
private long cachedHash;
private volatile boolean isHashCached;
private volatile boolean loaded;
private volatile boolean closed;
private volatile boolean isFlushingAllowed;
private final Object hashCacheLock = new Object();
private final Object accessLock = new Object();
private final Object closeLock = new Object();
public EntryReference(JCWDatabase.EntryReferenceTools db, long entryId, long hash, DBTypeParser<T> parser) {
this.loaded = false;
this.isHashCached = false;
this.db = db;
this.entryIndex = entryId;
this.parser = parser;
this.value = null;
}
public EntryReference(JCWDatabase.EntryReferenceTools db, long entryId, long hash, DBTypeParser<T> parser, T value) {
this.loaded = true;
this.isHashCached = true;
this.db = db;
this.entryIndex = entryId;
this.parser = parser;
this.cachedHash = hash;
this.value = value;
}
public DBTypeParser<T> getParser() {
return parser;
}
public long getIndex() {
return entryIndex;
}
public long calculateHash() {
synchronized(accessLock) {
load();
synchronized(hashCacheLock) {
if (isHashCached) {
return cachedHash;
}
}
return parser.calculateHash(this.value);
}
}
public boolean isClosed() {
return closed;
}
/**
* Note that this method won't be called when closing without saving
*/
public void save() {
this.save(false);
}
public void saveAndFlush() {
this.save(true);
}
private void save(boolean flush) {
synchronized(accessLock) {
if (loaded && !closed) {
try {
if (value instanceof Saveable) {
if (flush) {
((Saveable)value).saveAndFlush();
} else {
((Saveable)value).save();
}
}
IndexDetails returnedDetails = this.db.write(entryIndex, parser.getWriter(value));
synchronized(hashCacheLock) {
this.cachedHash = returnedDetails.getHash();
this.isHashCached = true;
}
if (flush) {
if (!isFlushingAllowed) {
this.db.setFlushingAllowed(entryIndex, true);
this.isFlushingAllowed = true;
}
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
public void editValue(BiFunction<T, Saveable, T> editFunction) {
synchronized(accessLock) {
load();
this.value = editFunction.apply(this.value, this);
this.save(true);
}
}
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
public void editValue(Function<T, T> editFunction) {
synchronized(accessLock) {
load();
this.value = editFunction.apply(this.value);
this.save(true);
}
}
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
public void editValue(BiConsumer<T, Saveable> editFunction) {
synchronized(accessLock) {
load();
editFunction.accept(this.value, this);
this.save(true);
}
}
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
public void editValue(Consumer<T> editFunction) {
synchronized(accessLock) {
load();
editFunction.accept(this.value);
this.save(true);
}
}
/**
* Reccomended way to edit the value
* DO NOT EDIT THE VALUE
* @param viewFunction
* @throws IOException
*/
public void viewValue(Consumer<T> viewFunction) {
synchronized(accessLock) {
load();
viewFunction.accept(this.value);
}
}
/**
* Substitute the old value with a new one
* @param val
* @throws IOException
*/
public void setValue(T val) {
synchronized(accessLock) {
this.loaded = true;
this.value = val;
synchronized(hashCacheLock) {
this.isHashCached = false;
}
this.save(true);
}
}
/**
* Use editValue instead. READ ONLY!!
* @return
*/
@Deprecated()
public T getValue() {
return getValueReadOnlyUnsafe();
}
/**
* DO NOT ATTEMPT TO MODIFY THE VALUE RETURNED
* @return
*/
public T getValueReadOnlyUnsafe() {
synchronized(accessLock) {
load();
return this.value;
}
}
private void load() {
synchronized(accessLock) {
if (!loaded) {
try {
if (this.isFlushingAllowed) {
this.db.setFlushingAllowed(entryIndex, false);
this.isFlushingAllowed = false;
}
this.value = db.read(entryIndex, parser.getReader());
this.loaded = true;
} catch (IOException e) {
throw (NullPointerException) new NullPointerException(e.getLocalizedMessage()).initCause(e);
}
}
}
}
@SuppressWarnings("unchecked")
@Override
public <U> U cast() {
return (U) this;
}
protected void close() throws IOException {
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
save(true);
closed = true;
}
}
public void closeWithoutSaving() {
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
closed = true;
}
}
public Object getAccessLock() {
return accessLock;
}
}

147
src/main/java/org/warp/jcwdb/FileAllocator.java
View File

@ -1,147 +0,0 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.longs.*;
import it.unimi.dsi.fastutil.objects.ObjectBidirectionalIterator;
import java.io.IOException;
import java.nio.channels.SeekableByteChannel;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Stream;
public class FileAllocator implements AutoCloseable {
private static final int MAXIMUM_UNALLOCATED_ENTRIES = 50000;
private final SeekableByteChannel dataFileChannel;
private volatile long fileSize;
private volatile boolean closed;
private final Object closeLock = new Object();
private final Object allocateLock = new Object();
/**
* index -> free space size
*/
private final Long2IntMap freeBytes = new Long2IntLinkedOpenHashMap();
public FileAllocator(SeekableByteChannel dataFileChannel) throws IOException {
this.dataFileChannel = dataFileChannel;
this.fileSize = this.dataFileChannel.size();
}
public FileAllocator(SeekableByteChannel dataFileChannel, long fileSize, Long2IntMap freeBytes) throws IOException {
this.dataFileChannel = dataFileChannel;
this.fileSize = fileSize;
this.freeBytes.putAll(freeBytes);
}
/**
* TODO: not implemented
*
* @param size
* @return offset
*/
public long allocate(int size) {
checkClosed();
synchronized (allocateLock) {
long offset = allocateIntoUnusedParts(size);
if (offset == -1) {
return allocateToEnd(size);
} else {
return offset;
}
}
}
private long allocateIntoUnusedParts(int size) {
Stream<Map.Entry<Long,Integer>> sorted =
freeBytes.entrySet().stream()
.sorted(Collections.reverseOrder(Map.Entry.comparingByValue()));
final VariableWrapper<Long> holeOffset = new VariableWrapper<>(-1L);
final VariableWrapper<Integer> holeSize = new VariableWrapper<>(0);
sorted.anyMatch((entry) -> {
int currentHoleSize = entry.getValue();
if (currentHoleSize < size) {
return true;
}
holeOffset.var = entry.getKey();
holeSize.var = currentHoleSize;
return false;
});
if (holeOffset.var != -1L) {
freeBytes.remove(holeOffset.var);
if (holeSize.var > size) {
freeBytes.put(holeOffset.var + size, holeSize.var - size);
}
}
return holeOffset.var;
}
private long allocateToEnd(int size) {
long allocatedOffset = fileSize;
fileSize += size;
return allocatedOffset;
}
public void close() throws IOException {
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
closed = true;
}
}
/**
* Frees the unused bytes
*
* @param startPosition
* @param length
*/
public void markFree(long startPosition, int length) {
checkClosed();
if (freeBytes.containsKey(startPosition + length)) {
int secondLength = freeBytes.remove(startPosition + length);
freeBytes.put(startPosition, length + secondLength);
} else {
boolean addedToList = false;
for (Long2IntMap.Entry entry : freeBytes.long2IntEntrySet()) {
if (entry.getLongKey() + entry.getIntValue() == startPosition) {
freeBytes.put(entry.getLongKey(), entry.getIntValue() + length);
addedToList = true;
break;
}
}
if (!addedToList && length > 0) {
freeBytes.put(startPosition, length);
}
}
if (startPosition + length >= fileSize) {
fileSize = startPosition;
}
// Remove the smallest hole in the file
if (freeBytes.size() > MAXIMUM_UNALLOCATED_ENTRIES) {
Stream<Map.Entry<Long,Integer>> sorted =
freeBytes.entrySet().stream()
.sorted(Map.Entry.comparingByValue());
Optional<Map.Entry<Long, Integer>> first = sorted.findFirst();
if (first.isPresent()) {
freeBytes.remove(first.get().getKey());
}
}
}
private void checkClosed() {
if (closed) {
throw new RuntimeException("Index Manager is closed.");
}
}
}

539
src/main/java/org/warp/jcwdb/FileIndexManager.java
View File

@ -1,539 +0,0 @@
package org.warp.jcwdb;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import it.unimi.dsi.fastutil.longs.*;
import it.unimi.dsi.fastutil.objects.ObjectIterator;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
public class FileIndexManager implements IndexManager {
private final SeekableByteChannel dataFileChannel, metadataFileChannel;
private volatile long metadataFileChannelSize;
private final FileAllocator fileAllocator;
private final ByteBuffer metadataByteBuffer = ByteBuffer.allocateDirect(IndexDetails.TOTAL_BYTES);
private final ByteBuffer maskByteBuffer = ByteBuffer.allocateDirect(Integer.BYTES);
private volatile boolean closed;
private final Object closeLock = new Object();
private final Object metadataByteBufferLock = new Object();
private final Object maskByteBufferLock = new Object();
private final Object indicesMapsAccessLock = new Object();
/**
* Edit this using editIndex()
* Get using getIndexMetadata()
* This hashmap must contain all indices.
*/
private final Long2ObjectMap<IndexDetails> loadedIndices;
/**
* Edit this using editIndex()
*/
private final LongSet dirtyLoadedIndices, flushingAllowedIndices, removedIndices;
private long firstAllocableIndex;
public FileIndexManager(Path dataFile, Path metadataFile) throws IOException {
if (Cleaner.DISABLE_CLEANER) {
loadedIndices = new Long2ObjectOpenHashMap<>();
dirtyLoadedIndices = new LongOpenHashSet();
flushingAllowedIndices = new LongOpenHashSet();
removedIndices = new LongOpenHashSet();
} else {
loadedIndices = new Long2ObjectLinkedOpenHashMap<>();
dirtyLoadedIndices = new LongLinkedOpenHashSet();
flushingAllowedIndices = new LongLinkedOpenHashSet();
removedIndices = new LongLinkedOpenHashSet();
}
if (Files.notExists(dataFile)) {
Files.createFile(dataFile);
}
if (Files.notExists(metadataFile)) {
Files.createFile(metadataFile);
}
dataFileChannel = Files.newByteChannel(dataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
metadataFileChannel = Files.newByteChannel(metadataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
fileAllocator = createFileAllocator(dataFileChannel, metadataFileChannel.position(0));
metadataFileChannelSize = metadataFileChannel.size();
firstAllocableIndex = getMetadataFileChannelSize() / (long) IndexDetails.TOTAL_BYTES;
if (firstAllocableIndex == 0) {
firstAllocableIndex = 1;
}
}
private long getMetadataFileChannelSize() throws IOException {
return metadataFileChannelSize;
}
private FileAllocator createFileAllocator(final SeekableByteChannel dataFileChannel, final SeekableByteChannel metadataFileChannel) throws IOException {
Long2IntMap freeBytes = new Long2IntRBTreeMap();
Long2IntMap usedBytes = new Long2IntRBTreeMap();
long firstOffset = 0;
while (metadataFileChannel.position() + IndexDetails.TOTAL_BYTES <= getMetadataFileChannelSize()) {
IndexDetails indexDetails = readIndexDetailsAt(metadataFileChannel);
if (indexDetails != null) {
long offset = indexDetails.getOffset();
usedBytes.put(offset, indexDetails.getSize());
if (offset < firstOffset) {
firstOffset = offset;
}
}
}
long previousEntryOffset = 0;
long previousEntrySize = 0;
ObjectIterator<Long2IntMap.Entry> it = usedBytes.long2IntEntrySet().iterator();
while (it.hasNext()) {
final Long2IntMap.Entry entry = it.next();
final long entryOffset = entry.getLongKey();
final long entrySize = entry.getIntValue();
it.remove();
if (previousEntryOffset + previousEntrySize < entryOffset) {
freeBytes.put(previousEntryOffset + previousEntrySize, (int) (entryOffset - (previousEntryOffset + previousEntrySize)));
}
previousEntryOffset = entryOffset;
previousEntrySize = entrySize;
}
final long fileSize = previousEntryOffset + previousEntrySize;
return new FileAllocator(dataFileChannel, fileSize, freeBytes);
}
@Override
public <T> T get(long index, DBReader<T> reader) throws IOException {
checkClosed();
IndexDetails details = getIndexMetadata(index);
Input i = new Input(Channels.newInputStream(dataFileChannel.position(details.getOffset())));
T result = reader.read(i, details.getSize());
return result;
}
@Override
public int getType(long index) throws IOException {
return getIndexMetadata(index).getType();
}
@Override
public long getHash(long index) throws IOException {
return getIndexMetadata(index).getHash();
}
@Override
public <T> IndexDetails set(long index, DBDataOutput<T> data) throws IOException {
checkClosed();
final int dataSize = data.getSize();
IndexDetails indexDetails = getIndexMetadataUnsafe(index);
if (indexDetails == null || indexDetails.getSize() < dataSize) {
// Allocate new space
IndexDetails newDetails = allocateAndWrite(index, data);
if (indexDetails != null) {
// Mark free the old bytes
fileAllocator.markFree(indexDetails.getOffset(), indexDetails.getSize());
}
return newDetails;
} else {
// Check if size changed
if (dataSize < indexDetails.getSize()) {
// Mark free the unused bytes
fileAllocator.markFree(indexDetails.getOffset() + dataSize, dataSize);
}
// Update index details
indexDetails = editIndex(index, indexDetails, indexDetails.getOffset(), dataSize, indexDetails.getType(), data.calculateHash());
// Write data
writeExact(indexDetails, data);
// Before returning, return IndexDetails
return indexDetails;
}
}
@Override
public void setFlushingAllowed(long index, boolean isUnloadingAllowed) {
checkClosed();
if (isUnloadingAllowed) {
flushingAllowedIndices.add(index);
} else {
flushingAllowedIndices.remove(index);
}
}
@Override
public <T> long add(DBDataOutput<T> data) throws IOException {
checkClosed();
final int size = data.getSize();
final long offset = fileAllocator.allocate(size);
final int type = data.getType();
final long hash = data.calculateHash();
final IndexDetails indexDetails = new IndexDetails(offset, size, type, hash);
final long index = createIndexMetadata(indexDetails);
writeExact(indexDetails, data);
return index;
}
@Override
public <T> FullIndexDetails addAndGetDetails(DBDataOutput<T> data) throws IOException {
checkClosed();
final int size = data.getSize();
final long offset = fileAllocator.allocate(size);
final int type = data.getType();
final long hash = data.calculateHash();
final IndexDetails indexDetails = new IndexDetails(offset, size, type, hash);
final long index = createIndexMetadata(indexDetails);
writeExact(indexDetails, data);
return new FullIndexDetails(index, indexDetails);
}
/**
* Write the data at index.
* The input size must be equal to the index size!
*
* @param indexDetails
* @param data
* @throws IOException
*/
private void writeExact(final IndexDetails indexDetails, DBDataOutput<?> data) throws IOException {
final int dataSize = data.getSize();
if (indexDetails.getSize() != dataSize) {
throw new IOException("Unable to write " + dataSize + " in a space of " + indexDetails.getSize());
}
final long offset = indexDetails.getOffset();
final Output o = new Output(Channels.newOutputStream(dataFileChannel.position(offset)), dataSize);
data.getWriter().write(o);
o.flush();
}
private IndexDetails allocateAndWrite(final long index, DBDataOutput<?> w) throws IOException {
final int size = w.getSize();
final int type = w.getType();
final long hash = w.calculateHash();
final long offset = fileAllocator.allocate(size);
IndexDetails details = editIndex(index, offset, size, type, hash);
writeExact(details, w);
return details;
}
@Override
public void delete(long index) throws IOException {
checkClosed();
IndexDetails indexDetails = getIndexMetadataUnsafe(index);
if (indexDetails != null) {
fileAllocator.markFree(indexDetails.getOffset(), indexDetails.getSize());
}
synchronized (indicesMapsAccessLock) {
dirtyLoadedIndices.remove(index);
flushingAllowedIndices.remove(index);
loadedIndices.remove(index);
removedIndices.add(index);
}
}
public void flushAndUnload(long index) throws IOException {
if (removedIndices.contains(index)) {
synchronized (indicesMapsAccessLock) {
removedIndices.remove(index);
dirtyLoadedIndices.remove(index);
flushingAllowedIndices.remove(index);
loadedIndices.remove(index);
}
// Update indices metadata
SeekableByteChannel metadata = metadataFileChannel.position(index * IndexDetails.TOTAL_BYTES);
eraseIndexDetails(metadata);
}
boolean isDirty = false;
IndexDetails indexDetails = null;
synchronized (indicesMapsAccessLock) {
if (dirtyLoadedIndices.contains(index)) {
indexDetails = loadedIndices.get(index);
dirtyLoadedIndices.remove(index);
flushingAllowedIndices.remove(index);
}
}
if (isDirty) {
// Update indices metadata
long position = index * IndexDetails.TOTAL_BYTES;
resizeMetadataFileChannel(position);
SeekableByteChannel metadata = metadataFileChannel.position(position);
writeIndexDetails(metadata, indexDetails);
}
synchronized (indicesMapsAccessLock) {
loadedIndices.remove(index);
}
}
@Override
public boolean has(long index) {
checkClosed();
try {
return getIndexMetadataUnsafe(index) != null;
} catch (IOException ex) {
ex.printStackTrace();
return false;
}
}
/**
* Edit index data if a change is detected
* @param index
* @param oldData Old index data to check
* @param offset offset
* @param size size
* @param type type
* @param hash hash
* @return
*/
private IndexDetails editIndex(long index, IndexDetails oldData, long offset, int size, int type, long hash) {
if (oldData.getOffset() != offset || oldData.getSize() != size || oldData.getType() != type || oldData.getHash() != hash) {
return editIndex(index, offset, size, type, hash);
} else {
return oldData;
}
}
/**
* Edit index data
* @param index
* @param offset
* @param size
* @param type
* @param hash
* @return
*/
private IndexDetails editIndex(long index, long offset, int size, int type, long hash) {
IndexDetails indexDetails = new IndexDetails(offset, size, type, hash);
editIndex(index, indexDetails);
return indexDetails;
}
/**
* Edit index data
* @param index
* @param details
*/
private void editIndex(long index, IndexDetails details) {
synchronized (indicesMapsAccessLock) {
loadedIndices.put(index, details);
dirtyLoadedIndices.add(index);
flushingAllowedIndices.remove(index);
}
}
private long createIndexMetadata(IndexDetails indexDetails) {
synchronized (indicesMapsAccessLock) {
long newIndex = firstAllocableIndex++;
loadedIndices.put(newIndex, indexDetails);
dirtyLoadedIndices.add(newIndex);
flushingAllowedIndices.remove(newIndex);
removedIndices.remove(newIndex);
return newIndex;
}
}
private IndexDetails getIndexMetadataUnsafe(long index) throws IOException {
// Return index details if loaded
IndexDetails details;
synchronized (indicesMapsAccessLock) {
details = loadedIndices.getOrDefault(index, null);
}
if (details != null) return details;
// Try to load the details from file
final long metadataPosition = index * IndexDetails.TOTAL_BYTES;
if (metadataPosition + IndexDetails.TOTAL_BYTES > getMetadataFileChannelSize()) {
// Avoid underflow exception
return null;
}
SeekableByteChannel currentMetadataFileChannel = metadataFileChannel.position(metadataPosition);
IndexDetails indexDetails = readIndexDetailsAt(currentMetadataFileChannel);
if (indexDetails != null) {
editIndex(index, indexDetails);
return indexDetails;
}
// No results found. Returning null
return null;
}
private IndexDetails readIndexDetailsAt(SeekableByteChannel currentMetadataFileChannel) throws IOException {
IndexDetails indexDetails = null;
synchronized (metadataByteBufferLock) {
metadataByteBuffer.rewind();
currentMetadataFileChannel.read(metadataByteBuffer);
metadataByteBuffer.rewind();
// If it's not deleted continue
if ((metadataByteBuffer.getInt() & IndexDetails.MASK_DELETED) == 0) {
final long offset = metadataByteBuffer.getLong();
// final long sizeAndType = metadataByteBuffer.getLong();
// final int size = (int)(sizeAndType >> 32);
// final int type = (int)sizeAndType;
final int size = metadataByteBuffer.getInt();
final int type = metadataByteBuffer.getInt();
final long hash = metadataByteBuffer.getLong();
indexDetails = new IndexDetails(offset, size, type, hash);
}
}
return indexDetails;
}
private IndexDetails getIndexMetadata(long index) throws IOException {
IndexDetails details = getIndexMetadataUnsafe(index);
if (details == null)
throw new IOException("Index " + index + " not found");
else
return details;
}
@Override
public void close() throws IOException {
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
closed = true;
}
// Update indices metadata
flushAllFlushableIndices();
// Remove removed indices
removeRemovedIndices();
fileAllocator.close();
}
private void writeIndexDetails(SeekableByteChannel position, IndexDetails indexDetails) throws IOException {
synchronized (metadataByteBufferLock) {// FIXXXX cleaner3
final int size = indexDetails.getSize();
final int type = indexDetails.getType();
final long offset = indexDetails.getOffset();
final long hash = indexDetails.getHash();
metadataByteBuffer.rewind();
metadataByteBuffer.putInt(0);
metadataByteBuffer.putLong(offset);
metadataByteBuffer.putInt(size);
metadataByteBuffer.putInt(type);
//metadataByteBuffer.putLong((long)size << 32 | type & 0xFFFFFFFFL);
metadataByteBuffer.putLong(hash);
metadataByteBuffer.rewind();
position.write(metadataByteBuffer);
}
}
private void eraseIndexDetails(SeekableByteChannel position) throws IOException {
synchronized (maskByteBufferLock) {
maskByteBuffer.rewind();
maskByteBuffer.putInt(IndexDetails.MASK_DELETED);
maskByteBuffer.rewind();
position.write(maskByteBuffer);
}
}
private void checkClosed() {
if (closed) {
throw new RuntimeException("Index Manager is closed.");
}
}
@Override
public long clean() {
long cleaned = 0;
long tim1 = System.currentTimeMillis();
try {
cleaned += flushAllFlushableIndices();
} catch (IOException ex) {
ex.printStackTrace();
}
long tim2 = System.currentTimeMillis();
try {
cleaned += removeRemovedIndices();
} catch (IOException ex) {
ex.printStackTrace();
}
long tim3 = System.currentTimeMillis();
cleaned += cleanExtraIndices();
long tim4 = System.currentTimeMillis();
if (Cleaner.ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] FileIndexManager CLEAN_TIME: " + (tim2-tim1) + "," + (tim3-tim2) + "," + (tim4-tim3));
return cleaned;
}
private long flushAllFlushableIndices() throws IOException {
long flushedIndices = 0;
SeekableByteChannel metadata = metadataFileChannel;
long lastIndex = -2;
synchronized (indicesMapsAccessLock) {
for (long index : dirtyLoadedIndices) {
if (!flushingAllowedIndices.contains(index)) {
IndexDetails indexDetails = loadedIndices.get(index);
long position = index * IndexDetails.TOTAL_BYTES;
resizeMetadataFileChannel(position);
if (index - lastIndex != 1) {
metadata = metadata.position(position);
}
writeIndexDetails(metadata, indexDetails);
lastIndex = index;
flushedIndices++;
}
}
dirtyLoadedIndices.clear();
dirtyLoadedIndices.addAll(flushingAllowedIndices);
flushingAllowedIndices.clear();
}
return flushedIndices;
}
private void resizeMetadataFileChannel(long position) {
if (position + IndexDetails.TOTAL_BYTES > metadataFileChannelSize) {
metadataFileChannelSize = position + IndexDetails.TOTAL_BYTES;
}
}
private long removeRemovedIndices() throws IOException {
SeekableByteChannel metadata = metadataFileChannel;
synchronized (indicesMapsAccessLock) {
long removed = this.removedIndices.size();
for (long index : this.removedIndices) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
eraseIndexDetails(metadata);
}
this.removedIndices.clear();
return removed;
}
}
private long cleanExtraIndices() {
long removedIndices = 0;
LongArrayList toUnload = new LongArrayList();
synchronized (indicesMapsAccessLock) {
if (loadedIndices.size() > JCWDatabase.MAX_LOADED_INDICES) {
long count = loadedIndices.size();
LongIterator it = loadedIndices.keySet().iterator();
while (it.hasNext()) {
long loadedIndex = it.nextLong();
if (count < JCWDatabase.MAX_LOADED_INDICES * 3l / 2l) {
break;
}
toUnload.add(loadedIndex);
removedIndices++;
count--;
}
}
}
for (long index : toUnload.elements()) {
try {
flushAndUnload(index);
} catch (IOException e) {
e.printStackTrace();
}
}
return removedIndices;
}
}

14
src/main/java/org/warp/jcwdb/FullIndexDetails.java
View File

@ -1,14 +0,0 @@
package org.warp.jcwdb;
public class FullIndexDetails extends IndexDetails {
private final long index;
public FullIndexDetails(long index, IndexDetails details) {
super(details);
this.index = index;
}
public long getIndex() {
return index;
}
}

88
src/main/java/org/warp/jcwdb/IndexDetails.java
View File

@ -1,88 +0,0 @@
package org.warp.jcwdb;
import java.util.Objects;
public class IndexDetails {
/**
* The bitmask is used to determine if an index has been deleted
*/
public static final int BITMASK_SIZE = Integer.BYTES;
public static final int OFFSET_BYTES = Long.BYTES;
public static final int DATA_SIZE_BYTES = Integer.BYTES;
public static final int TYPE_BYTES = Integer.BYTES;
public static final int HASH_BYTES = Long.BYTES;
public static final int TOTAL_BYTES = BITMASK_SIZE + OFFSET_BYTES + DATA_SIZE_BYTES + TYPE_BYTES + HASH_BYTES;
public static final int MASK_DELETED = 0b00000001;
private final long offset;
private final int size;
private final int type;
private final long hash;
public IndexDetails(long offset, int size, int type, long hash) {
this.offset = offset;
this.size = size;
this.type = type;
this.hash = hash;
}
public IndexDetails(IndexDetails indexDetails) {
this.offset = indexDetails.offset;
this.size = indexDetails.size;
this.type = indexDetails.type;
this.hash = indexDetails.hash;
}
public long getOffset() {
return offset;
}
public int getSize() {
return size;
}
public int getType() {
return type;
}
public long getHash() {
return hash;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (hash ^ (hash >>> 32));
result = prime * result + (int) (offset ^ (offset >>> 32));
result = prime * result + size;
result = prime * result + type;
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
IndexDetails other = (IndexDetails) obj;
if (hash != other.hash)
return false;
if (offset != other.offset)
return false;
if (size != other.size)
return false;
if (type != other.type)
return false;
return true;
}
@Override
public String toString() {
return "IndexDetails [offset=" + offset + ", size=" + size + ", type=" + type + ", hash=" + hash + "]";
}
}

19
src/main/java/org/warp/jcwdb/IndexManager.java
View File

@ -1,19 +0,0 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.util.function.BiConsumer;
import java.util.function.BiPredicate;
import java.util.function.Consumer;
public interface IndexManager extends Cleanable {
<T> T get(long index, DBReader<T> reader) throws IOException;
int getType(long index) throws IOException;
long getHash(long index) throws IOException;
<T> long add(DBDataOutput<T> writer) throws IOException;
<T> FullIndexDetails addAndGetDetails(DBDataOutput<T> writer) throws IOException;
<T> IndexDetails set(long index, DBDataOutput<T> writer) throws IOException;
void setFlushingAllowed(long index, boolean isUnloadingAllowed);
void delete(long index) throws IOException;
boolean has(long index);
void close() throws IOException;
}

197
src/main/java/org/warp/jcwdb/JCWDatabase.java
View File

@ -1,197 +0,0 @@
package org.warp.jcwdb;
import java.io.IOError;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.function.Supplier;
public class JCWDatabase implements AutoCloseable, Cleanable {
public final static long MAX_LOADED_INDICES = 1000;
private final TypesManager typesManager;
private final MixedIndexDatabase indices;
private final Cleaner databaseCleaner;
private final EntryReferenceTools entryReferenceTools = new EntryReferenceTools();
private volatile boolean closed;
private final Object closeLock = new Object();
private final Object indicesAccessLock = new Object();
private final LinkedList<EntryReference<?>> usedReferences = new LinkedList<>();
public JCWDatabase(Path dataFile, Path metadataFile) throws IOException {
this.typesManager = new TypesManager(this);
this.indices = new MixedIndexDatabase(dataFile, metadataFile);
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
JCWDatabase.this.close();
} catch (Exception e) {
e.printStackTrace();
}
}));
this.databaseCleaner = new Cleaner(this);
this.databaseCleaner.start();
}
public <T> EntryReference<LightList<T>> getRoot() {
try {
checkClosed();
if (exists(0)) {
return get(0);
} else {
LightList<T> newRoot = new LightBigList<>(this);
return set(0, newRoot);
}
} catch (IOException e) {
throw new IOError(e);
}
}
@SuppressWarnings("unchecked")
public <T> EntryReference<T> getRootItem(int index) {
return ((LightList<T>) getRoot().getValueReadOnlyUnsafe()).getReference(index);
}
@SuppressWarnings("unchecked")
public <T> EntryReference<T> getRootItem(int index, Supplier<T> defaultValue) {
return ((LightList<T>) getRoot().getValueReadOnlyUnsafe()).getReferenceOrInitialize(index, defaultValue);
}
public <T> EntryReference<LightList<T>> getRoot(Class<T> clazz) {
return getRoot().cast();
}
public <T> EntryReference<T> get(long index) throws IOException {
checkClosed();
int type;
long hash;
synchronized (indicesAccessLock) {
type = this.indices.getType(index);
hash = this.indices.getHash(index);
}
DBTypeParser<T> typeParser = this.typesManager.get(type);
return new EntryReference<>(entryReferenceTools, index, hash, typeParser);
}
protected <T> EntryReference<T> add(T value) throws IOException {
checkClosed();
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
long index;
long hash;
synchronized (indicesAccessLock) {
FullIndexDetails fullIndexDetails = indices.addAndGetDetails(typeParser.getWriter(value));
index = fullIndexDetails.getIndex();
hash = fullIndexDetails.getHash();
}
return new EntryReference<>(entryReferenceTools, index, hash, typeParser, value);
}
protected boolean exists(long index) {
checkClosed();
synchronized (indicesAccessLock) {
return this.indices.has(index);
}
}
protected <T> EntryReference<T> set(long index, T value) throws IOException {
checkClosed();
EntryReference<T> ref;
if (exists(index)) {
ref = get(index);
ref.setValue(value);
return ref;
} else {
@SuppressWarnings("unchecked")
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
long hash;
synchronized (indicesAccessLock) {
IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value));
hash = returnedDetails.getHash();
}
return new EntryReference<>(entryReferenceTools, index, hash, typeParser);
}
}
public <U> void registerType(Class<U> clazz, short type, DBTypeParser<U> parser) {
final int addition = 0xEFFF8000;
int extendedType = addition | (type & 0x7FFF);
typesManager.registerType(clazz, extendedType, parser);
}
public <U> void registerClass(Class<U> clazz, int type) {
typesManager.registerGenericClass(clazz, type);
}
public boolean isOpen() {
return !closed;
}
@Override
public void close() throws IOException {
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
closed = true;
}
this.databaseCleaner.stop();
synchronized (indicesAccessLock) {
this.indices.close();
}
System.out.println("Database closed.");
}
private void checkClosed() {
if (closed) {
throw new RuntimeException("Index Manager is closed.");
}
}
@Override
public long clean() {
long removedItems = indices.clean();
Iterator<EntryReference<?>> usedReferencesIterator = usedReferences.iterator();
while(usedReferencesIterator.hasNext()) {
EntryReference<?> entryReference = usedReferencesIterator.next();
if (entryReference.isClosed()) {
usedReferencesIterator.remove();
removedItems += 1;
}
}
return removedItems;
}
public class EntryReferenceTools {
private EntryReferenceTools() {
}
public <T> T read(long index, DBReader<T> reader) throws IOException {
return indices.get(index, reader);
}
public <T> IndexDetails write(long index, DBDataOutput<T> writer) throws IOException {
return indices.set(index, writer);
}
public void setFlushingAllowed(long index, boolean isFlushingAllowed) {
indices.setFlushingAllowed(index, isFlushingAllowed);
}
public <T> void setUsed(EntryReference<T> ref) {
usedReferences.add(ref);
}
}
@SuppressWarnings("unchecked")
protected <T> long calculateHash(T o) {
return ((DBTypeParser<T>) typesManager.get(o.getClass())).calculateHash(o);
}
}

455
src/main/java/org/warp/jcwdb/LightArrayList.java
View File

@ -1,455 +0,0 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import java.io.IOError;
import java.io.IOException;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Predicate;
public class LightArrayList<T> implements LightList<T> {
public final LongArrayList internalList;
private final transient JCWDatabase db;
/**
* @param db Database reference
*/
public LightArrayList(JCWDatabase db) {
this.db = db;
this.internalList = new LongArrayList();
}
/**
* @param db Database reference
* @param elements Elements to add
*/
public LightArrayList(JCWDatabase db, LongArrayList elements) {
this.db = db;
this.internalList = new LongArrayList(elements);
}
@Override
public int size() {
return internalList.size();
}
@Override
public boolean isEmpty() {
return internalList.isEmpty();
}
@Override
public boolean contains(Object o) {
if (o != null) {
for (long element : internalList) {
EntryReference<T> ref = null;
try {
ref = db.get(element);
if (o.equals(ref.getValueReadOnlyUnsafe())) {
return true;
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
return false;
}
/**
* Use iteratorReferences()
*/
@Deprecated
@SuppressWarnings("unchecked")
@Override
public Iterator<T> iterator() {
System.out.println("WARNING! YOU ARE USING iterator()! PLEASE USE ITERATORREFERENCES TO AVOID OUTOFMEMORY!");
final ArrayList<T> elements = new ArrayList<>();
for (long element : internalList) {
try {
elements.add((T) db.get(element).getValueReadOnlyUnsafe());
} catch (IOException e) {
e.printStackTrace();
}
}
return elements.iterator();
}
@SuppressWarnings("unchecked")
@Override
public Iterator<EntryReference<T>> iteratorReferences() {
final ArrayList<EntryReference<T>> elements = new ArrayList<>();
for (long element : internalList) {
try {
elements.add(db.get(element));
} catch (IOException e) {
e.printStackTrace();
}
}
return elements.iterator();
}
/**
* USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
*
* @param action
*/
@Deprecated
@Override
public void forEach(Consumer<? super T> action) {
Objects.requireNonNull(action);
for (T t : this) {
action.accept(t);
}
}
@Override
public void forEachReference(Consumer<? super EntryReference<T>> action) {
Objects.requireNonNull(action);
for (long index : this.internalList) {
try {
action.accept(db.get(index));
} catch (IOException e) {
throw new IOError(e);
}
}
}
@SuppressWarnings("unchecked")
@Override
public T[] toArray() {
final T[] elements = (T[]) new Object[internalList.size()];
for (int i = 0; i < elements.length; i++) {
try {
T element = (T) db.get(internalList.getLong(i)).getValueReadOnlyUnsafe();
elements[i] = element;
} catch (IOException e) {
e.printStackTrace();
}
}
return elements;
}
@SuppressWarnings("unchecked")
@Override
public <T1> T1[] toArray(T1[] a) {
final T1[] elements = (T1[]) new Objects[internalList.size()];
for (int i = 0; i < elements.length; i++) {
try {
elements[i] = (T1) db.get(internalList.getLong(i)).getValueReadOnlyUnsafe();
} catch (IOException e) {
e.printStackTrace();
}
}
return elements;
}
@Override
public boolean add(T o) {
EntryReference<T> ref = addEntry(o);
return ref != null;
}
@Override
public EntryReference<T> addEntry(T o) {
EntryReference<T> ref = addToDatabase(o);
if (internalList.add(ref.getIndex())) {
return ref;
} else {
return null;
}
}
@Override
public boolean remove(Object o) {
int removeIndex = indexOf(o);
if (removeIndex >= 0) {
internalList.removeLong(removeIndex);
return true;
}
return false;
}
@Override
public boolean remove(EntryReference<T> ref) {
int removeIndex = indexOfEntry(ref);
if (removeIndex >= 0) {
internalList.removeLong(removeIndex);
return true;
}
return false;
}
@Override
public boolean containsAll(Collection<?> c) {
for (Object o : c) {
int objIndex = indexOf(o);
if (objIndex < 0) {
return false;
}
}
return true;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(Collection<? extends T> c) {
boolean result = false;
for (Object o : c) {
result |= add((T) o);
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(int index, Collection<? extends T> c) {
boolean result = false;
int delta = 0;
for (Object o : c) {
add(index + delta, (T) o);
result = true;
delta++;
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeAll(Collection<?> c) {
boolean result = false;
for (Object o : c) {
result |= remove((T) o);
}
return result;
}
@Override
public boolean retainAll(Collection<?> c) {
boolean result = false;
LongArrayList collectionHashes = new LongArrayList();
ObjectArrayList<Object> collection = new ObjectArrayList<>();
collection.addAll(c);
for (Object o : c) {
collectionHashes.add(db.calculateHash(o));
}
for (int i = 0; i < internalList.size(); i++) {
long hash = internalList.getLong(i);
int positionInCollection = collectionHashes.indexOf(hash);
if (positionInCollection == -1) {
remove(collection.get(positionInCollection));
result = true;
}
}
return result;
}
@Override
public void clear() {
internalList.clear();
}
/**
* Use getReference or getReadOnlyValue
*/
@Deprecated
@Override
public T get(int index) {
return getReadOnlyValue(index);
}
@SuppressWarnings("unchecked")
public T getReadOnlyValue(int index) {
try {
return (T) db.get(internalList.getLong(index)).getValueReadOnlyUnsafe();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@Override
public EntryReference<T> getReference(int index) {
try {
return db.get(internalList.getLong(index)).cast();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@SuppressWarnings("unchecked")
@Override
public T set(int index, T element) {
EntryReference<T> ref = addToDatabase(element);
long oldIndex = internalList.set(index, ref.getIndex());
try {
ref.close();
return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnlyUnsafe();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
@Override
public void add(int index, T element) {
EntryReference<T> ref = addToDatabase(element);
internalList.add(index, ref.getIndex());
}
@SuppressWarnings("unchecked")
@Override
public T remove(int index) {
long oldIndex = internalList.removeLong(index);
try {
return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnlyUnsafe();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
@Override
public int indexOf(Object o) {
EntryReference<?> ref = addToDatabase(o);
long objToRemoveHash = ref.calculateHash();
LongArrayList hashes = new LongArrayList();
for (int i = 0; i < hashes.size(); i++) {
long hash = hashes.getLong(i);
if (objToRemoveHash == hash) {
try {
if (ref.equals(db.get(internalList.getLong(i)))) {
return i;
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
}
return -1;
}
@Override
public int indexOfEntry(EntryReference<T> ref) {
for (int i = 0; i < internalList.size(); i++) {
long index = internalList.getLong(i);
try {
EntryReference<?> ref2 = db.get(index);
if (ref.getValueReadOnlyUnsafe().equals(ref2.getValueReadOnlyUnsafe())) {
return i;
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
return -1;
}
@Override
public void appendIndex(long elementIndex) {
internalList.add(elementIndex);
}
@Override
public int lastIndexOf(Object o) {
EntryReference<T> ref = addToDatabase(o).cast();
return lastIndexOfEntry(ref);
}
@Override
public int lastIndexOfEntry(EntryReference<T> ref) {
long objToRemoveHash = ref.calculateHash();
int lastValue = -1;
for (int i = 0; i < internalList.size(); i++) {
long index2 = internalList.getLong(i);
try {
EntryReference<?> ref2 = db.get(index2);
if (objToRemoveHash == ref2.calculateHash()) {
if (ref.getValueReadOnlyUnsafe().equals(ref2.getValueReadOnlyUnsafe())) {
lastValue = i;
}
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
return lastValue;
}
@Deprecated
@Override
public ListIterator<T> listIterator() {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public ListIterator<T> listIterator(int index) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public List<T> subList(int fromIndex, int toIndex) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
private <U> EntryReference<U> addToDatabase(U obj) {
EntryReference<U> ref;
try {
ref = db.add(obj);
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
return ref;
}
@Deprecated
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((internalList == null) ? 0 : internalList.hashCode());
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeIf(Predicate<? super T> filter) {
Objects.requireNonNull(filter);
boolean removed = false;
for (int i = 0; i < internalList.size(); ) {
T obj;
try {
obj = ((EntryReference<T>) (db.get(internalList.getLong(i)).cast())).getValueReadOnlyUnsafe();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
if (filter.test(obj)) {
internalList.removeLong(i);
removed = true;
} else {
i++;
}
}
return removed;
}
@Override
public String toString() {
return "LightArrayList{" +
"internalList=" + internalList +
", db=" + db +
'}';
}
}

607
src/main/java/org/warp/jcwdb/LightBigList.java
View File

@ -1,607 +0,0 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import java.io.IOException;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.function.Supplier;
public class LightBigList<T> implements LightList<T>, Saveable {
public static final int MAX_ELEMENTS_PER_CHUNK = 200000;
public final LongArrayList chunks;
public final IntArrayList chunkSizes;
private final JCWDatabase db;
private LightArrayList<T> cachedChunk;
private EntryReference<LightArrayList<T>> cachedChunkRef;
private long cachedChunkIndex = -1;
private int cachedChunkNumber = -1;
private final Object cachedChunkLock = new Object();
/**
* @param db Database reference
*/
public LightBigList(JCWDatabase db) {
this.db = db;
this.chunks = new LongArrayList();
this.chunkSizes = new IntArrayList();
}
/**
* @param db Database reference
* @param elements Elements to add
*/
public LightBigList(JCWDatabase db, LongArrayList elements) {
this.db = db;
this.chunks = new LongArrayList();
this.chunkSizes = new IntArrayList();
elements.forEach((long element) -> {
this.appendIndex(element);
});
}
public LightBigList(JCWDatabase db, LongArrayList chunks, IntArrayList chunkSizes) {
this.db = db;
this.chunks = chunks;
this.chunkSizes = chunkSizes;
if (this.chunks.size() > 0) {
prepareAccessToChunk(0);
}
}
private void prepareAccessToChunk(int chunkNumber) {
if (this.cachedChunkRef != null) {
this.cachedChunkRef.save();
}
this.cachedChunkNumber = chunkNumber;
this.cachedChunkIndex = this.chunks.getLong(chunkNumber);
try {
this.cachedChunkRef = db.get(this.cachedChunkIndex);
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
this.cachedChunk = this.cachedChunkRef.getValueReadOnlyUnsafe();
}
/**
* Append an index to the first free chunk
* @param elementIndex
*/
public void appendIndex(long elementIndex) {
for (int i = 0; i < chunks.size(); i++) {
final int chunkNumber = i;
if (MAX_ELEMENTS_PER_CHUNK - chunkSizes.getInt(i) > 0) {
synchronized (cachedChunkLock) {
if (cachedChunkNumber != i) {
prepareAccessToChunk(i);
}
cachedChunk.appendIndex(elementIndex);
chunkSizes.set(chunkNumber, cachedChunk.size());
return;
}
}
}
LightList<T> newChunk = new LightArrayList<>(db);
newChunk.appendIndex(elementIndex);
long newChunkIndex;
try {
newChunkIndex = db.add(newChunk).getIndex();
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
chunks.add(newChunkIndex);
chunkSizes.add(1);
}
/**
* Get the elements count
* @return the size of the list
*/
@Override
public int size() {
int size = 0;
for (int chunkSize : this.chunkSizes) {
size += chunkSize;
}
return size;
}
/**
*
* @return the count of chunks
*/
public int chunksCount() {
return this.chunkSizes.size();
}
/**
* Check if the list is empty
* @return true if the list is empty
*/
@Override
public boolean isEmpty() {
return this.size() <= 0;
}
@Override
public boolean contains(Object o) {
if (o != null) {
for (long chunkIndex : chunks) {
try {
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
LightArrayList<T> chunk = chunkRef.getValueReadOnlyUnsafe();
if (chunk.contains(o)) {
return true;
}
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
}
return false;
}
/**
* Use iteratorReferences()
*/
@Deprecated
@Override
public Iterator<T> iterator()
{
throw new RuntimeException("iterator() isn't implemented!");
}
@Deprecated
@Override
public Iterator<EntryReference<T>> iteratorReferences() {
throw new RuntimeException("iteratorReferences() isn't implemented!");
}
/**
* USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
*
* @param action
*/
@Deprecated
@Override
public void forEach(Consumer<? super T> action) {
throw new RuntimeException("forEach() isn't implemented! Use forEachReferences() instead");
}
@Override
public void forEachReference(Consumer<? super EntryReference<T>> action) {
Objects.requireNonNull(action);
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
synchronized (cachedChunkLock) {
if (cachedChunkNumber != i) {
prepareAccessToChunk(i);
}
cachedChunk.forEachReference(action);
}
}
}
/**
* toArray() isn't implemented! DO NOT USE IT.
* @return
*/
@SuppressWarnings("unchecked")
@Deprecated
@Override
public T[] toArray() {
if (true) {
throw new RuntimeException("toArray() isn't implemented!");
}
T[] result = (T[]) new Object[this.size()];
long currentOffset = 0;
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
final int currentChunkSize = chunkSizes.getInt(i);
final long chunkStartOffset = currentOffset;
currentOffset += currentChunkSize;
// Get chunk index
final long chunkIndex = chunks.getLong(i);
try {
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
LightArrayList<T> chunk = chunkRef.getValueReadOnlyUnsafe();
for (int i1 = 0; i1 < chunk.size(); i1++) {
result[(int)(chunkStartOffset + i1)] = chunk.get(i);
}
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public <T1> T1[] toArray(T1[] a) {
throw new RuntimeException("toArray() isn't implemented!");
}
/**
* Use addEntry(o)
* @param o
* @return
*/
@Deprecated
@Override
public boolean add(T o) {
EntryReference<T> ref = addEntry(o);
return ref != null;
}
@Override
public EntryReference<T> addEntry(T o) {
EntryReference<T> ref = addToDatabase(o);
appendIndex(ref.getIndex());
return ref;
}
@Override
public boolean remove(Object o) {
final int removeOffset = indexOf(o);
return removeAt(removeOffset) != null;
}
@Override
public boolean remove(EntryReference<T> ref) {
final int removeOffset = indexOfEntry(ref);
return removeAt(removeOffset) != null;
}
private T removeAt(int removeOffset) {
final VariableWrapper<T> result = new VariableWrapper<>(null);
long currentOffset = 0;
if (removeOffset >= 0) {
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
final int currentChunkSize = chunkSizes.getInt(i);
final long chunkStartOffset = currentOffset;
currentOffset += currentChunkSize;
// If the offset to remove is in the current chunk
if (currentOffset > removeOffset) {
// Get chunk index
final long chunkIndex = chunks.getLong(i);
// Get the offset relative to the current chunk
final int relativeOffset = (int) (removeOffset - chunkStartOffset);
if (relativeOffset < 0) {
return null;
}
try {
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
chunkRef.editValue((chunk) -> {
result.var = chunk.remove(relativeOffset);
});
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
chunkSizes.set(removeOffset, currentChunkSize - 1);
break;
}
}
return result.var;
}
return null;
}
@Override
public boolean containsAll(Collection<?> c) {
for (Object o : c) {
int objIndex = indexOf(o);
if (objIndex < 0) {
return false;
}
}
return true;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(Collection<? extends T> c) {
boolean result = false;
for (Object o : c) {
result |= add((T) o);
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(int index, Collection<? extends T> c) {
boolean result = false;
int delta = 0;
for (Object o : c) {
add(index + delta, (T) o);
result = true;
delta++;
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeAll(Collection<?> c) {
boolean result = false;
for (Object o : c) {
result |= remove((T) o);
}
return result;
}
@Override
public boolean retainAll(Collection<?> c) {
boolean result = false;
LongArrayList collectionHashes = new LongArrayList();
ObjectArrayList<Object> collection = new ObjectArrayList<>();
collection.addAll(c);
for (Object o : c) {
collectionHashes.add(db.calculateHash(o));
}
for (int i = 0; i < chunks.size(); i++) {
long hash = chunks.getLong(i);
int positionInCollection = collectionHashes.indexOf(hash);
if (positionInCollection == -1) {
remove(collection.get(positionInCollection));
result = true;
}
}
return result;
}
@Override
public void clear() {
chunks.clear();
chunkSizes.clear();
}
/**
* Use getReference or getReadOnlyValue
*/
@Deprecated
@Override
public T get(int index) {
return getReadOnlyValue(index);
}
@SuppressWarnings("unchecked")
public T getReadOnlyValue(int index) {
try {
return (T) db.get(chunks.getLong(index)).getValueReadOnlyUnsafe();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@Override
public EntryReference<T> getReference(int index) {
return getReferenceUnsafe(index, true);
}
@Override
public EntryReference<T> getReferenceOrInitialize(int index, Supplier<T> initializer) {
EntryReference<T> value = getReferenceUnsafe(index, false);
if (value != null) {
return value;
} else {
T initializedData = initializer.get();
EntryReference<T> initializedDataRef = addToDatabase(initializedData);
return set(index, initializedDataRef);
}
}
private EntryReference<T> getReferenceUnsafe(int index, boolean throwError) {
try {
return db.get(chunks.getLong(index)).cast();
} catch (IOException e) {
if (throwError) e.printStackTrace();
return null;
}
}
@SuppressWarnings("unchecked")
@Override
public T set(int setOffset, final T element) {
return set(setOffset, addToDatabase(element)).getValueReadOnlyUnsafe();
}
@SuppressWarnings("unchecked")
@Override
public EntryReference<T> set(int setOffset, final EntryReference<T> element) {
long nextChunkOffset = 0;
VariableWrapper<EntryReference<T>> wrapper = new VariableWrapper<>(null);
if (setOffset >= 0) {
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
final int currentChunkSize = chunkSizes.getInt(i);
final long chunkStartOffset = nextChunkOffset;
nextChunkOffset += currentChunkSize;
// If the offset to remove is in the current chunk
if (nextChunkOffset > setOffset) {
// Get chunk index
final long chunkIndex = chunks.getLong(i);
// Get the offset relative to the current chunk
final int relativeOffset = (int) (setOffset - chunkStartOffset);
if (relativeOffset < 0) {
throw new NullPointerException("Relative Offset < 0");
}
try {
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
chunkRef.editValue((chunk) -> {
wrapper.var = chunk.set(relativeOffset, element);
});
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
break;
}
}
return wrapper.var;
}
return null;
}
@Override
public void add(int index, T element) {
throw new RuntimeException("add() isn't implemented!");
}
@SuppressWarnings("unchecked")
@Override
public T remove(int index) {
return this.removeAt(index);
}
@Override
public int indexOf(Object o) {
EntryReference<T> ref = addToDatabase(o).cast();
return indexOfEntry(ref);
}
@Override
public int indexOfEntry(EntryReference<T> ref) {
int currentOffset = 0;
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
try {
final int currentChunkSize = chunkSizes.getInt(i);
// If the offset to remove is in the current chunk
// Get chunk index
final long chunkIndex = chunks.getLong(i);
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
final int foundIndex = chunkRef.getValueReadOnlyUnsafe().indexOfEntry(ref);
if (foundIndex >= 0) {
return currentOffset + foundIndex;
}
currentOffset += currentChunkSize;
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
return -1;
}
@Override
public int lastIndexOf(Object o) {
return lastIndexOfEntry(addToDatabase(o).cast());
}
@Override
public int lastIndexOfEntry(EntryReference<T> ref) {
int currentOffset = 0;
// Iterate through all chunks
for (int i = chunks.size() - 1; i >= 0; i--) {
try {
final int currentChunkSize = chunkSizes.getInt(i);
// If the offset to remove is in the current chunk
// Get chunk index
final long chunkIndex = chunks.getLong(i);
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
final int foundIndex = chunkRef.getValueReadOnlyUnsafe().lastIndexOfEntry(ref);
if (foundIndex >= 0) {
return currentOffset + foundIndex;
}
currentOffset += currentChunkSize;
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
return -1;
}
@Deprecated
@Override
public ListIterator<T> listIterator() {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public ListIterator<T> listIterator(int index) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public List<T> subList(int fromIndex, int toIndex) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
private <U> EntryReference<U> addToDatabase(U obj) {
EntryReference<U> ref;
try {
ref = db.add(obj);
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
return ref;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((chunks == null) ? 0 : chunks.hashCode());
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeIf(Predicate<? super T> filter) {
Objects.requireNonNull(filter);
boolean result = false;
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
synchronized (cachedChunkLock) {
if (cachedChunkNumber != i) {
prepareAccessToChunk(i);
}
if (cachedChunk.removeIf(filter)) {
result = true;
chunkSizes.set(cachedChunkNumber, cachedChunk.size());
}
}
}
return result;
}
@Override
public String toString() {
return "LightBigList{" +
"chunks=" + chunks +
", chunkSizes=" + chunkSizes +
", db=" + db +
'}';
}
@Override
public void save() {
if (this.cachedChunkRef != null) {
this.cachedChunkRef.save();
}
}
@Override
public void saveAndFlush() {
save();
}
}

29
src/main/java/org/warp/jcwdb/LightList.java
View File

@ -1,29 +0,0 @@
package org.warp.jcwdb;
import java.util.Iterator;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Supplier;
public interface LightList<T> extends List<T> {
Iterator<EntryReference<T>> iteratorReferences();
void forEachReference(Consumer<? super EntryReference<T>> action);
EntryReference<T> addEntry(T o);
EntryReference<T> set(int setOffset, final EntryReference<T> element);
boolean remove(EntryReference<T> ref);
EntryReference<T> getReference(int index);
EntryReference<T> getReferenceOrInitialize(int index, Supplier<T> initializer);
int indexOfEntry(EntryReference<T> ref);
int lastIndexOfEntry(EntryReference<T> ref);
void appendIndex(long elementIndex);
}

92
src/main/java/org/warp/jcwdb/MixedIndexDatabase.java
View File

@ -1,92 +0,0 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.nio.file.Path;
public class MixedIndexDatabase implements IndexManager {
private final FileIndexManager fileIndices;
private final CacheIndexManager cacheIndices;
public MixedIndexDatabase(Path dataFile, Path metadataFile) throws IOException {
this.fileIndices = new FileIndexManager(dataFile, metadataFile);
this.cacheIndices = new CacheIndexManager();
}
@Override
public <T> T get(long index, DBReader<T> reader) throws IOException {
if (cacheIndices.has(index)) {
return cacheIndices.get(index, reader);
} else {
return fileIndices.get(index, reader);
}
}
@Override
public int getType(long index) throws IOException {
if (cacheIndices.has(index)) {
return cacheIndices.getType(index);
} else {
return fileIndices.getType(index);
}
}
@Override
public long getHash(long index) throws IOException {
if (cacheIndices.has(index)) {
return cacheIndices.getHash(index);
} else {
return fileIndices.getHash(index);
}
}
@Override
public <T> long add(DBDataOutput<T> writer) throws IOException {
return fileIndices.add(writer);
}
@Override
public <T> FullIndexDetails addAndGetDetails(DBDataOutput<T> writer) throws IOException {
return fileIndices.addAndGetDetails(writer);
}
@Override
public <T> IndexDetails set(long index, DBDataOutput<T> writer) throws IOException {
if (cacheIndices.has(index)) {
return cacheIndices.set(index, writer);
} else {
return fileIndices.set(index, writer);
}
}
@Override
public void setFlushingAllowed(long index, boolean isFlushingAllowed) {
if (cacheIndices.has(index)) {
cacheIndices.setFlushingAllowed(index, isFlushingAllowed);
} else {
fileIndices.setFlushingAllowed(index, isFlushingAllowed);
}
}
@Override
public void delete(long index) throws IOException {
cacheIndices.delete(index);
fileIndices.delete(index);
}
@Override
public boolean has(long index) {
return cacheIndices.has(index) || fileIndices.has(index);
}
@Override
public void close() throws IOException {
// TODO: move all cached indices to filesIndices before closing.
this.cacheIndices.close();
this.fileIndices.close();
}
@Override
public long clean() {
return fileIndices.clean()
+ cacheIndices.clean();
}
}

14
src/main/java/org/warp/jcwdb/NoParserFoundException.java
View File

@ -1,14 +0,0 @@
package org.warp.jcwdb;
public class NoParserFoundException extends NullPointerException {
public NoParserFoundException(String string) {
super(string);
}
/**
*
*/
private static final long serialVersionUID = 701010818132241139L;
}

8
src/main/java/org/warp/jcwdb/Saveable.java
View File

@ -1,8 +0,0 @@
package org.warp.jcwdb;
import java.io.IOException;
public interface Saveable {
void save();
void saveAndFlush();
}

59
src/main/java/org/warp/jcwdb/TypesManager.java
View File

@ -1,59 +0,0 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
public class TypesManager {
private final Int2ObjectMap<DBTypeParser<?>> types;
private final Object2ObjectMap<Class<?>, DBTypeParser<?>> typesByClass;
private DBTypedObjectParser<?> fallbackParser;
public TypesManager(JCWDatabase db) {
types = new Int2ObjectOpenHashMap<>();
typesByClass = new Object2ObjectOpenHashMap<>();
DBStandardTypes.registerStandardTypes(db, this);
}
public <T> void registerType(Class<T> clazz, int type, DBTypeParser<T> parser) {
this.types.put(type, parser);
this.typesByClass.put(clazz, parser);
}
/**
* Use this method with the most used classes to save disk space.
* @param clazz
* @param id
* @param <T>
*/
public <T> void registerGenericClass(Class<T> clazz, int id) {
this.fallbackParser.registerClass(clazz, id);
}
public void registerTypeFallback(DBTypedObjectParser<?> parser) {
this.fallbackParser = parser;
}
public <T> DBTypeParser<T> get(int type) {
if (types.containsKey(type) == false) {
if (fallbackParser == null) {
throw new NoParserFoundException("The type " + type + " can't be parsed.");
} else {
return fallbackParser.cast();
}
}
return types.get(type).cast();
}
public <T> DBTypeParser<T> get(Class<T> type) {
if (typesByClass.containsKey(type) == false) {
if (fallbackParser == null) {
throw new NoParserFoundException("The class " + type.getSimpleName() + " can't be parsed.");
} else {
return fallbackParser.cast();
}
}
return typesByClass.get(type).cast();
}
}

2
src/main/java/org/warp/jcwdb/VariableWrapper.java
View File

@ -1,7 +1,5 @@
package org.warp.jcwdb;
import java.nio.channels.SeekableByteChannel;
public class VariableWrapper<T> {
public T var;

15
src/main/java/org/warp/jcwdb/ann/ConsumerWithIO.java Normal file
View File

@ -0,0 +1,15 @@
package org.warp.jcwdb.ann;
import java.io.IOException;
import java.util.Objects;
@FunctionalInterface
public interface ConsumerWithIO<T> {
void accept(T t) throws IOException;
default ConsumerWithIO<T> andThen(ConsumerWithIO<? super T> after) {
Objects.requireNonNull(after);
return (T t) -> { accept(t); after.accept(t); };
}
}

13
src/main/java/org/warp/jcwdb/ann/DBDataType.java Normal file
View File

@ -0,0 +1,13 @@
package org.warp.jcwdb.ann;
public enum DBDataType {
DATABASE_OBJECT,
OBJECT,
BOOLEAN,
BYTE,
SHORT,
CHAR,
INTEGER,
LONG,
REFERENCES_LIST
}

13
src/main/java/org/warp/jcwdb/ann/DBField.java Normal file
View File

@ -0,0 +1,13 @@
package org.warp.jcwdb.ann;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.FIELD})
public @interface DBField {
int id();
DBDataType type() default DBDataType.OBJECT;
}

13
src/main/java/org/warp/jcwdb/ann/DBPropertyGetter.java Normal file
View File

@ -0,0 +1,13 @@
package org.warp.jcwdb.ann;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
public @interface DBPropertyGetter {
int id();
DBDataType type() default DBDataType.OBJECT;
}

13
src/main/java/org/warp/jcwdb/ann/DBPropertySetter.java Normal file
View File

@ -0,0 +1,13 @@
package org.warp.jcwdb.ann;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
public @interface DBPropertySetter {
int id();
DBDataType type() default DBDataType.OBJECT;
}

19
src/main/java/org/warp/jcwdb/ann/RunnableWithIO.java Normal file
View File

@ -0,0 +1,19 @@
package org.warp.jcwdb.ann;
import java.io.IOException;
@FunctionalInterface
public interface RunnableWithIO {
/**
* When an object implementing interface <code>Runnable</code> is used
* to create a thread, starting the thread causes the object's
* <code>run</code> method to be called in that separately executing
* thread.
* <p>
* The general contract of the method <code>run</code> is that it may
* take any action whatsoever.
*
* @see java.lang.Thread#run()
*/
public abstract void run() throws IOException;
}

8
src/main/java/org/warp/jcwdb/ann/SupplierWithIO.java Normal file
View File

@ -0,0 +1,8 @@
package org.warp.jcwdb.ann;
import java.io.IOException;
@FunctionalInterface
public interface SupplierWithIO<T> {
public T getWithIO() throws IOException;
}

9
src/main/java/org/warp/jcwdb/exampleimpl/Animal.java
View File

@ -1,9 +0,0 @@
package org.warp.jcwdb.exampleimpl;
public abstract class Animal {
protected int legsCount;
public static boolean hasFourLegs(Animal a) {
return a.legsCount == 4;
}
}

101
src/main/java/org/warp/jcwdb/exampleimpl/App.java
View File

@ -1,101 +0,0 @@
package org.warp.jcwdb.exampleimpl;
import org.warp.jcwdb.EntryReference;
import org.warp.jcwdb.JCWDatabase;
import org.warp.jcwdb.LightList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import it.unimi.dsi.fastutil.objects.ObjectList;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.function.Predicate;
public class App {
static long time3;
public static void main(String[] args) throws IOException {
if (args.length > 2 && Boolean.parseBoolean(args[2])) {
Files.delete(Paths.get(args[0]));
Files.delete(Paths.get(args[1]));
}
System.out.println("Loading database...");
long time0 = System.currentTimeMillis();
JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1]));
db.registerClass(StrangeAnimal.class, 0);
try {
long time01 = System.currentTimeMillis();
System.out.println("Time elapsed: " + (time01 - time0));
System.out.println("Loading root...");
EntryReference<LightList<Animal>> rootRef = db.getRoot(Animal.class);
rootRef.editValue((root, saver) -> {
long time1 = System.currentTimeMillis();
System.out.println("Time elapsed: " + (time1 - time01));
System.out.println("Root size: " + root.size());
System.out.println("Root:");
// for (int i = 0; i < root.size(); i++) {
// System.out.println(" - " + root.get(i));
// }
long prectime = System.currentTimeMillis();
for (int i = 0; i < 2000000/* 2000000 */; i++) {
Animal animal = new StrangeAnimal(i % 40);
root.addEntry(animal);
if (i > 0 && i % 200000 == 0) {
long precprectime = prectime;
prectime = System.currentTimeMillis();
System.out.println("Element " + i + " (" + (prectime - precprectime) + "ms)" + " Total Time: " + (prectime - time1));
}
}
long time2 = System.currentTimeMillis();
saver.save();
System.out.println("Root size: " + root.size());
System.out.println("Time elapsed: " + (time2 - time1));
System.out.println("Used memory: "
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
long time2_0 = System.currentTimeMillis();
System.out.println("Filtering strings...");
long oldSize = root.size();
root.removeIf(Animal::hasFourLegs);
long time2_1 = System.currentTimeMillis();
System.out.println("RemoveIf(x) removed items: " + (oldSize - root.size()));
System.out.println("Time elapsed: " + (time2_1 - time2_0));
ObjectList<Animal> results = new ObjectArrayList<>();
System.out.println("Retrieving items...");
root.forEachReference((valueReference) -> {
Animal value = valueReference.getValueReadOnlyUnsafe();
if (Animal.hasFourLegs(value)) {
results.add(value);
}
//System.out.println("val:" + value);
});
long time2_2 = System.currentTimeMillis();
System.out.println("Matches: " + results.size());
System.out.println("Time elapsed: " + (time2_2 - time2_1));
System.out.println("Used memory: "
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
System.out.println("Cleaning database (to reduce the amount of used memory and detect memory leaks)...");
db.clean();
time3 = System.currentTimeMillis();
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
System.out.println("Time elapsed: " + (time3 - time2_2));
System.out.println("Saving database...");
System.out.println("Root size: " + root.size());
});
db.close();
long time4 = System.currentTimeMillis();
System.out.println("Time elapsed: " + (time4 - time3));
} catch (Exception ex) {
ex.printStackTrace();
} finally {
if (db.isOpen()) {
db.close();
}
}
}
public static <T> Predicate<T> not(Predicate<T> t) {
return t.negate();
}
}

12
src/main/java/org/warp/jcwdb/exampleimpl/Cat.java
View File

@ -1,12 +0,0 @@
package org.warp.jcwdb.exampleimpl;
public class Cat extends Animal {
public Cat() {
this.legsCount = 12;
}
@Override
public String toString() {
return "Cat [legsCount=" + legsCount + "]";
}
}

12
src/main/java/org/warp/jcwdb/exampleimpl/Dog.java
View File

@ -1,12 +0,0 @@
package org.warp.jcwdb.exampleimpl;
public class Dog extends Animal {
public Dog() {
this.legsCount = 4;
}
@Override
public String toString() {
return "Dog [legsCount=" + legsCount + "]";
}
}

15
src/main/java/org/warp/jcwdb/exampleimpl/StrangeAnimal.java
View File

@ -1,15 +0,0 @@
package org.warp.jcwdb.exampleimpl;
public class StrangeAnimal extends Animal {
public StrangeAnimal() {
super();
}
public StrangeAnimal(int legs) {
super();
this.legsCount = legs;
}
@Override
public String toString() {
return "StrangeAnimal [legsCount=" + legsCount + "]";
}
}

18
src/test/java/org/warp/jcwdb/AppTest.java
View File

@ -1,18 +0,0 @@
package org.warp.jcwdb;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
/**
* Unit test for simple App.
*/
public class AppTest {
/**
* Rigorous Test :-)
*/
@Test
public void shouldAnswerWithTrue() {
assertTrue(true);
}
}

51
src/test/java/org/warp/jcwdb/FileAllocatorTest.java
View File

@ -1,51 +0,0 @@
package org.warp.jcwdb;
import org.junit.Test;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class FileAllocatorTest {
@Test
public void shouldAllocateAtZero() throws IOException {
Path tempFile = Files.createTempFile("", "");
SeekableByteChannel byteCh = Files.newByteChannel(tempFile);
FileAllocator allocator = new FileAllocator(byteCh);
long offset1 = allocator.allocate(512);
assertEquals(0, offset1);
}
@Test
public void shouldAllocateAt512() throws IOException {
Path tempFile = Files.createTempFile("", "");
SeekableByteChannel byteCh = Files.newByteChannel(tempFile, StandardOpenOption.WRITE);
byteCh.write(ByteBuffer.wrap(new byte[512]));
FileAllocator allocator = new FileAllocator(byteCh);
long offset1 = allocator.allocate(512);
assertEquals(512, offset1);
}
@Test
public void shouldAllocateUnusedSpace() throws IOException {
Path tempFile = Files.createTempFile("", "");
SeekableByteChannel byteCh = Files.newByteChannel(tempFile, StandardOpenOption.WRITE);
FileAllocator allocator = new FileAllocator(byteCh);
long offset1 = allocator.allocate(512);
allocator.markFree(offset1, 512);
long offset2 = allocator.allocate(128);
long offset3 = allocator.allocate(512-128);
long offset4 = allocator.allocate(128);
assertEquals(0, offset2);
assertEquals(128, offset3);
assertEquals(512, offset4);
}
}

90
src/test/java/org/warp/jcwdb/tests/NDBMultipleEnhancedObjects.java Normal file
View File

@ -0,0 +1,90 @@
package org.warp.jcwdb.tests;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.warp.cowdb.EnhancedObject;
import org.warp.cowdb.IDatabase;
import org.warp.jcwdb.ann.DBDataType;
import org.warp.jcwdb.ann.DBField;
import org.warp.jcwdb.ann.DBPropertyGetter;
import org.warp.jcwdb.ann.DBPropertySetter;
import org.warp.jcwdb.utils.NTestUtils;
import java.io.IOException;
public class NDBMultipleEnhancedObjects {
private NTestUtils.WrappedDb db;
private RootTwoClasses root;
@Before
public void setUp() throws Exception {
db = NTestUtils.wrapDb().create((db) -> {
root = db.get().loadRoot(RootTwoClasses.class);
});
root.class1 = new NTestUtils.RootClass(db.get());
db.setRootClassValues(root.class1);
root.class2 = new NTestUtils.RootClass(db.get());
db.setRootClassValues(root.class2);
root.setClass3(new NTestUtils.RootClass(db.get()));
db.setRootClassValues(root.getClass3());
root.setClass4(new NTestUtils.RootClass(db.get()));
db.setRootClassValues(root.getClass4());
db.closeAndReopen();
}
@Test
public void shouldMatchMultipleEnhancedObjects() {
db.testRootClassValues(root.class1);
db.testRootClassValues(root.class2);
db.testRootClassValues(root.getClass3());
db.testRootClassValues(root.getClass4());
}
@After
public void tearDown() throws Exception {
db.delete();
}
public static class RootTwoClasses extends EnhancedObject {
@DBField(id = 0, type = DBDataType.DATABASE_OBJECT)
public NTestUtils.RootClass class1;
@DBField(id = 1, type = DBDataType.DATABASE_OBJECT)
public NTestUtils.RootClass class2;
public RootTwoClasses() {
super();
}
public RootTwoClasses(IDatabase database) throws IOException {
super(database);
}
@DBPropertyGetter(id = 0, type = DBDataType.DATABASE_OBJECT)
public NTestUtils.RootClass getClass3() {
return getProperty();
}
@DBPropertySetter(id = 0, type = DBDataType.DATABASE_OBJECT)
public void setClass3(NTestUtils.RootClass value) {
setProperty(value);
}
@DBPropertyGetter(id = 1, type = DBDataType.DATABASE_OBJECT)
public NTestUtils.RootClass getClass4() {
return getProperty();
}
@DBPropertySetter(id = 1, type = DBDataType.DATABASE_OBJECT)
public void setClass4(NTestUtils.RootClass value) {
setProperty(value);
}
@Override
public void initialize() throws IOException {
}
}
}

32
src/test/java/org/warp/jcwdb/utils/ConsumerWithIO.java Normal file
View File

@ -0,0 +1,32 @@
package org.warp.jcwdb.utils;
import java.io.IOException;
import java.util.Objects;
@FunctionalInterface
public interface ConsumerWithIO<T> {
/**
* Performs this operation on the given argument.
*
* @param t the input argument
*/
void accept(T t) throws IOException;
/**
* Returns a composed {@code Consumer} that performs, in sequence, this
* operation followed by the {@code after} operation. If performing either
* operation throws an exception, it is relayed to the caller of the
* composed operation. If performing this operation throws an exception,
* the {@code after} operation will not be performed.
*
* @param after the operation to perform after this operation
* @return a composed {@code Consumer} that performs in sequence this
* operation followed by the {@code after} operation
* @throws NullPointerException if {@code after} is null
*/
default ConsumerWithIO<T> andThen(ConsumerWithIO<? super T> after) {
Objects.requireNonNull(after);
return (T t) -> { accept(t); after.accept(t); };
}
}

27
src/test/java/org/warp/jcwdb/utils/NSimplestClass.java Normal file
View File

@ -0,0 +1,27 @@
package org.warp.jcwdb.utils;
import org.warp.cowdb.Database;
import org.warp.cowdb.EnhancedObject;
import org.warp.jcwdb.ann.DBDataType;
import org.warp.jcwdb.ann.DBField;
import java.io.IOException;
public class NSimplestClass extends EnhancedObject {
@DBField(id = 0, type = DBDataType.BOOLEAN)
public boolean field1;
public NSimplestClass() {
}
public NSimplestClass(Database database) throws IOException {
super(database);
}
@Override
public void initialize() throws IOException {
field1 = true;
}
}

383
src/test/java/org/warp/jcwdb/utils/NTestUtils.java Normal file
View File

@ -0,0 +1,383 @@
package org.warp.jcwdb.utils;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.warp.cowdb.Database;
import org.warp.cowdb.EnhancedObject;
import org.warp.cowdb.IDatabase;
import org.warp.jcwdb.ann.*;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Comparator;
import static org.junit.Assert.*;
public class NTestUtils {
public static WrappedDb wrapDb() {
return new WrappedDb();
}
public static class WrappedDb {
private Database db;
private Path tempDir;
private RunnableWithIO r;
private WrappedDb() {
}
public WrappedDb create() throws IOException {
tempDir = Files.createTempDirectory("tests-");
db = openDatabase();
if (r != null) {
r.run();
}
return this;
}
public WrappedDb create(ConsumerWithIO<NTestUtils.WrappedDb> r) throws IOException {
this.r = () -> r.accept(WrappedDb.this);
this.create();
return this;
}
private Database openDatabase() throws IOException {
return new Database(tempDir.resolve(Paths.get("data.db")), tempDir.resolve(Paths.get("blocks.dat")), tempDir.resolve(Paths.get("references.dat")));
}
public void delete() throws IOException {
db.close();
deleteDir(tempDir);
}
public Database get() {
return db;
}
private void deleteDir(Path p) throws IOException {
Files.walk(p)
.sorted(Comparator.reverseOrder())
.map(Path::toFile)
.forEach(File::delete);
}
public void closeAndReopen() throws IOException {
db.close();
db = openDatabase();
r.run();
}
public void setRootClassValues(RootClass root) throws IOException {
setRootClassFields(root);
setRootClassProperties(root);
}
public void setRootClassFields(RootClass root) throws IOException {
root.field1 = true;
root.field2 = 2;
root.field3 = 3;
root.field4 = 4;
root.field5 = 5;
root.field6 = 6;
root.field7 = "Test";
root.field8 = new LongArrayList();
root.field8.add(0);
root.field8.add(1);
root.field8.add(2);
root.field8.add(Long.MAX_VALUE/2);
root.field8.add(Long.MIN_VALUE/2);
root.field8.add(Long.MAX_VALUE);
root.field8.add(Long.MIN_VALUE);
root.field9 = new NSimplestClass(db);
root.field9.field1 = true;
}
public void setRootClassProperties(RootClass root) throws IOException {
root.set1(true);
root.set2((byte)2);
root.set3((short)3);
root.set4((char)4);
root.set5(5);
root.set6(6);
root.set7("Test");
LongArrayList lArrayList = new LongArrayList();
lArrayList.add(0);
lArrayList.add(1);
lArrayList.add(2);
lArrayList.add(Long.MAX_VALUE/2);
lArrayList.add(Long.MIN_VALUE/2);
lArrayList.add(Long.MAX_VALUE);
lArrayList.add(Long.MIN_VALUE);
root.set8(lArrayList);
NSimplestClass simplestClass9 = new NSimplestClass(db);
simplestClass9.field1 = true;
root.set9(simplestClass9);
}
public void testRootClassValues(RootClass root) {
testRootClassFields(root);
testRootClassProperties(root);
}
public void testRootClassFields(RootClass root) {
shouldGetFieldBoolean(root);
shouldGetFieldByte(root);
shouldGetFieldShort(root);
shouldGetFieldCharacter(root);
shouldGetFieldInteger(root);
shouldGetFieldLong(root);
shouldGetFieldObject(root);
shouldGetFieldUID(root);
shouldGetFieldDBObject(root);
}
public void testRootClassProperties(RootClass root) {
shouldGetPropertyBoolean(root);
shouldGetPropertyByte(root);
shouldGetPropertyShort(root);
shouldGetPropertyCharacter(root);
shouldGetPropertyInteger(root);
shouldGetPropertyLong(root);
shouldGetPropertyObject(root);
shouldGetPropertyUID(root);
shouldGetPropertyDBObject(root);
}
private void shouldGetFieldBoolean(RootClass root) {
assertTrue(root.field1);
}
private void shouldGetPropertyBoolean(RootClass root) {
assertTrue(root.get1());
}
private void shouldGetFieldByte(RootClass root) {
assertEquals(2, root.field2);
}
private void shouldGetPropertyByte(RootClass root) {
assertEquals(2, root.get2());
}
private void shouldGetFieldShort(RootClass root) {
assertEquals(3, root.field3);
}
private void shouldGetPropertyShort(RootClass root) {
assertEquals(3, root.get3());
}
private void shouldGetFieldCharacter(RootClass root) {
assertEquals(4, root.field4);
}
private void shouldGetPropertyCharacter(RootClass root) {
assertEquals(4, root.get4());
}
private void shouldGetFieldInteger(RootClass root) {
assertEquals(5, root.field5);
}
private void shouldGetPropertyInteger(RootClass root) {
assertEquals(5, root.get5());
}
private void shouldGetFieldLong(RootClass root) {
assertEquals(6, root.field6);
}
private void shouldGetPropertyLong(RootClass root) {
assertEquals(6, root.get6());
}
private void shouldGetFieldObject(RootClass root) {
shouldGetObject(root.field7);
}
private void shouldGetPropertyObject(RootClass root) {
shouldGetObject(root.get7());
}
private void shouldGetFieldDBObject(RootClass root) {
assertTrue(root.field9.field1);
}
private void shouldGetPropertyDBObject(RootClass root) {
assertTrue(root.get9().field1);
}
private void shouldGetObject(String val) {
assertNotNull(val);
assertEquals("Test", val);
}
private void shouldGetDBObject(NSimplestClass val) {
assertNotNull(val);
assertTrue(val.field1);
}
private void shouldGetFieldUID(RootClass root) {
shouldGetUID(root.field8);
}
private void shouldGetPropertyUID(RootClass root) {
shouldGetUID(root.get8());
}
private void shouldGetUID(LongArrayList val) {
assertNotNull(val);
assertEquals(7, val.size());
assertEquals(0, val.getLong(0));
assertEquals(val.getLong(5), Long.MAX_VALUE);
assertEquals(val.getLong(6), Long.MIN_VALUE);
}
public void onLoad(RunnableWithIO r) {
this.r = r;
}
}
public static class RootClass extends EnhancedObject {
@DBField(id = 0, type = DBDataType.BOOLEAN)
public boolean field1;
@DBField(id = 1, type = DBDataType.BYTE)
public byte field2;
@DBField(id = 2, type = DBDataType.SHORT)
public short field3;
@DBField(id = 3, type = DBDataType.CHAR)
public char field4;
@DBField(id = 4, type = DBDataType.INTEGER)
public int field5;
@DBField(id = 5, type = DBDataType.LONG)
public long field6;
@DBField(id = 6, type = DBDataType.OBJECT)
public String field7;
@DBField(id = 7, type = DBDataType.REFERENCES_LIST)
public LongArrayList field8;
@DBField(id = 8, type = DBDataType.DATABASE_OBJECT)
public NSimplestClass field9;
public RootClass() {
}
public RootClass(IDatabase database) throws IOException {
super(database);
}
@DBPropertyGetter(id = 0, type = DBDataType.BOOLEAN)
public boolean get1() {
return getProperty();
}
@DBPropertyGetter(id = 1, type = DBDataType.BYTE)
public byte get2() {
return getProperty();
}
@DBPropertyGetter(id = 2, type = DBDataType.SHORT)
public short get3() {
return getProperty();
}
@DBPropertyGetter(id = 3, type = DBDataType.CHAR)
public char get4() {
return getProperty();
}
@DBPropertyGetter(id = 4, type = DBDataType.INTEGER)
public int get5() {
return getProperty();
}
@DBPropertyGetter(id = 5, type = DBDataType.LONG)
public long get6() {
return getProperty();
}
@DBPropertyGetter(id = 6, type = DBDataType.OBJECT)
public String get7() {
return getProperty();
}
@DBPropertyGetter(id = 7, type = DBDataType.REFERENCES_LIST)
public LongArrayList get8() {
return getProperty();
}
@DBPropertyGetter(id = 8, type = DBDataType.DATABASE_OBJECT)
public NSimplestClass get9() {
return getProperty();
}
@DBPropertySetter(id = 0, type = DBDataType.BOOLEAN)
public void set1(boolean val) {
setProperty(val);
}
@DBPropertySetter(id = 1, type = DBDataType.BYTE)
public void set2(byte val) {
setProperty(val);
}
@DBPropertySetter(id = 2, type = DBDataType.SHORT)
public void set3(short val) {
setProperty(val);
}
@DBPropertySetter(id = 3, type = DBDataType.CHAR)
public void set4(char val) {
setProperty(val);
}
@DBPropertySetter(id = 4, type = DBDataType.INTEGER)
public void set5(int val) {
setProperty(val);
}
@DBPropertySetter(id = 5, type = DBDataType.LONG)
public void set6(long val) {
setProperty(val);
}
@DBPropertySetter(id = 6, type = DBDataType.OBJECT)
public void set7(String val) {
setProperty(val);
}
@DBPropertySetter(id = 7, type = DBDataType.REFERENCES_LIST)
public void set8(LongArrayList val) {
setProperty(val);
}
@DBPropertySetter(id = 8, type = DBDataType.DATABASE_OBJECT)
public void set9(NSimplestClass val) {
setProperty(val);
}
public boolean test() {
return true;
}
@Override
public void initialize() throws IOException {
}
}
}