andreacavalli
/
strangedb
1
1
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Merge branch 'new-impl'

This commit is contained in:
Andrea Cavalli 2019-02-01 00:05:13 +01:00
parent 9ca8fea292 b06ee07ad2
commit e68d29e64e
66 changed files with 2314 additions and 3361 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
pom.xml
View File

@ -1,80 +1,100 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.warp</groupId>
<artifactId>jcwdb</artifactId>
<version>1.0-SNAPSHOT</version>
<groupId>org.warp</groupId>
<artifactId>jcwdb</artifactId>
<version>1.3</version>
<name>jcwdb</name>
<!-- FIXME change it to the project's website -->
<url>http://www.example.com</url>
<name>jcwdb</name>
<!-- FIXME change it to the project's website -->
<url>http://www.example.com</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
</properties>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>10</maven.compiler.source>
<maven.compiler.target>10</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
<version>8.2.2</version>
</dependency>
<dependency>
<groupId>com.esotericsoftware</groupId>
<artifactId>kryo</artifactId>
<version>5.0.0-RC1</version>
</dependency>
<dependency>
<groupId>net.openhft</groupId>
<artifactId>zero-allocation-hashing</artifactId>
<version>0.8</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>sonatype-snapshots</id>
<name>sonatype snapshots repo</name>
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
</repository>
</repositories>
<build>
<pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
<plugins>
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>3.0.0</version>
</plugin>
<!-- see http://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.1</version>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
</plugins>
</pluginManagement>
</build>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
<version>8.2.2</version>
</dependency>
<dependency>
<groupId>com.esotericsoftware</groupId>
<artifactId>kryo</artifactId>
<version>5.0.0-RC1</version>
</dependency>
<dependency>
<groupId>net.openhft</groupId>
<artifactId>zero-allocation-hashing</artifactId>
<version>0.8</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.5</version>
</dependency>
<dependency>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
<version>RELEASE</version>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
<plugins>
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>3.0.0</version>
</plugin>
<!-- see http://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.1</version>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
</plugins>
</pluginManagement>
</build>
</project>

44
src/main/java/org/warp/cowdb/BlockInfo.java Normal file
View File

@ -0,0 +1,44 @@
package org.warp.cowdb;
import java.util.Objects;
import java.util.StringJoiner;
public class BlockInfo {
private final long index;
private final int size;
public BlockInfo(long index, int size) {
this.index = index;
this.size = size;
}
public long getIndex() {
return index;
}
public int getSize() {
return size;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
BlockInfo blockInfo = (BlockInfo) o;
return index == blockInfo.index &&
size == blockInfo.size;
}
@Override
public int hashCode() {
return Objects.hash(index, size);
}
@Override
public String toString() {
return new StringJoiner(", ", BlockInfo.class.getSimpleName() + "[", "]")
.add("index=" + index)
.add("size=" + size)
.toString();
}
}

33
src/main/java/org/warp/cowdb/ByteBufferBackedInputStream.java Normal file
View File

@ -0,0 +1,33 @@
package org.warp.cowdb;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
public class ByteBufferBackedInputStream extends InputStream {
ByteBuffer buf;
public ByteBufferBackedInputStream(ByteBuffer buf) {
this.buf = buf;
}
public int read() throws IOException {
if (!buf.hasRemaining()) {
return -1;
}
return buf.get() & 0xFF;
}
public int read(byte[] bytes, int off, int len)
throws IOException {
if (!buf.hasRemaining()) {
return -1;
}
len = Math.min(len, buf.remaining());
buf.get(bytes, off, len);
return len;
}
}

906
src/main/java/org/warp/cowdb/Database.java Normal file
View File

@ -0,0 +1,906 @@
package org.warp.cowdb;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.ByteBufferInput;
import com.esotericsoftware.kryo.io.ByteBufferInputStream;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import it.unimi.dsi.fastutil.booleans.BooleanArrayList;
import it.unimi.dsi.fastutil.bytes.ByteArrayList;
import it.unimi.dsi.fastutil.chars.CharArrayList;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.shorts.ShortArrayList;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.warp.jcwdb.ann.*;
import java.io.ByteArrayOutputStream;
import java.io.IOError;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.*;
import java.util.function.Supplier;
import static org.warp.cowdb.IBlocksMetadata.EMPTY_BLOCK_ID;
import static org.warp.cowdb.IBlocksMetadata.EMPTY_BLOCK_INFO;
public class Database implements IDatabase {
private final DatabaseFileIO fileIO;
private final DatabaseBlocksIO blocksIO;
private final DatabaseBlocksMetadata blocksMetadata;
private final DatabaseReferencesIO referencesIO;
private final DatabaseReferencesMetadata referencesMetadata;
private final DatabaseObjectsIO objectsIO;
private final DatabaseDataInitializer dataInitializer;
private EnhancedObject loadedRootObject;
public Database(Path dataFile, Path blocksMetaFile, Path referencesMetaFile) throws IOException {
if (Files.notExists(dataFile)) {
Files.createFile(dataFile);
}
if (Files.notExists(blocksMetaFile)) {
Files.createFile(blocksMetaFile);
}
if (Files.notExists(referencesMetaFile)) {
Files.createFile(referencesMetaFile);
}
this.fileIO = new DatabaseFileIO(dataFile);
this.blocksMetadata = new DatabaseBlocksMetadata(blocksMetaFile);
this.blocksIO = new DatabaseBlocksIO(fileIO, blocksMetadata);
this.referencesMetadata = new DatabaseReferencesMetadata(referencesMetaFile);
this.referencesIO = new DatabaseReferencesIO(blocksIO, referencesMetadata);
this.objectsIO = new DatabaseObjectsIO(this, referencesIO);
this.dataInitializer = new DatabaseDataInitializer(objectsIO);
}
@Override
public IDataInitializer getDataInitializer() {
return dataInitializer;
}
@Override
public IObjectsIO getObjectsIO() {
return objectsIO;
}
@Override
public void close() throws IOException {
this.objectsIO.setEnhancedObject(0, loadedRootObject);
this.referencesMetadata.close();
this.blocksMetadata.close();
this.fileIO.close();
}
public <T extends EnhancedObject> T loadRoot(Class<T> type) throws IOException {
return loadRoot(type, () -> {
T obj = objectsIO.instantiateEnhancedObject(type);
dataInitializer.initializeDBObject(obj);
return obj;
});
}
public <T extends EnhancedObject> T loadRoot(Class<T> type, SupplierWithIO<T> ifAbsent) throws IOException {
if (loadedRootObject != null) {
throw new RuntimeException("Root already set!");
}
T root;
if (referencesMetadata.firstFreeReference > 0) {
root = objectsIO.loadEnhancedObject(0, type);
} else {
if (objectsIO.newNullObject() != 0) {
throw new IOException("Can't allocate root!");
} else {
root = ifAbsent.getWithIO();
objectsIO.setEnhancedObject(0, root);
}
}
loadedRootObject = root;
return root;
}
protected void registerClass(Class<?> type, int id) {
this.objectsIO.registerClass(type, id);
}
public static class DatabaseDataInitializer implements IDataInitializer {
private final DatabaseObjectsIO objectsIO;
public DatabaseDataInitializer(DatabaseObjectsIO objectsIO) {
this.objectsIO = objectsIO;
}
@Override
public void initializeDBObject(EnhancedObject obj) throws IOException {
initializeDBObjectFields(obj);
initializeDBObjectProperties(obj);
obj.initialize();
}
private void initializeDBObjectFields(EnhancedObject obj) throws IOException {
// Declare the variables needed to get the biggest field Id
Field[] unorderedFields = objectsIO.getFields(obj);
// Find the biggest field Id
int biggestFieldId = objectsIO.getBiggestFieldId(unorderedFields);
// Allocate new UIDs
long[] fieldUIDs = objectsIO.allocateNewUIDs(biggestFieldId + 1);
// Declare the other variables
Field[] fields = new Field[biggestFieldId + 1];
DBDataType[] orderedFieldTypes = new DBDataType[biggestFieldId + 1];
// Load all fields metadata and load them
for (Field field : unorderedFields) {
DBField fieldAnnotation = field.getAnnotation(DBField.class);
int fieldId = fieldAnnotation.id();
DBDataType fieldType = fieldAnnotation.type();
objectsIO.loadField(obj, field, fieldType, fieldUIDs[fieldId]);
fields[fieldId] = field;
orderedFieldTypes[fieldId] = fieldType;
}
// Set fields metadata
obj.setFields(fields, orderedFieldTypes, fieldUIDs);
}
private void initializeDBObjectProperties(EnhancedObject obj) throws IOException {
// Declare the variables needed to get the biggest property Id
Method[] unorderedPropertyGetters = obj.getPropertyGetters();
Method[] unorderedPropertySetters = obj.getPropertySetters();
// Find the biggest property Id
int biggestGetter = objectsIO.getBiggestPropertyGetterId(unorderedPropertyGetters);
int biggestSetter = objectsIO.getBiggestPropertySetterId(unorderedPropertySetters);
int biggestPropertyId = biggestGetter > biggestSetter ? biggestGetter : biggestSetter;
// Allocate new UIDs
long[] propertyUIDs = objectsIO.allocateNewUIDs(biggestPropertyId + 1);
for (Method property : unorderedPropertySetters) {
DBPropertySetter fieldAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
// Declare the other variables
DBDataType[] propertyTypes = new DBDataType[biggestPropertyId + 1];
Method[] propertyGetters = new Method[biggestPropertyId + 1];
Method[] propertySetters = new Method[biggestPropertyId + 1];
Map<String, DBPropertySetter> setterMethods = new LinkedHashMap<>();
Map<String, DBPropertyGetter> getterMethods = new LinkedHashMap<>();
// Load the properties metadata
for (Method property : unorderedPropertyGetters) {
DBPropertyGetter propertyAnnotation = property.getAnnotation(DBPropertyGetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertyGetters[propertyId] = property;
getterMethods.put(property.getName(), propertyAnnotation);
}
for (Method property : unorderedPropertySetters) {
DBPropertySetter propertyAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertySetters[propertyId] = property;
setterMethods.put(property.getName(), propertyAnnotation);
}
// Set properties metadata
obj.setProperties(propertyGetters, propertySetters, propertyTypes, propertyUIDs, setterMethods, getterMethods);
}
}
public static class DatabaseObjectsIO implements IObjectsIO {
private final IDatabase database;
private final DatabaseReferencesIO referencesIO;
private Kryo kryo = new Kryo();
private DatabaseObjectsIO(IDatabase database, DatabaseReferencesIO referencesIO) {
this.database = database;
this.referencesIO = referencesIO;
kryo.setRegistrationRequired(false);
int id = -90;
registerClass(boolean[].class, id++);
registerClass(byte[].class, id++);
registerClass(short[].class, id++);
registerClass(char[].class, id++);
registerClass(int[].class, id++);
registerClass(long[].class, id++);
registerClass(Boolean[].class, id++);
registerClass(Byte[].class, id++);
registerClass(Short[].class, id++);
registerClass(Character[].class, id++);
registerClass(Integer[].class, id++);
registerClass(Long[].class, id++);
registerClass(String.class, id++);
registerClass(String[].class, id++);
registerClass(Boolean.class, id++);
registerClass(Byte.class, id++);
registerClass(Short.class, id++);
registerClass(Character.class, id++);
registerClass(Integer.class, id++);
registerClass(Class.class, id++);
registerClass(Object.class, id++);
registerClass(Object[].class, id++);
registerClass(Long.class, id++);
registerClass(String.class, id++);
registerClass(String[].class, id++);
registerClass(boolean[][].class, id++);
registerClass(byte[][].class, id++);
registerClass(short[][].class, id++);
registerClass(char[][].class, id++);
registerClass(int[][].class, id++);
registerClass(long[][].class, id++);
registerClass(String[][].class, id++);
registerClass(List.class, id++);
registerClass(ArrayList.class, id++);
registerClass(LinkedList.class, id++);
registerClass(Set.class, id++);
registerClass(HashSet.class, id++);
registerClass(LinkedHashSet.class, id++);
registerClass(Map.class, id++);
registerClass(HashMap.class, id++);
registerClass(LinkedHashMap.class, id++);
registerClass(TreeMap.class, id++);
registerClass(BooleanArrayList.class, id++);
registerClass(ByteArrayList.class, id++);
registerClass(ShortArrayList.class, id++);
registerClass(CharArrayList.class, id++);
registerClass(IntArrayList.class, id++);
registerClass(LongArrayList.class, id++);
registerClass(TreeSet.class, id++);
registerClass(SortedSet.class, id++);
registerClass(SortedMap.class, id++);
}
@Override
public <T extends EnhancedObject> T loadEnhancedObject(long reference, Class<T> objectType) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return null;
}
int fieldsCount = buffer.getInt();
int methodsCount = buffer.getInt();
long[] fieldRefs = new long[fieldsCount];
long[] methodRefs = new long[methodsCount];
for (int i = 0; i < fieldsCount; i++) {
fieldRefs[i] = buffer.getLong();
}
for (int i = 0; i < methodsCount; i++) {
methodRefs[i] = buffer.getLong();
}
return preloadEnhancedObject(objectType, fieldRefs, methodRefs);
}
@SuppressWarnings("unchecked")
private Object loadData(DBDataType propertyType, long dataReference, Supplier<Class<?>> returnType) throws IOException {
switch (propertyType) {
case DATABASE_OBJECT:
return loadEnhancedObject(dataReference, (Class<? extends EnhancedObject>) returnType.get());
case OBJECT:
return loadObject(dataReference);
case REFERENCES_LIST:
return loadReferencesList(dataReference);
case BOOLEAN:
return loadBoolean(dataReference);
case BYTE:
return loadByte(dataReference);
case SHORT:
return loadShort(dataReference);
case CHAR:
return loadChar(dataReference);
case INTEGER:
return loadInt(dataReference);
case LONG:
return loadLong(dataReference);
default:
throw new NullPointerException("Unknown data type");
}
}
private <T> void setData(long reference, DBDataType propertyType, T loadedPropertyValue) throws IOException {
switch (propertyType) {
case BOOLEAN:
setBoolean(reference, loadedPropertyValue != null && (boolean) loadedPropertyValue);
break;
case BYTE:
setByte(reference, loadedPropertyValue == null ? 0 : (byte) loadedPropertyValue);
break;
case SHORT:
setShort(reference, loadedPropertyValue == null ? 0 : (short) loadedPropertyValue);
break;
case CHAR:
setChar(reference, loadedPropertyValue == null ? 0 : (char) loadedPropertyValue);
break;
case INTEGER:
setInt(reference, loadedPropertyValue == null ? 0 : (int) loadedPropertyValue);
break;
case LONG:
setLong(reference, loadedPropertyValue == null ? 0 : (long) loadedPropertyValue);
break;
case OBJECT:
setObject(reference, loadedPropertyValue);
break;
case REFERENCES_LIST:
setReferencesList(reference, (LongArrayList) loadedPropertyValue);
break;
case DATABASE_OBJECT:
setEnhancedObject(reference, (EnhancedObject) loadedPropertyValue);
break;
}
}
@Override
public <T extends EnhancedObject> void setEnhancedObject(long reference, T value) throws IOException {
if (value != null) {
EnhancedObjectFullInfo objectFullInfo = value.getAllInfo();
int totalSize = Integer.BYTES * 2 + objectFullInfo.getFieldReferences().length * Long.BYTES + objectFullInfo.getPropertyReferences().length * Long.BYTES;
ByteBuffer buffer = ByteBuffer.allocate(totalSize);
buffer.putInt(objectFullInfo.getFieldReferences().length);
buffer.putInt(objectFullInfo.getPropertyReferences().length);
for (int i = 0; i < objectFullInfo.getFieldReferences().length; i++) {
buffer.putLong(objectFullInfo.getFieldReferences()[i]);
}
for (int i = 0; i < objectFullInfo.getPropertyReferences().length; i++) {
buffer.putLong(objectFullInfo.getPropertyReferences()[i]);
}
for (int i = 0; i < objectFullInfo.getFieldReferences().length; i++) {
try {
setData(objectFullInfo.getFieldReferences()[i], objectFullInfo.getFieldTypes()[i], objectFullInfo.getFields()[i].get(value));
} catch (IllegalAccessException e) {
throw new IOException(e);
}
}
for (int i = 0; i < objectFullInfo.getPropertyReferences().length; i++) {
setData(objectFullInfo.getPropertyReferences()[i], objectFullInfo.getPropertyTypes()[i], objectFullInfo.getLoadedPropertyValues()[i]);
}
buffer.flip();
referencesIO.writeToReference(reference, totalSize, buffer);
} else {
referencesIO.writeToReference(reference, 0, null);
}
}
@SuppressWarnings("unchecked")
@Override
public <T> T loadObject(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return null;
}
buffer.rewind();
byte[] data = buffer.array();
return (T) kryo.readClassAndObject(new Input(data));
}
@Override
public LongArrayList loadReferencesList(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return null;
}
int itemsCount = buffer.getInt();
LongArrayList arrayList = new LongArrayList();
for (int i = 0; i < itemsCount; i++) {
arrayList.add(buffer.getLong());
}
return arrayList;
}
@Override
public boolean loadBoolean(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return false;
}
return buffer.get() == 1;
}
@Override
public byte loadByte(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return 0;
}
return buffer.get();
}
@Override
public short loadShort(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return 0;
}
return buffer.getShort();
}
@Override
public char loadChar(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return 0;
}
return buffer.getChar();
}
@Override
public int loadInt(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return 0;
}
return buffer.getInt();
}
@Override
public long loadLong(long reference) throws IOException {
ByteBuffer buffer = referencesIO.readFromReference(reference);
if (buffer.limit() == 0) {
return 0;
}
return buffer.getLong();
}
@Override
public <T> void setObject(long reference, T value) throws IOException {
if (value != null) {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
Output output = new Output(outputStream);
kryo.writeClassAndObject(output, value);
output.flush();
byte[] data = outputStream.toByteArray();
ByteBuffer dataByteBuffer = ByteBuffer.wrap(data);
referencesIO.writeToReference(reference, data.length, dataByteBuffer);
} else {
referencesIO.writeToReference(reference, 0, null);
}
}
@Override
public void setReferencesList(long reference, LongArrayList value) throws IOException {
if (value != null) {
int items = value.size();
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * items + Integer.BYTES);
buffer.putInt(items);
for (int i = 0; i < items; i++) {
buffer.putLong(value.getLong(i));
}
buffer.flip();
referencesIO.writeToReference(reference, buffer.limit(), buffer);
} else {
referencesIO.writeToReference(reference, 0, null);
}
}
@Override
public void setBoolean(long reference, boolean value) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(Byte.BYTES);
buffer.put(value ? (byte) 1 : (byte) 0);
buffer.flip();
referencesIO.writeToReference(reference, Byte.BYTES, buffer);
}
@Override
public void setByte(long reference, byte value) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(Byte.BYTES);
buffer.put(value);
buffer.flip();
referencesIO.writeToReference(reference, Byte.BYTES, buffer);
}
@Override
public void setShort(long reference, short value) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(Short.BYTES);
buffer.putShort(value);
buffer.flip();
referencesIO.writeToReference(reference, Short.BYTES, buffer);
}
@Override
public void setChar(long reference, char value) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(Character.BYTES);
buffer.putChar(value);
buffer.flip();
referencesIO.writeToReference(reference, Character.BYTES, buffer);
}
@Override
public void setInt(long reference, int value) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
buffer.putInt(value);
buffer.flip();
referencesIO.writeToReference(reference, Integer.BYTES, buffer);
}
@Override
public void setLong(long reference, long value) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
buffer.putLong(value);
buffer.flip();
referencesIO.writeToReference(reference, Long.BYTES, buffer);
}
@Override
public long newNullObject() throws IOException {
return referencesIO.allocateReference();
}
@Override
public void loadProperty(EnhancedObject obj, int propertyId, Method property, DBDataType propertyType, long propertyUID) throws IOException {
obj.setProperty(propertyId, loadData(propertyType, propertyUID, property::getReturnType));
}
@Override
public void registerClass(Class<?> type, int id) {
if (id < -100) {
throw new IllegalArgumentException();
}
kryo.register(type, 100 + id);
}
private <T extends EnhancedObject> void preloadEnhancedObjectProperties(T obj, long[] propertyReferences) {
// Declare the variables needed to get the biggest property Id
Method[] unorderedPropertyGetters = obj.getPropertyGetters();
Method[] unorderedPropertySetters = obj.getPropertySetters();
// Find the biggest property Id
int biggestGetter = getBiggestPropertyGetterId(unorderedPropertyGetters);
int biggestSetter = getBiggestPropertySetterId(unorderedPropertySetters);
int biggestPropertyId = biggestGetter > biggestSetter ? biggestGetter : biggestSetter;
for (Method property : unorderedPropertySetters) {
DBPropertySetter fieldAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
// Declare the other variables
DBDataType[] propertyTypes = new DBDataType[biggestPropertyId + 1];
Method[] propertyGetters = new Method[biggestPropertyId + 1];
Method[] propertySetters = new Method[biggestPropertyId + 1];
Map<String, DBPropertySetter> setterMethods = new LinkedHashMap<>();
Map<String, DBPropertyGetter> getterMethods = new LinkedHashMap<>();
// Load the properties metadata
for (Method property : unorderedPropertyGetters) {
DBPropertyGetter propertyAnnotation = property.getAnnotation(DBPropertyGetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertyGetters[propertyId] = property;
getterMethods.put(property.getName(), propertyAnnotation);
}
for (Method property : unorderedPropertySetters) {
DBPropertySetter propertyAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = propertyAnnotation.id();
DBDataType propertyType = propertyAnnotation.type();
propertyTypes[propertyId] = propertyType;
propertySetters[propertyId] = property;
setterMethods.put(property.getName(), propertyAnnotation);
}
// Set properties metadata
obj.setProperties(propertyGetters, propertySetters, propertyTypes, propertyReferences, setterMethods, getterMethods);
}
private int getBiggestPropertyGetterId(Method[] unorderedPropertyGetters) {
int biggestPropertyId = -1;
for (Method property : unorderedPropertyGetters) {
DBPropertyGetter fieldAnnotation = property.getAnnotation(DBPropertyGetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
return biggestPropertyId;
}
private int getBiggestPropertySetterId(Method[] unorderedPropertySetters) {
int biggestPropertyId = -1;
for (Method property : unorderedPropertySetters) {
DBPropertySetter fieldAnnotation = property.getAnnotation(DBPropertySetter.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestPropertyId) {
biggestPropertyId = propertyId;
}
}
return biggestPropertyId;
}
private <T extends EnhancedObject> void preloadEnhancedObjectFields(T obj, long[] fieldReferences) throws IOException {
// Declare the variables needed to get the biggest field Id
Field[] unorderedFields = getFields(obj);
// Find the biggest field Id
int biggestFieldId = getBiggestFieldId(unorderedFields);
// Declare the other variables
Field[] fields = new Field[biggestFieldId + 1];
DBDataType[] orderedFieldTypes = new DBDataType[biggestFieldId + 1];
// Load all fields metadata and load them
for (Field field : unorderedFields) {
DBField fieldAnnotation = field.getAnnotation(DBField.class);
int fieldId = fieldAnnotation.id();
DBDataType fieldType = fieldAnnotation.type();
loadField(obj, field, fieldType, fieldReferences[fieldId]);
fields[fieldId] = field;
orderedFieldTypes[fieldId] = fieldType;
}
// Set fields metadata
obj.setFields(fields, orderedFieldTypes, fieldReferences);
}
private <T extends EnhancedObject> void loadField(T obj, Field field, DBDataType fieldType, long fieldReference) throws IOException {
Object data = loadData(fieldType, fieldReference, field::getType);
try {
if (fieldType == DBDataType.OBJECT && data != null) {
if (!field.getType().isInstance(data)) {
throw new IOException("There is an attempt to load an object of type " + data.getClass() + " into a field of type " + field.getType());
}
}
FieldUtils.writeField(field, obj, data, true);
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
}
private <T extends EnhancedObject> Field[] getFields(T obj) {
return FieldUtils.getFieldsWithAnnotation(obj.getClass(), DBField.class);
}
private int getBiggestFieldId(Field[] unorderedFields) {
int biggestFieldId = -1;
for (Field field : unorderedFields) {
DBField fieldAnnotation = field.getAnnotation(DBField.class);
int propertyId = fieldAnnotation.id();
if (propertyId > biggestFieldId) {
biggestFieldId = propertyId;
}
}
return biggestFieldId;
}
private <T extends EnhancedObject> T instantiateEnhancedObject(Class<T> type) throws IOException {
try {
T obj = type.getConstructor().newInstance();
obj.database = database;
return obj;
} catch (NoSuchMethodException e) {
throw new IOException("You must declare a public empty constructor in class " + type + ": public " + type.getSimpleName() + "()", e);
} catch (IllegalAccessException | InvocationTargetException | InstantiationException e) {
throw new IOException(e);
}
}
private <T extends EnhancedObject> T preloadEnhancedObject(Class<T> objectType, long[] fieldRefs, long[] methodRefs) throws IOException {
T obj = instantiateEnhancedObject(objectType);
preloadEnhancedObjectFields(obj, fieldRefs);
preloadEnhancedObjectProperties(obj, methodRefs);
return obj;
}
public long[] allocateNewUIDs(int quantity) throws IOException {
long[] ids = new long[quantity];
for (int i = 0; i < quantity; i++) {
ids[i] = newNullObject();
}
return ids;
}
}
public static class DatabaseReferencesIO implements IReferencesIO {
private final DatabaseBlocksIO blocksIO;
private final DatabaseReferencesMetadata referencesMetadata;
public DatabaseReferencesIO(DatabaseBlocksIO blocksIO, DatabaseReferencesMetadata referencesMetadata) {
this.blocksIO = blocksIO;
this.referencesMetadata = referencesMetadata;
}
@Override
public long allocateReference() throws IOException {
return referencesMetadata.newReference(EMPTY_BLOCK_ID);
}
@Override
public long allocateReference(int size, ByteBuffer data) throws IOException {
long blockId = (size == 0) ? EMPTY_BLOCK_ID : blocksIO.newBlock(size, data);
return referencesMetadata.newReference(blockId);
}
@Override
public void writeToReference(long reference, int size, ByteBuffer data) throws IOException {
long blockId = (size == 0) ? EMPTY_BLOCK_ID : blocksIO.newBlock(size, data);
referencesMetadata.editReference(reference, blockId);
}
@Override
public ByteBuffer readFromReference(long reference) throws IOException {
long blockId = referencesMetadata.getReference(reference);
return blocksIO.readBlock(blockId);
}
}
public static class DatabaseReferencesMetadata implements IReferencesMetadata {
private final SeekableByteChannel metaFileChannel;
private final int REF_META_BYTES_COUNT = Long.BYTES;
private long firstFreeReference;
private DatabaseReferencesMetadata(Path refMetaFile) throws IOException {
metaFileChannel = Files.newByteChannel(refMetaFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
firstFreeReference = metaFileChannel.size() / REF_META_BYTES_COUNT;
}
@Override
public long getReference(long reference) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(REF_META_BYTES_COUNT);
if (reference >= firstFreeReference) {
return EMPTY_BLOCK_ID;
}
SeekableByteChannel currentFileChannel = metaFileChannel.position(reference * REF_META_BYTES_COUNT);
currentFileChannel.read(buffer);
buffer.flip();
return buffer.getLong();
}
@Override
public long newReference(long blockId) throws IOException {
long newReference = firstFreeReference++;
editReference(newReference, blockId);
return newReference;
}
@Override
public void editReference(long reference, long blockId) throws IOException {
ByteBuffer data = ByteBuffer.allocate(REF_META_BYTES_COUNT);
data.putLong(blockId);
SeekableByteChannel currentFileChannel = metaFileChannel.position(reference * REF_META_BYTES_COUNT);
data.flip();
currentFileChannel.write(data);
}
@Override
public void close() throws IOException {
metaFileChannel.close();
}
}
public static class DatabaseBlocksIO implements IBlocksIO {
private final DatabaseFileIO fileIO;
private final IBlocksMetadata blocksMetadata;
private DatabaseBlocksIO(DatabaseFileIO fileIO, IBlocksMetadata blocksMetadata) {
this.fileIO = fileIO;
this.blocksMetadata = blocksMetadata;
}
@Override
public long newBlock(int size, ByteBuffer data) throws IOException {
long index = fileIO.writeAtEnd(size, data);
return blocksMetadata.newBlock(index, size);
}
@Override
public ByteBuffer readBlock(long blockId) throws IOException {
if (blockId == EMPTY_BLOCK_ID) {
return ByteBuffer.wrap(new byte[0]);
}
BlockInfo blockInfo = blocksMetadata.getBlockInfo(blockId);
return fileIO.readAt(blockInfo.getIndex(), blockInfo.getSize());
}
@Override
public void close() {
}
}
public static class DatabaseBlocksMetadata implements IBlocksMetadata {
private final SeekableByteChannel metaFileChannel;
private final int BLOCK_META_BYTES_COUNT = Long.BYTES + Integer.BYTES;
private long firstFreeBlock;
private DatabaseBlocksMetadata(Path metaFile) throws IOException {
metaFileChannel = Files.newByteChannel(metaFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
firstFreeBlock = metaFileChannel.size() / BLOCK_META_BYTES_COUNT;
}
@Override
public BlockInfo getBlockInfo(long blockId) throws IOException {
if (blockId == EMPTY_BLOCK_ID) {
return EMPTY_BLOCK_INFO;
}
ByteBuffer buffer = ByteBuffer.allocate(BLOCK_META_BYTES_COUNT);
metaFileChannel.position(blockId * BLOCK_META_BYTES_COUNT).read(buffer);
buffer.flip();
long index = buffer.getLong();
int size = buffer.getInt();
return new BlockInfo(index, size);
}
@Override
public long newBlock(long index, int size) throws IOException {
long newBlockId = firstFreeBlock++;
ByteBuffer data = ByteBuffer.allocate(BLOCK_META_BYTES_COUNT);
data.putLong(index);
data.putInt(size);
data.flip();
metaFileChannel.position(newBlockId * BLOCK_META_BYTES_COUNT).write(data);
return newBlockId;
}
@Override
public void close() throws IOException {
metaFileChannel.close();
}
}
public static class DatabaseFileIO implements IFileIO {
private final SeekableByteChannel dataFileChannel;
private final Object dataAccessLock = new Object();
private long firstFreeIndex;
private DatabaseFileIO(Path dataFile) throws IOException {
synchronized (dataAccessLock) {
dataFileChannel = Files.newByteChannel(dataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
firstFreeIndex = dataFileChannel.size();
}
}
@Override
public ByteBuffer readAt(long index, int length) throws IOException {
ByteBuffer dataBuffer = ByteBuffer.allocate(length);
dataFileChannel.position(index).read(dataBuffer);
dataBuffer.flip();
return dataBuffer;
}
@Override
public void writeAt(long index, int length, ByteBuffer data) throws IOException {
synchronized (dataAccessLock) {
if (data.position() != 0) {
throw new IOException("You didn't flip the ByteBuffer!");
}
if (firstFreeIndex < index + length) {
firstFreeIndex = index + length;
}
dataFileChannel.position(index).write(data);
}
}
@Override
public long writeAtEnd(int length, ByteBuffer data) throws IOException {
synchronized (dataAccessLock) {
long index = firstFreeIndex;
firstFreeIndex += length;
writeAt(index, length, data);
return index;
}
}
@Override
public void close() throws IOException {
synchronized (dataAccessLock) {
dataFileChannel.close();
}
}
}
}

101
src/main/java/org/warp/cowdb/EnhancedObject.java Normal file
View File

@ -0,0 +1,101 @@
package org.warp.cowdb;
import org.apache.commons.lang3.reflect.MethodUtils;
import org.warp.jcwdb.ann.DBDataType;
import org.warp.jcwdb.ann.DBPropertyGetter;
import org.warp.jcwdb.ann.DBPropertySetter;
import java.io.IOError;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.Map;
public abstract class EnhancedObject {
protected IDatabase database;
private Field[] fields;
private DBDataType[] fieldTypes;
private long[] fieldReferences;
private Method[] propertyGetters;
private Method[] propertySetters;
private DBDataType[] propertyTypes;
private long[] propertyReferences;
private boolean[] loadedProperties;
private Object[] loadedPropertyValues;
private Map<String, DBPropertySetter> setterMethods;
private Map<String, DBPropertyGetter> getterMethods;
public EnhancedObject() {
}
public EnhancedObject(IDatabase database) throws IOException {
this.database = database;
database.getDataInitializer().initializeDBObject(this);
}
public abstract void initialize() throws IOException;
public void setFields(Field[] fields, DBDataType[] fieldTypes, long[] fieldReferences) {
this.fields = fields;
this.fieldTypes = fieldTypes;
this.fieldReferences = fieldReferences;
}
public Method[] getPropertyGetters() {
return MethodUtils.getMethodsWithAnnotation(this.getClass(), DBPropertyGetter.class);
}
public Method[] getPropertySetters() {
return MethodUtils.getMethodsWithAnnotation(this.getClass(), DBPropertySetter.class);
}
public void setProperties(Method[] propertyGetters, Method[] propertySetters, DBDataType[] propertyTypes, long[] propertyReferences, Map<String, DBPropertySetter> setterMethods, Map<String, DBPropertyGetter> getterMethods) {
this.propertyGetters = propertyGetters;
this.propertySetters = propertySetters;
this.propertyTypes = propertyTypes;
this.propertyReferences = propertyReferences;
this.loadedProperties = new boolean[this.propertyReferences.length];
this.loadedPropertyValues = new Object[this.propertyReferences.length];
this.setterMethods = setterMethods;
this.getterMethods = getterMethods;
}
public EnhancedObjectFullInfo getAllInfo() {
return new EnhancedObjectFullInfo(fieldReferences, fieldTypes, fields, propertyReferences, propertyTypes, loadedPropertyValues);
}
public <T> T getProperty() {
StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
StackWalker.StackFrame stackFrame = walker.walk(f -> f.skip(1).findFirst().orElse(null));
try {
int propertyId = stackFrame.getDeclaringClass().getDeclaredMethod(stackFrame.getMethodName()).getAnnotation(DBPropertyGetter.class).id();
return getProperty(propertyId);
} catch (IOException | NoSuchMethodException e) {
throw new RuntimeException(e);
}
}
@SuppressWarnings("unchecked")
private <T> T getProperty(int propertyId) throws IOException {
if (!loadedProperties[propertyId]) {
long propertyUID = propertyReferences[propertyId];
database.getObjectsIO().loadProperty(this, propertyId, propertyGetters[propertyId], propertyTypes[propertyId], propertyUID);
}
return (T) loadedPropertyValues[propertyId];
}
public <T> void setProperty(T value) {
StackWalker walker = StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);
StackWalker.StackFrame stackFrame = walker.walk(f -> f.skip(1).findFirst().orElse(null));
DBPropertySetter propertyAnnotation = setterMethods.get(stackFrame.getMethodName());
setProperty(propertyAnnotation.id(), value);
}
public <T> void setProperty(int propertyId, T value) {
loadedPropertyValues[propertyId] = value;
loadedProperties[propertyId] = true;
}
}

47
src/main/java/org/warp/cowdb/EnhancedObjectFullInfo.java Normal file
View File

@ -0,0 +1,47 @@
package org.warp.cowdb;
import org.warp.jcwdb.ann.DBDataType;
import java.lang.reflect.Field;
public class EnhancedObjectFullInfo {
private final long[] fieldReferences;
private final DBDataType[] fieldTypes;
private final Field[] fields;
private final long[] propertyReferences;
private final DBDataType[] propertyTypes;
private final Object[] loadedPropertyValues;
EnhancedObjectFullInfo(long[] fieldReferences, DBDataType[] fieldTypes, Field[] fields, long[] propertyReferences, DBDataType[] propertyTypes, Object[] loadedPropertyValues) {
this.fieldReferences = fieldReferences;
this.fieldTypes = fieldTypes;
this.fields = fields;
this.propertyReferences = propertyReferences;
this.propertyTypes = propertyTypes;
this.loadedPropertyValues = loadedPropertyValues;
}
long[] getFieldReferences() {
return fieldReferences;
}
DBDataType[] getFieldTypes() {
return fieldTypes;
}
public Field[] getFields() {
return fields;
}
long[] getPropertyReferences() {
return propertyReferences;
}
DBDataType[] getPropertyTypes() {
return propertyTypes;
}
Object[] getLoadedPropertyValues() {
return loadedPropertyValues;
}
}

28
src/main/java/org/warp/cowdb/IBlocksIO.java Normal file
View File

@ -0,0 +1,28 @@
package org.warp.cowdb;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
public interface IBlocksIO {
/**
* Allocate a block
* @param size block size
* @param data block data
* @return the block id
*/
long newBlock(int size, ByteBuffer data) throws IOException;
/**
* Read a block
* @param blockId block id
* @return block data
*/
ByteBuffer readBlock(long blockId) throws IOException;
/**
* Close file
*/
void close();
}

44
src/main/java/org/warp/cowdb/IBlocksMetadata.java Normal file
View File

@ -0,0 +1,44 @@
package org.warp.cowdb;
import java.io.IOException;
public interface IBlocksMetadata {
long EMPTY_BLOCK_ID = -1;
BlockInfo EMPTY_BLOCK_INFO = new BlockInfo(0, 0);
/**
* Get block info
* @param blockId block id
* @return block metadata
*/
BlockInfo getBlockInfo(long blockId) throws IOException;
/**
* New empty block info
* @return block id
*/
default long newBlock() {
return EMPTY_BLOCK_ID;
}
/**
* Set block info
* @param index block index
* @param size block size
* @return block id
*/
long newBlock(long index, int size) throws IOException;
/**
* Set block info
* @param blockInfo block info
* @return block id
*/
default long newBlock(BlockInfo blockInfo) throws IOException {
return this.newBlock(blockInfo.getIndex(), blockInfo.getSize());
}
/**
* Close file
*/
void close() throws IOException;
}

7
src/main/java/org/warp/cowdb/IDataInitializer.java Normal file
View File

@ -0,0 +1,7 @@
package org.warp.cowdb;
import java.io.IOException;
public interface IDataInitializer {
void initializeDBObject(EnhancedObject obj) throws IOException;
}

10
src/main/java/org/warp/cowdb/IDatabase.java Normal file
View File

@ -0,0 +1,10 @@
package org.warp.cowdb;
import java.io.IOException;
public interface IDatabase {
void close() throws IOException;
IDataInitializer getDataInitializer();
IObjectsIO getObjectsIO();
}

35
src/main/java/org/warp/cowdb/IFileIO.java Normal file
View File

@ -0,0 +1,35 @@
package org.warp.cowdb;
import java.io.*;
import java.nio.ByteBuffer;
public interface IFileIO {
/**
* Read *length* bytes in position *index*
* @param index index
* @param length length
* @return bytes
*/
ByteBuffer readAt(long index, int length) throws IOException;
/**
* Write *length* bytes in position *index*
* @param index index
* @param length length
* @param data bytes
*/
void writeAt(long index, int length, ByteBuffer data) throws IOException;
/**
* Write *length* bytes in position *index*
* @param length length
* @param data bytes
* @return index
*/
long writeAtEnd(int length, ByteBuffer data) throws IOException;
/**
* Close the file
*/
void close() throws IOException;
}

111
src/main/java/org/warp/cowdb/IObjectsIO.java Normal file
View File

@ -0,0 +1,111 @@
package org.warp.cowdb;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.warp.jcwdb.ann.DBDataType;
import java.io.IOException;
import java.lang.reflect.Method;
public interface IObjectsIO {
<T extends EnhancedObject> T loadEnhancedObject(long reference, Class<T> objectType) throws IOException;
<T> T loadObject(long reference) throws IOException;
LongArrayList loadReferencesList(long reference) throws IOException;
boolean loadBoolean(long reference) throws IOException;
byte loadByte(long reference) throws IOException;
short loadShort(long reference) throws IOException;
char loadChar(long reference) throws IOException;
int loadInt(long reference) throws IOException;
long loadLong(long reference) throws IOException;
<T extends EnhancedObject> void setEnhancedObject(long reference, T value) throws IOException;
<T> void setObject(long reference, T value) throws IOException;
void setReferencesList(long reference, LongArrayList value) throws IOException;
void setBoolean(long reference, boolean value) throws IOException;
void setByte(long reference, byte value) throws IOException;
void setShort(long reference, short value) throws IOException;
void setChar(long reference, char value) throws IOException;
void setInt(long reference, int value) throws IOException;
void setLong(long reference, long value) throws IOException;
long newNullObject() throws IOException;
default <T extends EnhancedObject> long newEnhancedObject(T value) throws IOException {
long reference = newNullObject();
if (value != null) {
setEnhancedObject(reference, value);
}
return reference;
}
default <T> long newObject(T value) throws IOException {
long reference = newNullObject();
if (value != null) {
setObject(reference, value);
}
return reference;
}
default long newReferencesList(LongArrayList value) throws IOException {
long reference = newNullObject();
if (value != null) {
setReferencesList(reference, value);
}
return reference;
}
default long newBoolean(boolean value) throws IOException {
long reference = newNullObject();
setBoolean(reference, value);
return reference;
}
default long newByte(byte value) throws IOException {
long reference = newNullObject();
setByte(reference, value);
return reference;
}
default long newShort(short value) throws IOException {
long reference = newNullObject();
setShort(reference, value);
return reference;
}
default long newChar(char value) throws IOException {
long reference = newNullObject();
setChar(reference, value);
return reference;
}
default long newInt(int value) throws IOException {
long reference = newNullObject();
setInt(reference, value);
return reference;
}
default long newLong(long value) throws IOException {
long reference = newNullObject();
setLong(reference, value);
return reference;
}
void loadProperty(EnhancedObject enhancedObject, int propertyId, Method propertyGetter, DBDataType propertyType, long propertyUID) throws IOException;
void registerClass(Class<?> type, int id);
}

37
src/main/java/org/warp/cowdb/IReferencesIO.java Normal file
View File

@ -0,0 +1,37 @@
package org.warp.cowdb;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
public interface IReferencesIO {
/**
* Allocate a new empty reference
* @return the new reference
*/
long allocateReference() throws IOException;
/**
* Allocate a new reference with that data
* @param size data size
* @param data bytes
* @return the new reference
*/
long allocateReference(int size, ByteBuffer data) throws IOException;
/**
* Write some data to the reference
* @param reference reference
* @param size data size
* @param data bytes
*/
void writeToReference(long reference, int size, ByteBuffer data) throws IOException;
/**
* Read data from the reference
* @param reference reference
* @return bytes
*/
ByteBuffer readFromReference(long reference) throws IOException;
}

31
src/main/java/org/warp/cowdb/IReferencesMetadata.java Normal file
View File

@ -0,0 +1,31 @@
package org.warp.cowdb;
import java.io.IOException;
public interface IReferencesMetadata {
/**
* Get block of reference
* @param reference reference
* @return block id
*/
long getReference(long reference) throws IOException;
/**
* Allocate a block for a new reference
* @param blockId block id
* @return reference
*/
long newReference(long blockId) throws IOException;
/**
* Change reference size
* @param reference reference
* @param blockId block id
*/
void editReference(long reference, long blockId) throws IOException;
/**
* Close file
*/
void close() throws IOException;
}

101
src/main/java/org/warp/cowdb/lists/CowList.java Normal file
View File

@ -0,0 +1,101 @@
package org.warp.cowdb.lists;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.warp.cowdb.EnhancedObject;
import org.warp.cowdb.IDatabase;
import org.warp.jcwdb.ann.DBDataType;
import org.warp.jcwdb.ann.DBField;
import java.io.IOException;
import java.util.StringJoiner;
public abstract class CowList<T> extends EnhancedObject {
private final Object indicesAccessLock = new Object();
@DBField(id = 0, type = DBDataType.REFERENCES_LIST)
private LongArrayList indices;
public CowList() {
}
public CowList(IDatabase database) throws IOException {
super(database);
}
@Override
public void initialize() throws IOException {
indices = new LongArrayList();
}
public T get(int index) throws IOException {
synchronized (indicesAccessLock) {
long uid = indices.getLong(index);
return loadItem(uid);
}
}
public void add(T value) throws IOException {
long uid = database.getObjectsIO().newNullObject();
synchronized (indicesAccessLock) {
indices.add(uid);
writeItemToDisk(uid, value);
}
}
public void update(int index, T value) throws IOException {
synchronized (indicesAccessLock) {
set(index, value);
}
}
public void set(int index, T value) throws IOException {
long uid = database.getObjectsIO().newNullObject();
synchronized (indicesAccessLock) {
indices.set(index, uid);
writeItemToDisk(uid, value);
}
}
public void add(int index, T value) throws IOException {
long uid = database.getObjectsIO().newNullObject();
synchronized (indicesAccessLock) {
indices.add(index, uid);
writeItemToDisk(uid, value);
}
}
public T getLast() throws IOException {
synchronized (indicesAccessLock) {
if (indices.size() > 0) {
return get(indices.size() - 1);
} else {
return null;
}
}
}
public boolean isEmpty() {
synchronized (indicesAccessLock) {
return indices.size() <= 0;
}
}
public int size() {
synchronized (indicesAccessLock) {
return indices.size();
}
}
protected abstract T loadItem(long uid) throws IOException;
protected abstract void writeItemToDisk(long uid, T item) throws IOException;
@Override
public String toString() {
return new StringJoiner(", ", CowList.class.getSimpleName() + "[", "]")
.add(indices.size() + " items")
.toString();
}
}

33
src/main/java/org/warp/cowdb/lists/EnhancedObjectCowList.java Normal file
View File

@ -0,0 +1,33 @@
package org.warp.cowdb.lists;
import org.warp.cowdb.EnhancedObject;
import org.warp.cowdb.IDatabase;
import org.warp.jcwdb.ann.DBDataType;
import org.warp.jcwdb.ann.DBField;
import java.io.IOException;
public class EnhancedObjectCowList<T extends EnhancedObject> extends CowList<T> {
@DBField(id = 1, type = DBDataType.OBJECT)
private Class<T> type;
public EnhancedObjectCowList() {
super();
}
public EnhancedObjectCowList(IDatabase database, Class<T> type) throws IOException {
super(database);
this.type = type;
}
@Override
protected T loadItem(long uid) throws IOException {
return database.getObjectsIO().loadEnhancedObject(uid, type);
}
@Override
protected void writeItemToDisk(long uid, T item) throws IOException {
database.getObjectsIO().setEnhancedObject(uid, item);
}
}

29
src/main/java/org/warp/cowdb/lists/ObjectCowList.java Normal file
View File

@ -0,0 +1,29 @@
package org.warp.cowdb.lists;
import org.warp.cowdb.EnhancedObject;
import org.warp.cowdb.IDatabase;
import org.warp.jcwdb.ann.DBDataType;
import org.warp.jcwdb.ann.DBField;
import java.io.IOException;
public class ObjectCowList<T> extends CowList<T> {
public ObjectCowList() {
super();
}
public ObjectCowList(IDatabase database) throws IOException {
super(database);
}
@Override
protected T loadItem(long uid) throws IOException {
return database.getObjectsIO().loadObject(uid);
}
@Override
protected void writeItemToDisk(long uid, T item) throws IOException {
database.getObjectsIO().setObject(uid, item);
}
}

73
src/main/java/org/warp/jcwdb/CacheIndexManager.java
View File

@ -1,73 +0,0 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
public class CacheIndexManager implements IndexManager {
public CacheIndexManager() {
}
@Override
public <T> T get(long index, DBReader<T> reader) {
// TODO: implement
return null;
}
@Override
public int getType(long index) {
// TODO: implement
return 0;
}
@Override
public long getHash(long index) {
// TODO: implement
return 0;
}
@Override
public <T> long add(DBDataOutput<T> writer) {
// TODO: implement
return 0;
}
@Override
public <T> FullIndexDetails addAndGetDetails(DBDataOutput<T> writer) {
// TODO: implement
return null;
}
@Override
public <T> IndexDetails set(long index, DBDataOutput<T> writer) {
// TODO: implement
return null;
}
@Override
public void setFlushingAllowed(long index, boolean isUnloadingAllowed) {
// TODO: implement
}
@Override
public void delete(long index) {
// TODO: implement
}
@Override
public boolean has(long index) {
// TODO: implement
return false;
}
@Override
public void close() {
// TODO: implement
}
@Override
public long clean() {
return 0;
}
}

5
src/main/java/org/warp/jcwdb/Castable.java
View File

@ -1,5 +0,0 @@
package org.warp.jcwdb;
public interface Castable {
<T> T cast();
}

9
src/main/java/org/warp/jcwdb/Cleanable.java
View File

@ -1,9 +0,0 @@
package org.warp.jcwdb;
public interface Cleanable {
/**
* Clean the object
* @return the approximated number of cleaned items
*/
public long clean();
}

122
src/main/java/org/warp/jcwdb/Cleaner.java
View File

@ -1,122 +0,0 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.nio.channels.ClosedChannelException;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
public class Cleaner {
public static final boolean DISABLE_CLEANER = false;
public static final boolean ENABLE_CLEANER_LOGGING = false;
private static final double MAXIMUM_SLEEP_INTERVAL = 8d * 1000d; // 8 seconds
private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second
private static final double NORMAL_REMOVED_ITEMS = 2500l;
private static final double REMOVED_ITEMS_RATIO = 2.5d; // 250%
private final Cleanable[] objectsToClean;
private final Thread cleanerThread;
private int sleepInterval = (int) MINIMUM_SLEEP_INTERVAL;
private volatile boolean stopRequest = false;
public Cleaner(Cleanable... objectsToClean) {
this.objectsToClean = objectsToClean;
this.cleanerThread = new Thread(new CleanLoop());
this.cleanerThread.setName("Cleaner thread");
this.cleanerThread.setDaemon(true);
}
public void start() {
if (!DISABLE_CLEANER) {
this.cleanerThread.start();
}
}
/**
* Clean
* @return number of removed items
*/
private long clean() {
long cleanedItems = 0;
for (Cleanable cleanable : objectsToClean) {
cleanedItems += cleanable.clean();
}
//System.gc();
return cleanedItems;
}
public void stop() {
if (cleanerThread != null) {
stopRequest = true;
while (cleanerThread.isAlive()) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
private class CleanLoop implements Runnable {
@Override
public void run() {
while(!stopRequest) {
try {
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Waiting " + sleepInterval + "ms.");
sleepFor(sleepInterval);
final long time1 = System.currentTimeMillis();
final double removedItems = clean();
final long time2 = System.currentTimeMillis();
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1));
double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2;
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems);
if (removedItems > 0) {
final double removedItemsRatio = removedItems / NORMAL_REMOVED_ITEMS;
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] REMOVED_ITEMS_RATIO: " + removedItemsRatio);
if (removedItemsRatio < 1d / REMOVED_ITEMS_RATIO || removedItemsRatio >= REMOVED_ITEMS_RATIO) {
suggestedExecutionTimeByItemsCalculations = sleepInterval / removedItemsRatio;
}
}
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Items: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + suggestedExecutionTimeByItemsCalculations + "ms");
double newSleepInterval = suggestedExecutionTimeByItemsCalculations;
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Total: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
if (newSleepInterval > MAXIMUM_SLEEP_INTERVAL) {
sleepInterval = (int) MAXIMUM_SLEEP_INTERVAL;
} else if (newSleepInterval < MINIMUM_SLEEP_INTERVAL) {
sleepInterval = (int) MINIMUM_SLEEP_INTERVAL;
} else {
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] CHANGED SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
sleepInterval = (int) newSleepInterval;
}
if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Cleaned " + removedItems + " items.");
}catch (InterruptedException e) {
}
}
}
private void sleepFor(int sleepInterval) throws InterruptedException {
int lastI = (int) Math.ceil(((double) sleepInterval) / 1000d);
for (int i = 0; i < lastI; i++) {
if (stopRequest) {
return;
}
if (i == lastI) {
Thread.sleep(sleepInterval % 1000);
} else {
Thread.sleep(lastI);
}
Thread.sleep(sleepInterval);
}
}
}
}

34
src/main/java/org/warp/jcwdb/DBDataOutput.java
View File

@ -1,34 +0,0 @@
package org.warp.jcwdb;
public interface DBDataOutput<T> {
int getSize();
int getType();
long calculateHash();
DBWriter<T> getWriter();
static <T> DBDataOutput<T> create(DBWriter<T> writer, int type, int size, long hash) {
return new DBDataOutput<T>() {
@Override
public int getSize() {
return size;
}
@Override
public int getType() {
return type;
}
@Override
public long calculateHash() {
return hash;
}
@Override
public DBWriter<T> getWriter() {
return writer;
}
};
}
}

58
src/main/java/org/warp/jcwdb/DBGenericObjectParser.java
View File

@ -1,58 +0,0 @@
package org.warp.jcwdb;
import java.io.ByteArrayOutputStream;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Output;
import net.openhft.hashing.LongHashFunction;
public class DBGenericObjectParser extends DBTypeParserImpl<Object> implements DBTypedObjectParser<Object> {
private static final LongHashFunction hashFunction = net.openhft.hashing.LongHashFunction.xx();
private static final Kryo kryo = new Kryo();
static {
kryo.setRegistrationRequired(false);
}
private static final DBReader<Object> defaultReader = (i, size) -> {
return kryo.readClassAndObject(i);
};
public DBReader<Object> getReader() {
return defaultReader;
}
public DBDataOutput<Object> getWriter(final Object value) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
Output tmpO = new Output(baos);
kryo.writeClassAndObject(tmpO, value);
tmpO.flush();
final byte[] bytes = baos.toByteArray();
final long hash = hashFunction.hashBytes(bytes);
tmpO.close();
return DBDataOutput.create((o) -> {
o.write(bytes);
}, DBStandardTypes.GENERIC_OBJECT, bytes.length, hash);
}
@Override
public long calculateHash(Object value) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
Output tmpO = new Output(baos);
kryo.writeClassAndObject(tmpO, value);
tmpO.flush();
final byte[] bytes = baos.toByteArray();
final long hash = hashFunction.hashBytes(bytes);
tmpO.close();
return hash;
}
@Override
public <U> void registerClass(Class<U> clazz, int id) {
if (id >= Integer.MAX_VALUE - 100) {
throw new IndexOutOfBoundsException();
}
kryo.register(clazz, id + 100);
}
}

45
src/main/java/org/warp/jcwdb/DBLightArrayListParser.java
View File

@ -1,45 +0,0 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.longs.LongArrayList;
public class DBLightArrayListParser<T> extends DBTypeParserImpl<LightArrayList<T>> {
private final JCWDatabase db;
public DBLightArrayListParser(JCWDatabase db) {
this.db = db;
}
public DBReader<LightArrayList<T>> getReader() {
return (i, size) -> {
LongArrayList internalList = new LongArrayList();
long max = size / Long.BYTES;
for (int item = 0; item < max; item++) {
long itm = i.readLong();
internalList.add(itm);
}
return new LightArrayList<T>(db, internalList);
};
}
public DBDataOutput<LightArrayList<T>> getWriter(final LightArrayList<T> value) {
final int elementsCount = value.size();
return DBDataOutput.create((o) -> {
LongArrayList list = value.internalList;
for (int i = 0; i < elementsCount; i++) {
o.writeLong(list.getLong(i));
}
}, DBStandardTypes.LIGHT_LIST_ARRAY, elementsCount * Long.BYTES, calculateHash(value));
}
@Override
public long calculateHash(LightArrayList<T> value) {
return value.internalList.hashCode();
}
@Override
public String toString() {
return "DBLightArrayListParser{" +
"db=" + db +
'}';
}
}

44
src/main/java/org/warp/jcwdb/DBLightBigListParser.java
View File

@ -1,44 +0,0 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
public class DBLightBigListParser<T> extends DBTypeParserImpl<LightBigList<T>> {
private final JCWDatabase db;
public DBLightBigListParser(JCWDatabase db) {
this.db = db;
}
public DBReader<LightBigList<T>> getReader() {
return (i, size) -> {
LongArrayList chunks = new LongArrayList();
IntArrayList chunkSizes = new IntArrayList();
long max = size / (Long.BYTES + Integer.BYTES);
for (int item = 0; item < max; item++) {
long itm = i.readLong();
int itm2 = i.readInt();
chunks.add(itm);
chunkSizes.add(itm2);
}
return new LightBigList<>(db, chunks, chunkSizes);
};
}
public DBDataOutput<LightBigList<T>> getWriter(final LightBigList<T> value) {
final int elementsCount = value.chunksCount();
return DBDataOutput.create((o) -> {
LongArrayList list = value.chunks;
IntArrayList list2 = value.chunkSizes;
for (int i = 0; i < elementsCount; i++) {
o.writeLong(list.getLong(i));
o.writeInt(list2.getInt(i));
}
}, DBStandardTypes.LIGHT_LIST_BIG, elementsCount * (Long.BYTES + Integer.BYTES), calculateHash(value));
}
@Override
public long calculateHash(LightBigList<T> value) {
return (((long)value.chunks.hashCode()) << 32) | (value.chunkSizes.hashCode() & 0xffffffffL);
}
}

7
src/main/java/org/warp/jcwdb/DBReader.java
View File

@ -1,7 +0,0 @@
package org.warp.jcwdb;
import com.esotericsoftware.kryo.io.Input;
public interface DBReader<T> {
T read(Input i, int size);
}

24
src/main/java/org/warp/jcwdb/DBStandardTypes.java
View File

@ -1,24 +0,0 @@
package org.warp.jcwdb;
public class DBStandardTypes {
private static final int STD = 0xFFFFF000;
public static final int BOOLEAN = STD| 0x000;
public static final int BYTE = STD| 0x001;
public static final int SHORT = STD| 0x002;
public static final int CHAR = STD| 0x003;
public static final int INTEGER = STD| 0x004;
public static final int FLOAT = STD| 0x005;
public static final int DOUBLE = STD| 0x006;
public static final int STRING = STD| 0x007;
public static final int BYTE_ARRAY = STD| 0x008;
public static final int LIGHT_LIST_ARRAY = STD| 0x009;
public static final int LIGHT_LIST_BIG = STD| 0x00A;
public static final int GENERIC_OBJECT = STD| 0x00B;
public static void registerStandardTypes(JCWDatabase db, TypesManager typesManager) {
typesManager.registerType(String.class, STRING, new DBStringParser());
typesManager.registerType(LightArrayList.class, LIGHT_LIST_ARRAY, new DBLightArrayListParser(db));
typesManager.registerType(LightBigList.class, LIGHT_LIST_BIG, new DBLightBigListParser(db));
typesManager.registerTypeFallback(new DBGenericObjectParser());
}
}

27
src/main/java/org/warp/jcwdb/DBStringParser.java
View File

@ -1,27 +0,0 @@
package org.warp.jcwdb;
import java.nio.charset.StandardCharsets;
import net.openhft.hashing.LongHashFunction;
public class DBStringParser extends DBTypeParserImpl<String> {
private static final LongHashFunction hashFunction = net.openhft.hashing.LongHashFunction.xx();
private static final DBReader<String> defaultReader = (i, size) -> {
return new String(i.readBytes(size), StandardCharsets.UTF_16LE);
};
public DBReader<String> getReader() {
return defaultReader;
}
public DBDataOutput<String> getWriter(final String value) {
return DBDataOutput.create((o) -> {
o.write(value.getBytes(StandardCharsets.UTF_16LE));
}, DBStandardTypes.STRING, value.length() * 2, calculateHash(value));
}
@Override
public long calculateHash(String value) {
return hashFunction.hashBytes(value.getBytes(StandardCharsets.UTF_16LE));
}
}

7
src/main/java/org/warp/jcwdb/DBTypeParser.java
View File

@ -1,7 +0,0 @@
package org.warp.jcwdb;
public interface DBTypeParser<T> extends Castable {
DBReader<T> getReader();
DBDataOutput<T> getWriter(final T value);
long calculateHash(final T value);
}

9
src/main/java/org/warp/jcwdb/DBTypeParserImpl.java
View File

@ -1,9 +0,0 @@
package org.warp.jcwdb;
public abstract class DBTypeParserImpl<T> implements DBTypeParser<T> {
@SuppressWarnings("unchecked")
@Override
public T cast() {
return (T) this;
}
}

5
src/main/java/org/warp/jcwdb/DBTypedObjectParser.java
View File

@ -1,5 +0,0 @@
package org.warp.jcwdb;
public interface DBTypedObjectParser<T> extends DBTypeParser<T> {
public <U> void registerClass(Class<U> clazz, int type);
}

7
src/main/java/org/warp/jcwdb/DBWriter.java
View File

@ -1,7 +0,0 @@
package org.warp.jcwdb;
import com.esotericsoftware.kryo.io.Output;
public interface DBWriter<T> {
void write(Output o);
}

60
src/main/java/org/warp/jcwdb/Editable.java
View File

@ -1,60 +0,0 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
public interface Editable<T> {
/**
* Reccomended way to edit the value
*
* @param editFunction
* @throws IOException
*/
void editValue(BiFunction<T, Saveable, T> editFunction);
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
void editValue(Function<T, T> editFunction);
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
void editValue(BiConsumer<T, Saveable> editFunction);
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
void editValue(Consumer<T> editFunction);
/**
* Reccomended way to view the value
* @param viewFunction
* @throws IOException
*/
void viewValue(Consumer<T> viewFunction);
/**
* Substitute the old value with a new one
* @param val
* @throws IOException
*/
void setValue(T val);
/**
* DO NOT ATTEMPT TO MODIFY THE VALUE RETURNED
* @return
*/
T getValueReadOnlyUnsafe();
}

266
src/main/java/org/warp/jcwdb/EntryReference.java
View File

@ -1,266 +0,0 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
/**
* You must have only a maximum of 1 reference for each index
* @param <T>
*/
public class EntryReference<T> implements Editable<T>, Saveable, Castable {
private final JCWDatabase.EntryReferenceTools db;
private final long entryIndex;
private final DBTypeParser<T> parser;
private T value;
private long cachedHash;
private volatile boolean isHashCached;
private volatile boolean loaded;
private volatile boolean closed;
private volatile boolean isFlushingAllowed;
private final Object hashCacheLock = new Object();
private final Object accessLock = new Object();
private final Object closeLock = new Object();
public EntryReference(JCWDatabase.EntryReferenceTools db, long entryId, long hash, DBTypeParser<T> parser) {
this.loaded = false;
this.isHashCached = false;
this.db = db;
this.entryIndex = entryId;
this.parser = parser;
this.value = null;
}
public EntryReference(JCWDatabase.EntryReferenceTools db, long entryId, long hash, DBTypeParser<T> parser, T value) {
this.loaded = true;
this.isHashCached = true;
this.db = db;
this.entryIndex = entryId;
this.parser = parser;
this.cachedHash = hash;
this.value = value;
}
public DBTypeParser<T> getParser() {
return parser;
}
public long getIndex() {
return entryIndex;
}
public long calculateHash() {
synchronized(accessLock) {
load();
synchronized(hashCacheLock) {
if (isHashCached) {
return cachedHash;
}
}
return parser.calculateHash(this.value);
}
}
public boolean isClosed() {
return closed;
}
/**
* Note that this method won't be called when closing without saving
*/
public void save() {
this.save(false);
}
public void saveAndFlush() {
this.save(true);
}
private void save(boolean flush) {
synchronized(accessLock) {
if (loaded && !closed) {
try {
if (value instanceof Saveable) {
if (flush) {
((Saveable)value).saveAndFlush();
} else {
((Saveable)value).save();
}
}
IndexDetails returnedDetails = this.db.write(entryIndex, parser.getWriter(value));
synchronized(hashCacheLock) {
this.cachedHash = returnedDetails.getHash();
this.isHashCached = true;
}
if (flush) {
if (!isFlushingAllowed) {
this.db.setFlushingAllowed(entryIndex, true);
this.isFlushingAllowed = true;
}
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
public void editValue(BiFunction<T, Saveable, T> editFunction) {
synchronized(accessLock) {
load();
this.value = editFunction.apply(this.value, this);
this.save(true);
}
}
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
public void editValue(Function<T, T> editFunction) {
synchronized(accessLock) {
load();
this.value = editFunction.apply(this.value);
this.save(true);
}
}
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
public void editValue(BiConsumer<T, Saveable> editFunction) {
synchronized(accessLock) {
load();
editFunction.accept(this.value, this);
this.save(true);
}
}
/**
* Reccomended way to edit the value
* @param editFunction
* @throws IOException
*/
public void editValue(Consumer<T> editFunction) {
synchronized(accessLock) {
load();
editFunction.accept(this.value);
this.save(true);
}
}
/**
* Reccomended way to edit the value
* DO NOT EDIT THE VALUE
* @param viewFunction
* @throws IOException
*/
public void viewValue(Consumer<T> viewFunction) {
synchronized(accessLock) {
load();
viewFunction.accept(this.value);
}
}
/**
* Substitute the old value with a new one
* @param val
* @throws IOException
*/
public void setValue(T val) {
synchronized(accessLock) {
this.loaded = true;
this.value = val;
synchronized(hashCacheLock) {
this.isHashCached = false;
}
this.save(true);
}
}
/**
* Use editValue instead. READ ONLY!!
* @return
*/
@Deprecated()
public T getValue() {
return getValueReadOnlyUnsafe();
}
/**
* DO NOT ATTEMPT TO MODIFY THE VALUE RETURNED
* @return
*/
public T getValueReadOnlyUnsafe() {
synchronized(accessLock) {
load();
return this.value;
}
}
private void load() {
synchronized(accessLock) {
if (!loaded) {
try {
if (this.isFlushingAllowed) {
this.db.setFlushingAllowed(entryIndex, false);
this.isFlushingAllowed = false;
}
this.value = db.read(entryIndex, parser.getReader());
this.loaded = true;
} catch (IOException e) {
throw (NullPointerException) new NullPointerException(e.getLocalizedMessage()).initCause(e);
}
}
}
}
@SuppressWarnings("unchecked")
@Override
public <U> U cast() {
return (U) this;
}
protected void close() throws IOException {
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
save(true);
closed = true;
}
}
public void closeWithoutSaving() {
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
closed = true;
}
}
public Object getAccessLock() {
return accessLock;
}
}

147
src/main/java/org/warp/jcwdb/FileAllocator.java
View File

@ -1,147 +0,0 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.longs.*;
import it.unimi.dsi.fastutil.objects.ObjectBidirectionalIterator;
import java.io.IOException;
import java.nio.channels.SeekableByteChannel;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Stream;
public class FileAllocator implements AutoCloseable {
private static final int MAXIMUM_UNALLOCATED_ENTRIES = 50000;
private final SeekableByteChannel dataFileChannel;
private volatile long fileSize;
private volatile boolean closed;
private final Object closeLock = new Object();
private final Object allocateLock = new Object();
/**
* index -> free space size
*/
private final Long2IntMap freeBytes = new Long2IntLinkedOpenHashMap();
public FileAllocator(SeekableByteChannel dataFileChannel) throws IOException {
this.dataFileChannel = dataFileChannel;
this.fileSize = this.dataFileChannel.size();
}
public FileAllocator(SeekableByteChannel dataFileChannel, long fileSize, Long2IntMap freeBytes) throws IOException {
this.dataFileChannel = dataFileChannel;
this.fileSize = fileSize;
this.freeBytes.putAll(freeBytes);
}
/**
* TODO: not implemented
*
* @param size
* @return offset
*/
public long allocate(int size) {
checkClosed();
synchronized (allocateLock) {
long offset = allocateIntoUnusedParts(size);
if (offset == -1) {
return allocateToEnd(size);
} else {
return offset;
}
}
}
private long allocateIntoUnusedParts(int size) {
Stream<Map.Entry<Long,Integer>> sorted =
freeBytes.entrySet().stream()
.sorted(Collections.reverseOrder(Map.Entry.comparingByValue()));
final VariableWrapper<Long> holeOffset = new VariableWrapper<>(-1L);
final VariableWrapper<Integer> holeSize = new VariableWrapper<>(0);
sorted.anyMatch((entry) -> {
int currentHoleSize = entry.getValue();
if (currentHoleSize < size) {
return true;
}
holeOffset.var = entry.getKey();
holeSize.var = currentHoleSize;
return false;
});
if (holeOffset.var != -1L) {
freeBytes.remove(holeOffset.var);
if (holeSize.var > size) {
freeBytes.put(holeOffset.var + size, holeSize.var - size);
}
}
return holeOffset.var;
}
private long allocateToEnd(int size) {
long allocatedOffset = fileSize;
fileSize += size;
return allocatedOffset;
}
public void close() throws IOException {
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
closed = true;
}
}
/**
* Frees the unused bytes
*
* @param startPosition
* @param length
*/
public void markFree(long startPosition, int length) {
checkClosed();
if (freeBytes.containsKey(startPosition + length)) {
int secondLength = freeBytes.remove(startPosition + length);
freeBytes.put(startPosition, length + secondLength);
} else {
boolean addedToList = false;
for (Long2IntMap.Entry entry : freeBytes.long2IntEntrySet()) {
if (entry.getLongKey() + entry.getIntValue() == startPosition) {
freeBytes.put(entry.getLongKey(), entry.getIntValue() + length);
addedToList = true;
break;
}
}
if (!addedToList && length > 0) {
freeBytes.put(startPosition, length);
}
}
if (startPosition + length >= fileSize) {
fileSize = startPosition;
}
// Remove the smallest hole in the file
if (freeBytes.size() > MAXIMUM_UNALLOCATED_ENTRIES) {
Stream<Map.Entry<Long,Integer>> sorted =
freeBytes.entrySet().stream()
.sorted(Map.Entry.comparingByValue());
Optional<Map.Entry<Long, Integer>> first = sorted.findFirst();
if (first.isPresent()) {
freeBytes.remove(first.get().getKey());
}
}
}
private void checkClosed() {
if (closed) {
throw new RuntimeException("Index Manager is closed.");
}
}
}

539
src/main/java/org/warp/jcwdb/FileIndexManager.java
View File

@ -1,539 +0,0 @@
package org.warp.jcwdb;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import it.unimi.dsi.fastutil.longs.*;
import it.unimi.dsi.fastutil.objects.ObjectIterator;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
public class FileIndexManager implements IndexManager {
private final SeekableByteChannel dataFileChannel, metadataFileChannel;
private volatile long metadataFileChannelSize;
private final FileAllocator fileAllocator;
private final ByteBuffer metadataByteBuffer = ByteBuffer.allocateDirect(IndexDetails.TOTAL_BYTES);
private final ByteBuffer maskByteBuffer = ByteBuffer.allocateDirect(Integer.BYTES);
private volatile boolean closed;
private final Object closeLock = new Object();
private final Object metadataByteBufferLock = new Object();
private final Object maskByteBufferLock = new Object();
private final Object indicesMapsAccessLock = new Object();
/**
* Edit this using editIndex()
* Get using getIndexMetadata()
* This hashmap must contain all indices.
*/
private final Long2ObjectMap<IndexDetails> loadedIndices;
/**
* Edit this using editIndex()
*/
private final LongSet dirtyLoadedIndices, flushingAllowedIndices, removedIndices;
private long firstAllocableIndex;
public FileIndexManager(Path dataFile, Path metadataFile) throws IOException {
if (Cleaner.DISABLE_CLEANER) {
loadedIndices = new Long2ObjectOpenHashMap<>();
dirtyLoadedIndices = new LongOpenHashSet();
flushingAllowedIndices = new LongOpenHashSet();
removedIndices = new LongOpenHashSet();
} else {
loadedIndices = new Long2ObjectLinkedOpenHashMap<>();
dirtyLoadedIndices = new LongLinkedOpenHashSet();
flushingAllowedIndices = new LongLinkedOpenHashSet();
removedIndices = new LongLinkedOpenHashSet();
}
if (Files.notExists(dataFile)) {
Files.createFile(dataFile);
}
if (Files.notExists(metadataFile)) {
Files.createFile(metadataFile);
}
dataFileChannel = Files.newByteChannel(dataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
metadataFileChannel = Files.newByteChannel(metadataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
fileAllocator = createFileAllocator(dataFileChannel, metadataFileChannel.position(0));
metadataFileChannelSize = metadataFileChannel.size();
firstAllocableIndex = getMetadataFileChannelSize() / (long) IndexDetails.TOTAL_BYTES;
if (firstAllocableIndex == 0) {
firstAllocableIndex = 1;
}
}
private long getMetadataFileChannelSize() throws IOException {
return metadataFileChannelSize;
}
private FileAllocator createFileAllocator(final SeekableByteChannel dataFileChannel, final SeekableByteChannel metadataFileChannel) throws IOException {
Long2IntMap freeBytes = new Long2IntRBTreeMap();
Long2IntMap usedBytes = new Long2IntRBTreeMap();
long firstOffset = 0;
while (metadataFileChannel.position() + IndexDetails.TOTAL_BYTES <= getMetadataFileChannelSize()) {
IndexDetails indexDetails = readIndexDetailsAt(metadataFileChannel);
if (indexDetails != null) {
long offset = indexDetails.getOffset();
usedBytes.put(offset, indexDetails.getSize());
if (offset < firstOffset) {
firstOffset = offset;
}
}
}
long previousEntryOffset = 0;
long previousEntrySize = 0;
ObjectIterator<Long2IntMap.Entry> it = usedBytes.long2IntEntrySet().iterator();
while (it.hasNext()) {
final Long2IntMap.Entry entry = it.next();
final long entryOffset = entry.getLongKey();
final long entrySize = entry.getIntValue();
it.remove();
if (previousEntryOffset + previousEntrySize < entryOffset) {
freeBytes.put(previousEntryOffset + previousEntrySize, (int) (entryOffset - (previousEntryOffset + previousEntrySize)));
}
previousEntryOffset = entryOffset;
previousEntrySize = entrySize;
}
final long fileSize = previousEntryOffset + previousEntrySize;
return new FileAllocator(dataFileChannel, fileSize, freeBytes);
}
@Override
public <T> T get(long index, DBReader<T> reader) throws IOException {
checkClosed();
IndexDetails details = getIndexMetadata(index);
Input i = new Input(Channels.newInputStream(dataFileChannel.position(details.getOffset())));
T result = reader.read(i, details.getSize());
return result;
}
@Override
public int getType(long index) throws IOException {
return getIndexMetadata(index).getType();
}
@Override
public long getHash(long index) throws IOException {
return getIndexMetadata(index).getHash();
}
@Override
public <T> IndexDetails set(long index, DBDataOutput<T> data) throws IOException {
checkClosed();
final int dataSize = data.getSize();
IndexDetails indexDetails = getIndexMetadataUnsafe(index);
if (indexDetails == null || indexDetails.getSize() < dataSize) {
// Allocate new space
IndexDetails newDetails = allocateAndWrite(index, data);
if (indexDetails != null) {
// Mark free the old bytes
fileAllocator.markFree(indexDetails.getOffset(), indexDetails.getSize());
}
return newDetails;
} else {
// Check if size changed
if (dataSize < indexDetails.getSize()) {
// Mark free the unused bytes
fileAllocator.markFree(indexDetails.getOffset() + dataSize, dataSize);
}
// Update index details
indexDetails = editIndex(index, indexDetails, indexDetails.getOffset(), dataSize, indexDetails.getType(), data.calculateHash());
// Write data
writeExact(indexDetails, data);
// Before returning, return IndexDetails
return indexDetails;
}
}
@Override
public void setFlushingAllowed(long index, boolean isUnloadingAllowed) {
checkClosed();
if (isUnloadingAllowed) {
flushingAllowedIndices.add(index);
} else {
flushingAllowedIndices.remove(index);
}
}
@Override
public <T> long add(DBDataOutput<T> data) throws IOException {
checkClosed();
final int size = data.getSize();
final long offset = fileAllocator.allocate(size);
final int type = data.getType();
final long hash = data.calculateHash();
final IndexDetails indexDetails = new IndexDetails(offset, size, type, hash);
final long index = createIndexMetadata(indexDetails);
writeExact(indexDetails, data);
return index;
}
@Override
public <T> FullIndexDetails addAndGetDetails(DBDataOutput<T> data) throws IOException {
checkClosed();
final int size = data.getSize();
final long offset = fileAllocator.allocate(size);
final int type = data.getType();
final long hash = data.calculateHash();
final IndexDetails indexDetails = new IndexDetails(offset, size, type, hash);
final long index = createIndexMetadata(indexDetails);
writeExact(indexDetails, data);
return new FullIndexDetails(index, indexDetails);
}
/**
* Write the data at index.
* The input size must be equal to the index size!
*
* @param indexDetails
* @param data
* @throws IOException
*/
private void writeExact(final IndexDetails indexDetails, DBDataOutput<?> data) throws IOException {
final int dataSize = data.getSize();
if (indexDetails.getSize() != dataSize) {
throw new IOException("Unable to write " + dataSize + " in a space of " + indexDetails.getSize());
}
final long offset = indexDetails.getOffset();
final Output o = new Output(Channels.newOutputStream(dataFileChannel.position(offset)), dataSize);
data.getWriter().write(o);
o.flush();
}
private IndexDetails allocateAndWrite(final long index, DBDataOutput<?> w) throws IOException {
final int size = w.getSize();
final int type = w.getType();
final long hash = w.calculateHash();
final long offset = fileAllocator.allocate(size);
IndexDetails details = editIndex(index, offset, size, type, hash);
writeExact(details, w);
return details;
}
@Override
public void delete(long index) throws IOException {
checkClosed();
IndexDetails indexDetails = getIndexMetadataUnsafe(index);
if (indexDetails != null) {
fileAllocator.markFree(indexDetails.getOffset(), indexDetails.getSize());
}
synchronized (indicesMapsAccessLock) {
dirtyLoadedIndices.remove(index);
flushingAllowedIndices.remove(index);
loadedIndices.remove(index);
removedIndices.add(index);
}
}
public void flushAndUnload(long index) throws IOException {
if (removedIndices.contains(index)) {
synchronized (indicesMapsAccessLock) {
removedIndices.remove(index);
dirtyLoadedIndices.remove(index);
flushingAllowedIndices.remove(index);
loadedIndices.remove(index);
}
// Update indices metadata
SeekableByteChannel metadata = metadataFileChannel.position(index * IndexDetails.TOTAL_BYTES);
eraseIndexDetails(metadata);
}
boolean isDirty = false;
IndexDetails indexDetails = null;
synchronized (indicesMapsAccessLock) {
if (dirtyLoadedIndices.contains(index)) {
indexDetails = loadedIndices.get(index);
dirtyLoadedIndices.remove(index);
flushingAllowedIndices.remove(index);
}
}
if (isDirty) {
// Update indices metadata
long position = index * IndexDetails.TOTAL_BYTES;
resizeMetadataFileChannel(position);
SeekableByteChannel metadata = metadataFileChannel.position(position);
writeIndexDetails(metadata, indexDetails);
}
synchronized (indicesMapsAccessLock) {
loadedIndices.remove(index);
}
}
@Override
public boolean has(long index) {
checkClosed();
try {
return getIndexMetadataUnsafe(index) != null;
} catch (IOException ex) {
ex.printStackTrace();
return false;
}
}
/**
* Edit index data if a change is detected
* @param index
* @param oldData Old index data to check
* @param offset offset
* @param size size
* @param type type
* @param hash hash
* @return
*/
private IndexDetails editIndex(long index, IndexDetails oldData, long offset, int size, int type, long hash) {
if (oldData.getOffset() != offset || oldData.getSize() != size || oldData.getType() != type || oldData.getHash() != hash) {
return editIndex(index, offset, size, type, hash);
} else {
return oldData;
}
}
/**
* Edit index data
* @param index
* @param offset
* @param size
* @param type
* @param hash
* @return
*/
private IndexDetails editIndex(long index, long offset, int size, int type, long hash) {
IndexDetails indexDetails = new IndexDetails(offset, size, type, hash);
editIndex(index, indexDetails);
return indexDetails;
}
/**
* Edit index data
* @param index
* @param details
*/
private void editIndex(long index, IndexDetails details) {
synchronized (indicesMapsAccessLock) {
loadedIndices.put(index, details);
dirtyLoadedIndices.add(index);
flushingAllowedIndices.remove(index);
}
}
private long createIndexMetadata(IndexDetails indexDetails) {
synchronized (indicesMapsAccessLock) {
long newIndex = firstAllocableIndex++;
loadedIndices.put(newIndex, indexDetails);
dirtyLoadedIndices.add(newIndex);
flushingAllowedIndices.remove(newIndex);
removedIndices.remove(newIndex);
return newIndex;
}
}
private IndexDetails getIndexMetadataUnsafe(long index) throws IOException {
// Return index details if loaded
IndexDetails details;
synchronized (indicesMapsAccessLock) {
details = loadedIndices.getOrDefault(index, null);
}
if (details != null) return details;
// Try to load the details from file
final long metadataPosition = index * IndexDetails.TOTAL_BYTES;
if (metadataPosition + IndexDetails.TOTAL_BYTES > getMetadataFileChannelSize()) {
// Avoid underflow exception
return null;
}
SeekableByteChannel currentMetadataFileChannel = metadataFileChannel.position(metadataPosition);
IndexDetails indexDetails = readIndexDetailsAt(currentMetadataFileChannel);
if (indexDetails != null) {
editIndex(index, indexDetails);
return indexDetails;
}
// No results found. Returning null
return null;
}
private IndexDetails readIndexDetailsAt(SeekableByteChannel currentMetadataFileChannel) throws IOException {
IndexDetails indexDetails = null;
synchronized (metadataByteBufferLock) {
metadataByteBuffer.rewind();
currentMetadataFileChannel.read(metadataByteBuffer);
metadataByteBuffer.rewind();
// If it's not deleted continue
if ((metadataByteBuffer.getInt() & IndexDetails.MASK_DELETED) == 0) {
final long offset = metadataByteBuffer.getLong();
// final long sizeAndType = metadataByteBuffer.getLong();
// final int size = (int)(sizeAndType >> 32);
// final int type = (int)sizeAndType;
final int size = metadataByteBuffer.getInt();
final int type = metadataByteBuffer.getInt();
final long hash = metadataByteBuffer.getLong();
indexDetails = new IndexDetails(offset, size, type, hash);
}
}
return indexDetails;
}
private IndexDetails getIndexMetadata(long index) throws IOException {
IndexDetails details = getIndexMetadataUnsafe(index);
if (details == null)
throw new IOException("Index " + index + " not found");
else
return details;
}
@Override
public void close() throws IOException {
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
closed = true;
}
// Update indices metadata
flushAllFlushableIndices();
// Remove removed indices
removeRemovedIndices();
fileAllocator.close();
}
private void writeIndexDetails(SeekableByteChannel position, IndexDetails indexDetails) throws IOException {
synchronized (metadataByteBufferLock) {// FIXXXX cleaner3
final int size = indexDetails.getSize();
final int type = indexDetails.getType();
final long offset = indexDetails.getOffset();
final long hash = indexDetails.getHash();
metadataByteBuffer.rewind();
metadataByteBuffer.putInt(0);
metadataByteBuffer.putLong(offset);
metadataByteBuffer.putInt(size);
metadataByteBuffer.putInt(type);
//metadataByteBuffer.putLong((long)size << 32 | type & 0xFFFFFFFFL);
metadataByteBuffer.putLong(hash);
metadataByteBuffer.rewind();
position.write(metadataByteBuffer);
}
}
private void eraseIndexDetails(SeekableByteChannel position) throws IOException {
synchronized (maskByteBufferLock) {
maskByteBuffer.rewind();
maskByteBuffer.putInt(IndexDetails.MASK_DELETED);
maskByteBuffer.rewind();
position.write(maskByteBuffer);
}
}
private void checkClosed() {
if (closed) {
throw new RuntimeException("Index Manager is closed.");
}
}
@Override
public long clean() {
long cleaned = 0;
long tim1 = System.currentTimeMillis();
try {
cleaned += flushAllFlushableIndices();
} catch (IOException ex) {
ex.printStackTrace();
}
long tim2 = System.currentTimeMillis();
try {
cleaned += removeRemovedIndices();
} catch (IOException ex) {
ex.printStackTrace();
}
long tim3 = System.currentTimeMillis();
cleaned += cleanExtraIndices();
long tim4 = System.currentTimeMillis();
if (Cleaner.ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] FileIndexManager CLEAN_TIME: " + (tim2-tim1) + "," + (tim3-tim2) + "," + (tim4-tim3));
return cleaned;
}
private long flushAllFlushableIndices() throws IOException {
long flushedIndices = 0;
SeekableByteChannel metadata = metadataFileChannel;
long lastIndex = -2;
synchronized (indicesMapsAccessLock) {
for (long index : dirtyLoadedIndices) {
if (!flushingAllowedIndices.contains(index)) {
IndexDetails indexDetails = loadedIndices.get(index);
long position = index * IndexDetails.TOTAL_BYTES;
resizeMetadataFileChannel(position);
if (index - lastIndex != 1) {
metadata = metadata.position(position);
}
writeIndexDetails(metadata, indexDetails);
lastIndex = index;
flushedIndices++;
}
}
dirtyLoadedIndices.clear();
dirtyLoadedIndices.addAll(flushingAllowedIndices);
flushingAllowedIndices.clear();
}
return flushedIndices;
}
private void resizeMetadataFileChannel(long position) {
if (position + IndexDetails.TOTAL_BYTES > metadataFileChannelSize) {
metadataFileChannelSize = position + IndexDetails.TOTAL_BYTES;
}
}
private long removeRemovedIndices() throws IOException {
SeekableByteChannel metadata = metadataFileChannel;
synchronized (indicesMapsAccessLock) {
long removed = this.removedIndices.size();
for (long index : this.removedIndices) {
metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
eraseIndexDetails(metadata);
}
this.removedIndices.clear();
return removed;
}
}
private long cleanExtraIndices() {
long removedIndices = 0;
LongArrayList toUnload = new LongArrayList();
synchronized (indicesMapsAccessLock) {
if (loadedIndices.size() > JCWDatabase.MAX_LOADED_INDICES) {
long count = loadedIndices.size();
LongIterator it = loadedIndices.keySet().iterator();
while (it.hasNext()) {
long loadedIndex = it.nextLong();
if (count < JCWDatabase.MAX_LOADED_INDICES * 3l / 2l) {
break;
}
toUnload.add(loadedIndex);
removedIndices++;
count--;
}
}
}
for (long index : toUnload.elements()) {
try {
flushAndUnload(index);
} catch (IOException e) {
e.printStackTrace();
}
}
return removedIndices;
}
}

14
src/main/java/org/warp/jcwdb/FullIndexDetails.java
View File

@ -1,14 +0,0 @@
package org.warp.jcwdb;
public class FullIndexDetails extends IndexDetails {
private final long index;
public FullIndexDetails(long index, IndexDetails details) {
super(details);
this.index = index;
}
public long getIndex() {
return index;
}
}

88
src/main/java/org/warp/jcwdb/IndexDetails.java
View File

@ -1,88 +0,0 @@
package org.warp.jcwdb;
import java.util.Objects;
public class IndexDetails {
/**
* The bitmask is used to determine if an index has been deleted
*/
public static final int BITMASK_SIZE = Integer.BYTES;
public static final int OFFSET_BYTES = Long.BYTES;
public static final int DATA_SIZE_BYTES = Integer.BYTES;
public static final int TYPE_BYTES = Integer.BYTES;
public static final int HASH_BYTES = Long.BYTES;
public static final int TOTAL_BYTES = BITMASK_SIZE + OFFSET_BYTES + DATA_SIZE_BYTES + TYPE_BYTES + HASH_BYTES;
public static final int MASK_DELETED = 0b00000001;
private final long offset;
private final int size;
private final int type;
private final long hash;
public IndexDetails(long offset, int size, int type, long hash) {
this.offset = offset;
this.size = size;
this.type = type;
this.hash = hash;
}
public IndexDetails(IndexDetails indexDetails) {
this.offset = indexDetails.offset;
this.size = indexDetails.size;
this.type = indexDetails.type;
this.hash = indexDetails.hash;
}
public long getOffset() {
return offset;
}
public int getSize() {
return size;
}
public int getType() {
return type;
}
public long getHash() {
return hash;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (hash ^ (hash >>> 32));
result = prime * result + (int) (offset ^ (offset >>> 32));
result = prime * result + size;
result = prime * result + type;
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
IndexDetails other = (IndexDetails) obj;
if (hash != other.hash)
return false;
if (offset != other.offset)
return false;
if (size != other.size)
return false;
if (type != other.type)
return false;
return true;
}
@Override
public String toString() {
return "IndexDetails [offset=" + offset + ", size=" + size + ", type=" + type + ", hash=" + hash + "]";
}
}

19
src/main/java/org/warp/jcwdb/IndexManager.java
View File

@ -1,19 +0,0 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.util.function.BiConsumer;
import java.util.function.BiPredicate;
import java.util.function.Consumer;
public interface IndexManager extends Cleanable {
<T> T get(long index, DBReader<T> reader) throws IOException;
int getType(long index) throws IOException;
long getHash(long index) throws IOException;
<T> long add(DBDataOutput<T> writer) throws IOException;
<T> FullIndexDetails addAndGetDetails(DBDataOutput<T> writer) throws IOException;
<T> IndexDetails set(long index, DBDataOutput<T> writer) throws IOException;
void setFlushingAllowed(long index, boolean isUnloadingAllowed);
void delete(long index) throws IOException;
boolean has(long index);
void close() throws IOException;
}

197
src/main/java/org/warp/jcwdb/JCWDatabase.java
View File

@ -1,197 +0,0 @@
package org.warp.jcwdb;
import java.io.IOError;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.function.Supplier;
public class JCWDatabase implements AutoCloseable, Cleanable {
public final static long MAX_LOADED_INDICES = 1000;
private final TypesManager typesManager;
private final MixedIndexDatabase indices;
private final Cleaner databaseCleaner;
private final EntryReferenceTools entryReferenceTools = new EntryReferenceTools();
private volatile boolean closed;
private final Object closeLock = new Object();
private final Object indicesAccessLock = new Object();
private final LinkedList<EntryReference<?>> usedReferences = new LinkedList<>();
public JCWDatabase(Path dataFile, Path metadataFile) throws IOException {
this.typesManager = new TypesManager(this);
this.indices = new MixedIndexDatabase(dataFile, metadataFile);
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
JCWDatabase.this.close();
} catch (Exception e) {
e.printStackTrace();
}
}));
this.databaseCleaner = new Cleaner(this);
this.databaseCleaner.start();
}
public <T> EntryReference<LightList<T>> getRoot() {
try {
checkClosed();
if (exists(0)) {
return get(0);
} else {
LightList<T> newRoot = new LightBigList<>(this);
return set(0, newRoot);
}
} catch (IOException e) {
throw new IOError(e);
}
}
@SuppressWarnings("unchecked")
public <T> EntryReference<T> getRootItem(int index) {
return ((LightList<T>) getRoot().getValueReadOnlyUnsafe()).getReference(index);
}
@SuppressWarnings("unchecked")
public <T> EntryReference<T> getRootItem(int index, Supplier<T> defaultValue) {
return ((LightList<T>) getRoot().getValueReadOnlyUnsafe()).getReferenceOrInitialize(index, defaultValue);
}
public <T> EntryReference<LightList<T>> getRoot(Class<T> clazz) {
return getRoot().cast();
}
public <T> EntryReference<T> get(long index) throws IOException {
checkClosed();
int type;
long hash;
synchronized (indicesAccessLock) {
type = this.indices.getType(index);
hash = this.indices.getHash(index);
}
DBTypeParser<T> typeParser = this.typesManager.get(type);
return new EntryReference<>(entryReferenceTools, index, hash, typeParser);
}
protected <T> EntryReference<T> add(T value) throws IOException {
checkClosed();
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
long index;
long hash;
synchronized (indicesAccessLock) {
FullIndexDetails fullIndexDetails = indices.addAndGetDetails(typeParser.getWriter(value));
index = fullIndexDetails.getIndex();
hash = fullIndexDetails.getHash();
}
return new EntryReference<>(entryReferenceTools, index, hash, typeParser, value);
}
protected boolean exists(long index) {
checkClosed();
synchronized (indicesAccessLock) {
return this.indices.has(index);
}
}
protected <T> EntryReference<T> set(long index, T value) throws IOException {
checkClosed();
EntryReference<T> ref;
if (exists(index)) {
ref = get(index);
ref.setValue(value);
return ref;
} else {
@SuppressWarnings("unchecked")
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
long hash;
synchronized (indicesAccessLock) {
IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value));
hash = returnedDetails.getHash();
}
return new EntryReference<>(entryReferenceTools, index, hash, typeParser);
}
}
public <U> void registerType(Class<U> clazz, short type, DBTypeParser<U> parser) {
final int addition = 0xEFFF8000;
int extendedType = addition | (type & 0x7FFF);
typesManager.registerType(clazz, extendedType, parser);
}
public <U> void registerClass(Class<U> clazz, int type) {
typesManager.registerGenericClass(clazz, type);
}
public boolean isOpen() {
return !closed;
}
@Override
public void close() throws IOException {
if (closed) {
return;
}
synchronized (closeLock) {
if (closed) {
return;
}
closed = true;
}
this.databaseCleaner.stop();
synchronized (indicesAccessLock) {
this.indices.close();
}
System.out.println("Database closed.");
}
private void checkClosed() {
if (closed) {
throw new RuntimeException("Index Manager is closed.");
}
}
@Override
public long clean() {
long removedItems = indices.clean();
Iterator<EntryReference<?>> usedReferencesIterator = usedReferences.iterator();
while(usedReferencesIterator.hasNext()) {
EntryReference<?> entryReference = usedReferencesIterator.next();
if (entryReference.isClosed()) {
usedReferencesIterator.remove();
removedItems += 1;
}
}
return removedItems;
}
public class EntryReferenceTools {
private EntryReferenceTools() {
}
public <T> T read(long index, DBReader<T> reader) throws IOException {
return indices.get(index, reader);
}
public <T> IndexDetails write(long index, DBDataOutput<T> writer) throws IOException {
return indices.set(index, writer);
}
public void setFlushingAllowed(long index, boolean isFlushingAllowed) {
indices.setFlushingAllowed(index, isFlushingAllowed);
}
public <T> void setUsed(EntryReference<T> ref) {
usedReferences.add(ref);
}
}
@SuppressWarnings("unchecked")
protected <T> long calculateHash(T o) {
return ((DBTypeParser<T>) typesManager.get(o.getClass())).calculateHash(o);
}
}

455
src/main/java/org/warp/jcwdb/LightArrayList.java
View File

@ -1,455 +0,0 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import java.io.IOError;
import java.io.IOException;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Predicate;
public class LightArrayList<T> implements LightList<T> {
public final LongArrayList internalList;
private final transient JCWDatabase db;
/**
* @param db Database reference
*/
public LightArrayList(JCWDatabase db) {
this.db = db;
this.internalList = new LongArrayList();
}
/**
* @param db Database reference
* @param elements Elements to add
*/
public LightArrayList(JCWDatabase db, LongArrayList elements) {
this.db = db;
this.internalList = new LongArrayList(elements);
}
@Override
public int size() {
return internalList.size();
}
@Override
public boolean isEmpty() {
return internalList.isEmpty();
}
@Override
public boolean contains(Object o) {
if (o != null) {
for (long element : internalList) {
EntryReference<T> ref = null;
try {
ref = db.get(element);
if (o.equals(ref.getValueReadOnlyUnsafe())) {
return true;
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
return false;
}
/**
* Use iteratorReferences()
*/
@Deprecated
@SuppressWarnings("unchecked")
@Override
public Iterator<T> iterator() {
System.out.println("WARNING! YOU ARE USING iterator()! PLEASE USE ITERATORREFERENCES TO AVOID OUTOFMEMORY!");
final ArrayList<T> elements = new ArrayList<>();
for (long element : internalList) {
try {
elements.add((T) db.get(element).getValueReadOnlyUnsafe());
} catch (IOException e) {
e.printStackTrace();
}
}
return elements.iterator();
}
@SuppressWarnings("unchecked")
@Override
public Iterator<EntryReference<T>> iteratorReferences() {
final ArrayList<EntryReference<T>> elements = new ArrayList<>();
for (long element : internalList) {
try {
elements.add(db.get(element));
} catch (IOException e) {
e.printStackTrace();
}
}
return elements.iterator();
}
/**
* USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
*
* @param action
*/
@Deprecated
@Override
public void forEach(Consumer<? super T> action) {
Objects.requireNonNull(action);
for (T t : this) {
action.accept(t);
}
}
@Override
public void forEachReference(Consumer<? super EntryReference<T>> action) {
Objects.requireNonNull(action);
for (long index : this.internalList) {
try {
action.accept(db.get(index));
} catch (IOException e) {
throw new IOError(e);
}
}
}
@SuppressWarnings("unchecked")
@Override
public T[] toArray() {
final T[] elements = (T[]) new Object[internalList.size()];
for (int i = 0; i < elements.length; i++) {
try {
T element = (T) db.get(internalList.getLong(i)).getValueReadOnlyUnsafe();
elements[i] = element;
} catch (IOException e) {
e.printStackTrace();
}
}
return elements;
}
@SuppressWarnings("unchecked")
@Override
public <T1> T1[] toArray(T1[] a) {
final T1[] elements = (T1[]) new Objects[internalList.size()];
for (int i = 0; i < elements.length; i++) {
try {
elements[i] = (T1) db.get(internalList.getLong(i)).getValueReadOnlyUnsafe();
} catch (IOException e) {
e.printStackTrace();
}
}
return elements;
}
@Override
public boolean add(T o) {
EntryReference<T> ref = addEntry(o);
return ref != null;
}
@Override
public EntryReference<T> addEntry(T o) {
EntryReference<T> ref = addToDatabase(o);
if (internalList.add(ref.getIndex())) {
return ref;
} else {
return null;
}
}
@Override
public boolean remove(Object o) {
int removeIndex = indexOf(o);
if (removeIndex >= 0) {
internalList.removeLong(removeIndex);
return true;
}
return false;
}
@Override
public boolean remove(EntryReference<T> ref) {
int removeIndex = indexOfEntry(ref);
if (removeIndex >= 0) {
internalList.removeLong(removeIndex);
return true;
}
return false;
}
@Override
public boolean containsAll(Collection<?> c) {
for (Object o : c) {
int objIndex = indexOf(o);
if (objIndex < 0) {
return false;
}
}
return true;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(Collection<? extends T> c) {
boolean result = false;
for (Object o : c) {
result |= add((T) o);
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(int index, Collection<? extends T> c) {
boolean result = false;
int delta = 0;
for (Object o : c) {
add(index + delta, (T) o);
result = true;
delta++;
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeAll(Collection<?> c) {
boolean result = false;
for (Object o : c) {
result |= remove((T) o);
}
return result;
}
@Override
public boolean retainAll(Collection<?> c) {
boolean result = false;
LongArrayList collectionHashes = new LongArrayList();
ObjectArrayList<Object> collection = new ObjectArrayList<>();
collection.addAll(c);
for (Object o : c) {
collectionHashes.add(db.calculateHash(o));
}
for (int i = 0; i < internalList.size(); i++) {
long hash = internalList.getLong(i);
int positionInCollection = collectionHashes.indexOf(hash);
if (positionInCollection == -1) {
remove(collection.get(positionInCollection));
result = true;
}
}
return result;
}
@Override
public void clear() {
internalList.clear();
}
/**
* Use getReference or getReadOnlyValue
*/
@Deprecated
@Override
public T get(int index) {
return getReadOnlyValue(index);
}
@SuppressWarnings("unchecked")
public T getReadOnlyValue(int index) {
try {
return (T) db.get(internalList.getLong(index)).getValueReadOnlyUnsafe();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@Override
public EntryReference<T> getReference(int index) {
try {
return db.get(internalList.getLong(index)).cast();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@SuppressWarnings("unchecked")
@Override
public T set(int index, T element) {
EntryReference<T> ref = addToDatabase(element);
long oldIndex = internalList.set(index, ref.getIndex());
try {
ref.close();
return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnlyUnsafe();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
@Override
public void add(int index, T element) {
EntryReference<T> ref = addToDatabase(element);
internalList.add(index, ref.getIndex());
}
@SuppressWarnings("unchecked")
@Override
public T remove(int index) {
long oldIndex = internalList.removeLong(index);
try {
return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnlyUnsafe();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
@Override
public int indexOf(Object o) {
EntryReference<?> ref = addToDatabase(o);
long objToRemoveHash = ref.calculateHash();
LongArrayList hashes = new LongArrayList();
for (int i = 0; i < hashes.size(); i++) {
long hash = hashes.getLong(i);
if (objToRemoveHash == hash) {
try {
if (ref.equals(db.get(internalList.getLong(i)))) {
return i;
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
}
return -1;
}
@Override
public int indexOfEntry(EntryReference<T> ref) {
for (int i = 0; i < internalList.size(); i++) {
long index = internalList.getLong(i);
try {
EntryReference<?> ref2 = db.get(index);
if (ref.getValueReadOnlyUnsafe().equals(ref2.getValueReadOnlyUnsafe())) {
return i;
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
return -1;
}
@Override
public void appendIndex(long elementIndex) {
internalList.add(elementIndex);
}
@Override
public int lastIndexOf(Object o) {
EntryReference<T> ref = addToDatabase(o).cast();
return lastIndexOfEntry(ref);
}
@Override
public int lastIndexOfEntry(EntryReference<T> ref) {
long objToRemoveHash = ref.calculateHash();
int lastValue = -1;
for (int i = 0; i < internalList.size(); i++) {
long index2 = internalList.getLong(i);
try {
EntryReference<?> ref2 = db.get(index2);
if (objToRemoveHash == ref2.calculateHash()) {
if (ref.getValueReadOnlyUnsafe().equals(ref2.getValueReadOnlyUnsafe())) {
lastValue = i;
}
}
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
}
return lastValue;
}
@Deprecated
@Override
public ListIterator<T> listIterator() {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public ListIterator<T> listIterator(int index) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public List<T> subList(int fromIndex, int toIndex) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
private <U> EntryReference<U> addToDatabase(U obj) {
EntryReference<U> ref;
try {
ref = db.add(obj);
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
return ref;
}
@Deprecated
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((internalList == null) ? 0 : internalList.hashCode());
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeIf(Predicate<? super T> filter) {
Objects.requireNonNull(filter);
boolean removed = false;
for (int i = 0; i < internalList.size(); ) {
T obj;
try {
obj = ((EntryReference<T>) (db.get(internalList.getLong(i)).cast())).getValueReadOnlyUnsafe();
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
if (filter.test(obj)) {
internalList.removeLong(i);
removed = true;
} else {
i++;
}
}
return removed;
}
@Override
public String toString() {
return "LightArrayList{" +
"internalList=" + internalList +
", db=" + db +
'}';
}
}

607
src/main/java/org/warp/jcwdb/LightBigList.java
View File

@ -1,607 +0,0 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import java.io.IOException;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.function.Supplier;
public class LightBigList<T> implements LightList<T>, Saveable {
public static final int MAX_ELEMENTS_PER_CHUNK = 200000;
public final LongArrayList chunks;
public final IntArrayList chunkSizes;
private final JCWDatabase db;
private LightArrayList<T> cachedChunk;
private EntryReference<LightArrayList<T>> cachedChunkRef;
private long cachedChunkIndex = -1;
private int cachedChunkNumber = -1;
private final Object cachedChunkLock = new Object();
/**
* @param db Database reference
*/
public LightBigList(JCWDatabase db) {
this.db = db;
this.chunks = new LongArrayList();
this.chunkSizes = new IntArrayList();
}
/**
* @param db Database reference
* @param elements Elements to add
*/
public LightBigList(JCWDatabase db, LongArrayList elements) {
this.db = db;
this.chunks = new LongArrayList();
this.chunkSizes = new IntArrayList();
elements.forEach((long element) -> {
this.appendIndex(element);
});
}
public LightBigList(JCWDatabase db, LongArrayList chunks, IntArrayList chunkSizes) {
this.db = db;
this.chunks = chunks;
this.chunkSizes = chunkSizes;
if (this.chunks.size() > 0) {
prepareAccessToChunk(0);
}
}
private void prepareAccessToChunk(int chunkNumber) {
if (this.cachedChunkRef != null) {
this.cachedChunkRef.save();
}
this.cachedChunkNumber = chunkNumber;
this.cachedChunkIndex = this.chunks.getLong(chunkNumber);
try {
this.cachedChunkRef = db.get(this.cachedChunkIndex);
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
this.cachedChunk = this.cachedChunkRef.getValueReadOnlyUnsafe();
}
/**
* Append an index to the first free chunk
* @param elementIndex
*/
public void appendIndex(long elementIndex) {
for (int i = 0; i < chunks.size(); i++) {
final int chunkNumber = i;
if (MAX_ELEMENTS_PER_CHUNK - chunkSizes.getInt(i) > 0) {
synchronized (cachedChunkLock) {
if (cachedChunkNumber != i) {
prepareAccessToChunk(i);
}
cachedChunk.appendIndex(elementIndex);
chunkSizes.set(chunkNumber, cachedChunk.size());
return;
}
}
}
LightList<T> newChunk = new LightArrayList<>(db);
newChunk.appendIndex(elementIndex);
long newChunkIndex;
try {
newChunkIndex = db.add(newChunk).getIndex();
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
chunks.add(newChunkIndex);
chunkSizes.add(1);
}
/**
* Get the elements count
* @return the size of the list
*/
@Override
public int size() {
int size = 0;
for (int chunkSize : this.chunkSizes) {
size += chunkSize;
}
return size;
}
/**
*
* @return the count of chunks
*/
public int chunksCount() {
return this.chunkSizes.size();
}
/**
* Check if the list is empty
* @return true if the list is empty
*/
@Override
public boolean isEmpty() {
return this.size() <= 0;
}
@Override
public boolean contains(Object o) {
if (o != null) {
for (long chunkIndex : chunks) {
try {
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
LightArrayList<T> chunk = chunkRef.getValueReadOnlyUnsafe();
if (chunk.contains(o)) {
return true;
}
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
}
return false;
}
/**
* Use iteratorReferences()
*/
@Deprecated
@Override
public Iterator<T> iterator()
{
throw new RuntimeException("iterator() isn't implemented!");
}
@Deprecated
@Override
public Iterator<EntryReference<T>> iteratorReferences() {
throw new RuntimeException("iteratorReferences() isn't implemented!");
}
/**
* USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
*
* @param action
*/
@Deprecated
@Override
public void forEach(Consumer<? super T> action) {
throw new RuntimeException("forEach() isn't implemented! Use forEachReferences() instead");
}
@Override
public void forEachReference(Consumer<? super EntryReference<T>> action) {
Objects.requireNonNull(action);
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
synchronized (cachedChunkLock) {
if (cachedChunkNumber != i) {
prepareAccessToChunk(i);
}
cachedChunk.forEachReference(action);
}
}
}
/**
* toArray() isn't implemented! DO NOT USE IT.
* @return
*/
@SuppressWarnings("unchecked")
@Deprecated
@Override
public T[] toArray() {
if (true) {
throw new RuntimeException("toArray() isn't implemented!");
}
T[] result = (T[]) new Object[this.size()];
long currentOffset = 0;
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
final int currentChunkSize = chunkSizes.getInt(i);
final long chunkStartOffset = currentOffset;
currentOffset += currentChunkSize;
// Get chunk index
final long chunkIndex = chunks.getLong(i);
try {
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
LightArrayList<T> chunk = chunkRef.getValueReadOnlyUnsafe();
for (int i1 = 0; i1 < chunk.size(); i1++) {
result[(int)(chunkStartOffset + i1)] = chunk.get(i);
}
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public <T1> T1[] toArray(T1[] a) {
throw new RuntimeException("toArray() isn't implemented!");
}
/**
* Use addEntry(o)
* @param o
* @return
*/
@Deprecated
@Override
public boolean add(T o) {
EntryReference<T> ref = addEntry(o);
return ref != null;
}
@Override
public EntryReference<T> addEntry(T o) {
EntryReference<T> ref = addToDatabase(o);
appendIndex(ref.getIndex());
return ref;
}
@Override
public boolean remove(Object o) {
final int removeOffset = indexOf(o);
return removeAt(removeOffset) != null;
}
@Override
public boolean remove(EntryReference<T> ref) {
final int removeOffset = indexOfEntry(ref);
return removeAt(removeOffset) != null;
}
private T removeAt(int removeOffset) {
final VariableWrapper<T> result = new VariableWrapper<>(null);
long currentOffset = 0;
if (removeOffset >= 0) {
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
final int currentChunkSize = chunkSizes.getInt(i);
final long chunkStartOffset = currentOffset;
currentOffset += currentChunkSize;
// If the offset to remove is in the current chunk
if (currentOffset > removeOffset) {
// Get chunk index
final long chunkIndex = chunks.getLong(i);
// Get the offset relative to the current chunk
final int relativeOffset = (int) (removeOffset - chunkStartOffset);
if (relativeOffset < 0) {
return null;
}
try {
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
chunkRef.editValue((chunk) -> {
result.var = chunk.remove(relativeOffset);
});
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
chunkSizes.set(removeOffset, currentChunkSize - 1);
break;
}
}
return result.var;
}
return null;
}
@Override
public boolean containsAll(Collection<?> c) {
for (Object o : c) {
int objIndex = indexOf(o);
if (objIndex < 0) {
return false;
}
}
return true;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(Collection<? extends T> c) {
boolean result = false;
for (Object o : c) {
result |= add((T) o);
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean addAll(int index, Collection<? extends T> c) {
boolean result = false;
int delta = 0;
for (Object o : c) {
add(index + delta, (T) o);
result = true;
delta++;
}
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeAll(Collection<?> c) {
boolean result = false;
for (Object o : c) {
result |= remove((T) o);
}
return result;
}
@Override
public boolean retainAll(Collection<?> c) {
boolean result = false;
LongArrayList collectionHashes = new LongArrayList();
ObjectArrayList<Object> collection = new ObjectArrayList<>();
collection.addAll(c);
for (Object o : c) {
collectionHashes.add(db.calculateHash(o));
}
for (int i = 0; i < chunks.size(); i++) {
long hash = chunks.getLong(i);
int positionInCollection = collectionHashes.indexOf(hash);
if (positionInCollection == -1) {
remove(collection.get(positionInCollection));
result = true;
}
}
return result;
}
@Override
public void clear() {
chunks.clear();
chunkSizes.clear();
}
/**
* Use getReference or getReadOnlyValue
*/
@Deprecated
@Override
public T get(int index) {
return getReadOnlyValue(index);
}
@SuppressWarnings("unchecked")
public T getReadOnlyValue(int index) {
try {
return (T) db.get(chunks.getLong(index)).getValueReadOnlyUnsafe();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
@Override
public EntryReference<T> getReference(int index) {
return getReferenceUnsafe(index, true);
}
@Override
public EntryReference<T> getReferenceOrInitialize(int index, Supplier<T> initializer) {
EntryReference<T> value = getReferenceUnsafe(index, false);
if (value != null) {
return value;
} else {
T initializedData = initializer.get();
EntryReference<T> initializedDataRef = addToDatabase(initializedData);
return set(index, initializedDataRef);
}
}
private EntryReference<T> getReferenceUnsafe(int index, boolean throwError) {
try {
return db.get(chunks.getLong(index)).cast();
} catch (IOException e) {
if (throwError) e.printStackTrace();
return null;
}
}
@SuppressWarnings("unchecked")
@Override
public T set(int setOffset, final T element) {
return set(setOffset, addToDatabase(element)).getValueReadOnlyUnsafe();
}
@SuppressWarnings("unchecked")
@Override
public EntryReference<T> set(int setOffset, final EntryReference<T> element) {
long nextChunkOffset = 0;
VariableWrapper<EntryReference<T>> wrapper = new VariableWrapper<>(null);
if (setOffset >= 0) {
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
final int currentChunkSize = chunkSizes.getInt(i);
final long chunkStartOffset = nextChunkOffset;
nextChunkOffset += currentChunkSize;
// If the offset to remove is in the current chunk
if (nextChunkOffset > setOffset) {
// Get chunk index
final long chunkIndex = chunks.getLong(i);
// Get the offset relative to the current chunk
final int relativeOffset = (int) (setOffset - chunkStartOffset);
if (relativeOffset < 0) {
throw new NullPointerException("Relative Offset < 0");
}
try {
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
chunkRef.editValue((chunk) -> {
wrapper.var = chunk.set(relativeOffset, element);
});
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
break;
}
}
return wrapper.var;
}
return null;
}
@Override
public void add(int index, T element) {
throw new RuntimeException("add() isn't implemented!");
}
@SuppressWarnings("unchecked")
@Override
public T remove(int index) {
return this.removeAt(index);
}
@Override
public int indexOf(Object o) {
EntryReference<T> ref = addToDatabase(o).cast();
return indexOfEntry(ref);
}
@Override
public int indexOfEntry(EntryReference<T> ref) {
int currentOffset = 0;
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
try {
final int currentChunkSize = chunkSizes.getInt(i);
// If the offset to remove is in the current chunk
// Get chunk index
final long chunkIndex = chunks.getLong(i);
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
final int foundIndex = chunkRef.getValueReadOnlyUnsafe().indexOfEntry(ref);
if (foundIndex >= 0) {
return currentOffset + foundIndex;
}
currentOffset += currentChunkSize;
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
return -1;
}
@Override
public int lastIndexOf(Object o) {
return lastIndexOfEntry(addToDatabase(o).cast());
}
@Override
public int lastIndexOfEntry(EntryReference<T> ref) {
int currentOffset = 0;
// Iterate through all chunks
for (int i = chunks.size() - 1; i >= 0; i--) {
try {
final int currentChunkSize = chunkSizes.getInt(i);
// If the offset to remove is in the current chunk
// Get chunk index
final long chunkIndex = chunks.getLong(i);
EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
final int foundIndex = chunkRef.getValueReadOnlyUnsafe().lastIndexOfEntry(ref);
if (foundIndex >= 0) {
return currentOffset + foundIndex;
}
currentOffset += currentChunkSize;
} catch (IOException ex) {
throw (NullPointerException) new NullPointerException().initCause(ex);
}
}
return -1;
}
@Deprecated
@Override
public ListIterator<T> listIterator() {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public ListIterator<T> listIterator(int index) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
@Deprecated
@Override
public List<T> subList(int fromIndex, int toIndex) {
// TODO: implement
throw new RuntimeException("Not implemented!");
}
private <U> EntryReference<U> addToDatabase(U obj) {
EntryReference<U> ref;
try {
ref = db.add(obj);
} catch (IOException e) {
throw (NullPointerException) new NullPointerException().initCause(e);
}
return ref;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((chunks == null) ? 0 : chunks.hashCode());
return result;
}
@SuppressWarnings("unchecked")
@Override
public boolean removeIf(Predicate<? super T> filter) {
Objects.requireNonNull(filter);
boolean result = false;
// Iterate through all chunks
for (int i = 0; i < chunks.size(); i++) {
synchronized (cachedChunkLock) {
if (cachedChunkNumber != i) {
prepareAccessToChunk(i);
}
if (cachedChunk.removeIf(filter)) {
result = true;
chunkSizes.set(cachedChunkNumber, cachedChunk.size());
}
}
}
return result;
}
@Override
public String toString() {
return "LightBigList{" +
"chunks=" + chunks +
", chunkSizes=" + chunkSizes +
", db=" + db +
'}';
}
@Override
public void save() {
if (this.cachedChunkRef != null) {
this.cachedChunkRef.save();
}
}
@Override
public void saveAndFlush() {
save();
}
}

29
src/main/java/org/warp/jcwdb/LightList.java
View File

@ -1,29 +0,0 @@
package org.warp.jcwdb;
import java.util.Iterator;
import java.util.List;
import java.util.function.Consumer;
import java.util.function.Supplier;
public interface LightList<T> extends List<T> {
Iterator<EntryReference<T>> iteratorReferences();
void forEachReference(Consumer<? super EntryReference<T>> action);
EntryReference<T> addEntry(T o);
EntryReference<T> set(int setOffset, final EntryReference<T> element);
boolean remove(EntryReference<T> ref);
EntryReference<T> getReference(int index);
EntryReference<T> getReferenceOrInitialize(int index, Supplier<T> initializer);
int indexOfEntry(EntryReference<T> ref);
int lastIndexOfEntry(EntryReference<T> ref);
void appendIndex(long elementIndex);
}

92
src/main/java/org/warp/jcwdb/MixedIndexDatabase.java
View File

@ -1,92 +0,0 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.nio.file.Path;
public class MixedIndexDatabase implements IndexManager {
private final FileIndexManager fileIndices;
private final CacheIndexManager cacheIndices;
public MixedIndexDatabase(Path dataFile, Path metadataFile) throws IOException {
this.fileIndices = new FileIndexManager(dataFile, metadataFile);
this.cacheIndices = new CacheIndexManager();
}
@Override
public <T> T get(long index, DBReader<T> reader) throws IOException {
if (cacheIndices.has(index)) {
return cacheIndices.get(index, reader);
} else {
return fileIndices.get(index, reader);
}
}
@Override
public int getType(long index) throws IOException {
if (cacheIndices.has(index)) {
return cacheIndices.getType(index);
} else {
return fileIndices.getType(index);
}
}
@Override
public long getHash(long index) throws IOException {
if (cacheIndices.has(index)) {
return cacheIndices.getHash(index);
} else {
return fileIndices.getHash(index);
}
}
@Override
public <T> long add(DBDataOutput<T> writer) throws IOException {
return fileIndices.add(writer);
}
@Override
public <T> FullIndexDetails addAndGetDetails(DBDataOutput<T> writer) throws IOException {
return fileIndices.addAndGetDetails(writer);
}
@Override
public <T> IndexDetails set(long index, DBDataOutput<T> writer) throws IOException {
if (cacheIndices.has(index)) {
return cacheIndices.set(index, writer);
} else {
return fileIndices.set(index, writer);
}
}
@Override
public void setFlushingAllowed(long index, boolean isFlushingAllowed) {
if (cacheIndices.has(index)) {
cacheIndices.setFlushingAllowed(index, isFlushingAllowed);
} else {
fileIndices.setFlushingAllowed(index, isFlushingAllowed);
}
}
@Override
public void delete(long index) throws IOException {
cacheIndices.delete(index);
fileIndices.delete(index);
}
@Override
public boolean has(long index) {
return cacheIndices.has(index) || fileIndices.has(index);
}
@Override
public void close() throws IOException {
// TODO: move all cached indices to filesIndices before closing.
this.cacheIndices.close();
this.fileIndices.close();
}
@Override
public long clean() {
return fileIndices.clean()
+ cacheIndices.clean();
}
}

14
src/main/java/org/warp/jcwdb/NoParserFoundException.java
View File

@ -1,14 +0,0 @@
package org.warp.jcwdb;
public class NoParserFoundException extends NullPointerException {
public NoParserFoundException(String string) {
super(string);
}
/**
*
*/
private static final long serialVersionUID = 701010818132241139L;
}

8
src/main/java/org/warp/jcwdb/Saveable.java
View File

@ -1,8 +0,0 @@
package org.warp.jcwdb;
import java.io.IOException;
public interface Saveable {
void save();
void saveAndFlush();
}

59
src/main/java/org/warp/jcwdb/TypesManager.java
View File

@ -1,59 +0,0 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
public class TypesManager {
private final Int2ObjectMap<DBTypeParser<?>> types;
private final Object2ObjectMap<Class<?>, DBTypeParser<?>> typesByClass;
private DBTypedObjectParser<?> fallbackParser;
public TypesManager(JCWDatabase db) {
types = new Int2ObjectOpenHashMap<>();
typesByClass = new Object2ObjectOpenHashMap<>();
DBStandardTypes.registerStandardTypes(db, this);
}
public <T> void registerType(Class<T> clazz, int type, DBTypeParser<T> parser) {
this.types.put(type, parser);
this.typesByClass.put(clazz, parser);
}
/**
* Use this method with the most used classes to save disk space.
* @param clazz
* @param id
* @param <T>
*/
public <T> void registerGenericClass(Class<T> clazz, int id) {
this.fallbackParser.registerClass(clazz, id);
}
public void registerTypeFallback(DBTypedObjectParser<?> parser) {
this.fallbackParser = parser;
}
public <T> DBTypeParser<T> get(int type) {
if (types.containsKey(type) == false) {
if (fallbackParser == null) {
throw new NoParserFoundException("The type " + type + " can't be parsed.");
} else {
return fallbackParser.cast();
}
}
return types.get(type).cast();
}
public <T> DBTypeParser<T> get(Class<T> type) {
if (typesByClass.containsKey(type) == false) {
if (fallbackParser == null) {
throw new NoParserFoundException("The class " + type.getSimpleName() + " can't be parsed.");
} else {
return fallbackParser.cast();
}
}
return typesByClass.get(type).cast();
}
}

2
src/main/java/org/warp/jcwdb/VariableWrapper.java
View File

@ -1,7 +1,5 @@
package org.warp.jcwdb;
import java.nio.channels.SeekableByteChannel;
public class VariableWrapper<T> {
public T var;

15
src/main/java/org/warp/jcwdb/ann/ConsumerWithIO.java Normal file
View File

@ -0,0 +1,15 @@
package org.warp.jcwdb.ann;
import java.io.IOException;
import java.util.Objects;
@FunctionalInterface
public interface ConsumerWithIO<T> {
void accept(T t) throws IOException;
default ConsumerWithIO<T> andThen(ConsumerWithIO<? super T> after) {
Objects.requireNonNull(after);
return (T t) -> { accept(t); after.accept(t); };
}
}

13
src/main/java/org/warp/jcwdb/ann/DBDataType.java Normal file
View File

@ -0,0 +1,13 @@
package org.warp.jcwdb.ann;
public enum DBDataType {
DATABASE_OBJECT,
OBJECT,
BOOLEAN,
BYTE,
SHORT,
CHAR,
INTEGER,
LONG,
REFERENCES_LIST
}

13
src/main/java/org/warp/jcwdb/ann/DBField.java Normal file
View File

@ -0,0 +1,13 @@
package org.warp.jcwdb.ann;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.FIELD})
public @interface DBField {
int id();
DBDataType type() default DBDataType.OBJECT;
}

13
src/main/java/org/warp/jcwdb/ann/DBPropertyGetter.java Normal file
View File

@ -0,0 +1,13 @@
package org.warp.jcwdb.ann;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
public @interface DBPropertyGetter {
int id();
DBDataType type() default DBDataType.OBJECT;
}

13
src/main/java/org/warp/jcwdb/ann/DBPropertySetter.java Normal file
View File

@ -0,0 +1,13 @@
package org.warp.jcwdb.ann;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
public @interface DBPropertySetter {
int id();
DBDataType type() default DBDataType.OBJECT;
}

19
src/main/java/org/warp/jcwdb/ann/RunnableWithIO.java Normal file
View File

@ -0,0 +1,19 @@
package org.warp.jcwdb.ann;
import java.io.IOException;
@FunctionalInterface
public interface RunnableWithIO {
/**
* When an object implementing interface <code>Runnable</code> is used
* to create a thread, starting the thread causes the object's
* <code>run</code> method to be called in that separately executing
* thread.
* <p>
* The general contract of the method <code>run</code> is that it may
* take any action whatsoever.
*
* @see java.lang.Thread#run()
*/
public abstract void run() throws IOException;
}

8
src/main/java/org/warp/jcwdb/ann/SupplierWithIO.java Normal file
View File

@ -0,0 +1,8 @@
package org.warp.jcwdb.ann;
import java.io.IOException;
@FunctionalInterface
public interface SupplierWithIO<T> {
public T getWithIO() throws IOException;
}

9
src/main/java/org/warp/jcwdb/exampleimpl/Animal.java
View File

@ -1,9 +0,0 @@
package org.warp.jcwdb.exampleimpl;
public abstract class Animal {
protected int legsCount;
public static boolean hasFourLegs(Animal a) {
return a.legsCount == 4;
}
}

101
src/main/java/org/warp/jcwdb/exampleimpl/App.java
View File

@ -1,101 +0,0 @@
package org.warp.jcwdb.exampleimpl;
import org.warp.jcwdb.EntryReference;
import org.warp.jcwdb.JCWDatabase;
import org.warp.jcwdb.LightList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import it.unimi.dsi.fastutil.objects.ObjectList;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.function.Predicate;
public class App {
static long time3;
public static void main(String[] args) throws IOException {
if (args.length > 2 && Boolean.parseBoolean(args[2])) {
Files.delete(Paths.get(args[0]));
Files.delete(Paths.get(args[1]));
}
System.out.println("Loading database...");
long time0 = System.currentTimeMillis();
JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1]));
db.registerClass(StrangeAnimal.class, 0);
try {
long time01 = System.currentTimeMillis();
System.out.println("Time elapsed: " + (time01 - time0));
System.out.println("Loading root...");
EntryReference<LightList<Animal>> rootRef = db.getRoot(Animal.class);
rootRef.editValue((root, saver) -> {
long time1 = System.currentTimeMillis();
System.out.println("Time elapsed: " + (time1 - time01));
System.out.println("Root size: " + root.size());
System.out.println("Root:");
// for (int i = 0; i < root.size(); i++) {
// System.out.println(" - " + root.get(i));
// }
long prectime = System.currentTimeMillis();
for (int i = 0; i < 2000000/* 2000000 */; i++) {
Animal animal = new StrangeAnimal(i % 40);
root.addEntry(animal);
if (i > 0 && i % 200000 == 0) {
long precprectime = prectime;
prectime = System.currentTimeMillis();
System.out.println("Element " + i + " (" + (prectime - precprectime) + "ms)" + " Total Time: " + (prectime - time1));
}
}
long time2 = System.currentTimeMillis();
saver.save();
System.out.println("Root size: " + root.size());
System.out.println("Time elapsed: " + (time2 - time1));
System.out.println("Used memory: "
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
long time2_0 = System.currentTimeMillis();
System.out.println("Filtering strings...");
long oldSize = root.size();
root.removeIf(Animal::hasFourLegs);
long time2_1 = System.currentTimeMillis();
System.out.println("RemoveIf(x) removed items: " + (oldSize - root.size()));
System.out.println("Time elapsed: " + (time2_1 - time2_0));
ObjectList<Animal> results = new ObjectArrayList<>();
System.out.println("Retrieving items...");
root.forEachReference((valueReference) -> {
Animal value = valueReference.getValueReadOnlyUnsafe();
if (Animal.hasFourLegs(value)) {
results.add(value);
}
//System.out.println("val:" + value);
});
long time2_2 = System.currentTimeMillis();
System.out.println("Matches: " + results.size());
System.out.println("Time elapsed: " + (time2_2 - time2_1));
System.out.println("Used memory: "
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
System.out.println("Cleaning database (to reduce the amount of used memory and detect memory leaks)...");
db.clean();
time3 = System.currentTimeMillis();
System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
System.out.println("Time elapsed: " + (time3 - time2_2));
System.out.println("Saving database...");
System.out.println("Root size: " + root.size());
});
db.close();
long time4 = System.currentTimeMillis();
System.out.println("Time elapsed: " + (time4 - time3));
} catch (Exception ex) {
ex.printStackTrace();
} finally {
if (db.isOpen()) {
db.close();
}
}
}
public static <T> Predicate<T> not(Predicate<T> t) {
return t.negate();
}
}

12
src/main/java/org/warp/jcwdb/exampleimpl/Cat.java
View File

@ -1,12 +0,0 @@
package org.warp.jcwdb.exampleimpl;
public class Cat extends Animal {
public Cat() {
this.legsCount = 12;
}
@Override
public String toString() {
return "Cat [legsCount=" + legsCount + "]";
}
}

12
src/main/java/org/warp/jcwdb/exampleimpl/Dog.java
View File

@ -1,12 +0,0 @@
package org.warp.jcwdb.exampleimpl;
public class Dog extends Animal {
public Dog() {
this.legsCount = 4;
}
@Override
public String toString() {
return "Dog [legsCount=" + legsCount + "]";
}
}

15
src/main/java/org/warp/jcwdb/exampleimpl/StrangeAnimal.java
View File

@ -1,15 +0,0 @@
package org.warp.jcwdb.exampleimpl;
public class StrangeAnimal extends Animal {
public StrangeAnimal() {
super();
}
public StrangeAnimal(int legs) {
super();
this.legsCount = legs;
}
@Override
public String toString() {
return "StrangeAnimal [legsCount=" + legsCount + "]";
}
}

18
src/test/java/org/warp/jcwdb/AppTest.java
View File

@ -1,18 +0,0 @@
package org.warp.jcwdb;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
/**
* Unit test for simple App.
*/
public class AppTest {
/**
* Rigorous Test :-)
*/
@Test
public void shouldAnswerWithTrue() {
assertTrue(true);
}
}

51
src/test/java/org/warp/jcwdb/FileAllocatorTest.java
View File

@ -1,51 +0,0 @@
package org.warp.jcwdb;
import org.junit.Test;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class FileAllocatorTest {
@Test
public void shouldAllocateAtZero() throws IOException {
Path tempFile = Files.createTempFile("", "");
SeekableByteChannel byteCh = Files.newByteChannel(tempFile);
FileAllocator allocator = new FileAllocator(byteCh);
long offset1 = allocator.allocate(512);
assertEquals(0, offset1);
}
@Test
public void shouldAllocateAt512() throws IOException {
Path tempFile = Files.createTempFile("", "");
SeekableByteChannel byteCh = Files.newByteChannel(tempFile, StandardOpenOption.WRITE);
byteCh.write(ByteBuffer.wrap(new byte[512]));
FileAllocator allocator = new FileAllocator(byteCh);
long offset1 = allocator.allocate(512);
assertEquals(512, offset1);
}
@Test
public void shouldAllocateUnusedSpace() throws IOException {
Path tempFile = Files.createTempFile("", "");
SeekableByteChannel byteCh = Files.newByteChannel(tempFile, StandardOpenOption.WRITE);
FileAllocator allocator = new FileAllocator(byteCh);
long offset1 = allocator.allocate(512);
allocator.markFree(offset1, 512);
long offset2 = allocator.allocate(128);
long offset3 = allocator.allocate(512-128);
long offset4 = allocator.allocate(128);
assertEquals(0, offset2);
assertEquals(128, offset3);
assertEquals(512, offset4);
}
}

90
src/test/java/org/warp/jcwdb/tests/NDBMultipleEnhancedObjects.java Normal file
View File

@ -0,0 +1,90 @@
package org.warp.jcwdb.tests;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.warp.cowdb.EnhancedObject;
import org.warp.cowdb.IDatabase;
import org.warp.jcwdb.ann.DBDataType;
import org.warp.jcwdb.ann.DBField;
import org.warp.jcwdb.ann.DBPropertyGetter;
import org.warp.jcwdb.ann.DBPropertySetter;
import org.warp.jcwdb.utils.NTestUtils;
import java.io.IOException;
public class NDBMultipleEnhancedObjects {
private NTestUtils.WrappedDb db;
private RootTwoClasses root;
@Before
public void setUp() throws Exception {
db = NTestUtils.wrapDb().create((db) -> {
root = db.get().loadRoot(RootTwoClasses.class);
});
root.class1 = new NTestUtils.RootClass(db.get());
db.setRootClassValues(root.class1);
root.class2 = new NTestUtils.RootClass(db.get());
db.setRootClassValues(root.class2);
root.setClass3(new NTestUtils.RootClass(db.get()));
db.setRootClassValues(root.getClass3());
root.setClass4(new NTestUtils.RootClass(db.get()));
db.setRootClassValues(root.getClass4());
db.closeAndReopen();
}
@Test
public void shouldMatchMultipleEnhancedObjects() {
db.testRootClassValues(root.class1);
db.testRootClassValues(root.class2);
db.testRootClassValues(root.getClass3());
db.testRootClassValues(root.getClass4());
}
@After
public void tearDown() throws Exception {
db.delete();
}
public static class RootTwoClasses extends EnhancedObject {
@DBField(id = 0, type = DBDataType.DATABASE_OBJECT)
public NTestUtils.RootClass class1;
@DBField(id = 1, type = DBDataType.DATABASE_OBJECT)
public NTestUtils.RootClass class2;
public RootTwoClasses() {
super();
}
public RootTwoClasses(IDatabase database) throws IOException {
super(database);
}
@DBPropertyGetter(id = 0, type = DBDataType.DATABASE_OBJECT)
public NTestUtils.RootClass getClass3() {
return getProperty();
}
@DBPropertySetter(id = 0, type = DBDataType.DATABASE_OBJECT)
public void setClass3(NTestUtils.RootClass value) {
setProperty(value);
}
@DBPropertyGetter(id = 1, type = DBDataType.DATABASE_OBJECT)
public NTestUtils.RootClass getClass4() {
return getProperty();
}
@DBPropertySetter(id = 1, type = DBDataType.DATABASE_OBJECT)
public void setClass4(NTestUtils.RootClass value) {
setProperty(value);
}
@Override
public void initialize() throws IOException {
}
}
}

32
src/test/java/org/warp/jcwdb/utils/ConsumerWithIO.java Normal file
View File

@ -0,0 +1,32 @@
package org.warp.jcwdb.utils;
import java.io.IOException;
import java.util.Objects;
@FunctionalInterface
public interface ConsumerWithIO<T> {
/**
* Performs this operation on the given argument.
*
* @param t the input argument
*/
void accept(T t) throws IOException;
/**
* Returns a composed {@code Consumer} that performs, in sequence, this
* operation followed by the {@code after} operation. If performing either
* operation throws an exception, it is relayed to the caller of the
* composed operation. If performing this operation throws an exception,
* the {@code after} operation will not be performed.
*
* @param after the operation to perform after this operation
* @return a composed {@code Consumer} that performs in sequence this
* operation followed by the {@code after} operation
* @throws NullPointerException if {@code after} is null
*/
default ConsumerWithIO<T> andThen(ConsumerWithIO<? super T> after) {
Objects.requireNonNull(after);
return (T t) -> { accept(t); after.accept(t); };
}
}

27
src/test/java/org/warp/jcwdb/utils/NSimplestClass.java Normal file
View File

@ -0,0 +1,27 @@
package org.warp.jcwdb.utils;
import org.warp.cowdb.Database;
import org.warp.cowdb.EnhancedObject;
import org.warp.jcwdb.ann.DBDataType;
import org.warp.jcwdb.ann.DBField;
import java.io.IOException;
public class NSimplestClass extends EnhancedObject {
@DBField(id = 0, type = DBDataType.BOOLEAN)
public boolean field1;
public NSimplestClass() {
}
public NSimplestClass(Database database) throws IOException {
super(database);
}
@Override
public void initialize() throws IOException {
field1 = true;
}
}

383
src/test/java/org/warp/jcwdb/utils/NTestUtils.java Normal file
View File

@ -0,0 +1,383 @@
package org.warp.jcwdb.utils;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.warp.cowdb.Database;
import org.warp.cowdb.EnhancedObject;
import org.warp.cowdb.IDatabase;
import org.warp.jcwdb.ann.*;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Comparator;
import static org.junit.Assert.*;
public class NTestUtils {
public static WrappedDb wrapDb() {
return new WrappedDb();
}
public static class WrappedDb {
private Database db;
private Path tempDir;
private RunnableWithIO r;
private WrappedDb() {
}
public WrappedDb create() throws IOException {
tempDir = Files.createTempDirectory("tests-");
db = openDatabase();
if (r != null) {
r.run();
}
return this;
}
public WrappedDb create(ConsumerWithIO<NTestUtils.WrappedDb> r) throws IOException {
this.r = () -> r.accept(WrappedDb.this);
this.create();
return this;
}
private Database openDatabase() throws IOException {
return new Database(tempDir.resolve(Paths.get("data.db")), tempDir.resolve(Paths.get("blocks.dat")), tempDir.resolve(Paths.get("references.dat")));
}
public void delete() throws IOException {
db.close();
deleteDir(tempDir);
}
public Database get() {
return db;
}
private void deleteDir(Path p) throws IOException {
Files.walk(p)
.sorted(Comparator.reverseOrder())
.map(Path::toFile)
.forEach(File::delete);
}
public void closeAndReopen() throws IOException {
db.close();
db = openDatabase();
r.run();
}
public void setRootClassValues(RootClass root) throws IOException {
setRootClassFields(root);
setRootClassProperties(root);
}
public void setRootClassFields(RootClass root) throws IOException {
root.field1 = true;
root.field2 = 2;
root.field3 = 3;
root.field4 = 4;
root.field5 = 5;
root.field6 = 6;
root.field7 = "Test";
root.field8 = new LongArrayList();
root.field8.add(0);
root.field8.add(1);
root.field8.add(2);
root.field8.add(Long.MAX_VALUE/2);
root.field8.add(Long.MIN_VALUE/2);
root.field8.add(Long.MAX_VALUE);
root.field8.add(Long.MIN_VALUE);
root.field9 = new NSimplestClass(db);
root.field9.field1 = true;
}
public void setRootClassProperties(RootClass root) throws IOException {
root.set1(true);
root.set2((byte)2);
root.set3((short)3);
root.set4((char)4);
root.set5(5);
root.set6(6);
root.set7("Test");
LongArrayList lArrayList = new LongArrayList();
lArrayList.add(0);
lArrayList.add(1);
lArrayList.add(2);
lArrayList.add(Long.MAX_VALUE/2);
lArrayList.add(Long.MIN_VALUE/2);
lArrayList.add(Long.MAX_VALUE);
lArrayList.add(Long.MIN_VALUE);
root.set8(lArrayList);
NSimplestClass simplestClass9 = new NSimplestClass(db);
simplestClass9.field1 = true;
root.set9(simplestClass9);
}
public void testRootClassValues(RootClass root) {
testRootClassFields(root);
testRootClassProperties(root);
}
public void testRootClassFields(RootClass root) {
shouldGetFieldBoolean(root);
shouldGetFieldByte(root);
shouldGetFieldShort(root);
shouldGetFieldCharacter(root);
shouldGetFieldInteger(root);
shouldGetFieldLong(root);
shouldGetFieldObject(root);
shouldGetFieldUID(root);
shouldGetFieldDBObject(root);
}
public void testRootClassProperties(RootClass root) {
shouldGetPropertyBoolean(root);
shouldGetPropertyByte(root);
shouldGetPropertyShort(root);
shouldGetPropertyCharacter(root);
shouldGetPropertyInteger(root);
shouldGetPropertyLong(root);
shouldGetPropertyObject(root);
shouldGetPropertyUID(root);
shouldGetPropertyDBObject(root);
}
private void shouldGetFieldBoolean(RootClass root) {
assertTrue(root.field1);
}
private void shouldGetPropertyBoolean(RootClass root) {
assertTrue(root.get1());
}
private void shouldGetFieldByte(RootClass root) {
assertEquals(2, root.field2);
}
private void shouldGetPropertyByte(RootClass root) {
assertEquals(2, root.get2());
}
private void shouldGetFieldShort(RootClass root) {
assertEquals(3, root.field3);
}
private void shouldGetPropertyShort(RootClass root) {
assertEquals(3, root.get3());
}
private void shouldGetFieldCharacter(RootClass root) {
assertEquals(4, root.field4);
}
private void shouldGetPropertyCharacter(RootClass root) {
assertEquals(4, root.get4());
}
private void shouldGetFieldInteger(RootClass root) {
assertEquals(5, root.field5);
}
private void shouldGetPropertyInteger(RootClass root) {
assertEquals(5, root.get5());
}
private void shouldGetFieldLong(RootClass root) {
assertEquals(6, root.field6);
}
private void shouldGetPropertyLong(RootClass root) {
assertEquals(6, root.get6());
}
private void shouldGetFieldObject(RootClass root) {
shouldGetObject(root.field7);
}
private void shouldGetPropertyObject(RootClass root) {
shouldGetObject(root.get7());
}
private void shouldGetFieldDBObject(RootClass root) {
assertTrue(root.field9.field1);
}
private void shouldGetPropertyDBObject(RootClass root) {
assertTrue(root.get9().field1);
}
private void shouldGetObject(String val) {
assertNotNull(val);
assertEquals("Test", val);
}
private void shouldGetDBObject(NSimplestClass val) {
assertNotNull(val);
assertTrue(val.field1);
}
private void shouldGetFieldUID(RootClass root) {
shouldGetUID(root.field8);
}
private void shouldGetPropertyUID(RootClass root) {
shouldGetUID(root.get8());
}
private void shouldGetUID(LongArrayList val) {
assertNotNull(val);
assertEquals(7, val.size());
assertEquals(0, val.getLong(0));
assertEquals(val.getLong(5), Long.MAX_VALUE);
assertEquals(val.getLong(6), Long.MIN_VALUE);
}
public void onLoad(RunnableWithIO r) {
this.r = r;
}
}
public static class RootClass extends EnhancedObject {
@DBField(id = 0, type = DBDataType.BOOLEAN)
public boolean field1;
@DBField(id = 1, type = DBDataType.BYTE)
public byte field2;
@DBField(id = 2, type = DBDataType.SHORT)
public short field3;
@DBField(id = 3, type = DBDataType.CHAR)
public char field4;
@DBField(id = 4, type = DBDataType.INTEGER)
public int field5;
@DBField(id = 5, type = DBDataType.LONG)
public long field6;
@DBField(id = 6, type = DBDataType.OBJECT)
public String field7;
@DBField(id = 7, type = DBDataType.REFERENCES_LIST)
public LongArrayList field8;
@DBField(id = 8, type = DBDataType.DATABASE_OBJECT)
public NSimplestClass field9;
public RootClass() {
}
public RootClass(IDatabase database) throws IOException {
super(database);
}
@DBPropertyGetter(id = 0, type = DBDataType.BOOLEAN)
public boolean get1() {
return getProperty();
}
@DBPropertyGetter(id = 1, type = DBDataType.BYTE)
public byte get2() {
return getProperty();
}
@DBPropertyGetter(id = 2, type = DBDataType.SHORT)
public short get3() {
return getProperty();
}
@DBPropertyGetter(id = 3, type = DBDataType.CHAR)
public char get4() {
return getProperty();
}
@DBPropertyGetter(id = 4, type = DBDataType.INTEGER)
public int get5() {
return getProperty();
}
@DBPropertyGetter(id = 5, type = DBDataType.LONG)
public long get6() {
return getProperty();
}
@DBPropertyGetter(id = 6, type = DBDataType.OBJECT)
public String get7() {
return getProperty();
}
@DBPropertyGetter(id = 7, type = DBDataType.REFERENCES_LIST)
public LongArrayList get8() {
return getProperty();
}
@DBPropertyGetter(id = 8, type = DBDataType.DATABASE_OBJECT)
public NSimplestClass get9() {
return getProperty();
}
@DBPropertySetter(id = 0, type = DBDataType.BOOLEAN)
public void set1(boolean val) {
setProperty(val);
}
@DBPropertySetter(id = 1, type = DBDataType.BYTE)
public void set2(byte val) {
setProperty(val);
}
@DBPropertySetter(id = 2, type = DBDataType.SHORT)
public void set3(short val) {
setProperty(val);
}
@DBPropertySetter(id = 3, type = DBDataType.CHAR)
public void set4(char val) {
setProperty(val);
}
@DBPropertySetter(id = 4, type = DBDataType.INTEGER)
public void set5(int val) {
setProperty(val);
}
@DBPropertySetter(id = 5, type = DBDataType.LONG)
public void set6(long val) {
setProperty(val);
}
@DBPropertySetter(id = 6, type = DBDataType.OBJECT)
public void set7(String val) {
setProperty(val);
}
@DBPropertySetter(id = 7, type = DBDataType.REFERENCES_LIST)
public void set8(LongArrayList val) {
setProperty(val);
}
@DBPropertySetter(id = 8, type = DBDataType.DATABASE_OBJECT)
public void set9(NSimplestClass val) {
setProperty(val);
}
public boolean test() {
return true;
}
@Override
public void initialize() throws IOException {
}
}
}