andreacavalli
/
strangedb
1
1
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Removed reference storage

This commit is contained in:
Andrea Cavalli 2018-12-15 01:01:23 +01:00
parent b179ee4636
commit 73ae46e7b4
6 changed files with 85 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/org/warp/jcwdb/DBGenericObjectParser.java
View File

@ -7,7 +7,7 @@ import com.esotericsoftware.kryo.io.Output;
import net.openhft.hashing.LongHashFunction;
public class DBGenericObjectParser extends DBTypeParserImpl<Object> {
public class DBGenericObjectParser extends DBTypeParserImpl<Object> implements DBTypedObjectParser<Object> {
private static final LongHashFunction hashFunction = net.openhft.hashing.LongHashFunction.xx();
private static final Kryo kryo = new Kryo();
static {
@ -46,4 +46,13 @@ public class DBGenericObjectParser extends DBTypeParserImpl<Object> {
tmpO.close();
return hash;
}
@Override
public <U> void registerClass(Class<U> clazz, int id) {
if (id >= Integer.MAX_VALUE - 100) {
throw new IndexOutOfBoundsException();
}
kryo.register(clazz, id + 100);
}
}

5
src/main/java/org/warp/jcwdb/DBTypedObjectParser.java Normal file
View File

@ -0,0 +1,5 @@
package org.warp.jcwdb;
public interface DBTypedObjectParser<T> extends DBTypeParser<T> {
public <U> void registerClass(Class<U> clazz, int type);
}

4
src/main/java/org/warp/jcwdb/FileIndexManager.java
View File

@ -373,12 +373,12 @@ public class FileIndexManager implements IndexManager {
long removedIndices = 0;
LongArrayList toUnload = new LongArrayList();
synchronized (indicesMapsAccessLock) {
if (loadedIndices.size() > JCWDatabase.MAX_LOADED_REFERENCES) {
if (loadedIndices.size() > JCWDatabase.MAX_LOADED_INDICES) {
long count = loadedIndices.size();
LongIterator it = loadedIndices.keySet().iterator();
while (it.hasNext()) {
long loadedIndex = it.nextLong();
if (count < JCWDatabase.MAX_LOADED_REFERENCES * 3l / 2l) {
if (count < JCWDatabase.MAX_LOADED_INDICES * 3l / 2l) {
break;
}
toUnload.add(loadedIndex);

173
src/main/java/org/warp/jcwdb/JCWDatabase.java
View File

@ -1,36 +1,24 @@
package org.warp.jcwdb;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.function.Consumer;
import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.objects.ObjectIterator;
import java.io.IOException;
import java.nio.file.Path;
public class JCWDatabase implements AutoCloseable, Cleanable {
public final static long MAX_LOADED_REFERENCES = 1000;
public final static long MAX_LOADED_INDICES = 10000;
private final TypesManager typesManager;
private final MixedIndexDatabase indices;
private final Cleaner databaseCleaner;
private final EntryReferenceTools entryReferenceTools = new EntryReferenceTools();
private final Long2ObjectMap<WeakReference<EntryReference<?>>> references;
private volatile boolean closed;
private final Object closeLock = new Object();
private final Object indicesAccessLock = new Object();
private final Object referencesAccessLock = new Object();
public JCWDatabase(Path dataFile, Path metadataFile) throws IOException {
this.typesManager = new TypesManager(this);
this.indices = new MixedIndexDatabase(typesManager, dataFile, metadataFile);
this.references = new Long2ObjectLinkedOpenHashMap<>();
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
JCWDatabase.this.close();
@ -39,14 +27,14 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
}
}));
this.databaseCleaner = new Cleaner(this);
//this.databaseCleaner.start();
}
public <T> EntryReference<LightList<T>> getRoot() throws IOException {
return getRoot(Object.class).cast();
}
public <T> EntryReference<LightList<T>> getRoot(Class<T> clazz) throws IOException {
checkClosed();
if (exists(0)) {
@ -59,74 +47,64 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
public <T> EntryReference<T> get(long index) throws IOException {
checkClosed();
synchronized (referencesAccessLock) {
WeakReference<EntryReference<?>> refRef = this.references.getOrDefault(index, null);
EntryReference<T> ref;
if (refRef == null || (ref = (EntryReference<T>) refRef.get()) == null) {
int type;
long hash;
synchronized (indicesAccessLock) {
type = this.indices.getType(index);
hash = this.indices.getHash(index);
}
DBTypeParser<T> typeParser = this.typesManager.get(type);
ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser);
refRef = new WeakReference<>(ref);
this.references.put(index, refRef);
}
return ref;
int type;
long hash;
synchronized (indicesAccessLock) {
type = this.indices.getType(index);
hash = this.indices.getHash(index);
}
DBTypeParser<T> typeParser = this.typesManager.get(type);
return new EntryReference<>(entryReferenceTools, index, hash, typeParser);
}
protected <T> EntryReference<T> add(T value) throws IOException {
checkClosed();
synchronized (referencesAccessLock) {
EntryReference<T> ref;
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
long index;
long hash;
synchronized (indicesAccessLock) {
index = indices.add(typeParser.getWriter(value));
hash = indices.getHash(index);
}
ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser, value);
this.references.put(index, new WeakReference<>(ref));
return ref;
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
long index;
long hash;
synchronized (indicesAccessLock) {
index = indices.add(typeParser.getWriter(value));
hash = indices.getHash(index);
}
return new EntryReference<>(entryReferenceTools, index, hash, typeParser, value);
}
protected boolean exists(long index) {
checkClosed();
synchronized (referencesAccessLock) {
synchronized (indicesAccessLock) {
return this.references.containsKey(index) || this.indices.has(index);
}
synchronized (indicesAccessLock) {
return this.indices.has(index);
}
}
protected <T> EntryReference<T> set(long index, T value) throws IOException {
checkClosed();
synchronized (referencesAccessLock) {
EntryReference<T> ref;
if (exists(index)) {
ref = get(index);
ref.setValue(value);
return ref;
} else {
@SuppressWarnings("unchecked")
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
long hash;
synchronized (indicesAccessLock) {
IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value));
hash = returnedDetails.getHash();
}
ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser);
this.references.put(index, new WeakReference<EntryReference<?>>(ref));
return ref;
EntryReference<T> ref;
if (exists(index)) {
ref = get(index);
ref.setValue(value);
return ref;
} else {
@SuppressWarnings("unchecked")
DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
long hash;
synchronized (indicesAccessLock) {
IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value));
hash = returnedDetails.getHash();
}
return new EntryReference<>(entryReferenceTools, index, hash, typeParser);
}
}
public <U> void registerType(Class<U> clazz, short type, DBTypeParser<U> parser) {
final int addition = 0xEFFF8000;
int extendedType = addition | (type & 0x7FFF);
typesManager.registerType(clazz, extendedType, parser);
}
public <U> void registerClass(Class<U> clazz, int type) {
typesManager.registerGenericClass(clazz, type);
}
public boolean isOpen() {
return !closed;
}
@ -145,18 +123,6 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
this.databaseCleaner.stop();
synchronized (referencesAccessLock) {
ObjectIterator<WeakReference<EntryReference<?>>> iterator = references.values().iterator();
while (iterator.hasNext()) {
WeakReference<EntryReference<?>> referenceRef = iterator.next();
EntryReference<?> reference = referenceRef.get();
if (reference != null) {
reference.close();
iterator.remove();
}
}
}
synchronized (indicesAccessLock) {
this.indices.close();
}
@ -171,56 +137,9 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
@Override
public long clean() {
long removedItems = cleanEmptyReferences()
+ cleanExtraReferences()
+ indices.clean();
long removedItems = indices.clean();
return removedItems;
}
private long cleanEmptyReferences() {
long removed = 0;
synchronized(referencesAccessLock) {
ObjectIterator<Entry<WeakReference<EntryReference<?>>>> iterator = references.long2ObjectEntrySet().iterator();
while (iterator.hasNext()) {
Entry<WeakReference<EntryReference<?>>> entry = iterator.next();
if (entry.getValue().get() == null) {
iterator.remove();
removed++;
}
}
}
return removed;
}
private long cleanExtraReferences() {
long removedReferences = 0;
synchronized(referencesAccessLock) {
if (references.size() > MAX_LOADED_REFERENCES) {
long count = 0;
ObjectIterator<Entry<WeakReference<EntryReference<?>>>> iterator = references.long2ObjectEntrySet().iterator();
while (iterator.hasNext()) {
Entry<WeakReference<EntryReference<?>>> entry = iterator.next();
if (count > MAX_LOADED_REFERENCES * 3l / 2l) {
WeakReference<EntryReference<?>> weakRef = entry.getValue();
EntryReference<?> ref = weakRef.get();
if (ref != null) {
try {
ref.close();
} catch (IOException e) {
e.printStackTrace();
}
}
iterator.remove();
removedReferences++;
} else {
count++;
}
}
}
}
return removedReferences;
}
public class EntryReferenceTools {
private EntryReferenceTools() {

31
src/main/java/org/warp/jcwdb/TypesManager.java
View File

@ -1,26 +1,37 @@
package org.warp.jcwdb;
import java.util.Map;
import java.util.Map.Entry;
import java.util.HashMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
public class TypesManager {
private final Map<Integer, DBTypeParser<?>> types;
private final Map<Class<?>, DBTypeParser<?>> typesByClass;
private DBTypeParser<?> fallbackParser;
private final Int2ObjectMap<DBTypeParser<?>> types;
private final Object2ObjectMap<Class<?>, DBTypeParser<?>> typesByClass;
private DBTypedObjectParser<?> fallbackParser;
public TypesManager(JCWDatabase db) {
types = new HashMap<>();
typesByClass = new HashMap<>();
types = new Int2ObjectOpenHashMap<>();
typesByClass = new Object2ObjectOpenHashMap<>();
DBStandardTypes.registerStandardTypes(db, this);
}
public <T> void registerType(Class<T> clazz, int type, DBTypeParser<T> parser) {
this.types.put(type, parser);
this.typesByClass.put(clazz, parser);
}
public void registerTypeFallback(DBTypeParser<?> parser) {
/**
* Use this method with the most used classes to save disk space.
* @param clazz
* @param id
* @param <T>
*/
public <T> void registerGenericClass(Class<T> clazz, int id) {
this.fallbackParser.registerClass(clazz, id);
}
public void registerTypeFallback(DBTypedObjectParser<?> parser) {
this.fallbackParser = parser;
}

3
src/main/java/org/warp/jcwdb/exampleimpl/App.java
View File

@ -22,6 +22,7 @@ public class App {
System.out.println("Loading database...");
long time0 = System.currentTimeMillis();
JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1]));
db.registerClass(StrangeAnimal.class, 0);
try {
long time01 = System.currentTimeMillis();
System.out.println("Time elapsed: " + (time01 - time0));
@ -58,14 +59,12 @@ public class App {
System.out.println("Time elapsed: " + (time2_1 - time2_0));
ObjectList<Animal> results = new ObjectArrayList<>();
/*
root.forEach((value) -> {
if (Animal.hasFourLegs(value)) {
results.add(value);
}
//System.out.println("val:" + value);
});
*/
long time2_2 = System.currentTimeMillis();
System.out.println("Time elapsed: " + (time2_2 - time2_1));
System.out.println("Used memory: "