Removed reference storage

src/main/java/org/warp/jcwdb/DBGenericObjectParser.java

@@ -7,7 +7,7 @@ import com.esotericsoftware.kryo.io.Output;
 
 import net.openhft.hashing.LongHashFunction;
 
-public class DBGenericObjectParser extends DBTypeParserImpl<Object> {
+public class DBGenericObjectParser extends DBTypeParserImpl<Object> implements DBTypedObjectParser<Object> {
 	private static final LongHashFunction hashFunction = net.openhft.hashing.LongHashFunction.xx();
 	private static final Kryo kryo = new Kryo();
 	static {
@@ -46,4 +46,13 @@ public class DBGenericObjectParser extends DBTypeParserImpl<Object> {
 		tmpO.close();
 		return hash;
 	}
+
+
+	@Override
+	public <U> void registerClass(Class<U> clazz, int id) {
+		if (id >= Integer.MAX_VALUE - 100) {
+			throw new IndexOutOfBoundsException();
+		}
+		kryo.register(clazz, id + 100);
+	}
 }

src/main/java/org/warp/jcwdb/DBTypedObjectParser.java

@@ -0,0 +1,5 @@
+package org.warp.jcwdb;
+
+public interface DBTypedObjectParser<T> extends DBTypeParser<T> {
+	public <U> void registerClass(Class<U> clazz, int type);
+}

src/main/java/org/warp/jcwdb/FileIndexManager.java

@@ -373,12 +373,12 @@ public class FileIndexManager implements IndexManager {
 		long removedIndices = 0;
 		LongArrayList toUnload = new LongArrayList();
 		synchronized (indicesMapsAccessLock) {
-			if (loadedIndices.size() > JCWDatabase.MAX_LOADED_REFERENCES) {
+			if (loadedIndices.size() > JCWDatabase.MAX_LOADED_INDICES) {
 				long count = loadedIndices.size();
 				LongIterator it = loadedIndices.keySet().iterator();
 				while (it.hasNext()) {
 					long loadedIndex = it.nextLong();
-					if (count < JCWDatabase.MAX_LOADED_REFERENCES * 3l / 2l) {
+					if (count < JCWDatabase.MAX_LOADED_INDICES * 3l / 2l) {
 						break;
 					}
 					toUnload.add(loadedIndex);

src/main/java/org/warp/jcwdb/JCWDatabase.java

@@ -1,36 +1,24 @@
 package org.warp.jcwdb;
 
-import java.io.IOException;
-import java.lang.ref.WeakReference;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.function.Consumer;
-
-import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
-import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
-import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
 import it.unimi.dsi.fastutil.longs.LongArrayList;
-import it.unimi.dsi.fastutil.objects.ObjectIterator;
+
+import java.io.IOException;
+import java.nio.file.Path;
 
 public class JCWDatabase implements AutoCloseable, Cleanable {
-	public final static long MAX_LOADED_REFERENCES = 1000;
 	public final static long MAX_LOADED_INDICES = 10000;
-	
+
 	private final TypesManager typesManager;
 	private final MixedIndexDatabase indices;
 	private final Cleaner databaseCleaner;
 	private final EntryReferenceTools entryReferenceTools = new EntryReferenceTools();
-	private final Long2ObjectMap<WeakReference<EntryReference<?>>> references;
 	private volatile boolean closed;
 	private final Object closeLock = new Object();
 	private final Object indicesAccessLock = new Object();
-	private final Object referencesAccessLock = new Object();
 
 	public JCWDatabase(Path dataFile, Path metadataFile) throws IOException {
 		this.typesManager = new TypesManager(this);
 		this.indices = new MixedIndexDatabase(typesManager, dataFile, metadataFile);
-		this.references = new Long2ObjectLinkedOpenHashMap<>();
 		Runtime.getRuntime().addShutdownHook(new Thread(() -> {
 			try {
 				JCWDatabase.this.close();
@@ -39,14 +27,14 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
 			}
 		}));
 		this.databaseCleaner = new Cleaner(this);
-		
+
 		//this.databaseCleaner.start();
 	}
 
 	public <T> EntryReference<LightList<T>> getRoot() throws IOException {
 		return getRoot(Object.class).cast();
 	}
-	
+
 	public <T> EntryReference<LightList<T>> getRoot(Class<T> clazz) throws IOException {
 		checkClosed();
 		if (exists(0)) {
@@ -59,74 +47,64 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
 
 	public <T> EntryReference<T> get(long index) throws IOException {
 		checkClosed();
-		synchronized (referencesAccessLock) {
+		int type;
-			WeakReference<EntryReference<?>> refRef = this.references.getOrDefault(index, null);
+		long hash;
-			EntryReference<T> ref;
+		synchronized (indicesAccessLock) {
-			if (refRef == null || (ref = (EntryReference<T>) refRef.get()) == null) {
+			type = this.indices.getType(index);
-				int type;
+			hash = this.indices.getHash(index);
-				long hash;
-				synchronized (indicesAccessLock) {
-					type = this.indices.getType(index);
-					hash = this.indices.getHash(index);
-				}
-				DBTypeParser<T> typeParser = this.typesManager.get(type);
-				ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser);
-				refRef = new WeakReference<>(ref);
-				this.references.put(index, refRef);	
-			}
-			return ref;
 		}
+		DBTypeParser<T> typeParser = this.typesManager.get(type);
+		return new EntryReference<>(entryReferenceTools, index, hash, typeParser);
 	}
 
 	protected <T> EntryReference<T> add(T value) throws IOException {
 		checkClosed();
-		synchronized (referencesAccessLock) {
+		DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
-			EntryReference<T> ref;
+		long index;
-			DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
+		long hash;
-			long index;
+		synchronized (indicesAccessLock) {
-			long hash;
+			index = indices.add(typeParser.getWriter(value));
-			synchronized (indicesAccessLock) {
+			hash = indices.getHash(index);
-				index = indices.add(typeParser.getWriter(value));
-				hash = indices.getHash(index);
-			}
-			ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser, value);
-			this.references.put(index, new WeakReference<>(ref));
-			return ref;
 		}
+		return new EntryReference<>(entryReferenceTools, index, hash, typeParser, value);
 	}
 
 	protected boolean exists(long index) {
 		checkClosed();
-		synchronized (referencesAccessLock) {
+		synchronized (indicesAccessLock) {
-			synchronized (indicesAccessLock) {
+			return this.indices.has(index);
-				return this.references.containsKey(index) || this.indices.has(index);
-			}
 		}
 	}
 
 	protected <T> EntryReference<T> set(long index, T value) throws IOException {
 		checkClosed();
-		synchronized (referencesAccessLock) {
+		EntryReference<T> ref;
-			EntryReference<T> ref;
+		if (exists(index)) {
-			if (exists(index)) {
+			ref = get(index);
-				ref = get(index);
+			ref.setValue(value);
-				ref.setValue(value);
+			return ref;
-				return ref;
+		} else {
-			} else {
+			@SuppressWarnings("unchecked")
-				@SuppressWarnings("unchecked")
+			DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
-				DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
+			long hash;
-				long hash;
+			synchronized (indicesAccessLock) {
-				synchronized (indicesAccessLock) {
+				IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value));
-					IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value));
+				hash = returnedDetails.getHash();
-					hash = returnedDetails.getHash();
-				}
-				ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser);
-				this.references.put(index, new WeakReference<EntryReference<?>>(ref));
-				return ref;
 			}
+			return new EntryReference<>(entryReferenceTools, index, hash, typeParser);
 		}
 	}
 
+	public <U> void registerType(Class<U> clazz, short type, DBTypeParser<U> parser) {
+		final int addition = 0xEFFF8000;
+		int extendedType = addition | (type & 0x7FFF);
+		typesManager.registerType(clazz, extendedType, parser);
+	}
+
+	public <U> void registerClass(Class<U> clazz, int type) {
+		typesManager.registerGenericClass(clazz, type);
+	}
+
 	public boolean isOpen() {
 		return !closed;
 	}
@@ -145,18 +123,6 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
 
 		this.databaseCleaner.stop();
 
-		synchronized (referencesAccessLock) {
-			ObjectIterator<WeakReference<EntryReference<?>>> iterator = references.values().iterator();
-			while (iterator.hasNext()) {
-				WeakReference<EntryReference<?>> referenceRef = iterator.next();
-				EntryReference<?> reference = referenceRef.get();
-				if (reference != null) {
-					reference.close();
-					iterator.remove();
-				}
-
-			}
-		}
 		synchronized (indicesAccessLock) {
 			this.indices.close();
 		}
@@ -171,56 +137,9 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
 
 	@Override
 	public long clean() {
-		long removedItems = cleanEmptyReferences()
+		long removedItems = indices.clean();
-				+ cleanExtraReferences()
-				+ indices.clean();
 		return removedItems;
 	}
-	
-
-	private long cleanEmptyReferences() {
-		long removed = 0;
-		synchronized(referencesAccessLock) {
-			ObjectIterator<Entry<WeakReference<EntryReference<?>>>> iterator = references.long2ObjectEntrySet().iterator();
-			while (iterator.hasNext()) {
-				Entry<WeakReference<EntryReference<?>>> entry = iterator.next();
-				if (entry.getValue().get() == null) {
-					iterator.remove();
-					removed++;
-				}
-			}
-		}
-		return removed;
-	}
-
-	private long cleanExtraReferences() {
-		long removedReferences = 0;
-		synchronized(referencesAccessLock) {
-			if (references.size() > MAX_LOADED_REFERENCES) {
-				long count = 0;
-				ObjectIterator<Entry<WeakReference<EntryReference<?>>>> iterator = references.long2ObjectEntrySet().iterator();
-				while (iterator.hasNext()) {
-					Entry<WeakReference<EntryReference<?>>> entry = iterator.next();
-					if (count > MAX_LOADED_REFERENCES * 3l / 2l) {
-						WeakReference<EntryReference<?>> weakRef = entry.getValue();
-						EntryReference<?> ref = weakRef.get();
-						if (ref != null) {
-							try {
-								ref.close();
-							} catch (IOException e) {
-								e.printStackTrace();
-							}
-						}
-						iterator.remove();
-						removedReferences++;
-					} else {
-						count++;
-					}
-				}
-			}
-		}
-		return removedReferences;
-	}
 
 	public class EntryReferenceTools {
 		private EntryReferenceTools() {

src/main/java/org/warp/jcwdb/TypesManager.java

@@ -1,26 +1,37 @@
 package org.warp.jcwdb;
 
-import java.util.Map;
+import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
-import java.util.Map.Entry;
+import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
-import java.util.HashMap;
+import it.unimi.dsi.fastutil.objects.Object2ObjectMap;
+import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
 
 public class TypesManager {
-	private final Map<Integer, DBTypeParser<?>> types;
+	private final Int2ObjectMap<DBTypeParser<?>> types;
-	private final Map<Class<?>, DBTypeParser<?>> typesByClass;
+	private final Object2ObjectMap<Class<?>, DBTypeParser<?>> typesByClass;
-	private DBTypeParser<?> fallbackParser;
+	private DBTypedObjectParser<?> fallbackParser;
 
 	public TypesManager(JCWDatabase db) {
-		types = new HashMap<>();
+		types = new Int2ObjectOpenHashMap<>();
-		typesByClass = new HashMap<>();
+		typesByClass = new Object2ObjectOpenHashMap<>();
 		DBStandardTypes.registerStandardTypes(db, this);
 	}
-	
+
 	public <T> void registerType(Class<T> clazz, int type, DBTypeParser<T> parser) {
 		this.types.put(type, parser);
 		this.typesByClass.put(clazz, parser);
 	}
 
-	public void registerTypeFallback(DBTypeParser<?> parser) {
+	/**
+	 * Use this method with the most used classes to save disk space.
+	 * @param clazz
+	 * @param id
+	 * @param <T>
+	 */
+	public <T> void registerGenericClass(Class<T> clazz, int id) {
+		this.fallbackParser.registerClass(clazz, id);
+	}
+
+	public void registerTypeFallback(DBTypedObjectParser<?> parser) {
 		this.fallbackParser = parser;
 	}
 

src/main/java/org/warp/jcwdb/exampleimpl/App.java

@@ -22,6 +22,7 @@ public class App {
 			System.out.println("Loading database...");
 			long time0 = System.currentTimeMillis();
 			JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1]));
+			db.registerClass(StrangeAnimal.class, 0);
 			try {
 				long time01 = System.currentTimeMillis();
 				System.out.println("Time elapsed: " + (time01 - time0));
@@ -58,14 +59,12 @@ public class App {
 					System.out.println("Time elapsed: " + (time2_1 - time2_0));
 					ObjectList<Animal> results = new ObjectArrayList<>();
 
-					/*
 					root.forEach((value) -> {
 						if (Animal.hasFourLegs(value)) {
 							results.add(value);
 						}
 						//System.out.println("val:" + value);
 					});
-					*/
 					long time2_2 = System.currentTimeMillis();
 					System.out.println("Time elapsed: " + (time2_2 - time2_1));
 					System.out.println("Used memory: "