Merge branch 'better'

src/main/java/org/warp/jcwdb/Cleaner.java

@@ -8,7 +8,7 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
 
 public class Cleaner {
 
-	private static final double MAXIMUM_SLEEP_INTERVAL = 20d * 1000d; // 20 minutes
+	private static final double MAXIMUM_SLEEP_INTERVAL = 8d * 1000d; // 8 seconds
 	private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second
 	private static final double NORMAL_REMOVED_ITEMS = 1000l;
 	private static final double REMOVED_ITEMS_RATIO = 2.5d; // 250%
@@ -63,7 +63,10 @@ public class Cleaner {
 				try {
 					System.out.println("[CLEANER] Waiting " + sleepInterval + "ms.");
 					sleepFor(sleepInterval);
+					final long time1 = System.currentTimeMillis();
 					final double removedItems = clean();
+					final long time2 = System.currentTimeMillis();
+					System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1));
 					double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2;
 
 					System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems);

src/main/java/org/warp/jcwdb/DBGenericObjectParser.java

@@ -7,7 +7,7 @@ import com.esotericsoftware.kryo.io.Output;
 
 import net.openhft.hashing.LongHashFunction;
 
-public class DBGenericObjectParser extends DBTypeParserImpl<Object> {
+public class DBGenericObjectParser extends DBTypeParserImpl<Object> implements DBTypedObjectParser<Object> {
 	private static final LongHashFunction hashFunction = net.openhft.hashing.LongHashFunction.xx();
 	private static final Kryo kryo = new Kryo();
 	static {
@@ -46,4 +46,13 @@ public class DBGenericObjectParser extends DBTypeParserImpl<Object> {
 		tmpO.close();
 		return hash;
 	}
+
+
+	@Override
+	public <U> void registerClass(Class<U> clazz, int id) {
+		if (id >= Integer.MAX_VALUE - 100) {
+			throw new IndexOutOfBoundsException();
+		}
+		kryo.register(clazz, id + 100);
+	}
 }

src/main/java/org/warp/jcwdb/DBLightArrayListParser.java

@@ -1,40 +1,38 @@
 package org.warp.jcwdb;
 
-import java.util.ArrayList;
-
 import it.unimi.dsi.fastutil.longs.LongArrayList;
 
-public class DBLightListParser<T> extends DBTypeParserImpl<LightList<T>> {
+public class DBLightArrayListParser<T> extends DBTypeParserImpl<LightArrayList<T>> {
 	private final JCWDatabase db;
 
-	public DBLightListParser(JCWDatabase db) {
+	public DBLightArrayListParser(JCWDatabase db) {
 		this.db = db;
 	}
 
-	public DBReader<LightList<T>> getReader() {
+	public DBReader<LightArrayList<T>> getReader() {
 		return (i, size) -> {
 			LongArrayList internalList = new LongArrayList();
 			long max = size / Long.BYTES;
-			for (int item = 0; item < max; item++){
+			for (int item = 0; item < max; item++) {
 				long itm = i.readLong();
 				internalList.add(itm);
 			}
-			return new LightList<T>(db, internalList);
+			return new LightArrayList<T>(db, internalList);
 		};
 	}
 
-	public DBDataOutput<LightList<T>> getWriter(final LightList<T> value) {
+	public DBDataOutput<LightArrayList<T>> getWriter(final LightArrayList<T> value) {
 		final int elementsCount = value.size();
 		return DBDataOutput.create((o) -> {
 			LongArrayList list = value.internalList;
 			for (int i = 0; i < elementsCount; i++) {
 				o.writeLong(list.getLong(i));
 			}
-		}, DBStandardTypes.LIGHT_LIST, elementsCount * Long.BYTES, calculateHash(value));
+		}, DBStandardTypes.LIGHT_LIST_ARRAY, elementsCount * Long.BYTES, calculateHash(value));
 	}
 
 	@Override
-	public long calculateHash(LightList<T> value) {
+	public long calculateHash(LightArrayList<T> value) {
 		return value.internalList.hashCode();
 	}
 }

src/main/java/org/warp/jcwdb/DBLightBigListParser.java

@@ -0,0 +1,44 @@
+package org.warp.jcwdb;
+
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+
+public class DBLightBigListParser<T> extends DBTypeParserImpl<LightBigList<T>> {
+	private final JCWDatabase db;
+
+	public DBLightBigListParser(JCWDatabase db) {
+		this.db = db;
+	}
+
+	public DBReader<LightBigList<T>> getReader() {
+		return (i, size) -> {
+			LongArrayList chunks = new LongArrayList();
+			IntArrayList chunkSizes = new IntArrayList();
+			long max = size / (Long.BYTES + Integer.BYTES);
+			for (int item = 0; item < max; item++) {
+				long itm = i.readLong();
+				int itm2 = i.readInt();
+				chunks.add(itm);
+				chunkSizes.add(itm2);
+			}
+			return new LightBigList<T>(db, chunks, chunkSizes);
+		};
+	}
+
+	public DBDataOutput<LightBigList<T>> getWriter(final LightBigList<T> value) {
+		final int elementsCount = value.chunksCount();
+		return DBDataOutput.create((o) -> {
+			LongArrayList list = value.chunks;
+			IntArrayList list2 = value.chunkSizes;
+			for (int i = 0; i < elementsCount; i++) {
+				o.writeLong(list.getLong(i));
+				o.writeInt(list2.getInt(i));
+			}
+		}, DBStandardTypes.LIGHT_LIST_BIG, elementsCount * (Long.BYTES + Integer.BYTES), calculateHash(value));
+	}
+
+	@Override
+	public long calculateHash(LightBigList<T> value) {
+		return value.chunks.hashCode();
+	}
+}

src/main/java/org/warp/jcwdb/DBStandardTypes.java

@@ -11,12 +11,14 @@ public class DBStandardTypes {
 	public static final int DOUBLE =			STD|  6;
 	public static final int STRING =			STD|  7;
 	public static final int BYTE_ARRAY =		STD|  8;
-	public static final int LIGHT_LIST =		STD|  9;
-	public static final int GENERIC_OBJECT =	STD| 10;
+	public static final int LIGHT_LIST_ARRAY =	STD|  9;
+	public static final int LIGHT_LIST_BIG =	STD|  10;
+	public static final int GENERIC_OBJECT =	STD| 11;
 
 	public static void registerStandardTypes(JCWDatabase db, TypesManager typesManager) {
 		typesManager.registerType(String.class, STRING, new DBStringParser());
-		typesManager.registerType(LightList.class, LIGHT_LIST, new DBLightListParser(db));
+		typesManager.registerType(LightArrayList.class, LIGHT_LIST_ARRAY, new DBLightArrayListParser(db));
+		typesManager.registerType(LightBigList.class, LIGHT_LIST_BIG, new DBLightBigListParser(db));
 		typesManager.registerTypeFallback(new DBGenericObjectParser());
 	}
 }

src/main/java/org/warp/jcwdb/DBTypedObjectParser.java

@@ -0,0 +1,5 @@
+package org.warp.jcwdb;
+
+public interface DBTypedObjectParser<T> extends DBTypeParser<T> {
+	public <U> void registerClass(Class<U> clazz, int type);
+}

src/main/java/org/warp/jcwdb/EntryReference.java

@@ -86,7 +86,7 @@ public class EntryReference<T> implements Castable, Saveable {
 	 * @param editFunction
 	 * @throws IOException
 	 */
-	public void editValue(BiFunction<T, Saveable, T> editFunction) throws IOException {
+	public void editValue(BiFunction<T, Saveable, T> editFunction) {
 		synchronized(accessLock) {
 			load();
 			this.value = editFunction.apply(this.value, this);
@@ -99,7 +99,20 @@ public class EntryReference<T> implements Castable, Saveable {
 	 * @param editFunction
 	 * @throws IOException
 	 */
-	public void editValue(BiConsumer<T, Saveable> editFunction) throws IOException {
+	public void editValue(Function<T, T> editFunction) {
+		synchronized(accessLock) {
+			load();
+			this.value = editFunction.apply(this.value);
+			this.save();
+		}
+	}
+
+	/**
+	 * Reccomended way to edit the value
+	 * @param editFunction
+	 * @throws IOException
+	 */
+	public void editValue(BiConsumer<T, Saveable> editFunction) {
 		synchronized(accessLock) {
 			load();
 			editFunction.accept(this.value, this);
@@ -107,12 +120,25 @@ public class EntryReference<T> implements Castable, Saveable {
 		}
 	}
 
+	/**
+	 * Reccomended way to edit the value
+	 * @param editFunction
+	 * @throws IOException
+	 */
+	public void editValue(Consumer<T> editFunction) {
+		synchronized(accessLock) {
+			load();
+			editFunction.accept(this.value);
+			this.save();
+		}
+	}
+
 	/**
 	 * Substitute the old value with a new one
 	 * @param val
 	 * @throws IOException
 	 */
-	public void setValue(T val) throws IOException {
+	public void setValue(T val) {
 		synchronized(accessLock) {
 			this.loaded = true;
 			this.value = val;

src/main/java/org/warp/jcwdb/FileAllocator.java

@@ -1,62 +1,116 @@
 package org.warp.jcwdb;
 
-import com.esotericsoftware.kryo.io.Output;
+import it.unimi.dsi.fastutil.longs.Long2IntMap;
+import it.unimi.dsi.fastutil.longs.Long2IntRBTreeMap;
+import it.unimi.dsi.fastutil.objects.ObjectBidirectionalIterator;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
-import java.nio.channels.FileChannel;
 import java.nio.channels.SeekableByteChannel;
-import java.util.*;
 
 public class FileAllocator implements AutoCloseable {
+	private static final int MAXIMUM_UNALLOCATED_ENTRIES = 500000;
+
 	private final SeekableByteChannel dataFileChannel;
-	private volatile long allocableOffset;
+	private volatile long fileSize;
 	private volatile boolean closed;
 	private final Object closeLock = new Object();
 	private final Object allocateLock = new Object();
+	/**
+	 * index -> free space size
+	 */
+	private final Long2IntRBTreeMap freeBytes = new Long2IntRBTreeMap((a, b) -> (int) (a - b));
 
 	public FileAllocator(SeekableByteChannel dataFileChannel) throws IOException {
 		this.dataFileChannel = dataFileChannel;
-		this.allocableOffset = this.dataFileChannel.size();
+		this.fileSize = this.dataFileChannel.size();
 	}
 
 	/**
 	 * TODO: not implemented
+	 *
 	 * @param size
 	 * @return offset
 	 */
 	public long allocate(int size) {
 		checkClosed();
 		synchronized (allocateLock) {
-			long allocatedOffset = allocableOffset;
-			allocableOffset += size;
-			return allocatedOffset;
+			long offset = allocateIntoUnusedParts(size);
+			if (offset == -1) {
+				return allocateToEnd(size);
+			} else {
+				return offset;
+			}
 		}
 	}
 
+	private long allocateIntoUnusedParts(int size) {
+		ObjectBidirectionalIterator<Long2IntMap.Entry> it = freeBytes.long2IntEntrySet().iterator();
+		long holeOffset = -1;
+		int holeSize = 0;
+		while (it.hasNext()) {
+			Long2IntMap.Entry entry = it.next();
+			int currentHoleSize = entry.getIntValue();
+			if (currentHoleSize < size) {
+				freeBytes.remove(holeOffset);
+				if (holeSize > size) {
+					freeBytes.put(holeOffset + size, holeSize - size);
+				}
+				break;
+			}
+			holeOffset = entry.getLongKey();
+			holeSize = currentHoleSize;
+		}
+		return holeOffset;
+	}
+
+	private long allocateToEnd(int size) {
+		long allocatedOffset = fileSize;
+		fileSize += size;
+		return allocatedOffset;
+	}
+
 
 	public void close() throws IOException {
-        if (closed) {
-            return;
-        }
-        synchronized (closeLock) {
-            if (closed) {
-                return;
-            }
-            closed = true;
-        }
+		if (closed) {
+			return;
+		}
+		synchronized (closeLock) {
+			if (closed) {
+				return;
+			}
+			closed = true;
+		}
 	}
 
 	/**
 	 * Frees the unused bytes
+	 *
 	 * @param startPosition
 	 * @param length
 	 */
 	public void markFree(long startPosition, int length) {
 		checkClosed();
-		// TODO: advanced feature, not implemented.
+
+		if (freeBytes.containsKey(startPosition + length)) {
+			int secondLength = freeBytes.remove(startPosition + length);
+			freeBytes.put(startPosition, length + secondLength);
+		} else {
+			boolean addedToList = false;
+			for (Long2IntMap.Entry entry : freeBytes.long2IntEntrySet()) {
+				if (entry.getLongKey() + entry.getIntValue() == startPosition) {
+					freeBytes.put(entry.getLongKey(), entry.getIntValue() + length);
+					addedToList = true;
+					break;
+				}
+			}
+			if (!addedToList) {
+				freeBytes.put(startPosition, length);
+			}
+		}
+
+		if (freeBytes.size() > MAXIMUM_UNALLOCATED_ENTRIES) {
+			freeBytes.remove(freeBytes.lastLongKey());
+		}
 	}
 
 

src/main/java/org/warp/jcwdb/FileIndexManager.java

@@ -11,6 +11,7 @@ import java.nio.channels.SeekableByteChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import java.util.Iterator;
 import java.util.function.Consumer;
 
 public class FileIndexManager implements IndexManager {
@@ -81,7 +82,12 @@ public class FileIndexManager implements IndexManager {
 		final IndexDetails indexDetails = getIndexMetadataUnsafe(index);
 		if (indexDetails == null || indexDetails.getSize() < dataSize) {
 			// Allocate new space
-			return allocateAndWrite(index, data);
+			IndexDetails newDetails = allocateAndWrite(index, data);
+			if (indexDetails != null) {
+				// Mark free the old bytes
+				fileAllocator.markFree(indexDetails.getOffset(), indexDetails.getSize());
+			}
+			return newDetails;
 		} else {
 			// Check if size changed
 			if (dataSize < indexDetails.getSize()) {
@@ -233,7 +239,7 @@ public class FileIndexManager implements IndexManager {
 	 * @param details
 	 */
 	private void editIndex(long index, IndexDetails details) {
-		synchronized (indicesMapsAccessLock) {
+		synchronized (indicesMapsAccessLock) {// FIXXXX main3
 			loadedIndices.put(index, details);
 			dirtyLoadedIndices.add(index);
 		}
@@ -264,7 +270,8 @@ public class FileIndexManager implements IndexManager {
 			return null;
 		}
 		SeekableByteChannel currentMetadataFileChannel = metadataFileChannel.position(metadataPosition);
-		synchronized (metadataByteBufferLock) {
+		IndexDetails indexDetails = null;
+		synchronized (metadataByteBufferLock) {// FIXXXX main2
 			metadataByteBuffer.rewind();
 			currentMetadataFileChannel.read(metadataByteBuffer);
 			metadataByteBuffer.rewind();
@@ -277,12 +284,15 @@ public class FileIndexManager implements IndexManager {
 				final int size = metadataByteBuffer.getInt();
 				final int type = metadataByteBuffer.getInt();
 				final long hash = metadataByteBuffer.getLong();
-				final IndexDetails indexDetails = new IndexDetails(offset, size, type, hash);
-				editIndex(index, indexDetails);
-				return indexDetails;
+				indexDetails = new IndexDetails(offset, size, type, hash);
 			}
 		}
 
+		if (indexDetails != null) {
+			editIndex(index, indexDetails);
+			return indexDetails;
+		}
+
 		// No results found. Returning null
 		return null;
 	}
@@ -308,31 +318,15 @@ public class FileIndexManager implements IndexManager {
 		}
 
 		// Update indices metadata
-		SeekableByteChannel metadata = metadataFileChannel;
-		long lastIndex = -2;
-		synchronized (indicesMapsAccessLock) {
-			for (long index : dirtyLoadedIndices) {
-				IndexDetails indexDetails = loadedIndices.get(index);
-				if (index - lastIndex != 1) {
-					metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
-				}
-				writeIndexDetails(metadata, indexDetails);
-				lastIndex = index;
-			}
-		}
+		flushAllIndices();
 
 		// Remove removed indices
-		synchronized (indicesMapsAccessLock) {
-			for (long index : removedIndices) {
-				metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
-				eraseIndexDetails(metadata);
-			}
-		}
+		removeRemovedIndices();
 		fileAllocator.close();
 	}
 
 	private void writeIndexDetails(SeekableByteChannel position, IndexDetails indexDetails) throws IOException {
-		synchronized (metadataByteBufferLock) {
+		synchronized (metadataByteBufferLock) {// FIXXXX cleaner3
 			final int size = indexDetails.getSize();
 			final int type = indexDetails.getType();
 			final long offset = indexDetails.getOffset();
@@ -366,19 +360,63 @@ public class FileIndexManager implements IndexManager {
 
 	@Override
 	public long clean() {
-		return cleanExtraIndices();
+		long cleaned = 0;
+		try {
+			cleaned += flushAllIndices();
+		} catch (IOException ex) {
+			ex.printStackTrace();
+		}
+		try {
+			cleaned += removeRemovedIndices();
+		} catch (IOException ex) {
+			ex.printStackTrace();
+		}
+		cleaned += cleanExtraIndices();
+		return cleaned;
+	}
+
+	private long flushAllIndices() throws IOException {
+		long flushedIndices = 0;
+		SeekableByteChannel metadata = metadataFileChannel;
+		long lastIndex = -2;
+		synchronized (indicesMapsAccessLock) {
+			for (long index : dirtyLoadedIndices) {
+				IndexDetails indexDetails = loadedIndices.get(index);
+				if (index - lastIndex != 1) {
+					metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
+				}
+				writeIndexDetails(metadata, indexDetails);
+				lastIndex = index;
+				flushedIndices++;
+			}
+			dirtyLoadedIndices.clear();
+		}
+		return flushedIndices;
+	}
+
+	private long removeRemovedIndices() throws IOException {
+		SeekableByteChannel metadata = metadataFileChannel;
+		synchronized (indicesMapsAccessLock) {
+			long removed = this.removedIndices.size();
+			for (long index : this.removedIndices) {
+					metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
+					eraseIndexDetails(metadata);
+			}
+			this.removedIndices.clear();
+			return removed;
+		}
 	}
 
 	private long cleanExtraIndices() {
 		long removedIndices = 0;
 		LongArrayList toUnload = new LongArrayList();
 		synchronized (indicesMapsAccessLock) {
-			if (loadedIndices.size() > JCWDatabase.MAX_LOADED_REFERENCES) {
+			if (loadedIndices.size() > JCWDatabase.MAX_LOADED_INDICES) {
 				long count = loadedIndices.size();
 				LongIterator it = loadedIndices.keySet().iterator();
 				while (it.hasNext()) {
 					long loadedIndex = it.nextLong();
-					if (count < JCWDatabase.MAX_LOADED_REFERENCES * 3l / 2l) {
+					if (count < JCWDatabase.MAX_LOADED_INDICES * 3l / 2l) {
 						break;
 					}
 					toUnload.add(loadedIndex);

src/main/java/org/warp/jcwdb/JCWDatabase.java

@@ -1,36 +1,24 @@
 package org.warp.jcwdb;
 
-import java.io.IOException;
-import java.lang.ref.WeakReference;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.function.Consumer;
-
-import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
-import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
-import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
 import it.unimi.dsi.fastutil.longs.LongArrayList;
-import it.unimi.dsi.fastutil.objects.ObjectIterator;
+
+import java.io.IOException;
+import java.nio.file.Path;
 
 public class JCWDatabase implements AutoCloseable, Cleanable {
-	public final static long MAX_LOADED_REFERENCES = 1000;
 	public final static long MAX_LOADED_INDICES = 10000;
 
 	private final TypesManager typesManager;
 	private final MixedIndexDatabase indices;
 	private final Cleaner databaseCleaner;
 	private final EntryReferenceTools entryReferenceTools = new EntryReferenceTools();
-	private final Long2ObjectMap<WeakReference<EntryReference<?>>> references;
 	private volatile boolean closed;
 	private final Object closeLock = new Object();
 	private final Object indicesAccessLock = new Object();
-	private final Object referencesAccessLock = new Object();
 
 	public JCWDatabase(Path dataFile, Path metadataFile) throws IOException {
 		this.typesManager = new TypesManager(this);
-		this.indices = new MixedIndexDatabase(typesManager, dataFile, metadataFile);
-		this.references = new Long2ObjectLinkedOpenHashMap<>();
+		this.indices = new MixedIndexDatabase(dataFile, metadataFile);
 		Runtime.getRuntime().addShutdownHook(new Thread(() -> {
 			try {
 				JCWDatabase.this.close();
@@ -40,93 +28,83 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
 		}));
 		this.databaseCleaner = new Cleaner(this);
 
-		//this.databaseCleaner.start();
+		this.databaseCleaner.start();
 	}
 
 	public <T> EntryReference<LightList<T>> getRoot() throws IOException {
-		return getRoot(Object.class).cast();
-	}
-	
-	public <T> EntryReference<LightList<T>> getRoot(Class<T> clazz) throws IOException {
 		checkClosed();
 		if (exists(0)) {
 			return get(0);
 		} else {
-			LightList<T> newRoot = new LightList<T>(this, new LongArrayList());
+			LightList<T> newRoot = new LightArrayList<>(this);
 			return set(0, newRoot);
 		}
 	}
 
+	public <T> EntryReference<LightList<T>> getRoot(Class<T> clazz) throws IOException {
+		return getRoot().cast();
+	}
+
 	public <T> EntryReference<T> get(long index) throws IOException {
 		checkClosed();
-		synchronized (referencesAccessLock) {
-			WeakReference<EntryReference<?>> refRef = this.references.getOrDefault(index, null);
-			EntryReference<T> ref;
-			if (refRef == null || (ref = (EntryReference<T>) refRef.get()) == null) {
-				int type;
-				long hash;
-				synchronized (indicesAccessLock) {
-					type = this.indices.getType(index);
-					hash = this.indices.getHash(index);
-				}
-				DBTypeParser<T> typeParser = this.typesManager.get(type);
-				ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser);
-				refRef = new WeakReference<>(ref);
-				this.references.put(index, refRef);	
-			}
-			return ref;
+		int type;
+		long hash;
+		synchronized (indicesAccessLock) {
+			type = this.indices.getType(index);
+			hash = this.indices.getHash(index);
 		}
+		DBTypeParser<T> typeParser = this.typesManager.get(type);
+		return new EntryReference<>(entryReferenceTools, index, hash, typeParser);
 	}
 
 	protected <T> EntryReference<T> add(T value) throws IOException {
 		checkClosed();
-		synchronized (referencesAccessLock) {
-			EntryReference<T> ref;
-			DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
-			long index;
-			long hash;
-			synchronized (indicesAccessLock) {
-				index = indices.add(typeParser.getWriter(value));
-				hash = indices.getHash(index);
-			}
-			ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser, value);
-			this.references.put(index, new WeakReference<>(ref));
-			return ref;
+		DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
+		long index;
+		long hash;
+		synchronized (indicesAccessLock) {
+			index = indices.add(typeParser.getWriter(value));
+			hash = indices.getHash(index);
 		}
+		return new EntryReference<>(entryReferenceTools, index, hash, typeParser, value);
 	}
 
 	protected boolean exists(long index) {
 		checkClosed();
-		synchronized (referencesAccessLock) {
-			synchronized (indicesAccessLock) {
-				return this.references.containsKey(index) || this.indices.has(index);
-			}
+		synchronized (indicesAccessLock) {
+			return this.indices.has(index);
 		}
 	}
 
 	protected <T> EntryReference<T> set(long index, T value) throws IOException {
 		checkClosed();
-		synchronized (referencesAccessLock) {
-			EntryReference<T> ref;
-			if (exists(index)) {
-				ref = get(index);
-				ref.setValue(value);
-				return ref;
-			} else {
-				@SuppressWarnings("unchecked")
-				DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
-				long hash;
-				synchronized (indicesAccessLock) {
-					IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value));
-					hash = returnedDetails.getHash();
-				}
-				ref = new EntryReference<>(entryReferenceTools, index, hash, typeParser);
-				this.references.put(index, new WeakReference<EntryReference<?>>(ref));
-				return ref;
+		EntryReference<T> ref;
+		if (exists(index)) {
+			ref = get(index);
+			ref.setValue(value);
+			return ref;
+		} else {
+			@SuppressWarnings("unchecked")
+			DBTypeParser<T> typeParser = this.typesManager.get((Class<T>) value.getClass());
+			long hash;
+			synchronized (indicesAccessLock) {
+				IndexDetails returnedDetails = indices.set(index, typeParser.getWriter(value));
+				hash = returnedDetails.getHash();
 			}
+			return new EntryReference<>(entryReferenceTools, index, hash, typeParser);
 		}
 	}
 
+	public <U> void registerType(Class<U> clazz, short type, DBTypeParser<U> parser) {
+		final int addition = 0xEFFF8000;
+		int extendedType = addition | (type & 0x7FFF);
+		typesManager.registerType(clazz, extendedType, parser);
+	}
+
+	public <U> void registerClass(Class<U> clazz, int type) {
+		typesManager.registerGenericClass(clazz, type);
+	}
+
 	public boolean isOpen() {
 		return !closed;
 	}
@@ -145,18 +123,6 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
 
 		this.databaseCleaner.stop();
 
-		synchronized (referencesAccessLock) {
-			ObjectIterator<WeakReference<EntryReference<?>>> iterator = references.values().iterator();
-			while (iterator.hasNext()) {
-				WeakReference<EntryReference<?>> referenceRef = iterator.next();
-				EntryReference<?> reference = referenceRef.get();
-				if (reference != null) {
-					reference.close();
-					iterator.remove();
-				}
-
-			}
-		}
 		synchronized (indicesAccessLock) {
 			this.indices.close();
 		}
@@ -171,57 +137,10 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
 
 	@Override
 	public long clean() {
-		long removedItems = cleanEmptyReferences()
-				+ cleanExtraReferences()
-				+ indices.clean();
+		long removedItems = indices.clean();
 		return removedItems;
 	}
 
-
-	private long cleanEmptyReferences() {
-		long removed = 0;
-		synchronized(referencesAccessLock) {
-			ObjectIterator<Entry<WeakReference<EntryReference<?>>>> iterator = references.long2ObjectEntrySet().iterator();
-			while (iterator.hasNext()) {
-				Entry<WeakReference<EntryReference<?>>> entry = iterator.next();
-				if (entry.getValue().get() == null) {
-					iterator.remove();
-					removed++;
-				}
-			}
-		}
-		return removed;
-	}
-
-	private long cleanExtraReferences() {
-		long removedReferences = 0;
-		synchronized(referencesAccessLock) {
-			if (references.size() > MAX_LOADED_REFERENCES) {
-				long count = 0;
-				ObjectIterator<Entry<WeakReference<EntryReference<?>>>> iterator = references.long2ObjectEntrySet().iterator();
-				while (iterator.hasNext()) {
-					Entry<WeakReference<EntryReference<?>>> entry = iterator.next();
-					if (count > MAX_LOADED_REFERENCES * 3l / 2l) {
-						WeakReference<EntryReference<?>> weakRef = entry.getValue();
-						EntryReference<?> ref = weakRef.get();
-						if (ref != null) {
-							try {
-								ref.close();
-							} catch (IOException e) {
-								e.printStackTrace();
-							}
-						}
-						iterator.remove();
-						removedReferences++;
-					} else {
-						count++;
-					}
-				}
-			}
-		}
-		return removedReferences;
-	}
-
 	public class EntryReferenceTools {
 		private EntryReferenceTools() {
 

src/main/java/org/warp/jcwdb/LightArrayList.java

@@ -0,0 +1,445 @@
+package org.warp.jcwdb;
+
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+
+public class LightArrayList<T> implements LightList<T> {
+
+	public final LongArrayList internalList;
+	private final transient JCWDatabase db;
+
+	/**
+	 * @param db Database reference
+	 */
+	public LightArrayList(JCWDatabase db) {
+		this.db = db;
+		this.internalList = new LongArrayList();
+	}
+
+	/**
+	 * @param db       Database reference
+	 * @param elements Elements to add
+	 */
+	public LightArrayList(JCWDatabase db, LongArrayList elements) {
+		this.db = db;
+		this.internalList = new LongArrayList(elements);
+	}
+
+	@Override
+	public int size() {
+		return internalList.size();
+	}
+
+	@Override
+	public boolean isEmpty() {
+		return internalList.isEmpty();
+	}
+
+	@Override
+	public boolean contains(Object o) {
+		if (o != null) {
+			for (long element : internalList) {
+				EntryReference<T> ref = null;
+				try {
+					ref = db.get(element);
+					if (o.equals(ref.getValueReadOnly())) {
+						return true;
+					}
+				} catch (IOException e) {
+					e.printStackTrace();
+				}
+			}
+		}
+		return false;
+	}
+
+	/**
+	 * Use iteratorReferences()
+	 */
+	@Deprecated
+	@SuppressWarnings("unchecked")
+	@Override
+	public Iterator<T> iterator() {
+		System.out.println("WARNING! YOU ARE USING iterator()! PLEASE USE ITERATORREFERENCES TO AVOID OUTOFMEMORY!");
+		final ArrayList<T> elements = new ArrayList<>();
+		for (long element : internalList) {
+			try {
+				elements.add((T) db.get(element).getValueReadOnly());
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+		return elements.iterator();
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public Iterator<EntryReference<T>> iteratorReferences() {
+		final ArrayList<EntryReference<T>> elements = new ArrayList<>();
+		for (long element : internalList) {
+			try {
+				elements.add(db.get(element));
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+		return elements.iterator();
+	}
+
+	/**
+	 * USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
+	 *
+	 * @param action
+	 */
+	@Deprecated
+	@Override
+	public void forEach(Consumer<? super T> action) {
+		Objects.requireNonNull(action);
+		for (T t : this) {
+			action.accept(t);
+		}
+	}
+
+	@Override
+	public void forEachReference(Consumer<? super EntryReference<T>> action) {
+		Objects.requireNonNull(action);
+		for (long index : this.internalList) {
+			try {
+				action.accept(db.get(index));
+			} catch (IOException e) {
+				throw (RuntimeException) new RuntimeException().initCause(e);
+			}
+		}
+	}
+
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public T[] toArray() {
+		final T[] elements = (T[]) new Objects[internalList.size()];
+		for (int i = 0; i < elements.length; i++) {
+			try {
+				elements[i] = (T) db.get(internalList.getLong(i)).getValueReadOnly();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+		return elements;
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public <T1> T1[] toArray(T1[] a) {
+		final T1[] elements = (T1[]) new Objects[internalList.size()];
+		for (int i = 0; i < elements.length; i++) {
+			try {
+				elements[i] = (T1) db.get(internalList.getLong(i)).getValueReadOnly();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+		return elements;
+	}
+
+	@Override
+	public boolean add(T o) {
+		EntryReference<T> ref = addEntry(o);
+		return ref != null;
+	}
+
+	@Override
+	public EntryReference<T> addEntry(T o) {
+		EntryReference<T> ref = addToDatabase(o);
+		if (internalList.add(ref.getIndex())) {
+			return ref;
+		} else {
+			return null;
+		}
+	}
+
+	@Override
+	public boolean remove(Object o) {
+		int removeIndex = indexOf(o);
+		if (removeIndex >= 0) {
+			internalList.removeLong(removeIndex);
+			return true;
+		}
+		return false;
+	}
+
+	@Override
+	public boolean remove(EntryReference<T> ref) {
+		int removeIndex = indexOfEntry(ref);
+		if (removeIndex >= 0) {
+			internalList.removeLong(removeIndex);
+			return true;
+		}
+		return false;
+	}
+
+	@Override
+	public boolean containsAll(Collection<?> c) {
+		for (Object o : c) {
+			int objIndex = indexOf(o);
+			if (objIndex < 0) {
+				return false;
+			}
+		}
+		return true;
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public boolean addAll(Collection<? extends T> c) {
+		boolean result = false;
+		for (Object o : c) {
+			result |= add((T) o);
+		}
+		return result;
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public boolean addAll(int index, Collection<? extends T> c) {
+		boolean result = false;
+		int delta = 0;
+		for (Object o : c) {
+			add(index + delta, (T) o);
+			result = true;
+			delta++;
+		}
+		return result;
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public boolean removeAll(Collection<?> c) {
+		boolean result = false;
+		for (Object o : c) {
+			result |= remove((T) o);
+		}
+		return result;
+	}
+
+	@Override
+	public boolean retainAll(Collection<?> c) {
+		boolean result = false;
+		LongArrayList collectionHashes = new LongArrayList();
+		ObjectArrayList<Object> collection = new ObjectArrayList<>();
+		collection.addAll(c);
+		for (Object o : c) {
+			collectionHashes.add(db.calculateHash(o));
+		}
+		for (int i = 0; i < internalList.size(); i++) {
+			long hash = internalList.getLong(i);
+			int positionInCollection = collectionHashes.indexOf(hash);
+			if (positionInCollection == -1) {
+				remove(collection.get(positionInCollection));
+				result = true;
+			}
+		}
+		return result;
+	}
+
+	@Override
+	public void clear() {
+		internalList.clear();
+	}
+
+	/**
+	 * Use getReference or getReadOnlyValue
+	 */
+	@Deprecated
+	@Override
+	public T get(int index) {
+		return getReadOnlyValue(index);
+	}
+
+	@SuppressWarnings("unchecked")
+	public T getReadOnlyValue(int index) {
+		try {
+			return (T) db.get(internalList.getLong(index)).getValueReadOnly();
+		} catch (IOException e) {
+			e.printStackTrace();
+			return null;
+		}
+	}
+
+	@Override
+	public EntryReference<T> getReference(int index) {
+		try {
+			return db.get(internalList.getLong(index)).cast();
+		} catch (IOException e) {
+			e.printStackTrace();
+			return null;
+		}
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public T set(int index, T element) {
+		EntryReference<T> ref = addToDatabase(element);
+		long oldIndex = internalList.set(index, ref.getIndex());
+		try {
+			ref.close();
+			return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnly();
+		} catch (IOException e) {
+			throw (NullPointerException) new NullPointerException().initCause(e);
+		}
+	}
+
+	@Override
+	public void add(int index, T element) {
+		EntryReference<T> ref = addToDatabase(element);
+		internalList.add(index, ref.getIndex());
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public T remove(int index) {
+		long oldIndex = internalList.removeLong(index);
+		try {
+			return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnly();
+		} catch (IOException e) {
+			throw (NullPointerException) new NullPointerException().initCause(e);
+		}
+	}
+
+	@Override
+	public int indexOf(Object o) {
+		EntryReference<?> ref = addToDatabase(o);
+		long objToRemoveHash = ref.calculateHash();
+		LongArrayList hashes = new LongArrayList();
+
+
+		for (int i = 0; i < hashes.size(); i++) {
+			long hash = hashes.getLong(i);
+			if (objToRemoveHash == hash) {
+				try {
+					if (ref.equals(db.get(internalList.getLong(i)))) {
+						return i;
+					}
+				} catch (IOException e) {
+					throw (NullPointerException) new NullPointerException().initCause(e);
+				}
+			}
+		}
+		return -1;
+	}
+
+	@Override
+	public int indexOfEntry(EntryReference<T> ref) {
+		for (int i = 0; i < internalList.size(); i++) {
+			long index = internalList.getLong(i);
+			try {
+				EntryReference<?> ref2 = db.get(index);
+				if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) {
+					return i;
+				}
+			} catch (IOException e) {
+				throw (NullPointerException) new NullPointerException().initCause(e);
+			}
+		}
+		return -1;
+	}
+
+	@Override
+	public void appendIndex(long elementIndex) {
+		internalList.add(elementIndex);
+	}
+
+	@Override
+	public int lastIndexOf(Object o) {
+		EntryReference<T> ref = addToDatabase(o).cast();
+		return lastIndexOfEntry(ref);
+	}
+
+	@Override
+	public int lastIndexOfEntry(EntryReference<T> ref) {
+		long objToRemoveHash = ref.calculateHash();
+
+		int lastValue = -1;
+
+		for (int i = 0; i < internalList.size(); i++) {
+			long index2 = internalList.getLong(i);
+			try {
+				EntryReference<?> ref2 = db.get(index2);
+				if (objToRemoveHash == ref2.calculateHash()) {
+					if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) {
+						lastValue = i;
+					}
+				}
+			} catch (IOException e) {
+				throw (NullPointerException) new NullPointerException().initCause(e);
+			}
+		}
+		return lastValue;
+	}
+
+	@Deprecated
+	@Override
+	public ListIterator<T> listIterator() {
+		// TODO: implement
+		throw new RuntimeException("Not implemented!");
+	}
+
+	@Deprecated
+	@Override
+	public ListIterator<T> listIterator(int index) {
+		// TODO: implement
+		throw new RuntimeException("Not implemented!");
+	}
+
+	@Deprecated
+	@Override
+	public List<T> subList(int fromIndex, int toIndex) {
+		// TODO: implement
+		throw new RuntimeException("Not implemented!");
+	}
+
+	private <U> EntryReference<U> addToDatabase(U obj) {
+		EntryReference<U> ref;
+		try {
+			ref = db.add(obj);
+		} catch (IOException e) {
+			throw (NullPointerException) new NullPointerException().initCause(e);
+		}
+		return ref;
+	}
+
+	@Deprecated
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = 1;
+		result = prime * result + ((internalList == null) ? 0 : internalList.hashCode());
+		return result;
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public boolean removeIf(Predicate<? super T> filter) {
+		Objects.requireNonNull(filter);
+		boolean removed = false;
+		for (int i = 0; i < internalList.size(); ) {
+			T obj;
+			try {
+				obj = ((EntryReference<T>) (db.get(internalList.getLong(i)).cast())).getValueReadOnly();
+			} catch (IOException e) {
+				throw (NullPointerException) new NullPointerException().initCause(e);
+			}
+			if (filter.test(obj)) {
+				internalList.removeLong(i);
+				removed = true;
+			} else {
+				i++;
+			}
+		}
+		return removed;
+	}
+}

src/main/java/org/warp/jcwdb/LightBigList.java

@@ -0,0 +1,521 @@
+package org.warp.jcwdb;
+
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+
+public class LightBigList<T> implements LightList<T> {
+
+	public static final int MAX_ELEMENTS_PER_CHUNK = 10000;
+
+	public final LongArrayList chunks;
+	public final IntArrayList chunkSizes;
+	private final transient JCWDatabase db;
+
+	/**
+	 * @param db Database reference
+	 */
+	public LightBigList(JCWDatabase db) {
+		this.db = db;
+		this.chunks = new LongArrayList();
+		this.chunkSizes = new IntArrayList();
+	}
+
+	/**
+	 * @param db       Database reference
+	 * @param elements Elements to add
+	 */
+	public LightBigList(JCWDatabase db, LongArrayList elements) {
+		this.db = db;
+		this.chunks = new LongArrayList();
+		this.chunkSizes = new IntArrayList();
+		elements.forEach((long element) -> {
+			this.appendIndex(element);
+		});
+	}
+
+	public LightBigList(JCWDatabase db, LongArrayList chunks, IntArrayList chunkSizes) {
+		this.db = db;
+		this.chunks = chunks;
+		this.chunkSizes = chunkSizes;
+	}
+
+	/**
+	 * Append an index to the first free chunk
+	 * @param elementIndex
+	 */
+	public void appendIndex(long elementIndex) {
+		for (int i = 0; i < chunks.size(); i++) {
+			final int chunkNumber = i;
+			if (MAX_ELEMENTS_PER_CHUNK - chunkSizes.getInt(i) > 0) {
+				try {
+					final long chunkIndex = chunks.getLong(i);
+					final EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
+					chunkRef.editValue((chunk) -> {
+						chunk.appendIndex(elementIndex);
+						chunkSizes.set(chunkNumber, chunkSizes.getInt(chunkNumber) + 1);
+					});
+					return;
+				} catch (IOException ex) {
+					throw (NullPointerException) new NullPointerException().initCause(ex);
+				}
+			}
+		}
+		LightList<T> newChunk = new LightArrayList<>(db);
+		newChunk.appendIndex(elementIndex);
+		long newChunkIndex;
+		try {
+			newChunkIndex = db.add(newChunk).getIndex();
+		} catch (IOException ex) {
+			throw (NullPointerException) new NullPointerException().initCause(ex);
+		}
+		chunks.add(newChunkIndex);
+		chunkSizes.add(1);
+	}
+
+	/**
+	 * Get the elements count
+	 * @return the size of the list
+	 */
+	@Override
+	public int size() {
+		int size = 0;
+		for (int chunkSize : this.chunkSizes) {
+			size += chunkSize;
+		}
+		return size;
+	}
+
+	/**
+	 *
+	 * @return the count of chunks
+	 */
+	public int chunksCount() {
+		return this.chunkSizes.size();
+	}
+
+	/**
+	 * Check if the list is empty
+	 * @return true if the list is empty
+	 */
+	@Override
+	public boolean isEmpty() {
+		return this.size() <= 0;
+	}
+
+	@Override
+	public boolean contains(Object o) {
+		if (o != null) {
+			for (long chunkIndex : chunks) {
+				try {
+					EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
+					LightList<T> chunk = chunkRef.getValueReadOnly();
+					if (chunk.contains(o)) {
+						return true;
+					}
+				} catch (IOException ex) {
+					throw (NullPointerException) new NullPointerException().initCause(ex);
+				}
+			}
+		}
+		return false;
+	}
+
+	/**
+	 * Use iteratorReferences()
+	 */
+	@Deprecated
+	@Override
+	public Iterator<T> iterator() {
+		throw new RuntimeException("iterator() isn't implemented!");
+	}
+
+	@Deprecated
+	@Override
+	public Iterator<EntryReference<T>> iteratorReferences() {
+		throw new RuntimeException("iteratorReferences() isn't implemented!");
+	}
+
+	/**
+	 * USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
+	 *
+	 * @param action
+	 */
+	@Deprecated
+	@Override
+	public void forEach(Consumer<? super T> action) {
+		throw new RuntimeException("forEach() isn't implemented! Use forEachReferences() instead");
+	}
+
+	@Override
+	public void forEachReference(Consumer<? super EntryReference<T>> action) {
+		Objects.requireNonNull(action);
+		for (long chunkIndex : this.chunks) {
+			try {
+				EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
+				LightList<T> chunk = chunkRef.getValueReadOnly();
+				chunk.forEachReference(action);
+			} catch (IOException ex) {
+				throw (NullPointerException) new NullPointerException().initCause(ex);
+			}
+		}
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public T[] toArray() {
+		throw new RuntimeException("toArray() isn't implemented!");
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public <T1> T1[] toArray(T1[] a) {
+		throw new RuntimeException("toArray() isn't implemented!");
+	}
+
+	/**
+	 * Use addEntry(o)
+	 * @param o
+	 * @return
+	 */
+	@Deprecated
+	@Override
+	public boolean add(T o) {
+		EntryReference<T> ref = addEntry(o);
+		return ref != null;
+	}
+
+	@Override
+	public EntryReference<T> addEntry(T o) {
+		EntryReference<T> ref = addToDatabase(o);
+		appendIndex(ref.getIndex());
+		return ref;
+	}
+
+	@Override
+	public boolean remove(Object o) {
+		final int removeOffset = indexOf(o);
+		return removeAt(removeOffset) != null;
+	}
+
+	@Override
+	public boolean remove(EntryReference<T> ref) {
+		final int removeOffset = indexOfEntry(ref);
+		return removeAt(removeOffset) != null;
+	}
+
+	private T removeAt(int removeOffset) {
+		final VariableWrapper<T> result = new VariableWrapper<>(null);
+		long currentOffset = 0;
+		if (removeOffset >= 0) {
+			// Iterate through all chunks
+			for (int i = 0; i < chunks.size(); i++) {
+				final int currentChunkSize = chunkSizes.getInt(i);
+				final long chunkStartOffset = currentOffset;
+				currentOffset += currentChunkSize;
+				// If the offset to remove is in the current chunk
+				if (currentOffset > removeOffset) {
+					// Get chunk index
+					final long chunkIndex = chunks.getLong(i);
+					// Get the offset relative to the current chunk
+					final int relativeOffset = (int) (removeOffset - chunkStartOffset);
+
+					if (relativeOffset < 0) {
+						return null;
+					}
+
+					try {
+						EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
+						chunkRef.editValue((chunk) -> {
+							result.var = chunk.remove(relativeOffset);
+						});
+					} catch (IOException ex) {
+						throw (NullPointerException) new NullPointerException().initCause(ex);
+					}
+					chunkSizes.set(removeOffset, currentChunkSize - 1);
+					break;
+				}
+			}
+			return result.var;
+		}
+		return null;
+	}
+
+	@Override
+	public boolean containsAll(Collection<?> c) {
+		for (Object o : c) {
+			int objIndex = indexOf(o);
+			if (objIndex < 0) {
+				return false;
+			}
+		}
+		return true;
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public boolean addAll(Collection<? extends T> c) {
+		boolean result = false;
+		for (Object o : c) {
+			result |= add((T) o);
+		}
+		return result;
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public boolean addAll(int index, Collection<? extends T> c) {
+		boolean result = false;
+		int delta = 0;
+		for (Object o : c) {
+			add(index + delta, (T) o);
+			result = true;
+			delta++;
+		}
+		return result;
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public boolean removeAll(Collection<?> c) {
+		boolean result = false;
+		for (Object o : c) {
+			result |= remove((T) o);
+		}
+		return result;
+	}
+
+	@Override
+	public boolean retainAll(Collection<?> c) {
+		boolean result = false;
+		LongArrayList collectionHashes = new LongArrayList();
+		ObjectArrayList<Object> collection = new ObjectArrayList<>();
+		collection.addAll(c);
+		for (Object o : c) {
+			collectionHashes.add(db.calculateHash(o));
+		}
+		for (int i = 0; i < chunks.size(); i++) {
+			long hash = chunks.getLong(i);
+			int positionInCollection = collectionHashes.indexOf(hash);
+			if (positionInCollection == -1) {
+				remove(collection.get(positionInCollection));
+				result = true;
+			}
+		}
+		return result;
+	}
+
+	@Override
+	public void clear() {
+		chunks.clear();
+		chunkSizes.clear();
+	}
+
+	/**
+	 * Use getReference or getReadOnlyValue
+	 */
+	@Deprecated
+	@Override
+	public T get(int index) {
+		return getReadOnlyValue(index);
+	}
+
+	@SuppressWarnings("unchecked")
+	public T getReadOnlyValue(int index) {
+		try {
+			return (T) db.get(chunks.getLong(index)).getValueReadOnly();
+		} catch (IOException e) {
+			e.printStackTrace();
+			return null;
+		}
+	}
+
+	@Override
+	public EntryReference<T> getReference(int index) {
+		try {
+			return db.get(chunks.getLong(index)).cast();
+		} catch (IOException e) {
+			e.printStackTrace();
+			return null;
+		}
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public T set(int setOffset, final T element) {
+		long nextChunkOffset = 0;
+		VariableWrapper<T> wrapper = new VariableWrapper<>(null);
+		if (setOffset >= 0) {
+			// Iterate through all chunks
+			for (int i = 0; i < chunks.size(); i++) {
+				final int currentChunkSize = chunkSizes.getInt(i);
+				final long chunkStartOffset = nextChunkOffset;
+				nextChunkOffset += currentChunkSize;
+				// If the offset to remove is in the current chunk
+				if (nextChunkOffset > setOffset) {
+					// Get chunk index
+					final long chunkIndex = chunks.getLong(i);
+					// Get the offset relative to the current chunk
+					final int relativeOffset = (int) (setOffset - chunkStartOffset);
+
+					if (relativeOffset < 0) {
+						throw new NullPointerException("Relative Offset < 0");
+					}
+
+					try {
+						EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
+						chunkRef.editValue((chunk) -> {
+							chunk.set(relativeOffset, element);
+							wrapper.var = element;
+						});
+					} catch (IOException ex) {
+						throw (NullPointerException) new NullPointerException().initCause(ex);
+					}
+					break;
+				}
+			}
+			return wrapper.var;
+		}
+		return null;
+	}
+
+	@Override
+	public void add(int index, T element) {
+		throw new RuntimeException("add() isn't implemented!");
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public T remove(int index) {
+		return this.removeAt(index);
+	}
+
+	@Override
+	public int indexOf(Object o) {
+		EntryReference<T> ref = addToDatabase(o).cast();
+		return indexOfEntry(ref);
+	}
+
+	@Override
+	public int indexOfEntry(EntryReference<T> ref) {
+		int currentOffset = 0;
+		// Iterate through all chunks
+		for (int i = 0; i < chunks.size(); i++) {
+			try {
+				final int currentChunkSize = chunkSizes.getInt(i);
+				// If the offset to remove is in the current chunk
+
+				// Get chunk index
+				final long chunkIndex = chunks.getLong(i);
+				EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
+				final int foundIndex = chunkRef.getValueReadOnly().indexOfEntry(ref);
+				if (foundIndex >= 0) {
+					return currentOffset + foundIndex;
+				}
+				currentOffset += currentChunkSize;
+			} catch (IOException ex) {
+				throw (NullPointerException) new NullPointerException().initCause(ex);
+			}
+		}
+		return -1;
+	}
+
+	@Override
+	public int lastIndexOf(Object o) {
+	return lastIndexOfEntry(addToDatabase(o).cast());
+	}
+
+	@Override
+	public int lastIndexOfEntry(EntryReference<T> ref) {
+		int currentOffset = 0;
+		// Iterate through all chunks
+		for (int i = chunks.size() - 1; i >= 0; i--) {
+			try {
+				final int currentChunkSize = chunkSizes.getInt(i);
+				// If the offset to remove is in the current chunk
+
+				// Get chunk index
+				final long chunkIndex = chunks.getLong(i);
+				EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
+				final int foundIndex = chunkRef.getValueReadOnly().lastIndexOfEntry(ref);
+				if (foundIndex >= 0) {
+					return currentOffset + foundIndex;
+				}
+				currentOffset += currentChunkSize;
+			} catch (IOException ex) {
+				throw (NullPointerException) new NullPointerException().initCause(ex);
+			}
+		}
+		return -1;
+	}
+
+	@Deprecated
+	@Override
+	public ListIterator<T> listIterator() {
+		// TODO: implement
+		throw new RuntimeException("Not implemented!");
+	}
+
+	@Deprecated
+	@Override
+	public ListIterator<T> listIterator(int index) {
+		// TODO: implement
+		throw new RuntimeException("Not implemented!");
+	}
+
+	@Deprecated
+	@Override
+	public List<T> subList(int fromIndex, int toIndex) {
+		// TODO: implement
+		throw new RuntimeException("Not implemented!");
+	}
+
+	private <U> EntryReference<U> addToDatabase(U obj) {
+		EntryReference<U> ref;
+		try {
+			ref = db.add(obj);
+		} catch (IOException e) {
+			throw (NullPointerException) new NullPointerException().initCause(e);
+		}
+		return ref;
+	}
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = 1;
+		result = prime * result + ((chunks == null) ? 0 : chunks.hashCode());
+		return result;
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public boolean removeIf(Predicate<? super T> filter) {
+		Objects.requireNonNull(filter);
+		final VariableWrapper<Boolean> result = new VariableWrapper(false);
+		// Iterate through all chunks
+		for (int i = 0; i < chunks.size(); i++) {
+			try {
+				final int chunkOffset = i;
+				// Get chunk index
+				final long chunkIndex = chunks.getLong(i);
+				EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
+				chunkRef.editValue((chunk) -> {
+					 boolean removed = chunk.removeIf(filter);
+					 if (removed) {
+						 result.var = true;
+						 chunkSizes.set(chunkOffset, chunk.size());
+					 }
+				});
+			} catch (IOException ex) {
+				throw (NullPointerException) new NullPointerException().initCause(ex);
+			}
+		}
+		return result.var;
+	}
+}

src/main/java/org/warp/jcwdb/LightList.java

@@ -1,390 +1,24 @@
 package org.warp.jcwdb;
 
-import java.io.IOException;
-import java.util.*;
-import java.util.function.Predicate;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Consumer;
 
-import it.unimi.dsi.fastutil.longs.LongArrayList;
-import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+public interface LightList<T> extends List<T> {
 
-public class LightList<T> implements List<T> {
+	Iterator<EntryReference<T>> iteratorReferences();
 
-	public final LongArrayList internalList;
-	private final transient JCWDatabase db;
+	void forEachReference(Consumer<? super EntryReference<T>> action);
 
-	public LightList(JCWDatabase db, LongArrayList internalList) {
-		this.internalList = internalList;
-		this.db = db;
-	}
+	EntryReference<T> addEntry(T o);
 
-	@Override
-	public int size() {
-		return internalList.size();
-	}
+	boolean remove(EntryReference<T> ref);
 
-	@Override
-	public boolean isEmpty() {
-		return internalList.isEmpty();
-	}
+	EntryReference<T> getReference(int index);
 
-	@Override
-	public boolean contains(Object o) {
-		if (o != null) {
-			for (Long element : internalList) {
-				EntryReference<T> ref = null;
-				try {
-					ref = db.get(element);
-					if (o.equals(ref.getValueReadOnly())) {
-						return true;
-					}
-				} catch (IOException e) {
-					e.printStackTrace();
-				}
-			}
-		}
-		return false;
-	}
+	int indexOfEntry(EntryReference<T> ref);
 
-	/**
-	 * Use iteratorReferences()
-	 */
-	@Deprecated
-	@SuppressWarnings("unchecked")
-	@Override
-	public Iterator<T> iterator() {
-		final ArrayList<T> elements = new ArrayList<>();
-		for (Long element : internalList) {
-			try {
-				elements.add((T) db.get(element).getValueReadOnly());
-			} catch (IOException e) {
-				e.printStackTrace();
-			}
-		}
-		return elements.iterator();
-	}
+	int lastIndexOfEntry(EntryReference<T> ref);
 
-	@SuppressWarnings("unchecked")
-	public Iterator<EntryReference<T>> iteratorReferences() {
-		final ArrayList<EntryReference<T>> elements = new ArrayList<>();
-		for (Long element : internalList) {
-			try {
-				elements.add(db.get(element));
-			} catch (IOException e) {
-				e.printStackTrace();
-			}
-		}
-		return elements.iterator();
-	}
-	
-
-	@SuppressWarnings("unchecked")
-	@Override
-	public T[] toArray() {
-		final T[] elements = (T[]) new Objects[internalList.size()];
-		for (int i = 0; i < elements.length; i++) {
-			try {
-				elements[i] = (T) db.get(internalList.getLong(i)).getValueReadOnly();
-			} catch (IOException e) {
-				e.printStackTrace();
-			}
-		}
-		return elements;
-	}
-
-	@SuppressWarnings("unchecked")
-	@Override
-	public <T1> T1[] toArray(T1[] a) {
-		final T1[] elements = (T1[]) new Objects[internalList.size()];
-		for (int i = 0; i < elements.length; i++) {
-			try {
-				elements[i] = (T1) db.get(internalList.getLong(i)).getValueReadOnly();
-			} catch (IOException e) {
-				e.printStackTrace();
-			}
-		}
-		return elements;
-	}
-
-	@Override
-	public boolean add(T o) {
-		EntryReference<T> ref = addEntry(o);
-		return ref != null;
-	}
-	
-	public EntryReference<T> addEntry(T o) {
-		EntryReference<T> ref = addToDatabase(o);
-		if (internalList.add(ref.getIndex())) {
-			return ref;
-		} else {
-			return null;
-		}
-	}
-
-	@Override
-	public boolean remove(Object o) {
-		int removeIndex = indexOf(o);
-		if (removeIndex >= 0) {
-			internalList.removeLong(removeIndex);
-			return true;
-		}
-		return false;
-	}
-	
-	public boolean remove(EntryReference<T> ref) {
-		int removeIndex = indexOfEntry(ref);
-		if (removeIndex >= 0) {
-			internalList.removeLong(removeIndex);
-			return true;
-		}
-		return false;
-	}
-
-	@Override
-	public boolean containsAll(Collection<?> c) {
-		for (Object o : c) {
-			int objIndex = indexOf(o);
-			if (objIndex < 0) {
-				return false;
-			}
-		}
-		return true;
-	}
-
-	@SuppressWarnings("unchecked")
-	@Override
-	public boolean addAll(Collection<? extends T> c) {
-		boolean result = false;
-		for (Object o : c) {
-			result |= add((T) o);
-		}
-		return result;
-	}
-
-	@SuppressWarnings("unchecked")
-	@Override
-	public boolean addAll(int index, Collection<? extends T> c) {
-		boolean result = false;
-		int delta = 0;
-		for (Object o : c) {
-			add(index + delta, (T) o);
-			result = true;
-			delta++;
-		}
-		return result;
-	}
-
-	@SuppressWarnings("unchecked")
-	@Override
-	public boolean removeAll(Collection<?> c) {
-		boolean result = false;
-		for (Object o : c) {
-			result |= remove((T) o);
-		}
-		return result;
-	}
-
-	@Override
-	public boolean retainAll(Collection<?> c) {
-		boolean result = false;
-		LongArrayList collectionHashes = new LongArrayList();
-		ObjectArrayList<Object> collection = new ObjectArrayList<>();
-		collection.addAll(c);
-		for (Object o : c) {
-			collectionHashes.add(db.calculateHash(o));
-		}
-		for (int i = 0; i < internalList.size(); i++) {
-			long hash = internalList.getLong(i);
-			int positionInCollection = collectionHashes.indexOf(hash);
-			if (positionInCollection == -1) {
-				remove(collection.get(positionInCollection));
-				result = true;
-			}
-		}
-		return result;
-	}
-
-	@Override
-	public void clear() {
-		internalList.clear();
-	}
-
-	/**
-	 * Use getReference or getReadOnlyValue
-	 */
-	@Deprecated
-	@Override
-	public T get(int index) {
-		return getReadOnlyValue(index);
-	}
-
-	@SuppressWarnings("unchecked")
-	public T getReadOnlyValue(int index) {
-		try {
-			return (T) db.get(internalList.getLong(index)).getValueReadOnly();
-		} catch (IOException e) {
-			e.printStackTrace();
-			return null;
-		}
-	}
-	
-	public EntryReference<T> getReference(int index) {
-		try {
-			return db.get(internalList.getLong(index)).cast();
-		} catch (IOException e) {
-			e.printStackTrace();
-			return null;
-		}
-	}
-
-	@SuppressWarnings("unchecked")
-	@Override
-	public T set(int index, T element) {
-		EntryReference<T> ref = addToDatabase(element);
-		long oldIndex = internalList.set(index, ref.getIndex());
-		try {
-			ref.close();
-			return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnly();
-		} catch (IOException e) {
-			throw (NullPointerException) new NullPointerException().initCause(e);
-		}
-	}
-
-	@Override
-	public void add(int index, T element) {
-		EntryReference<T> ref = addToDatabase(element);
-		internalList.add(index, ref.getIndex());
-	}
-
-	@SuppressWarnings("unchecked")
-	@Override
-	public T remove(int index) {
-		long oldIndex = internalList.removeLong(index);
-		try {
-			return ((EntryReference<T>) (db.get(oldIndex))).getValueReadOnly();
-		} catch (IOException e) {
-			throw (NullPointerException) new NullPointerException().initCause(e);
-		}
-	}
-	
-	@Override
-	public int indexOf(Object o) {
-		EntryReference<?> ref = addToDatabase(o);
-		long objToRemoveHash = ref.calculateHash();
-		LongArrayList hashes = new LongArrayList();
-		
-		
-		for (int i = 0; i < hashes.size(); i++) {
-			long hash = hashes.getLong(i);
-			if (objToRemoveHash == hash) {
-				try {
-					if (ref.equals(db.get(internalList.getLong(i)))) {
-						return i;
-					}
-				} catch (IOException e) {
-					throw (NullPointerException) new NullPointerException().initCause(e);
-				}
-			}
-		}
-		return -1;
-	}
-
-	public int indexOfEntry(EntryReference<T> ref) {
-		for (int i = 0; i < internalList.size(); i++) {
-			long index = internalList.getLong(i);
-			try {
-				EntryReference<?> ref2 = db.get(index);
-				if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) {
-					return i;
-				}
-			} catch (IOException e) {
-				throw (NullPointerException) new NullPointerException().initCause(e);
-			}
-		}
-		return -1;
-	}
-
-	@Override
-	public int lastIndexOf(Object o) {
-		EntryReference<?> ref = addToDatabase(o);
-		long objToRemoveHash = ref.calculateHash();
-
-		int lastValue = -1;
-
-		for (int i = 0; i < internalList.size(); i++) {
-			long index2 = internalList.getLong(i);
-			try {
-				EntryReference<?> ref2 = db.get(index2);
-				if (objToRemoveHash == ref2.calculateHash()) {
-					if (ref.getValueReadOnly().equals(ref2.getValueReadOnly())) {
-						lastValue = i;
-					}
-				}
-			} catch (IOException e) {
-				throw (NullPointerException) new NullPointerException().initCause(e);
-			}
-		}
-		return lastValue;
-	}
-
-	@Deprecated
-	@Override
-	public ListIterator<T> listIterator() {
-		// TODO: implement
-		throw new RuntimeException("Not implemented!");
-	}
-
-	@Deprecated
-	@Override
-	public ListIterator<T> listIterator(int index) {
-		// TODO: implement
-		throw new RuntimeException("Not implemented!");
-	}
-
-	@Deprecated
-	@Override
-	public List<T> subList(int fromIndex, int toIndex) {
-		// TODO: implement
-		throw new RuntimeException("Not implemented!");
-	}
-
-	private <U> EntryReference<U> addToDatabase(U obj) {
-		EntryReference<U> ref;
-		try {
-			ref = db.add(obj);
-		} catch (IOException e) {
-			throw (NullPointerException) new NullPointerException().initCause(e);
-		}
-		return ref;
-	}
-
-	@Deprecated
-	@Override
-	public int hashCode() {
-		final int prime = 31;
-		int result = 1;
-		result = prime * result + ((internalList == null) ? 0 : internalList.hashCode());
-		return result;
-	}
-
-	@SuppressWarnings("unchecked")
-	@Override
-	public boolean removeIf(Predicate<? super T> filter) {
-		Objects.requireNonNull(filter);
-		boolean removed = false;
-		for (int i = 0; i < internalList.size(); ) {
-			T obj;
-			try {
-				obj = ((EntryReference<T>) (db.get(internalList.getLong(i)).cast())).getValueReadOnly();
-			} catch (IOException e) {
-				throw (NullPointerException) new NullPointerException().initCause(e);
-			}
-			if (filter.test(obj)) {
-				internalList.removeLong(i);
-				removed = true;
-			} else {
-				i++;
-			}
-		}
-		return removed;
-	}
+	void appendIndex(long elementIndex);
 }

src/main/java/org/warp/jcwdb/MixedIndexDatabase.java

@@ -8,19 +8,16 @@ import java.nio.file.Path;
 import java.util.function.Consumer;
 
 public class MixedIndexDatabase implements IndexManager {
-	private final Long2LongMap mostAccessedIndices;
 	private final FileIndexManager fileIndices;
 	private final CacheIndexManager cacheIndices;
 
-	public MixedIndexDatabase(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException {
-		this.mostAccessedIndices = new Long2LongLinkedOpenHashMap();
+	public MixedIndexDatabase(Path dataFile, Path metadataFile) throws IOException {
 		this.fileIndices = new FileIndexManager(dataFile, metadataFile);
 		this.cacheIndices = new CacheIndexManager();
 	}
 
 	@Override
 	public <T> T get(long index, DBReader<T> reader) throws IOException {
-		incrementUsage(index);
 		if (cacheIndices.has(index)) {
 			return cacheIndices.get(index, reader);
 		} else {
@@ -71,10 +68,6 @@ public class MixedIndexDatabase implements IndexManager {
 		return cacheIndices.has(index) || fileIndices.has(index);
 	}
 
-	private void incrementUsage(long index) {
-		mostAccessedIndices.put(index, mostAccessedIndices.getOrDefault(index, 0) + 1);
-	}
-
 	@Override
 	public void close() throws IOException {
 		// TODO: move all cached indices to filesIndices before closing.

src/main/java/org/warp/jcwdb/TypesManager.java

@@ -1,17 +1,18 @@
 package org.warp.jcwdb;
 
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.HashMap;
+import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
+import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import it.unimi.dsi.fastutil.objects.Object2ObjectMap;
+import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
 
 public class TypesManager {
-	private final Map<Integer, DBTypeParser<?>> types;
-	private final Map<Class<?>, DBTypeParser<?>> typesByClass;
-	private DBTypeParser<?> fallbackParser;
+	private final Int2ObjectMap<DBTypeParser<?>> types;
+	private final Object2ObjectMap<Class<?>, DBTypeParser<?>> typesByClass;
+	private DBTypedObjectParser<?> fallbackParser;
 
 	public TypesManager(JCWDatabase db) {
-		types = new HashMap<>();
-		typesByClass = new HashMap<>();
+		types = new Int2ObjectOpenHashMap<>();
+		typesByClass = new Object2ObjectOpenHashMap<>();
 		DBStandardTypes.registerStandardTypes(db, this);
 	}
 
@@ -20,7 +21,17 @@ public class TypesManager {
 		this.typesByClass.put(clazz, parser);
 	}
 
-	public void registerTypeFallback(DBTypeParser<?> parser) {
+	/**
+	 * Use this method with the most used classes to save disk space.
+	 * @param clazz
+	 * @param id
+	 * @param <T>
+	 */
+	public <T> void registerGenericClass(Class<T> clazz, int id) {
+		this.fallbackParser.registerClass(clazz, id);
+	}
+
+	public void registerTypeFallback(DBTypedObjectParser<?> parser) {
 		this.fallbackParser = parser;
 	}
 

src/main/java/org/warp/jcwdb/VariableWrapper.java

@@ -0,0 +1,12 @@
+package org.warp.jcwdb;
+
+import java.nio.channels.SeekableByteChannel;
+
+public class VariableWrapper<T> {
+
+	public T var;
+
+	public VariableWrapper(T value) {
+		this.var = value;
+	}
+}

src/main/java/org/warp/jcwdb/exampleimpl/App.java

@@ -22,6 +22,7 @@ public class App {
 			System.out.println("Loading database...");
 			long time0 = System.currentTimeMillis();
 			JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1]));
+			db.registerClass(StrangeAnimal.class, 0);
 			try {
 				long time01 = System.currentTimeMillis();
 				System.out.println("Time elapsed: " + (time01 - time0));
@@ -58,15 +59,15 @@ public class App {
 					System.out.println("Time elapsed: " + (time2_1 - time2_0));
 					ObjectList<Animal> results = new ObjectArrayList<>();
 
-					/*
-					root.forEach((value) -> {
+					root.forEachReference((valueReference) -> {
+						Animal value = valueReference.getValueReadOnly();
 						if (Animal.hasFourLegs(value)) {
 							results.add(value);
 						}
 						//System.out.println("val:" + value);
 					});
-					*/
 					long time2_2 = System.currentTimeMillis();
+					System.out.println("Matches: " + results.size());
 					System.out.println("Time elapsed: " + (time2_2 - time2_1));
 					System.out.println("Used memory: "
 									+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");

src/test/java/org/warp/jcwdb/AppTest.java

@@ -26,93 +26,4 @@ public class AppTest
         assertTrue( true );
     }
 
-    /**
-     * 
-     * @throws IOException 
-     */
-    @Test
-    public void shouldReadCustomClasses() throws IOException
-    {
-    	CustomClass customClass = new CustomClass();
-    	customClass.primitive = 1;
-    	customClass.string = "test";
-    	
-    	Path path = Files.createTempFile("", ".db"), idx = Files.createTempFile("", ".idx");
-    	
-    	JCWDatabase db = new JCWDatabase(path, idx);
-    	EntryReference<LightList<CustomClass>> ref = db.getRoot();
-    	ref.editValue((root, saver) -> {
-        	root.add(customClass);
-    	});
-    	db.close();
-    	
-    	JCWDatabase db2 = new JCWDatabase(path, idx);
-    	EntryReference<LightList<CustomClass>> ref2 = db2.getRoot();
-    	CustomClass customClass2 = ref2.getValueReadOnly().getReadOnlyValue(0);
-    	assertTrue(customClass.equals(customClass2));
-    	assertTrue(customClass.string.equals(customClass2.string));
-    	db2.close();
-    	Files.delete(path);
-    	Files.delete(idx);
-    }
-    
-    public static class CustomClass {
-    	public String string;
-    	public int primitive;
-		@Override
-		public int hashCode() {
-			final int prime = 31;
-			int result = 1;
-			result = prime * result + primitive;
-			result = prime * result + ((string == null) ? 0 : string.hashCode());
-			return result;
-		}
-		@Override
-		public boolean equals(Object obj) {
-			if (this == obj)
-				return true;
-			if (obj == null)
-				return false;
-			if (getClass() != obj.getClass())
-				return false;
-			CustomClass other = (CustomClass) obj;
-			if (primitive != other.primitive)
-				return false;
-			if (string == null) {
-				if (other.string != null)
-					return false;
-			} else if (!string.equals(other.string))
-				return false;
-			return true;
-		}
-    	
-    	
-    }
-    
-    /**
-     * 
-     * @throws IOException 
-     */
-    @Test
-    public void shouldActAsAnArrayListWithEqualObjects() throws IOException
-    {
-    	Path path = Files.createTempFile("", ".db"), idx = Files.createTempFile("", ".idx");
-    	JCWDatabase db = new JCWDatabase(path, idx);
-    	EntryReference<LightList<String>> ref = db.getRoot();
-    	LightList<String> list1 = ref.getValueReadOnly();
-    	ArrayList<String> list2 = new ArrayList<String>();
-		String s = "a";
-    	for (int i = 0; i < 10; i++) {
-    		list1.add(s);
-    		list2.add(s);
-    	}
-    	assertTrue(list1.size() == list2.size());
-    	for (int i = 0; i < 10; i++) {
-    		assertTrue(list1.get(i) == list2.get(i));
-    		assertTrue(list2.get(i) == list1.get(i));
-    	}
-    	db.close();
-    	Files.delete(path);
-    	Files.delete(idx);
-    }
 }