Fixed BigList and Cleaner

src/main/java/org/warp/jcwdb/AdvancedSaveable.java

@@ -0,0 +1,5 @@
+package org.warp.jcwdb;
+
+public interface AdvancedSaveable extends Saveable {
+	public void save(boolean isEditFinished);
+}

src/main/java/org/warp/jcwdb/CacheIndexManager.java

@@ -45,6 +45,11 @@ public class CacheIndexManager implements IndexManager {
 		return null;
 	}
 
+	@Override
+	public void setFlushingAllowed(long index, boolean isUnloadingAllowed) {
+		// TODO: implement
+	}
+
 	@Override
 	public void delete(long index) {
 		// TODO: implement

src/main/java/org/warp/jcwdb/Cleaner.java

@@ -8,6 +8,8 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
 
 public class Cleaner {
 
+	public static final boolean DISABLE_CLEANER = false;
+	public static final boolean ENABLE_CLEANER_LOGGING = false;
 	private static final double MAXIMUM_SLEEP_INTERVAL = 8d * 1000d; // 8 seconds
 	private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second
 	private static final double NORMAL_REMOVED_ITEMS = 2500l;
@@ -26,7 +28,9 @@ public class Cleaner {
 	}
 
 	public void start() {
-		//this.cleanerThread.start();
+		if (!DISABLE_CLEANER) {
+			this.cleanerThread.start();
+		}
 	}
 	
 	/**
@@ -38,7 +42,7 @@ public class Cleaner {
 		for (Cleanable cleanable : objectsToClean) {
 			cleanedItems += cleanable.clean();
 		}
-		System.gc();
+		//System.gc();
 		return cleanedItems;
 	}
 
@@ -61,38 +65,38 @@ public class Cleaner {
 		public void run() {
 			while(!stopRequest) {
 				try {
-					System.out.println("[CLEANER] Waiting " + sleepInterval + "ms.");
+					if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Waiting " + sleepInterval + "ms.");
 					sleepFor(sleepInterval);
 					final long time1 = System.currentTimeMillis();
 					final double removedItems = clean();
 					final long time2 = System.currentTimeMillis();
-					System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1));
+					if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1));
 					double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2;
 
-					System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems);
+					if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems);
 					if (removedItems > 0) {
 						final double removedItemsRatio = removedItems / NORMAL_REMOVED_ITEMS;
-						System.out.println("[CLEANER] REMOVED_ITEMS_RATIO: " + removedItemsRatio);
+						if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] REMOVED_ITEMS_RATIO: " + removedItemsRatio);
 						if (removedItemsRatio < 1d / REMOVED_ITEMS_RATIO || removedItemsRatio >= REMOVED_ITEMS_RATIO) {
 							suggestedExecutionTimeByItemsCalculations = sleepInterval / removedItemsRatio;
 						}
 					}
 
-					System.out.println("[CLEANER] Items: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + suggestedExecutionTimeByItemsCalculations + "ms");
+					if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Items: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + suggestedExecutionTimeByItemsCalculations + "ms");
 
 					double newSleepInterval = suggestedExecutionTimeByItemsCalculations;
-					System.out.println("[CLEANER] Total: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
+					if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Total: SUGGESTING SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
 					if (newSleepInterval > MAXIMUM_SLEEP_INTERVAL) {
 						sleepInterval = (int) MAXIMUM_SLEEP_INTERVAL;
 					} else if (newSleepInterval < MINIMUM_SLEEP_INTERVAL) {
 						sleepInterval = (int) MINIMUM_SLEEP_INTERVAL;
 					} else {
-						System.out.println("[CLEANER] CHANGED SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
+						if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] CHANGED SLEEP_INTERVAL FROM " + sleepInterval + "ms TO " + newSleepInterval + "ms");
 						sleepInterval = (int) newSleepInterval;
 					}
 
 
-					System.out.println("[CLEANER] Cleaned " + removedItems + " items.");
+					if (ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] Cleaned " + removedItems + " items.");
 				}catch (InterruptedException e) {
 
 				}

src/main/java/org/warp/jcwdb/EntryReference.java

@@ -10,7 +10,7 @@ import java.util.function.Function;
  * You must have only a maximum of 1 reference for each index
  * @param <T>
  */
-public class EntryReference<T> implements Castable, Saveable {
+public class EntryReference<T> implements Castable, AdvancedSaveable {
 	private final JCWDatabase.EntryReferenceTools db;
 	private final long entryIndex;
 	private final DBTypeParser<T> parser;
@@ -19,6 +19,7 @@ public class EntryReference<T> implements Castable, Saveable {
 	private volatile boolean isHashCached;
 	private volatile boolean loaded;
 	private volatile boolean closed;
+	private volatile boolean isFlushingAllowed;
 	private final Object hashCacheLock = new Object();
 	private final Object accessLock = new Object();
 	private final Object closeLock = new Object();
@@ -66,17 +67,29 @@ public class EntryReference<T> implements Castable, Saveable {
 	 * Note that this method won't be called when closing without saving
 	 */
 	public void save() {
+		this.save(false);
+	}
+
+	public void save(boolean isEditFinished) {
 		synchronized(accessLock) {
 			if (loaded && !closed) {
 				try {
-					if (value instanceof Saveable) {
+					if (value instanceof AdvancedSaveable) {
+						((AdvancedSaveable)value).save(isEditFinished);
+					} else if (value instanceof Saveable) {
 						((Saveable)value).save();
 					}
-					IndexDetails returnedDetails = db.write(entryIndex, parser.getWriter(value));
+					IndexDetails returnedDetails = this.db.write(entryIndex, parser.getWriter(value));
 					synchronized(hashCacheLock) {
 						this.cachedHash = returnedDetails.getHash();
 						this.isHashCached = true;
 					}
+					if (isEditFinished) {
+						if (!isFlushingAllowed) {
+							this.db.setFlushingAllowed(entryIndex, true);
+							this.isFlushingAllowed = true;
+						}
+					}
 				} catch (IOException e) {
 					e.printStackTrace();
 				}
@@ -93,7 +106,7 @@ public class EntryReference<T> implements Castable, Saveable {
 		synchronized(accessLock) {
 			load();
 			this.value = editFunction.apply(this.value, this);
-			this.save();
+			this.save(true);
 		}
 	}
 
@@ -106,7 +119,7 @@ public class EntryReference<T> implements Castable, Saveable {
 		synchronized(accessLock) {
 			load();
 			this.value = editFunction.apply(this.value);
-			this.save();
+			this.save(true);
 		}
 	}
 
@@ -119,7 +132,7 @@ public class EntryReference<T> implements Castable, Saveable {
 		synchronized(accessLock) {
 			load();
 			editFunction.accept(this.value, this);
-			this.save();
+			this.save(true);
 		}
 	}
 
@@ -132,7 +145,7 @@ public class EntryReference<T> implements Castable, Saveable {
 		synchronized(accessLock) {
 			load();
 			editFunction.accept(this.value);
-			this.save();
+			this.save(true);
 		}
 	}
 
@@ -148,7 +161,7 @@ public class EntryReference<T> implements Castable, Saveable {
 			synchronized(hashCacheLock) {
 				this.isHashCached = false;
 			}
-			this.save();
+			this.save(true);
 		}
 	}
 
@@ -177,6 +190,10 @@ public class EntryReference<T> implements Castable, Saveable {
 		synchronized(accessLock) {
 			if (!loaded) {
 				try {
+					if (this.isFlushingAllowed) {
+						this.db.setFlushingAllowed(entryIndex, false);
+						this.isFlushingAllowed = false;
+					}
 					this.value = db.read(entryIndex, parser.getReader());
 					this.loaded = true;
 				} catch (IOException e) {
@@ -201,7 +218,7 @@ public class EntryReference<T> implements Castable, Saveable {
 				return;
 			}
 
-			save();
+			save(true);
 
 			closed = true;
 		}

src/main/java/org/warp/jcwdb/FileIndexManager.java

@@ -12,11 +12,10 @@ import java.nio.channels.SeekableByteChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
-import java.util.function.BiConsumer;
-import java.util.function.Consumer;
 
 public class FileIndexManager implements IndexManager {
 	private final SeekableByteChannel dataFileChannel, metadataFileChannel;
+	private volatile long metadataFileChannelSize;
 	private final FileAllocator fileAllocator;
 	private final ByteBuffer metadataByteBuffer = ByteBuffer.allocateDirect(IndexDetails.TOTAL_BYTES);
 	private final ByteBuffer maskByteBuffer = ByteBuffer.allocateDirect(Integer.BYTES);
@@ -35,13 +34,21 @@ public class FileIndexManager implements IndexManager {
 	/**
 	 * Edit this using editIndex()
 	 */
-	private final LongSet dirtyLoadedIndices, removedIndices;
+	private final LongSet dirtyLoadedIndices, flushingAllowedIndices, removedIndices;
 	private long firstAllocableIndex;
 
 	public FileIndexManager(Path dataFile, Path metadataFile) throws IOException {
-		loadedIndices = new Long2ObjectOpenHashMap<>();
+		if (Cleaner.DISABLE_CLEANER) {
-		dirtyLoadedIndices = new LongOpenHashSet();
+			loadedIndices = new Long2ObjectOpenHashMap<>();
-		removedIndices = new LongOpenHashSet();
+			dirtyLoadedIndices = new LongOpenHashSet();
+			flushingAllowedIndices = new LongOpenHashSet();
+			removedIndices = new LongOpenHashSet();
+		} else {
+			loadedIndices = new Long2ObjectLinkedOpenHashMap<>();
+			dirtyLoadedIndices = new LongLinkedOpenHashSet();
+			flushingAllowedIndices = new LongLinkedOpenHashSet();
+			removedIndices = new LongLinkedOpenHashSet();
+		}
 		if (Files.notExists(dataFile)) {
 			Files.createFile(dataFile);
 		}
@@ -51,17 +58,22 @@ public class FileIndexManager implements IndexManager {
 		dataFileChannel = Files.newByteChannel(dataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
 		metadataFileChannel = Files.newByteChannel(metadataFile, StandardOpenOption.READ, StandardOpenOption.WRITE);
 		fileAllocator = createFileAllocator(dataFileChannel, metadataFileChannel.position(0));
-		firstAllocableIndex = metadataFileChannel.size() / (long) IndexDetails.TOTAL_BYTES;
+		metadataFileChannelSize = metadataFileChannel.size();
+		firstAllocableIndex = getMetadataFileChannelSize() / (long) IndexDetails.TOTAL_BYTES;
 		if (firstAllocableIndex == 0) {
 			firstAllocableIndex = 1;
 		}
 	}
 
+	private long getMetadataFileChannelSize() throws IOException {
+		return metadataFileChannelSize;
+	}
+
 	private FileAllocator createFileAllocator(final SeekableByteChannel dataFileChannel, final SeekableByteChannel metadataFileChannel) throws IOException {
 		Long2IntMap freeBytes = new Long2IntRBTreeMap();
 		Long2IntMap usedBytes = new Long2IntRBTreeMap();
 		long firstOffset = 0;
-		while (metadataFileChannel.position() + IndexDetails.TOTAL_BYTES <= metadataFileChannel.size()) {
+		while (metadataFileChannel.position() + IndexDetails.TOTAL_BYTES <= getMetadataFileChannelSize()) {
 			IndexDetails indexDetails = readIndexDetailsAt(metadataFileChannel);
 			if (indexDetails != null) {
 				long offset = indexDetails.getOffset();
@@ -117,7 +129,7 @@ public class FileIndexManager implements IndexManager {
 	public <T> IndexDetails set(long index, DBDataOutput<T> data) throws IOException {
 		checkClosed();
 		final int dataSize = data.getSize();
-		final IndexDetails indexDetails = getIndexMetadataUnsafe(index);
+		IndexDetails indexDetails = getIndexMetadataUnsafe(index);
 		if (indexDetails == null || indexDetails.getSize() < dataSize) {
 			// Allocate new space
 			IndexDetails newDetails = allocateAndWrite(index, data);
@@ -133,7 +145,7 @@ public class FileIndexManager implements IndexManager {
 				fileAllocator.markFree(indexDetails.getOffset() + dataSize, dataSize);
 			}
 			// Update index details
-			editIndex(index, indexDetails, indexDetails.getOffset(), dataSize, indexDetails.getType(), data.calculateHash());
+			indexDetails = editIndex(index, indexDetails, indexDetails.getOffset(), dataSize, indexDetails.getType(), data.calculateHash());
 			// Write data
 			writeExact(indexDetails, data);
 			// Before returning, return IndexDetails
@@ -141,6 +153,16 @@ public class FileIndexManager implements IndexManager {
 		}
 	}
 
+	@Override
+	public void setFlushingAllowed(long index, boolean isUnloadingAllowed) {
+		checkClosed();
+		if (isUnloadingAllowed) {
+			flushingAllowedIndices.add(index);
+		} else {
+			flushingAllowedIndices.remove(index);
+		}
+	}
+
 	@Override
 	public <T> long add(DBDataOutput<T> data) throws IOException {
 		checkClosed();
@@ -206,6 +228,7 @@ public class FileIndexManager implements IndexManager {
 		}
 		synchronized (indicesMapsAccessLock) {
 			dirtyLoadedIndices.remove(index);
+			flushingAllowedIndices.remove(index);
 			loadedIndices.remove(index);
 			removedIndices.add(index);
 		}
@@ -216,6 +239,7 @@ public class FileIndexManager implements IndexManager {
 			synchronized (indicesMapsAccessLock) {
 				removedIndices.remove(index);
 				dirtyLoadedIndices.remove(index);
+				flushingAllowedIndices.remove(index);
 				loadedIndices.remove(index);
 			}
 			// Update indices metadata
@@ -228,11 +252,14 @@ public class FileIndexManager implements IndexManager {
 			if (dirtyLoadedIndices.contains(index)) {
 				indexDetails = loadedIndices.get(index);
 				dirtyLoadedIndices.remove(index);
+				flushingAllowedIndices.remove(index);
 			}
 		}
 		if (isDirty) {
 			// Update indices metadata
-			SeekableByteChannel metadata = metadataFileChannel.position(index * IndexDetails.TOTAL_BYTES);
+			long position = index * IndexDetails.TOTAL_BYTES;
+			resizeMetadataFileChannel(position);
+			SeekableByteChannel metadata = metadataFileChannel.position(position);
 			writeIndexDetails(metadata, indexDetails);
 		}
 		synchronized (indicesMapsAccessLock) {
@@ -293,6 +320,7 @@ public class FileIndexManager implements IndexManager {
 		synchronized (indicesMapsAccessLock) {
 			loadedIndices.put(index, details);
 			dirtyLoadedIndices.add(index);
+			flushingAllowedIndices.remove(index);
 		}
 	}
 
@@ -301,6 +329,7 @@ public class FileIndexManager implements IndexManager {
 			long newIndex = firstAllocableIndex++;
 			loadedIndices.put(newIndex, indexDetails);
 			dirtyLoadedIndices.add(newIndex);
+			flushingAllowedIndices.remove(newIndex);
 			removedIndices.remove(newIndex);
 			return newIndex;
 		}
@@ -316,7 +345,7 @@ public class FileIndexManager implements IndexManager {
 
 		// Try to load the details from file
 		final long metadataPosition = index * IndexDetails.TOTAL_BYTES;
-		if (metadataPosition + IndexDetails.TOTAL_BYTES > metadataFileChannel.size()) {
+		if (metadataPosition + IndexDetails.TOTAL_BYTES > getMetadataFileChannelSize()) {
 			// Avoid underflow exception
 			return null;
 		}
@@ -374,7 +403,7 @@ public class FileIndexManager implements IndexManager {
 		}
 
 		// Update indices metadata
-		flushAllIndices();
+		flushAllFlushableIndices();
 
 		// Remove removed indices
 		removeRemovedIndices();
@@ -417,39 +446,56 @@ public class FileIndexManager implements IndexManager {
 	@Override
 	public long clean() {
 		long cleaned = 0;
+		long tim1 = System.currentTimeMillis();
 		try {
-			cleaned += flushAllIndices();
+			cleaned += flushAllFlushableIndices();
 		} catch (IOException ex) {
 			ex.printStackTrace();
 		}
+		long tim2 = System.currentTimeMillis();
 		try {
 			cleaned += removeRemovedIndices();
 		} catch (IOException ex) {
 			ex.printStackTrace();
 		}
+		long tim3 = System.currentTimeMillis();
 		cleaned += cleanExtraIndices();
+		long tim4 = System.currentTimeMillis();
+		if (Cleaner.ENABLE_CLEANER_LOGGING) System.out.println("[CLEANER] FileIndexManager CLEAN_TIME: " + (tim2-tim1) + "," + (tim3-tim2) + "," + (tim4-tim3));
 		return cleaned;
 	}
 
-	private long flushAllIndices() throws IOException {
+	private long flushAllFlushableIndices() throws IOException {
 		long flushedIndices = 0;
 		SeekableByteChannel metadata = metadataFileChannel;
 		long lastIndex = -2;
 		synchronized (indicesMapsAccessLock) {
 			for (long index : dirtyLoadedIndices) {
-				IndexDetails indexDetails = loadedIndices.get(index);
+				if (!flushingAllowedIndices.contains(index)) {
-				if (index - lastIndex != 1) {
+					IndexDetails indexDetails = loadedIndices.get(index);
-					metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
+					long position = index * IndexDetails.TOTAL_BYTES;
+					resizeMetadataFileChannel(position);
+					if (index - lastIndex != 1) {
+						metadata = metadata.position(position);
+					}
+					writeIndexDetails(metadata, indexDetails);
+					lastIndex = index;
+					flushedIndices++;
 				}
-				writeIndexDetails(metadata, indexDetails);
-				lastIndex = index;
-				flushedIndices++;
 			}
 			dirtyLoadedIndices.clear();
+			dirtyLoadedIndices.addAll(flushingAllowedIndices);
+			flushingAllowedIndices.clear();
 		}
 		return flushedIndices;
 	}
 
+	private void resizeMetadataFileChannel(long position) {
+		if (position + IndexDetails.TOTAL_BYTES > metadataFileChannelSize) {
+			metadataFileChannelSize = position + IndexDetails.TOTAL_BYTES;
+		}
+	}
+
 	private long removeRemovedIndices() throws IOException {
 		SeekableByteChannel metadata = metadataFileChannel;
 		synchronized (indicesMapsAccessLock) {

src/main/java/org/warp/jcwdb/IndexManager.java

@@ -12,6 +12,7 @@ public interface IndexManager extends Cleanable {
 	<T> long add(DBDataOutput<T> writer) throws IOException;
 	<T> FullIndexDetails addAndGetDetails(DBDataOutput<T> writer) throws IOException;
 	<T> IndexDetails set(long index, DBDataOutput<T> writer) throws IOException;
+	void setFlushingAllowed(long index, boolean isUnloadingAllowed);
 	void delete(long index) throws IOException;
 	boolean has(long index);
 	void close() throws IOException;

src/main/java/org/warp/jcwdb/JCWDatabase.java

@@ -1,7 +1,5 @@
 package org.warp.jcwdb;
 
-import it.unimi.dsi.fastutil.longs.LongArrayList;
-
 import java.io.IOException;
 import java.nio.file.Path;
 
@@ -154,6 +152,10 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
 		public <T> IndexDetails write(long index, DBDataOutput<T> writer) throws IOException {
 			return indices.set(index, writer);
 		}
+
+		public void setFlushingAllowed(long index, boolean isFlushingAllowed) {
+			indices.setFlushingAllowed(index, isFlushingAllowed);
+		}
 	}
 
 	@SuppressWarnings("unchecked")

src/main/java/org/warp/jcwdb/LightArrayList.java

@@ -3,6 +3,7 @@ package org.warp.jcwdb;
 import it.unimi.dsi.fastutil.longs.LongArrayList;
 import it.unimi.dsi.fastutil.objects.ObjectArrayList;
 
+import java.io.IOError;
 import java.io.IOException;
 import java.util.*;
 import java.util.function.Consumer;
@@ -112,7 +113,7 @@ public class LightArrayList<T> implements LightList<T> {
 			try {
 				action.accept(db.get(index));
 			} catch (IOException e) {
-				throw (RuntimeException) new RuntimeException().initCause(e);
+				throw new IOError(e);
 			}
 		}
 	}

src/main/java/org/warp/jcwdb/LightBigList.java

@@ -4,6 +4,7 @@ import it.unimi.dsi.fastutil.ints.IntArrayList;
 import it.unimi.dsi.fastutil.longs.LongArrayList;
 import it.unimi.dsi.fastutil.objects.ObjectArrayList;
 
+import java.io.IOError;
 import java.io.IOException;
 import java.util.*;
 import java.util.function.Consumer;
@@ -16,8 +17,8 @@ public class LightBigList<T> implements LightList<T>, Saveable {
 	public final LongArrayList chunks;
 	public final IntArrayList chunkSizes;
 	private final JCWDatabase db;
-	private LightList<T> cachedChunk;
+	private LightArrayList<T> cachedChunk;
-	private EntryReference<LightList<T>> cachedChunkRef;
+	private EntryReference<LightArrayList<T>> cachedChunkRef;
 	private long cachedChunkIndex = -1;
 	private int cachedChunkNumber = -1;
 	private final Object cachedChunkLock = new Object();
@@ -132,8 +133,8 @@ public class LightBigList<T> implements LightList<T>, Saveable {
 		if (o != null) {
 			for (long chunkIndex : chunks) {
 				try {
-					EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
+					EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
-					LightList<T> chunk = chunkRef.getValueReadOnly();
+					LightArrayList<T> chunk = chunkRef.getValueReadOnly();
 					if (chunk.contains(o)) {
 						return true;
 					}
@@ -175,13 +176,13 @@ public class LightBigList<T> implements LightList<T>, Saveable {
 	@Override
 	public void forEachReference(Consumer<? super EntryReference<T>> action) {
 		Objects.requireNonNull(action);
-		for (long chunkIndex : this.chunks) {
+		// Iterate through all chunks
-			try {
+		for (int i = 0; i < chunks.size(); i++) {
-				EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
+			synchronized (cachedChunkLock) {
-				LightList<T> chunk = chunkRef.getValueReadOnly();
+				if (cachedChunkNumber != i) {
-				chunk.forEachReference(action);
+					prepareAccessToChunk(i);
-			} catch (IOException ex) {
+				}
-				throw (NullPointerException) new NullPointerException().initCause(ex);
+				cachedChunk.forEachReference(action);
 			}
 		}
 	}
@@ -194,6 +195,9 @@ public class LightBigList<T> implements LightList<T>, Saveable {
 	@Deprecated
 	@Override
 	public T[] toArray() {
+		if (true) {
+			throw new RuntimeException("toArray() isn't implemented!");
+		}
 		T[] result = (T[]) new Object[this.size()];
 
 		long currentOffset = 0;
@@ -206,8 +210,8 @@ public class LightBigList<T> implements LightList<T>, Saveable {
 			final long chunkIndex = chunks.getLong(i);
 
 			try {
-				EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
+				EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
-				LightList<T> chunk = chunkRef.getValueReadOnly();
+				LightArrayList<T> chunk = chunkRef.getValueReadOnly();
 				for (int i1 = 0; i1 < chunk.size(); i1++) {
 					result[(int)(chunkStartOffset + i1)] = chunk.get(i);
 				}
@@ -276,7 +280,7 @@ public class LightBigList<T> implements LightList<T>, Saveable {
 					}
 
 					try {
-						EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
+						EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
 						chunkRef.editValue((chunk) -> {
 							result.var = chunk.remove(relativeOffset);
 						});
@@ -414,7 +418,7 @@ public class LightBigList<T> implements LightList<T>, Saveable {
 					}
 
 					try {
-						EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
+						EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
 						chunkRef.editValue((chunk) -> {
 							chunk.set(relativeOffset, element);
 							wrapper.var = element;
@@ -458,7 +462,7 @@ public class LightBigList<T> implements LightList<T>, Saveable {
 
 				// Get chunk index
 				final long chunkIndex = chunks.getLong(i);
-				EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
+				EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
 				final int foundIndex = chunkRef.getValueReadOnly().indexOfEntry(ref);
 				if (foundIndex >= 0) {
 					return currentOffset + foundIndex;
@@ -487,7 +491,7 @@ public class LightBigList<T> implements LightList<T>, Saveable {
 
 				// Get chunk index
 				final long chunkIndex = chunks.getLong(i);
-				EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
+				EntryReference<LightArrayList<T>> chunkRef = db.get(chunkIndex);
 				final int foundIndex = chunkRef.getValueReadOnly().lastIndexOfEntry(ref);
 				if (foundIndex >= 0) {
 					return currentOffset + foundIndex;
@@ -543,26 +547,20 @@ public class LightBigList<T> implements LightList<T>, Saveable {
 	@Override
 	public boolean removeIf(Predicate<? super T> filter) {
 		Objects.requireNonNull(filter);
-		final VariableWrapper<Boolean> result = new VariableWrapper(false);
+		boolean result = false;
 		// Iterate through all chunks
 		for (int i = 0; i < chunks.size(); i++) {
-			try {
+			synchronized (cachedChunkLock) {
-				final int chunkOffset = i;
+				if (cachedChunkNumber != i) {
-				// Get chunk index
+					prepareAccessToChunk(i);
-				final long chunkIndex = chunks.getLong(i);
+				}
-				EntryReference<LightList<T>> chunkRef = db.get(chunkIndex);
+				if (cachedChunk.removeIf(filter)) {
-				chunkRef.editValue((chunk) -> {
+					result = true;
-					 boolean removed = chunk.removeIf(filter);
+					chunkSizes.set(cachedChunkNumber, cachedChunk.size());
-					 if (removed) {
+				}
-						 result.var = true;
-						 chunkSizes.set(chunkOffset, chunk.size());
-					 }
-				});
-			} catch (IOException ex) {
-				throw (NullPointerException) new NullPointerException().initCause(ex);
 			}
 		}
-		return result.var;
+		return result;
 	}
 
 	@Override

src/main/java/org/warp/jcwdb/MixedIndexDatabase.java

@@ -1,12 +1,7 @@
 package org.warp.jcwdb;
 
-import it.unimi.dsi.fastutil.longs.Long2LongLinkedOpenHashMap;
-import it.unimi.dsi.fastutil.longs.Long2LongMap;
-
 import java.io.IOException;
 import java.nio.file.Path;
-import java.util.function.BiConsumer;
-import java.util.function.Consumer;
 
 public class MixedIndexDatabase implements IndexManager {
 	private final FileIndexManager fileIndices;
@@ -62,6 +57,14 @@ public class MixedIndexDatabase implements IndexManager {
 			return fileIndices.set(index, writer);
 		}
 	}
+	@Override
+	public void setFlushingAllowed(long index, boolean isFlushingAllowed) {
+		if (cacheIndices.has(index)) {
+			cacheIndices.setFlushingAllowed(index, isFlushingAllowed);
+		} else {
+			fileIndices.setFlushingAllowed(index, isFlushingAllowed);
+		}
+	}
 
 	@Override
 	public void delete(long index) throws IOException {

src/main/java/org/warp/jcwdb/exampleimpl/App.java

@@ -7,92 +7,91 @@ import org.warp.jcwdb.LightList;
 import it.unimi.dsi.fastutil.objects.ObjectArrayList;
 import it.unimi.dsi.fastutil.objects.ObjectList;
 
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.function.Predicate;
 
 public class App {
 	static long time3;
-	public static void main(String[] args) {
+	public static void main(String[] args) throws IOException {
+		if (args.length > 2 && Boolean.parseBoolean(args[2])) {
+			Files.delete(Paths.get(args[0]));
+			Files.delete(Paths.get(args[1]));
+		}
+		System.out.println("Loading database...");
+		long time0 = System.currentTimeMillis();
+		JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1]));
+		db.registerClass(StrangeAnimal.class, 0);
 		try {
-			if (args.length > 2 && Boolean.parseBoolean(args[2])) {
+			long time01 = System.currentTimeMillis();
-				Files.delete(Paths.get(args[0]));
+			System.out.println("Time elapsed: " + (time01 - time0));
-				Files.delete(Paths.get(args[1]));
+			System.out.println("Loading root...");
-			}
+			EntryReference<LightList<Animal>> rootRef = db.getRoot(Animal.class);
-			System.out.println("Loading database...");
+			rootRef.editValue((root, saver) -> {
-			long time0 = System.currentTimeMillis();
+				long time1 = System.currentTimeMillis();
-			JCWDatabase db = new JCWDatabase(Paths.get(args[0]), Paths.get(args[1]));
+				System.out.println("Time elapsed: " + (time1 - time01));
-			db.registerClass(StrangeAnimal.class, 0);
+				System.out.println("Root size: " + root.size());
-			try {
+				System.out.println("Root:");
-				long time01 = System.currentTimeMillis();
+//		for (int i = 0; i < root.size(); i++) {
-				System.out.println("Time elapsed: " + (time01 - time0));
+//			System.out.println(" - " + root.get(i));
-				System.out.println("Loading root...");
+//		}
-				EntryReference<LightList<Animal>> rootRef = db.getRoot(Animal.class);
+				long prectime = System.currentTimeMillis();
-				rootRef.editValue((root, saver) -> {
+				for (int i = 0; i < 2000000/* 2000000 */; i++) {
-					long time1 = System.currentTimeMillis();
+					Animal animal = new StrangeAnimal(i % 40);
-					System.out.println("Time elapsed: " + (time1 - time01));
+					root.addEntry(animal);
-					System.out.println("Root size: " + root.size());
+					if (i > 0 && i % 200000 == 0) {
-					System.out.println("Root:");
+						long precprectime = prectime;
-	//		for (int i = 0; i < root.size(); i++) {
+						prectime = System.currentTimeMillis();
-	//			System.out.println(" - " + root.get(i));
+						System.out.println("Element " + i + " (" + (prectime - precprectime) + "ms)" + " Total Time: " + (prectime - time1));
-	//		}
-					long prectime = System.currentTimeMillis();
-					for (int i = 0; i < 20000000/* 2000000 */; i++) {
-						Animal animal = new StrangeAnimal(i % 40);
-						root.add(animal);
-						if (i > 0 && i % 200000 == 0) {
-							long precprectime = prectime;
-							prectime = System.currentTimeMillis();
-							System.out.println("Element " + i + " (" + (prectime - precprectime) + "ms)");
-						}
 					}
-					long time2 = System.currentTimeMillis();
-					saver.save();
-					System.out.println("Root size: " + root.size());
-					System.out.println("Time elapsed: " + (time2 - time1));
-					System.out.println("Used memory: "
-									+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
-					long time2_0 = System.currentTimeMillis();
-					System.out.println("Filtering strings...");
-					//root.removeIf(Animal::hasFourLegs);
-					long time2_1 = System.currentTimeMillis();
-					System.out.println("Time elapsed: " + (time2_1 - time2_0));
-					ObjectList<Animal> results = new ObjectArrayList<>();
-
-					root.forEachReference((valueReference) -> {
-						Animal value = valueReference.getValueReadOnly();
-						if (Animal.hasFourLegs(value)) {
-							results.add(value);
-						}
-						//System.out.println("val:" + value);
-					});
-					long time2_2 = System.currentTimeMillis();
-					System.out.println("Matches: " + results.size());
-					System.out.println("Time elapsed: " + (time2_2 - time2_1));
-					System.out.println("Used memory: "
-									+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
-					System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
-					System.out.println("Cleaning database (to reduce the amount of used memory and detect memory leaks)...");
-					long removedItems = 0;//db.clean();
-					time3 = System.currentTimeMillis();
-					System.out.println("Removed items: " + removedItems);
-					System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
-					System.out.println("Time elapsed: " + (time3 - time2_2));
-					System.out.println("Saving database...");
-					System.out.println("Root size: " + root.size());
-				});
-				db.close();
-				long time4 = System.currentTimeMillis();
-				System.out.println("Time elapsed: " + (time4 - time3));
-			} catch (Exception ex) {
-				ex.printStackTrace();
-			} finally {
-				if (db.isOpen()) {
-					db.close();
 				}
-			}
+				long time2 = System.currentTimeMillis();
+				saver.save();
+				System.out.println("Root size: " + root.size());
+				System.out.println("Time elapsed: " + (time2 - time1));
+				System.out.println("Used memory: "
+								+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
+				long time2_0 = System.currentTimeMillis();
+				System.out.println("Filtering strings...");
+				long oldSize = root.size();
+				root.removeIf(Animal::hasFourLegs);
+				long time2_1 = System.currentTimeMillis();
+				System.out.println("RemoveIf(x) removed items: " + (oldSize - root.size()));
+				System.out.println("Time elapsed: " + (time2_1 - time2_0));
+				ObjectList<Animal> results = new ObjectArrayList<>();
+
+				System.out.println("Retrieving items...");
+				root.forEachReference((valueReference) -> {
+					Animal value = valueReference.getValueReadOnly();
+					if (Animal.hasFourLegs(value)) {
+						results.add(value);
+					}
+					//System.out.println("val:" + value);
+				});
+				long time2_2 = System.currentTimeMillis();
+				System.out.println("Matches: " + results.size());
+				System.out.println("Time elapsed: " + (time2_2 - time2_1));
+				System.out.println("Used memory: "
+								+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
+				System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
+				System.out.println("Cleaning database (to reduce the amount of used memory and detect memory leaks)...");
+				db.clean();
+				time3 = System.currentTimeMillis();
+				System.out.println("Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");
+				System.out.println("Time elapsed: " + (time3 - time2_2));
+				System.out.println("Saving database...");
+				System.out.println("Root size: " + root.size());
+			});
+			db.close();
+			long time4 = System.currentTimeMillis();
+			System.out.println("Time elapsed: " + (time4 - time3));
 		} catch (Exception ex) {
 			ex.printStackTrace();
+		} finally {
+			if (db.isOpen()) {
+				db.close();
+			}
 		}
 	}