Removed indices usage map

src/main/java/org/warp/jcwdb/Cleaner.java

@@ -8,7 +8,7 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
 
 public class Cleaner {
 
-	private static final double MAXIMUM_SLEEP_INTERVAL = 20d * 1000d; // 20 minutes
+	private static final double MAXIMUM_SLEEP_INTERVAL = 8d * 1000d; // 8 seconds
 	private static final double MINIMUM_SLEEP_INTERVAL = 1d * 1000d; // 1 second
 	private static final double NORMAL_REMOVED_ITEMS = 1000l;
 	private static final double REMOVED_ITEMS_RATIO = 2.5d; // 250%
@@ -63,7 +63,10 @@ public class Cleaner {
 				try {
 					System.out.println("[CLEANER] Waiting " + sleepInterval + "ms.");
 					sleepFor(sleepInterval);
+					final long time1 = System.currentTimeMillis();
 					final double removedItems = clean();
+					final long time2 = System.currentTimeMillis();
+					System.out.println("[CLEANER] CLEAN_TIME " + (time2 - time1));
 					double suggestedExecutionTimeByItemsCalculations = (sleepInterval + MAXIMUM_SLEEP_INTERVAL) / 2;
 
 					System.out.println("[CLEANER] REMOVED_ITEMS: " + removedItems);

src/main/java/org/warp/jcwdb/FileIndexManager.java

@@ -11,6 +11,7 @@ import java.nio.channels.SeekableByteChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import java.util.Iterator;
 import java.util.function.Consumer;
 
 public class FileIndexManager implements IndexManager {
@@ -233,7 +234,7 @@ public class FileIndexManager implements IndexManager {
 	 * @param details
 	 */
 	private void editIndex(long index, IndexDetails details) {
-		synchronized (indicesMapsAccessLock) {
+		synchronized (indicesMapsAccessLock) {// FIXXXX main3
 			loadedIndices.put(index, details);
 			dirtyLoadedIndices.add(index);
 		}
@@ -264,7 +265,8 @@ public class FileIndexManager implements IndexManager {
 			return null;
 		}
 		SeekableByteChannel currentMetadataFileChannel = metadataFileChannel.position(metadataPosition);
-		synchronized (metadataByteBufferLock) {
+		IndexDetails indexDetails = null;
+		synchronized (metadataByteBufferLock) {// FIXXXX main2
 			metadataByteBuffer.rewind();
 			currentMetadataFileChannel.read(metadataByteBuffer);
 			metadataByteBuffer.rewind();
@@ -277,12 +279,15 @@ public class FileIndexManager implements IndexManager {
 				final int size = metadataByteBuffer.getInt();
 				final int type = metadataByteBuffer.getInt();
 				final long hash = metadataByteBuffer.getLong();
-				final IndexDetails indexDetails = new IndexDetails(offset, size, type, hash);
+				indexDetails = new IndexDetails(offset, size, type, hash);
-				editIndex(index, indexDetails);
-				return indexDetails;
 			}
 		}
 
+		if (indexDetails != null) {
+			editIndex(index, indexDetails);
+			return indexDetails;
+		}
+
 		// No results found. Returning null
 		return null;
 	}
@@ -308,31 +313,15 @@ public class FileIndexManager implements IndexManager {
 		}
 
 		// Update indices metadata
-		SeekableByteChannel metadata = metadataFileChannel;
+		flushAllIndices();
-		long lastIndex = -2;
-		synchronized (indicesMapsAccessLock) {
-			for (long index : dirtyLoadedIndices) {
-				IndexDetails indexDetails = loadedIndices.get(index);
-				if (index - lastIndex != 1) {
-					metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
-				}
-				writeIndexDetails(metadata, indexDetails);
-				lastIndex = index;
-			}
-		}
 
 		// Remove removed indices
-		synchronized (indicesMapsAccessLock) {
+		removeRemovedIndices();
-			for (long index : removedIndices) {
-				metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
-				eraseIndexDetails(metadata);
-			}
-		}
 		fileAllocator.close();
 	}
 
 	private void writeIndexDetails(SeekableByteChannel position, IndexDetails indexDetails) throws IOException {
-		synchronized (metadataByteBufferLock) {
+		synchronized (metadataByteBufferLock) {// FIXXXX cleaner3
 			final int size = indexDetails.getSize();
 			final int type = indexDetails.getType();
 			final long offset = indexDetails.getOffset();
@@ -366,7 +355,51 @@ public class FileIndexManager implements IndexManager {
 
 	@Override
 	public long clean() {
-		return cleanExtraIndices();
+		long cleaned = 0;
+		try {
+			cleaned += flushAllIndices();
+		} catch (IOException ex) {
+			ex.printStackTrace();
+		}
+		try {
+			cleaned += removeRemovedIndices();
+		} catch (IOException ex) {
+			ex.printStackTrace();
+		}
+		cleaned += cleanExtraIndices();
+		return cleaned;
+	}
+
+	private long flushAllIndices() throws IOException {
+		long flushedIndices = 0;
+		SeekableByteChannel metadata = metadataFileChannel;
+		long lastIndex = -2;
+		synchronized (indicesMapsAccessLock) {
+			for (long index : dirtyLoadedIndices) {
+				IndexDetails indexDetails = loadedIndices.get(index);
+				if (index - lastIndex != 1) {
+					metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
+				}
+				writeIndexDetails(metadata, indexDetails);
+				lastIndex = index;
+				flushedIndices++;
+			}
+			dirtyLoadedIndices.clear();
+		}
+		return flushedIndices;
+	}
+
+	private long removeRemovedIndices() throws IOException {
+		SeekableByteChannel metadata = metadataFileChannel;
+		synchronized (indicesMapsAccessLock) {
+			long removed = this.removedIndices.size();
+			for (long index : this.removedIndices) {
+					metadata = metadata.position(index * IndexDetails.TOTAL_BYTES);
+					eraseIndexDetails(metadata);
+			}
+			this.removedIndices.clear();
+			return removed;
+		}
 	}
 
 	private long cleanExtraIndices() {

src/main/java/org/warp/jcwdb/JCWDatabase.java

@@ -18,7 +18,7 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
 
 	public JCWDatabase(Path dataFile, Path metadataFile) throws IOException {
 		this.typesManager = new TypesManager(this);
-		this.indices = new MixedIndexDatabase(typesManager, dataFile, metadataFile);
+		this.indices = new MixedIndexDatabase(dataFile, metadataFile);
 		Runtime.getRuntime().addShutdownHook(new Thread(() -> {
 			try {
 				JCWDatabase.this.close();
@@ -28,7 +28,7 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
 		}));
 		this.databaseCleaner = new Cleaner(this);
 
-		//this.databaseCleaner.start();
+		this.databaseCleaner.start();
 	}
 
 	public <T> EntryReference<LightList<T>> getRoot() throws IOException {

src/main/java/org/warp/jcwdb/LightList.java

@@ -2,6 +2,7 @@ package org.warp.jcwdb;
 
 import java.io.IOException;
 import java.util.*;
+import java.util.function.Consumer;
 import java.util.function.Predicate;
 
 import it.unimi.dsi.fastutil.longs.LongArrayList;
@@ -52,6 +53,7 @@ public class LightList<T> implements List<T> {
 	@SuppressWarnings("unchecked")
 	@Override
 	public Iterator<T> iterator() {
+		System.out.println("WARNING! YOU ARE USING iterator()! PLEASE USE ITERATORREFERENCES TO AVOID OUTOFMEMORY!");
 		final ArrayList<T> elements = new ArrayList<>();
 		for (Long element : internalList) {
 			try {
@@ -75,6 +77,30 @@ public class LightList<T> implements List<T> {
 		}
 		return elements.iterator();
 	}
+
+	/**
+	 * USE forEachReference INSTEAD, TO AVOID OUTOFMEMORY
+	 * @param action
+	 */
+	@Deprecated
+	@Override
+	public void forEach(Consumer<? super T> action) {
+		Objects.requireNonNull(action);
+		for (T t : this) {
+			action.accept(t);
+		}
+	}
+
+	public void forEachReference(Consumer<? super EntryReference<T>> action) {
+		Objects.requireNonNull(action);
+		for (long index : this.internalList) {
+			try {
+				action.accept(db.get(index));
+			} catch (IOException e) {
+				throw (RuntimeException) new RuntimeException().initCause(e);
+			}
+		}
+	}
 	
 
 	@SuppressWarnings("unchecked")

src/main/java/org/warp/jcwdb/MixedIndexDatabase.java

@@ -8,19 +8,16 @@ import java.nio.file.Path;
 import java.util.function.Consumer;
 
 public class MixedIndexDatabase implements IndexManager {
-	private final Long2LongMap mostAccessedIndices;
 	private final FileIndexManager fileIndices;
 	private final CacheIndexManager cacheIndices;
 
-	public MixedIndexDatabase(TypesManager typesManager, Path dataFile, Path metadataFile) throws IOException {
+	public MixedIndexDatabase(Path dataFile, Path metadataFile) throws IOException {
-		this.mostAccessedIndices = new Long2LongLinkedOpenHashMap();
 		this.fileIndices = new FileIndexManager(dataFile, metadataFile);
 		this.cacheIndices = new CacheIndexManager();
 	}
 
 	@Override
 	public <T> T get(long index, DBReader<T> reader) throws IOException {
-		incrementUsage(index);
 		if (cacheIndices.has(index)) {
 			return cacheIndices.get(index, reader);
 		} else {
@@ -71,10 +68,6 @@ public class MixedIndexDatabase implements IndexManager {
 		return cacheIndices.has(index) || fileIndices.has(index);
 	}
 
-	private void incrementUsage(long index) {
-		mostAccessedIndices.put(index, mostAccessedIndices.getOrDefault(index, 0) + 1);
-	}
-
 	@Override
 	public void close() throws IOException {
 		// TODO: move all cached indices to filesIndices before closing.

src/main/java/org/warp/jcwdb/VariableWrapper.java

@@ -0,0 +1,12 @@
+package org.warp.jcwdb;
+
+import java.nio.channels.SeekableByteChannel;
+
+public class VariableWrapper<T> {
+
+	public T var;
+
+	public VariableWrapper(T value) {
+		this.var = value;
+	}
+}

src/main/java/org/warp/jcwdb/exampleimpl/App.java

@@ -37,7 +37,7 @@ public class App {
 	//			System.out.println(" - " + root.get(i));
 	//		}
 					long prectime = System.currentTimeMillis();
-					for (int i = 0; i < 20000/* 2000000 */; i++) {
+					for (int i = 0; i < 200000/* 2000000 */; i++) {
 						Animal animal = new StrangeAnimal(i % 40);
 						root.add(animal);
 						if (i > 0 && i % 200000 == 0) {
@@ -59,13 +59,15 @@ public class App {
 					System.out.println("Time elapsed: " + (time2_1 - time2_0));
 					ObjectList<Animal> results = new ObjectArrayList<>();
 
-					root.forEach((value) -> {
+					root.forEachReference((valueReference) -> {
+						Animal value = valueReference.getValueReadOnly();
 						if (Animal.hasFourLegs(value)) {
 							results.add(value);
 						}
 						//System.out.println("val:" + value);
 					});
 					long time2_2 = System.currentTimeMillis();
+					System.out.println("Matches: " + results.size());
 					System.out.println("Time elapsed: " + (time2_2 - time2_1));
 					System.out.println("Used memory: "
 									+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024) + "MB");