From e4fbd613dbdffed45494d18ff747a357bfc0f071 Mon Sep 17 00:00:00 2001
From: Andrea Cavalli <andrea@warp.ovh>
Date: Thu, 20 Dec 2018 01:12:46 +0100
Subject: [PATCH] Reusing unused space in database

---
 .../java/org/warp/jcwdb/FileAllocator.java    | 71 +++++++++++++++++--
 .../java/org/warp/jcwdb/FileIndexManager.java |  7 +-
 src/main/java/org/warp/jcwdb/JCWDatabase.java |  2 +-
 .../java/org/warp/jcwdb/exampleimpl/App.java  |  2 +-
 4 files changed, 73 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/warp/jcwdb/FileAllocator.java b/src/main/java/org/warp/jcwdb/FileAllocator.java
index 4dcd34a..94eb771 100644
--- a/src/main/java/org/warp/jcwdb/FileAllocator.java
+++ b/src/main/java/org/warp/jcwdb/FileAllocator.java
@@ -1,18 +1,28 @@
 package org.warp.jcwdb;
 
+import it.unimi.dsi.fastutil.longs.Long2IntMap;
+import it.unimi.dsi.fastutil.longs.Long2IntRBTreeMap;
+import it.unimi.dsi.fastutil.objects.ObjectBidirectionalIterator;
+
 import java.io.IOException;
 import java.nio.channels.SeekableByteChannel;
 
 public class FileAllocator implements AutoCloseable {
+	private static final int MAXIMUM_UNALLOCATED_ENTRIES = 500000;
+
 	private final SeekableByteChannel dataFileChannel;
-	private volatile long allocableOffset;
+	private volatile long fileSize;
 	private volatile boolean closed;
 	private final Object closeLock = new Object();
 	private final Object allocateLock = new Object();
+	/**
+	 * index -> free space size
+	 */
+	private final Long2IntRBTreeMap freeBytes = new Long2IntRBTreeMap((a, b) -> (int) (a - b));
 
 	public FileAllocator(SeekableByteChannel dataFileChannel) throws IOException {
 		this.dataFileChannel = dataFileChannel;
-		this.allocableOffset = this.dataFileChannel.size();
+		this.fileSize = this.dataFileChannel.size();
 	}
 
 	/**
@@ -24,12 +34,41 @@ public class FileAllocator implements AutoCloseable {
 	public long allocate(int size) {
 		checkClosed();
 		synchronized (allocateLock) {
-			long allocatedOffset = allocableOffset;
-			allocableOffset += size;
-			return allocatedOffset;
+			long offset = allocateIntoUnusedParts(size);
+			if (offset == -1) {
+				return allocateToEnd(size);
+			} else {
+				return offset;
+			}
 		}
 	}
 
+	private long allocateIntoUnusedParts(int size) {
+		ObjectBidirectionalIterator<Long2IntMap.Entry> it = freeBytes.long2IntEntrySet().iterator();
+		long holeOffset = -1;
+		int holeSize = 0;
+		while (it.hasNext()) {
+			Long2IntMap.Entry entry = it.next();
+			int currentHoleSize = entry.getIntValue();
+			if (currentHoleSize < size) {
+				freeBytes.remove(holeOffset);
+				if (holeSize > size) {
+					freeBytes.put(holeOffset + size, holeSize - size);
+				}
+				break;
+			}
+			holeOffset = entry.getLongKey();
+			holeSize = currentHoleSize;
+		}
+		return holeOffset;
+	}
+
+	private long allocateToEnd(int size) {
+		long allocatedOffset = fileSize;
+		fileSize += size;
+		return allocatedOffset;
+	}
+
 
 	public void close() throws IOException {
 		if (closed) {
@@ -51,7 +90,27 @@ public class FileAllocator implements AutoCloseable {
 	 */
 	public void markFree(long startPosition, int length) {
 		checkClosed();
-		// TODO: advanced feature, not implemented.
+
+		if (freeBytes.containsKey(startPosition + length)) {
+			int secondLength = freeBytes.remove(startPosition + length);
+			freeBytes.put(startPosition, length + secondLength);
+		} else {
+			boolean addedToList = false;
+			for (Long2IntMap.Entry entry : freeBytes.long2IntEntrySet()) {
+				if (entry.getLongKey() + entry.getIntValue() == startPosition) {
+					freeBytes.put(entry.getLongKey(), entry.getIntValue() + length);
+					addedToList = true;
+					break;
+				}
+			}
+			if (!addedToList) {
+				freeBytes.put(startPosition, length);
+			}
+		}
+
+		if (freeBytes.size() > MAXIMUM_UNALLOCATED_ENTRIES) {
+			freeBytes.remove(freeBytes.lastLongKey());
+		}
 	}
 
 
diff --git a/src/main/java/org/warp/jcwdb/FileIndexManager.java b/src/main/java/org/warp/jcwdb/FileIndexManager.java
index 5ea06d9..fe87653 100644
--- a/src/main/java/org/warp/jcwdb/FileIndexManager.java
+++ b/src/main/java/org/warp/jcwdb/FileIndexManager.java
@@ -82,7 +82,12 @@ public class FileIndexManager implements IndexManager {
 		final IndexDetails indexDetails = getIndexMetadataUnsafe(index);
 		if (indexDetails == null || indexDetails.getSize() < dataSize) {
 			// Allocate new space
-			return allocateAndWrite(index, data);
+			IndexDetails newDetails = allocateAndWrite(index, data);
+			if (indexDetails != null) {
+				// Mark free the old bytes
+				fileAllocator.markFree(indexDetails.getOffset(), indexDetails.getSize());
+			}
+			return newDetails;
 		} else {
 			// Check if size changed
 			if (dataSize < indexDetails.getSize()) {
diff --git a/src/main/java/org/warp/jcwdb/JCWDatabase.java b/src/main/java/org/warp/jcwdb/JCWDatabase.java
index 4fa8079..49425ec 100644
--- a/src/main/java/org/warp/jcwdb/JCWDatabase.java
+++ b/src/main/java/org/warp/jcwdb/JCWDatabase.java
@@ -36,7 +36,7 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
 		if (exists(0)) {
 			return get(0);
 		} else {
-			LightList<T> newRoot = new LightBigList<>(this);
+			LightList<T> newRoot = new LightArrayList<>(this);
 			return set(0, newRoot);
 		}
 	}
diff --git a/src/main/java/org/warp/jcwdb/exampleimpl/App.java b/src/main/java/org/warp/jcwdb/exampleimpl/App.java
index de5e5ab..4e0f70a 100644
--- a/src/main/java/org/warp/jcwdb/exampleimpl/App.java
+++ b/src/main/java/org/warp/jcwdb/exampleimpl/App.java
@@ -37,7 +37,7 @@ public class App {
 	//			System.out.println(" - " + root.get(i));
 	//		}
 					long prectime = System.currentTimeMillis();
-					for (int i = 0; i < 200000/* 2000000 */; i++) {
+					for (int i = 0; i < 20000/* 2000000 */; i++) {
 						Animal animal = new StrangeAnimal(i % 40);
 						root.add(animal);
 						if (i > 0 && i % 200000 == 0) {