Reusing unused space in database

This commit is contained in:
Andrea Cavalli 2018-12-20 01:12:46 +01:00
parent f9372a0aaf
commit e4fbd613db
4 changed files with 73 additions and 9 deletions

View File

@ -1,18 +1,28 @@
package org.warp.jcwdb;
import it.unimi.dsi.fastutil.longs.Long2IntMap;
import it.unimi.dsi.fastutil.longs.Long2IntRBTreeMap;
import it.unimi.dsi.fastutil.objects.ObjectBidirectionalIterator;
import java.io.IOException;
import java.nio.channels.SeekableByteChannel;
public class FileAllocator implements AutoCloseable {
private static final int MAXIMUM_UNALLOCATED_ENTRIES = 500000;
private final SeekableByteChannel dataFileChannel;
private volatile long allocableOffset;
private volatile long fileSize;
private volatile boolean closed;
private final Object closeLock = new Object();
private final Object allocateLock = new Object();
/**
* index -> free space size
*/
private final Long2IntRBTreeMap freeBytes = new Long2IntRBTreeMap((a, b) -> (int) (a - b));
public FileAllocator(SeekableByteChannel dataFileChannel) throws IOException {
this.dataFileChannel = dataFileChannel;
this.allocableOffset = this.dataFileChannel.size();
this.fileSize = this.dataFileChannel.size();
}
/**
@ -24,11 +34,40 @@ public class FileAllocator implements AutoCloseable {
public long allocate(int size) {
checkClosed();
synchronized (allocateLock) {
long allocatedOffset = allocableOffset;
allocableOffset += size;
return allocatedOffset;
long offset = allocateIntoUnusedParts(size);
if (offset == -1) {
return allocateToEnd(size);
} else {
return offset;
}
}
}
private long allocateIntoUnusedParts(int size) {
ObjectBidirectionalIterator<Long2IntMap.Entry> it = freeBytes.long2IntEntrySet().iterator();
long holeOffset = -1;
int holeSize = 0;
while (it.hasNext()) {
Long2IntMap.Entry entry = it.next();
int currentHoleSize = entry.getIntValue();
if (currentHoleSize < size) {
freeBytes.remove(holeOffset);
if (holeSize > size) {
freeBytes.put(holeOffset + size, holeSize - size);
}
break;
}
holeOffset = entry.getLongKey();
holeSize = currentHoleSize;
}
return holeOffset;
}
private long allocateToEnd(int size) {
long allocatedOffset = fileSize;
fileSize += size;
return allocatedOffset;
}
public void close() throws IOException {
@ -51,7 +90,27 @@ public class FileAllocator implements AutoCloseable {
*/
public void markFree(long startPosition, int length) {
checkClosed();
// TODO: advanced feature, not implemented.
if (freeBytes.containsKey(startPosition + length)) {
int secondLength = freeBytes.remove(startPosition + length);
freeBytes.put(startPosition, length + secondLength);
} else {
boolean addedToList = false;
for (Long2IntMap.Entry entry : freeBytes.long2IntEntrySet()) {
if (entry.getLongKey() + entry.getIntValue() == startPosition) {
freeBytes.put(entry.getLongKey(), entry.getIntValue() + length);
addedToList = true;
break;
}
}
if (!addedToList) {
freeBytes.put(startPosition, length);
}
}
if (freeBytes.size() > MAXIMUM_UNALLOCATED_ENTRIES) {
freeBytes.remove(freeBytes.lastLongKey());
}
}

View File

@ -82,7 +82,12 @@ public class FileIndexManager implements IndexManager {
final IndexDetails indexDetails = getIndexMetadataUnsafe(index);
if (indexDetails == null || indexDetails.getSize() < dataSize) {
// Allocate new space
return allocateAndWrite(index, data);
IndexDetails newDetails = allocateAndWrite(index, data);
if (indexDetails != null) {
// Mark free the old bytes
fileAllocator.markFree(indexDetails.getOffset(), indexDetails.getSize());
}
return newDetails;
} else {
// Check if size changed
if (dataSize < indexDetails.getSize()) {

View File

@ -36,7 +36,7 @@ public class JCWDatabase implements AutoCloseable, Cleanable {
if (exists(0)) {
return get(0);
} else {
LightList<T> newRoot = new LightBigList<>(this);
LightList<T> newRoot = new LightArrayList<>(this);
return set(0, newRoot);
}
}

View File

@ -37,7 +37,7 @@ public class App {
// System.out.println(" - " + root.get(i));
// }
long prectime = System.currentTimeMillis();
for (int i = 0; i < 200000/* 2000000 */; i++) {
for (int i = 0; i < 20000/* 2000000 */; i++) {
Animal animal = new StrangeAnimal(i % 40);
root.add(animal);
if (i > 0 && i % 200000 == 0) {